Release ipx 1.5.1 and ipx-directory 1.4.1

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.claude/skills/ipx-prod-check/SKILL.md

@@ -65,7 +65,7 @@ lines had no trace id, the access log's time was `ms`, and each event was logged
 | `ipx::http` | `method`, `path`, `route`, `status`, `duration_ms` | one per web request; `route` is the pattern, empty for an unrouted path |
 | `ipx::scan` | `ev` and the event's own: `feed`, `new`, `downloaded`, `failed`, `bytes`, `msg`, `url`, `feeds`, `reason`, `from`, `to`; on a failure `error.type` and, from an HTTP error, `http.response.status_code` | the daemon's events, one line each, in words; warnings are feed and download failures |
 | `ipx::io` | `ev`, `feeds`, `pending`, `downloaded` on the `status` reply | commands arriving (`-> {...}`) and the healthcheck's answer |
-| `ipx` | message, sometimes fields | start-up, shutdown, account and config messages |
+| `ipx::daemon`, `ipx::cli`, `ipx::catalogue`, `ipx::scan`, ... | message, sometimes fields | start-up, shutdown, account and config messages, under the module that logs them; plain `ipx` before 2026-10-24, when main.rs was split (#356) |
 | `ipx::client` | `ev`, `user`, `feed`, `enclosure`, `deadair`; `msg`, `src`, `line` on an error; `ms` on a timing | what the page reports from the browser (#190): `page_error`, `rejection` and `player_error` as warnings; `play_start` (ms from play to sound), `stall` (ms waiting mid-play) and `list_shown` (ms from a click to its list drawn, with `feed` the view and `filter`, #246) as info. Lines carry the trace of the action in the page: its API requests send `traceparent`, one trace id per tap or key |
 
 Events (`ev`): `feed_start`, `feed_done` (new, downloaded, failed, torrents), `feed_skip` (not due,

Cargo.lock

@@ -1860,7 +1860,7 @@ checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0"
 
 [[package]]
 name = "ipx"
-version = "1.5.0"
+version = "1.5.1"
 dependencies = [
  "ammonia",
  "anyhow",
@@ -1875,6 +1875,7 @@ dependencies = [
  "jsonwebtoken",
  "librqbit",
  "opentelemetry",
+ "opentelemetry_sdk",
  "opml",
  "percent-encoding",
  "reqwest",
@@ -1894,7 +1895,7 @@ dependencies = [
 
 [[package]]
 name = "ipx-directory"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "ammonia",
  "anyhow",

Cargo.toml

@@ -7,6 +7,10 @@ resolver = "3"
 [workspace.package]
 edition = "2024"
 
+# None of ours needs unsafe, and the style guide asks for none where safe code will do.
+[workspace.lints.rust]
+unsafe_code = "forbid"
+
 [workspace.dependencies]
 ipx-feed = { path = "crates/feed" }
 ipx-directory = { path = "crates/directory" }

crates/directory/CHANGELOG.md

@@ -10,6 +10,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.4.1] - 2026-10-24
+
+### Fixed
+
+- A feed larger than 50 MB is checked again, up to 200 MB, where it was marked as failing (#357).
+
 ## [1.4.0] - 2026-10-23
 
 ### Fixed
@@ -86,6 +92,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Traces: a span a request, joining the trace of the ipx server that called, and a trace a refresher round with a span a feed (#213).
 
 [unreleased]: https://git.sdf1.net/rays/ipx/compare/directory-v1.4.0...main
+[1.4.1]: https://git.sdf1.net/rays/ipx/compare/directory-v1.4.0...directory-v1.4.1
 [1.4.0]: https://git.sdf1.net/rays/ipx/compare/directory-v1.3.0...directory-v1.4.0
 [1.3.0]: https://git.sdf1.net/rays/ipx/compare/directory-v1.2.0...directory-v1.3.0
 [1.2.0]: https://git.sdf1.net/rays/ipx/compare/directory-v1.1.1...directory-v1.2.0

crates/directory/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ipx-directory"
-version = "1.4.0"
+version = "1.4.1"
 edition.workspace = true
 publish = false
 
@@ -8,6 +8,9 @@ publish = false
 name = "ipx-directory"
 path = "src/main.rs"
 
+[lints]
+workspace = true
+
 [dependencies]
 ammonia.workspace = true
 anyhow.workspace = true

crates/directory/src/api.rs

@@ -0,0 +1,428 @@
+//! The HTTP side: the router, the token check, and `/v1`, where ipx servers send snapshots and read the
+//! catalogue.
+
+use crate::config::Access;
+use crate::directory::Shared;
+use crate::wire::{Blocked, FeedPage, Instance, Listing, MAX_SNAPSHOT, Me, SnapshotEntry, SnapshotResult};
+use crate::{otel, site};
+use axum::extract::{Path, Query, Request, State};
+use axum::http::{StatusCode, header};
+use axum::middleware::Next;
+use axum::response::{IntoResponse, Response};
+use axum::routing::{delete, get, post};
+use axum::{Extension, Json, Router};
+use serde::Deserialize;
+use std::time::Instant;
+use tower_http::compression::CompressionLayer;
+use tower_http::decompression::RequestDecompressionLayer;
+use tracing::Instrument;
+
+/// Every route: the API under `/v1`, behind a token, and the public pages.
+pub fn router(dir: Shared) -> Router {
+    let v1 = Router::new()
+        .route("/v1/me", get(me))
+        .route("/v1/feeds", get(list))
+        .route("/v1/popular", get(popular))
+        .route("/v1/feeds/{id}", get(feed).delete(remove))
+        .route("/v1/feeds/{id}/refresh", post(refresh))
+        .route("/v1/snapshot", post(snapshot))
+        .route("/v1/blocked", get(blocked))
+        .route("/v1/blocked/{id}", delete(unblock))
+        .route("/v1/instances", get(instances))
+        .route("/v1/instances/{name}", delete(drop_instance))
+        .layer(axum::middleware::from_fn_with_state(dir.clone(), authenticate));
+    Router::new()
+        .route("/healthz", get(healthz))
+        .merge(v1)
+        // Anyone may browse, no token, read only (#215).
+        .merge(site::router())
+        // Brotli, zstd or gzip as the caller takes them: the feed list went to ipx as 603 KB of
+        // JSON, most of a Directory page's time (#253). Inside the access log, which records what
+        // was sent.
+        .layer(CompressionLayer::new().br(true).zstd(true).gzip(true))
+        // And takes a request's body gzipped: ipx sends its snapshot so, about 100 KB as JSON.
+        .layer(RequestDecompressionLayer::new().gzip(true))
+        .layer(axum::middleware::from_fn(access_log))
+        .with_state(dir)
+}
+
+pub(crate) struct ApiError(StatusCode, String);
+
+impl IntoResponse for ApiError {
+    fn into_response(self) -> Response {
+        (self.0, Json(serde_json::json!({ "error": self.1 }))).into_response()
+    }
+}
+
+impl From<anyhow::Error> for ApiError {
+    fn from(e: anyhow::Error) -> Self {
+        tracing::error!(error = %format!("{e:#}"), "request failed");
+        ApiError(StatusCode::INTERNAL_SERVER_ERROR, "something went wrong; the directory's log says what".into())
+    }
+}
+
+/// Who made the request, for the access log, set by `authenticate`.
+#[derive(Clone)]
+pub(crate) struct Who(String);
+
+/// One line a request, with the token's name and never the token: what the Grafana panels read.
+pub(crate) async fn access_log(req: Request, next: Next) -> Response {
+    let (method, path) = (req.method().clone(), req.uri().path().to_owned());
+    let start = Instant::now();
+    // A span for each request but the healthcheck's, a child of the caller's when it sent a
+    // traceparent, as ipx does on its calls here (#213).
+    let span = if path == "/healthz" {
+        tracing::Span::none()
+    } else {
+        tracing::info_span!(
+            "http",
+            otel.name = %format!("{method} {path}"),
+            http.request.method = %method,
+            url.path = %path,
+            http.response.status_code = tracing::field::Empty,
+        )
+    };
+    if let Some(parent) = otel::remote_parent(req.headers(), None) {
+        use tracing_opentelemetry::OpenTelemetrySpanExt;
+        let _ = span.set_parent(parent);
+    }
+    let resp = Instrument::instrument(next.run(req), span.clone()).await;
+    span.record("http.response.status_code", resp.status().as_u16());
+    let _in = span.enter();
+    let who = resp.extensions().get::<Who>().map_or("", |w| w.0.as_str()).to_owned();
+    let status = resp.status().as_u16();
+    if path != "/healthz" {
+        tracing::info!(
+            target: "ipx_directory::http",
+            method = %method, path = %path, status, duration_ms = start.elapsed().as_millis() as u64, token = %who,
+            "{method} {path} -> {status}"
+        );
+    }
+    resp
+}
+
+pub(crate) async fn authenticate(State(dir): State<Shared>, mut req: Request, next: Next) -> Response {
+    let presented = req
+        .headers()
+        .get(header::AUTHORIZATION)
+        .and_then(|v| v.to_str().ok())
+        .and_then(|v| v.strip_prefix("Bearer "))
+        .map(str::trim)
+        .filter(|t| !t.is_empty());
+    let Some(me) = presented.and_then(|t| dir.caller(t)) else {
+        return ApiError(StatusCode::UNAUTHORIZED, "a valid token is needed: Authorization: Bearer <token>".into())
+            .into_response();
+    };
+    let who = Who(me.name.clone());
+    req.extensions_mut().insert(me);
+    let mut resp = next.run(req).await;
+    resp.extensions_mut().insert(who);
+    resp
+}
+
+pub(crate) fn need(me: &Me, access: Access) -> Result<(), ApiError> {
+    if me.access >= access {
+        Ok(())
+    } else {
+        let what = match access {
+            Access::Ro => "browse",
+            Access::Rw => "send snapshots",
+            Access::Rwa => "administer the directory",
+        };
+        Err(ApiError(StatusCode::FORBIDDEN, format!("this token may not {what}")))
+    }
+}
+
+pub(crate) async fn healthz(State(dir): State<Shared>) -> Result<&'static str, ApiError> {
+    dir.store.ping().await?;
+    Ok("ok")
+}
+
+pub(crate) async fn me(Extension(me): Extension<Me>) -> Json<Me> {
+    Json(me)
+}
+
+pub(crate) async fn list(State(dir): State<Shared>) -> Result<Json<Vec<Listing>>, ApiError> {
+    Ok(Json(dir.store.list().await?))
+}
+
+#[derive(Deserialize)]
+pub(crate) struct Limit {
+    limit: Option<u64>,
+}
+
+pub(crate) async fn popular(State(dir): State<Shared>, Query(q): Query<Limit>) -> Result<Json<Vec<Listing>>, ApiError> {
+    Ok(Json(dir.store.popular(q.limit.unwrap_or(10).clamp(1, 100)).await?))
+}
+
+pub(crate) async fn feed(State(dir): State<Shared>, Path(id): Path<String>) -> Result<Json<FeedPage>, ApiError> {
+    dir.store
+        .feed(&id)
+        .await?
+        .map(Json)
+        .ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))
+}
+
+pub(crate) async fn snapshot(
+    State(dir): State<Shared>,
+    Extension(me): Extension<Me>,
+    Json(entries): Json<Vec<SnapshotEntry>>,
+) -> Result<Json<SnapshotResult>, ApiError> {
+    need(&me, Access::Rw)?;
+    if entries.len() > MAX_SNAPSHOT {
+        return Err(ApiError(StatusCode::PAYLOAD_TOO_LARGE, format!("a snapshot lists at most {MAX_SNAPSHOT} feeds")));
+    }
+    let r = dir.store.snapshot(&me.name, &entries).await?;
+    dir.wake.notify_one();
+    tracing::info!(
+        instance = %me.name, accepted = r.accepted, refused_private = r.refused_private,
+        refused_blocked = r.refused_blocked, refused_invalid = r.refused_invalid, "snapshot"
+    );
+    Ok(Json(r))
+}
+
+pub(crate) async fn remove(
+    State(dir): State<Shared>,
+    Extension(me): Extension<Me>,
+    Path(id): Path<String>,
+) -> Result<StatusCode, ApiError> {
+    need(&me, Access::Rwa)?;
+    dir.store.block(&id).await?;
+    tracing::info!(feed = %id, by = %me.name, "feed removed and blocked");
+    Ok(StatusCode::NO_CONTENT)
+}
+
+pub(crate) async fn blocked(
+    State(dir): State<Shared>,
+    Extension(me): Extension<Me>,
+) -> Result<Json<Vec<Blocked>>, ApiError> {
+    need(&me, Access::Rwa)?;
+    Ok(Json(dir.store.blocked().await?))
+}
+
+pub(crate) async fn unblock(
+    State(dir): State<Shared>,
+    Extension(me): Extension<Me>,
+    Path(id): Path<String>,
+) -> Result<StatusCode, ApiError> {
+    need(&me, Access::Rwa)?;
+    if !dir.store.unblock(&id).await? {
+        return Err(ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not blocked")));
+    }
+    tracing::info!(feed = %id, by = %me.name, "feed unblocked");
+    Ok(StatusCode::NO_CONTENT)
+}
+
+pub(crate) async fn instances(
+    State(dir): State<Shared>,
+    Extension(me): Extension<Me>,
+) -> Result<Json<Vec<Instance>>, ApiError> {
+    need(&me, Access::Rwa)?;
+    Ok(Json(dir.store.instances().await?))
+}
+
+pub(crate) async fn drop_instance(
+    State(dir): State<Shared>,
+    Extension(me): Extension<Me>,
+    Path(name): Path<String>,
+) -> Result<StatusCode, ApiError> {
+    need(&me, Access::Rwa)?;
+    let n = dir.store.drop_instance(&name).await?;
+    tracing::info!(instance = %name, feeds = n, by = %me.name, "snapshot dropped");
+    Ok(StatusCode::NO_CONTENT)
+}
+
+pub(crate) async fn refresh(
+    State(dir): State<Shared>,
+    Extension(me): Extension<Me>,
+    Path(id): Path<String>,
+) -> Result<Json<FeedPage>, ApiError> {
+    need(&me, Access::Rwa)?;
+    let due = dir
+        .store
+        .due_one(&id)
+        .await?
+        .ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))?;
+    dir.refresh(&due).await?;
+    feed(State(dir), Path(id)).await
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+    use crate::config;
+    use crate::config::Token;
+    use crate::directory::Directory;
+    use crate::store;
+    use crate::store::Store;
+    use axum::body::Body;
+    use tower::ServiceExt;
+
+    async fn app() -> (Router, Shared) {
+        let tokens = ["ro", "rw", "rwa"]
+            .iter()
+            .map(|n| Token {
+                name: format!("{n}-token"),
+                access: serde_json::from_value(serde_json::json!(n)).unwrap(),
+                sha256: config::hash(&format!("secret-{n}")),
+            })
+            .collect();
+        let dir = Directory::new(Store::scratch().await, tokens, Default::default()).unwrap();
+        (router(dir.clone()), dir)
+    }
+
+    async fn call(
+        app: &Router,
+        method: &str,
+        path: &str,
+        token: Option<&str>,
+        body: Option<serde_json::Value>,
+    ) -> (u16, serde_json::Value) {
+        let mut req = axum::http::Request::builder().method(method).uri(path);
+        if let Some(t) = token {
+            req = req.header("authorization", format!("Bearer secret-{t}"));
+        }
+        let req = match body {
+            Some(b) => req.header("content-type", "application/json").body(Body::from(b.to_string())),
+            None => req.body(Body::empty()),
+        }
+        .unwrap();
+        let resp = app.clone().oneshot(req).await.unwrap();
+        let status = resp.status().as_u16();
+        let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap();
+        (status, serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null))
+    }
+
+    #[tokio::test]
+    async fn a_snapshot_may_come_gzipped() {
+        use std::io::Write as _;
+        let (app, _) = app().await;
+        let json = serde_json::json!([{"url": "https://example.com/feed.xml", "subscribers": 2}]).to_string();
+        let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
+        gz.write_all(json.as_bytes()).unwrap();
+        // As ipx sends it (#253).
+        let req = axum::http::Request::builder()
+            .method("POST")
+            .uri("/v1/snapshot")
+            .header("authorization", "Bearer secret-rw")
+            .header("content-type", "application/json")
+            .header("content-encoding", "gzip")
+            .body(Body::from(gz.finish().unwrap()))
+            .unwrap();
+        let resp = app.clone().oneshot(req).await.unwrap();
+        assert_eq!(resp.status(), 200);
+        let got: serde_json::Value =
+            serde_json::from_slice(&axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap()).unwrap();
+        assert_eq!(got["accepted"], 1);
+    }
+
+    #[tokio::test]
+    async fn answers_come_compressed_as_the_caller_asks() {
+        let (app, _) = app().await;
+        for (asked, sent) in [("br", Some("br")), ("zstd", Some("zstd")), ("gzip", Some("gzip")), ("identity", None)] {
+            // A page, not an empty feed list: under 32 bytes is sent as it is.
+            let req =
+                axum::http::Request::builder().uri("/").header("accept-encoding", asked).body(Body::empty()).unwrap();
+            let resp = app.clone().oneshot(req).await.unwrap();
+            assert_eq!(resp.status(), 200);
+            let got = resp.headers().get("content-encoding").map(|v| v.to_str().unwrap().to_owned());
+            assert_eq!(got.as_deref(), sent, "asked for {asked} (#253)");
+        }
+    }
+
+    async fn get_page(app: &Router, path: &str) -> (u16, String, Option<String>) {
+        let req = axum::http::Request::builder().uri(path).body(Body::empty()).unwrap();
+        let resp = app.clone().oneshot(req).await.unwrap();
+        let status = resp.status().as_u16();
+        let csp = resp.headers().get("content-security-policy").map(|v| v.to_str().unwrap().to_owned());
+        let bytes = axum::body::to_bytes(resp.into_body(), 1 << 22).await.unwrap();
+        (status, String::from_utf8(bytes.to_vec()).unwrap(), csp)
+    }
+
+    #[tokio::test]
+    async fn anyone_browses_the_pages_and_the_api_still_needs_a_token() {
+        let (app, dir) = app().await;
+        let snap = serde_json::json!([{ "url": "https://x.test/a.xml", "subscribers": 3 }]);
+        call(&app, "POST", "/v1/snapshot", Some("rw"), Some(snap)).await;
+        store::tests::fetch_all(&dir.store).await;
+        let id = dir.store.list().await.unwrap()[0].id.clone();
+
+        let (s, home, csp) = get_page(&app, "/").await;
+        assert_eq!(s, 200);
+        assert!(csp.unwrap().contains("script-src 'self'"));
+        assert!(home.contains("Most subscribed") && home.contains(&format!("/feed/{id}")), "{home}");
+        assert!(get_page(&app, "/all").await.1.contains("3 people subscribe"));
+        let (s, page, _) = get_page(&app, &format!("/feed/{id}")).await;
+        assert_eq!(s, 200);
+        // Its address, to subscribe to anywhere.
+        assert!(page.contains("value=\"https://x.test/a.xml\""), "{page}");
+        assert!(!page.contains("<script>"), "the publisher's HTML is text here");
+        assert_eq!(get_page(&app, "/feed/nope").await.0, 404);
+        assert_eq!(get_page(&app, "/category/Nothing").await.0, 404);
+        assert_eq!(get_page(&app, "/search?q=").await.0, 303);
+        assert_eq!(get_page(&app, "/site.js").await.0, 200);
+        // Reading the pages takes no token; the API still does.
+        assert_eq!(call(&app, "GET", "/v1/feeds", None, None).await.0, 401);
+        assert_eq!(call(&app, "DELETE", &format!("/v1/feeds/{id}"), None, None).await.0, 401);
+        assert_eq!(call(&app, "DELETE", &format!("/feed/{id}"), None, None).await.0, 405);
+    }
+
+    #[tokio::test]
+    async fn each_access_level_does_what_it_may_and_no_more() {
+        let (app, dir) = app().await;
+        let snap = serde_json::json!([{ "url": "https://x.test/a.xml", "subscribers": 2, "category": "Comedy" }]);
+
+        assert_eq!(call(&app, "GET", "/v1/feeds", None, None).await.0, 401);
+        assert_eq!(call(&app, "GET", "/v1/feeds", Some("wrong"), None).await.0, 401);
+        assert_eq!(call(&app, "GET", "/healthz", None, None).await.0, 200);
+
+        let (s, me) = call(&app, "GET", "/v1/me", Some("ro"), None).await;
+        assert_eq!((s, me["name"].as_str(), me["access"].as_str()), (200, Some("ro-token"), Some("ro")));
+        assert_eq!(call(&app, "POST", "/v1/snapshot", Some("ro"), Some(snap.clone())).await.0, 403);
+
+        let (s, r) = call(&app, "POST", "/v1/snapshot", Some("rw"), Some(snap.clone())).await;
+        assert_eq!((s, r["accepted"].as_u64()), (200, Some(1)));
+        store::tests::fetch_all(&dir.store).await;
+        let (_, list) = call(&app, "GET", "/v1/feeds", Some("ro"), None).await;
+        let id = list[0]["id"].as_str().unwrap().to_owned();
+        assert_eq!(list[0]["subscribers"], 2);
+        assert_ne!(list[0]["category"], "Comedy", "the feed's own category wins over the sender's");
+        assert_eq!(call(&app, "GET", &format!("/v1/feeds/{id}"), Some("ro"), None).await.0, 200);
+        assert_eq!(call(&app, "GET", "/v1/feeds/nope", Some("ro"), None).await.0, 404);
+        assert_eq!(call(&app, "GET", "/v1/popular?limit=5", Some("ro"), None).await.1[0]["id"], id.as_str());
+
+        for (m, p) in [
+            ("DELETE", format!("/v1/feeds/{id}")),
+            ("GET", "/v1/blocked".into()),
+            ("DELETE", "/v1/instances/rw-token".into()),
+        ] {
+            assert_eq!(call(&app, m, &p, Some("rw"), None).await.0, 403, "{m} {p}");
+        }
+        assert_eq!(call(&app, "DELETE", &format!("/v1/feeds/{id}"), Some("rwa"), None).await.0, 204);
+        assert_eq!(call(&app, "GET", "/v1/blocked", Some("rwa"), None).await.1[0]["id"], id.as_str());
+        assert_eq!(call(&app, "POST", "/v1/snapshot", Some("rw"), Some(snap.clone())).await.1["refused_blocked"], 1);
+        assert_eq!(call(&app, "DELETE", &format!("/v1/blocked/{id}"), Some("rwa"), None).await.0, 204);
+        assert_eq!(call(&app, "DELETE", &format!("/v1/blocked/{id}"), Some("rwa"), None).await.0, 404);
+        call(&app, "POST", "/v1/snapshot", Some("rw"), Some(snap)).await;
+        assert_eq!(call(&app, "DELETE", "/v1/instances/rw-token", Some("rwa"), None).await.0, 204);
+        assert_eq!(call(&app, "GET", "/v1/feeds", Some("ro"), None).await.1, serde_json::json!([]));
+    }
+
+    #[tokio::test]
+    async fn a_revoked_token_loses_its_access_and_its_counts() {
+        let dir = Directory::new(Store::scratch().await, vec![], Default::default()).unwrap();
+        let t = |n: &str| Token { name: n.into(), access: Access::Rw, sha256: config::hash(&format!("secret-{n}")) };
+        dir.set_tokens(vec![t("a"), t("b")]).await.unwrap();
+        let app = router(dir.clone());
+        let snap = serde_json::json!([{ "url": "https://x.test/a.xml", "subscribers": 1 }]);
+        call(&app, "POST", "/v1/snapshot", Some("a"), Some(snap.clone())).await;
+        call(&app, "POST", "/v1/snapshot", Some("b"), Some(snap)).await;
+        store::tests::fetch_all(&dir.store).await;
+        assert_eq!(call(&app, "GET", "/v1/feeds", Some("a"), None).await.1[0]["subscribers"], 2);
+
+        dir.set_tokens(vec![t("a")]).await.unwrap();
+        assert_eq!(call(&app, "GET", "/v1/feeds", Some("b"), None).await.0, 401);
+        assert_eq!(call(&app, "GET", "/v1/feeds", Some("a"), None).await.1[0]["subscribers"], 1);
+    }
+}

crates/directory/src/apple.rs

@@ -13,6 +13,7 @@ use serde::Deserialize;
 use tracing::Instrument;
 
 use crate::{Shared, SnapshotEntry};
+use std::collections::{BTreeSet, HashSet};
 
 /// What the charts are filed under, as a server's snapshot is under its token's name.
 pub const INSTANCE: &str = "apple-podcasts";
@@ -83,7 +84,7 @@ async fn sync(dir: &Shared, client: &reqwest::Client, country: &str) -> Result<(
         }
         tokio::time::sleep(PAUSE).await;
     }
-    let mut seen = std::collections::HashSet::new();
+    let mut seen = HashSet::new();
     ids.retain(|id| seen.insert(id.clone()));
     anyhow::ensure!(!ids.is_empty(), "no chart had a show in it");
 
@@ -150,9 +151,8 @@ fn differ(theirs: &BTreeMap<String, Vec<String>>, ours: &[(&str, &[&str])]) -> (
     let flat = |c: &str, subs: &mut dyn Iterator<Item = String>| {
         std::iter::once(c.to_string()).chain(subs.map(|s| format!("{c} / {s}"))).collect::<Vec<_>>()
     };
-    let theirs: std::collections::BTreeSet<String> =
-        theirs.iter().flat_map(|(c, subs)| flat(c, &mut subs.iter().cloned())).collect();
-    let ours: std::collections::BTreeSet<String> =
+    let theirs: BTreeSet<String> = theirs.iter().flat_map(|(c, subs)| flat(c, &mut subs.iter().cloned())).collect();
+    let ours: BTreeSet<String> =
         ours.iter().flat_map(|(c, subs)| flat(c, &mut subs.iter().map(|s| s.to_string()))).collect();
     (theirs.difference(&ours).cloned().collect(), ours.difference(&theirs).cloned().collect())
 }

crates/directory/src/config.rs

@@ -4,11 +4,15 @@
 
 use anyhow::{Context, Result, bail};
 use serde::Deserialize;
+use std::collections::HashSet;
+use std::fs::File;
 use std::path::Path;
 
+/// `directory.toml`. See docs/directory.md.
 #[derive(Debug, Clone, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct Config {
+    /// Where the API and the public pages listen.
     #[serde(default = "default_bind")]
     pub bind: String,
     /// The environment variable holding the database URL, so the file holds no secret.
@@ -39,6 +43,7 @@ fn default_database_url_env() -> String {
     "DIRECTORY_DATABASE_URL".into()
 }
 
+/// A token as `directory.toml` lists it: a name, what it may do, and only the token's hash.
 #[derive(Debug, Clone, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct TokenCfg {
@@ -74,14 +79,17 @@ pub struct Token {
 }
 
 impl Config {
+    /// Reads and checks `path`; see `parse`.
     pub fn load(path: &Path) -> Result<Self> {
         let text = std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?;
         Self::parse(&text).with_context(|| format!("in {}", path.display()))
     }
 
+    /// Reads a config, refusing tokens with empty or repeated names, which would file two
+    /// instances' snapshots together.
     pub fn parse(text: &str) -> Result<Self> {
         let cfg: Config = toml::from_str(text)?;
-        let mut seen = std::collections::HashSet::new();
+        let mut seen = HashSet::new();
         for t in &cfg.tokens {
             if t.name.trim().is_empty() {
                 bail!("a token has an empty name");
@@ -105,7 +113,7 @@ impl Config {
     /// The tokens as the server checks them. A hash that is not 64 hex digits, or the same
     /// hash under two names, is an error: which name a request is would be ambiguous.
     pub fn tokens(&self) -> Result<Vec<Token>> {
-        let mut seen = std::collections::HashSet::new();
+        let mut seen = HashSet::new();
         self.tokens
             .iter()
             .map(|t| {
@@ -123,25 +131,26 @@ impl Config {
 }
 
 /// A token's SHA-256 in hex, as the config file holds it.
+#[must_use]
 pub fn hash(token: &str) -> String {
     use sha2::Digest;
     sha2::Sha256::digest(token.trim().as_bytes()).iter().map(|b| format!("{b:02x}")).collect()
 }
 
 /// Equal without saying, by how long it took, how much of the guess was right.
+#[must_use]
 pub fn same_secret(a: &str, b: &str) -> bool {
     let (a, b) = (a.as_bytes(), b.as_bytes());
     a.len() == b.len() && a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0
 }
 
 /// A new token: `ipxd_` and 256 random bits, the prefix so one found in a log says what it is.
+#[must_use]
 pub fn new_token() -> String {
     use std::io::Read;
     let mut bytes = [0u8; 32];
     // Falling back to the clock would make a guessable token. Better to fail.
-    std::fs::File::open("/dev/urandom")
-        .and_then(|mut f| f.read_exact(&mut bytes))
-        .expect("no source of randomness for a token");
+    File::open("/dev/urandom").and_then(|mut f| f.read_exact(&mut bytes)).expect("no source of randomness for a token");
     format!("ipxd_{}", bytes.iter().map(|b| format!("{b:02x}")).collect::<String>())
 }
 

crates/directory/src/directory.rs

@@ -0,0 +1,218 @@
+//! The running directory: its store, the tokens it accepts, the refresher that fetches its feeds, and
+//! the reloader that picks up a changed config.
+
+use crate::config::Token;
+use crate::wire::{Listing, Me};
+use crate::{config, store};
+use reqwest::redirect::Policy;
+use std::path::PathBuf;
+use std::sync::{Arc, RwLock};
+use std::time::{Duration, Instant};
+use store::{Refreshed, Store};
+use tokio::sync::Notify;
+use tracing::Instrument;
+
+/// How often each feed is fetched again. Metadata changes slowly, and every server subscribed
+/// to a feed fetches it itself anyway: every six hours was 3.9 GB of feeds a day (#243).
+pub const REFRESH_EVERY: i64 = 12 * 3600;
+
+/// The running directory: its database, the tokens it accepts and the client its refresher
+/// fetches feeds with.
+pub struct Directory {
+    pub store: Store,
+    pub(crate) tokens: RwLock<Vec<Token>>,
+    pub(crate) client: reqwest::Client,
+    /// Wakes the refresher, so a feed a snapshot brings in is fetched now, not in a minute.
+    pub(crate) wake: Notify,
+    /// The public pages' copy of the list, and when it was read.
+    pub(crate) pages_list: tokio::sync::Mutex<Option<(Instant, Arc<Vec<Listing>>)>>,
+}
+
+/// How long the public pages show the same copy of the list.
+pub(crate) const PAGES_LIST_FOR: Duration = Duration::from_secs(60);
+
+/// The directory as every handler and task holds it.
+pub type Shared = Arc<Directory>;
+
+impl Directory {
+    /// A directory on `store`, accepting `tokens`, whose refresher may reach the internal
+    /// addresses `fetch_internal` lists and no others.
+    pub fn new(
+        store: Store,
+        tokens: Vec<Token>,
+        fetch_internal: ipx_feed::guard::Allowed,
+    ) -> Result<Shared, reqwest::Error> {
+        // No redirects: ipx_feed::fetch follows them itself, so it sees each hop.
+        let client = reqwest::Client::builder()
+            .connect_timeout(Duration::from_secs(10))
+            .read_timeout(ipx_feed::FEED_SILENCE)
+            .redirect(Policy::none())
+            .user_agent(concat!("ipx-directory/", env!("CARGO_PKG_VERSION")))
+            // The feeds are named by whoever holds a write token, and redirect where their
+            // publishers like: nothing on this network is fetched unless listed (#329).
+            .connector_layer(ipx_feed::guard::Guard::new(fetch_internal))
+            .build()?;
+        Ok(Arc::new(Self {
+            store,
+            tokens: RwLock::new(tokens),
+            client,
+            wake: Notify::new(),
+            pages_list: Default::default(),
+        }))
+    }
+
+    /// The whole list for the public pages, read from the database at most once a minute. Anyone
+    /// may ask for a page, and each asked the database for the whole catalogue on the pool the
+    /// refresher and every server's /v1 calls share; a query string of its own got past any
+    /// cache in front (#341). The lock is held through the read, so a burst waits on one query.
+    pub async fn pages_list(&self) -> anyhow::Result<Arc<Vec<Listing>>> {
+        let mut held = self.pages_list.lock().await;
+        if let Some((at, list)) = held.as_ref()
+            && at.elapsed() < PAGES_LIST_FOR
+        {
+            return Ok(list.clone());
+        }
+        let list = Arc::new(self.store.list().await?);
+        *held = Some((Instant::now(), list.clone()));
+        Ok(list)
+    }
+
+    /// Takes the tokens of a config file read again, and drops the snapshots of any that went.
+    pub async fn set_tokens(&self, tokens: Vec<Token>) -> anyhow::Result<u64> {
+        let names: Vec<String> = tokens.iter().map(|t| t.name.clone()).collect();
+        *self.tokens.write().expect("a panic elsewhere poisoned this lock") = tokens;
+        self.store.keep_instances(&names).await
+    }
+
+    pub(crate) fn caller(&self, presented: &str) -> Option<Me> {
+        let h = config::hash(presented);
+        // Every token compared, whether or not one matched, so timing says nothing either way.
+        self.tokens.read().expect("a panic elsewhere poisoned this lock").iter().fold(None, |found, t| {
+            if config::same_secret(&t.sha256, &h) { Some(Me { name: t.name.clone(), access: t.access }) } else { found }
+        })
+    }
+
+    /// Fetches one feed and stores what it found.
+    pub async fn refresh(&self, due: &store::Due) -> anyhow::Result<()> {
+        let got = match ipx_feed::fetch(&self.client, &due.url, None, due.etag.as_deref(), due.last_modified.as_deref())
+            .await
+        {
+            Ok((ipx_feed::Fetched::NotModified, _)) => Refreshed::NotModified,
+            Ok((ipx_feed::Fetched::Body { bytes, etag, last_modified }, _)) => {
+                if ipx_feed::is_opml(&bytes) {
+                    // ipx sends the feeds inside an OPML, never the OPML.
+                    Refreshed::Failed("an OPML list, not a feed".into())
+                } else {
+                    match ipx_feed::parse(&bytes) {
+                        Ok(parsed) => Refreshed::Feed { parsed, etag, last_modified },
+                        Err(e) => Refreshed::Failed(format!("{e:#}")),
+                    }
+                }
+            }
+            Err(e) => Refreshed::Failed(format!("{e:#}")),
+        };
+        if let Refreshed::Failed(e) = &got {
+            tracing::info!(feed = %due.id, error = %ipx_feed::redact_text(e), "feed check failed");
+        }
+        // ponytail: a feed that moved for good is still asked for at its old address, which
+        // redirects; follow the move when a feed's id can change without losing its counts.
+        self.store.save(&due.id, got).await
+    }
+}
+
+/// Fetches whatever is due, a few at a time, forever.
+pub async fn refresher(dir: Shared) {
+    loop {
+        match dir.store.due(REFRESH_EVERY, 25).await {
+            Ok(due) => {
+                if !due.is_empty() {
+                    // A round is a trace, each feed's check a span in it (#213).
+                    let round = tracing::info_span!("refresh", feeds = due.len());
+                    Instrument::instrument(
+                        async {
+                            // A few at a time: in turn, two publishers timing out held a round
+                            // for a minute and more (#227).
+                            futures_util::StreamExt::for_each_concurrent(futures_util::stream::iter(&due), 5, |d| {
+                                let check = tracing::info_span!("feed", feed = %d.id);
+                                async {
+                                    if let Err(e) = Instrument::instrument(dir.refresh(d), check).await {
+                                        tracing::warn!(feed = %d.id, error = %format!("{e:#}"), "could not store a feed check");
+                                    }
+                                }
+                            })
+                            .await;
+                            // One line a round, for the dashboard's refresher panel (#210); failures have their own.
+                            tracing::info!(checked = due.len(), "feeds checked");
+                        },
+                        round,
+                    )
+                    .await;
+                    continue;
+                }
+            }
+            Err(e) => tracing::warn!(error = %format!("{e:#}"), "could not read which feeds are due"),
+        }
+        tokio::select! {
+            _ = tokio::time::sleep(Duration::from_secs(60)) => {}
+            _ = dir.wake.notified() => {}
+        }
+    }
+}
+
+/// Reads the config file again whenever it changes, checked each minute. A file that fails to
+/// load leaves the tokens as they were, so a typo locks nobody out.
+pub async fn reloader(dir: Shared, path: PathBuf) {
+    let mtime = |p: &std::path::Path| std::fs::metadata(p).and_then(|m| m.modified()).ok();
+    let mut last = mtime(&path);
+    loop {
+        tokio::time::sleep(Duration::from_secs(60)).await;
+        let now = mtime(&path);
+        if now == last {
+            continue;
+        }
+        last = now;
+        match config::Config::load(&path).and_then(|c| c.tokens()) {
+            Ok(tokens) => {
+                let n = tokens.len();
+                match dir.set_tokens(tokens).await {
+                    Ok(dropped) => tracing::info!(tokens = n, dropped, "config read again"),
+                    Err(e) => {
+                        tracing::warn!(error = %format!("{e:#}"), "config read again, but old snapshots could not be dropped")
+                    }
+                }
+            }
+            Err(e) => {
+                tracing::warn!(error = %format!("{e:#}"), "config changed but does not load; keeping the tokens as they were")
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    async fn scratch() -> Shared {
+        Directory::new(Store::scratch().await, vec![], Default::default()).unwrap()
+    }
+
+    #[tokio::test]
+    async fn the_refresher_fetches_nothing_on_this_network() {
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = listener.local_addr().unwrap();
+        tokio::spawn(async move { while listener.accept().await.is_ok() {} });
+        let dir = scratch().await;
+        let err = dir.client.get(format!("http://{addr}/feed.xml")).send().await.unwrap_err();
+        assert!(format!("{err:?}").contains("internal address"), "{err:?}");
+    }
+
+    #[tokio::test]
+    async fn the_public_pages_read_the_list_once_a_minute() {
+        let dir = scratch().await;
+        let first = dir.pages_list().await.unwrap();
+        assert!(Arc::ptr_eq(&first, &dir.pages_list().await.unwrap()), "a second page asks the database nothing");
+        // Once the copy is a minute old, the next page reads the list again.
+        dir.pages_list.lock().await.as_mut().unwrap().0 -= PAGES_LIST_FOR;
+        assert!(!Arc::ptr_eq(&first, &dir.pages_list().await.unwrap()));
+    }
+}

crates/directory/src/lib.rs

@@ -7,700 +7,16 @@
 //! The types here are the wire format, shared with ipx, which asks the directory on its
 //! people's behalf.
 
+mod api;
 pub mod apple;
 pub mod config;
+mod directory;
 pub mod metrics;
 pub mod otel;
 mod site;
 pub mod store;
+mod wire;
 
-use std::sync::{Arc, RwLock};
-use std::time::{Duration, Instant};
-
-use axum::extract::{Path, Query, Request, State};
-use axum::http::{StatusCode, header};
-use axum::middleware::Next;
-use axum::response::{IntoResponse, Response};
-use axum::routing::{delete, get, post};
-use axum::{Extension, Json, Router};
-use serde::{Deserialize, Serialize};
-
-use config::{Access, Token};
-use store::{Refreshed, Store};
-
-/// A feed as the directory lists it.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct Listing {
-    /// The first 64 bits of the SHA-256 of its normalised address (`store::feed_id`).
-    pub id: String,
-    pub url: String,
-    pub title: Option<String>,
-    pub image: Option<String>,
-    /// Apple's category and the subcategory within it, from the feed's own iTunes category.
-    pub category: Option<String>,
-    pub subcategory: Option<String>,
-    /// Any audio or video in its latest items.
-    pub podcast: bool,
-    /// Summed over every instance's latest snapshot.
-    pub subscribers: i64,
-}
-
-/// One feed's page: what it says it is, and its latest items.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct FeedPage {
-    #[serde(flatten)]
-    pub listing: Listing,
-    /// The feed's own text, HTML as often as not, as it was sent: whoever shows it sanitizes it.
-    pub description: Option<String>,
-    /// Why its last check failed, in words, when it did.
-    pub failing: Option<String>,
-    pub items: Vec<Item>,
-}
-
-/// One of a feed's latest items: what it is, never its files.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct Item {
-    pub guid: String,
-    pub title: Option<String>,
-    pub link: Option<String>,
-    pub published: Option<i64>,
-    pub duration: Option<i64>,
-    /// As the feed sent it, unsanitized.
-    pub description: Option<String>,
-}
-
-/// One feed in a snapshot: its address, and how many people on the sending server subscribe.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct SnapshotEntry {
-    pub url: String,
-    pub subscribers: i64,
-    /// The category the sending server's admin gave a feed that names none itself: used only
-    /// while the feed names none.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub category: Option<String>,
-}
-
-#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
-pub struct SnapshotResult {
-    pub accepted: usize,
-    /// Addresses that look like they carry a subscriber's own access.
-    pub refused_private: usize,
-    /// Feeds an admin removed from the directory.
-    pub refused_blocked: usize,
-    /// Not an http(s) address.
-    pub refused_invalid: usize,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct Me {
-    pub name: String,
-    pub access: Access,
-}
-
-/// A token whose snapshot the directory holds: how many feeds it gave, and how many people.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct Instance {
-    pub name: String,
-    pub feeds: i64,
-    pub subscribers: i64,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct Blocked {
-    pub id: String,
-    pub url: Option<String>,
-    pub title: Option<String>,
-    pub at: i64,
-}
-
-/// The most a snapshot may list: far more than one server's subscriptions, few enough that one
-/// request cannot hold the database for long.
-pub const MAX_SNAPSHOT: usize = 20_000;
-
-/// How often each feed is fetched again. Metadata changes slowly, and every server subscribed
-/// to a feed fetches it itself anyway: every six hours was 3.9 GB of feeds a day (#243).
-pub const REFRESH_EVERY: i64 = 12 * 3600;
-
-pub struct Directory {
-    pub store: Store,
-    tokens: RwLock<Vec<Token>>,
-    client: reqwest::Client,
-    /// Wakes the refresher, so a feed a snapshot brings in is fetched now, not in a minute.
-    wake: tokio::sync::Notify,
-    /// The public pages' copy of the list, and when it was read.
-    pages_list: tokio::sync::Mutex<Option<(Instant, Arc<Vec<Listing>>)>>,
-}
-
-/// How long the public pages show the same copy of the list.
-const PAGES_LIST_FOR: Duration = Duration::from_secs(60);
-
-pub type Shared = Arc<Directory>;
-
-impl Directory {
-    pub fn new(
-        store: Store,
-        tokens: Vec<Token>,
-        fetch_internal: ipx_feed::guard::Allowed,
-    ) -> Result<Shared, reqwest::Error> {
-        // No redirects: ipx_feed::fetch follows them itself, so it sees each hop.
-        let client = reqwest::Client::builder()
-            .connect_timeout(std::time::Duration::from_secs(10))
-            .read_timeout(ipx_feed::FEED_SILENCE)
-            .redirect(reqwest::redirect::Policy::none())
-            .user_agent(concat!("ipx-directory/", env!("CARGO_PKG_VERSION")))
-            // The feeds are named by whoever holds a write token, and redirect where their
-            // publishers like: nothing on this network is fetched unless listed (#329).
-            .connector_layer(ipx_feed::guard::Guard::new(fetch_internal))
-            .build()?;
-        Ok(Arc::new(Self {
-            store,
-            tokens: RwLock::new(tokens),
-            client,
-            wake: tokio::sync::Notify::new(),
-            pages_list: Default::default(),
-        }))
-    }
-
-    /// The whole list for the public pages, read from the database at most once a minute. Anyone
-    /// may ask for a page, and each asked the database for the whole catalogue on the pool the
-    /// refresher and every server's /v1 calls share; a query string of its own got past any
-    /// cache in front (#341). The lock is held through the read, so a burst waits on one query.
-    pub async fn pages_list(&self) -> anyhow::Result<Arc<Vec<Listing>>> {
-        let mut held = self.pages_list.lock().await;
-        if let Some((at, list)) = held.as_ref()
-            && at.elapsed() < PAGES_LIST_FOR
-        {
-            return Ok(list.clone());
-        }
-        let list = Arc::new(self.store.list().await?);
-        *held = Some((Instant::now(), list.clone()));
-        Ok(list)
-    }
-
-    /// Takes the tokens of a config file read again, and drops the snapshots of any that went.
-    pub async fn set_tokens(&self, tokens: Vec<Token>) -> anyhow::Result<u64> {
-        let names: Vec<String> = tokens.iter().map(|t| t.name.clone()).collect();
-        *self.tokens.write().unwrap() = tokens;
-        self.store.keep_instances(&names).await
-    }
-
-    fn caller(&self, presented: &str) -> Option<Me> {
-        let h = config::hash(presented);
-        // Every token compared, whether or not one matched, so timing says nothing either way.
-        self.tokens.read().unwrap().iter().fold(None, |found, t| {
-            if config::same_secret(&t.sha256, &h) { Some(Me { name: t.name.clone(), access: t.access }) } else { found }
-        })
-    }
-
-    /// Fetches one feed and stores what it found.
-    pub async fn refresh(&self, due: &store::Due) -> anyhow::Result<()> {
-        let got = match ipx_feed::fetch(&self.client, &due.url, None, due.etag.as_deref(), due.last_modified.as_deref())
-            .await
-        {
-            Ok((ipx_feed::Fetched::NotModified, _)) => Refreshed::NotModified,
-            Ok((ipx_feed::Fetched::Body { bytes, etag, last_modified }, _)) => {
-                if ipx_feed::is_opml(&bytes) {
-                    // ipx sends the feeds inside an OPML, never the OPML.
-                    Refreshed::Failed("an OPML list, not a feed".into())
-                } else {
-                    match ipx_feed::parse(&bytes) {
-                        Ok(parsed) => Refreshed::Feed { parsed, etag, last_modified },
-                        Err(e) => Refreshed::Failed(format!("{e:#}")),
-                    }
-                }
-            }
-            Err(e) => Refreshed::Failed(format!("{e:#}")),
-        };
-        if let Refreshed::Failed(e) = &got {
-            tracing::info!(feed = %due.id, error = %ipx_feed::redact_text(e), "feed check failed");
-        }
-        // ponytail: a feed that moved for good is still asked for at its old address, which
-        // redirects; follow the move when a feed's id can change without losing its counts.
-        self.store.save(&due.id, got).await
-    }
-}
-
-/// Fetches whatever is due, a few at a time, forever.
-pub async fn refresher(dir: Shared) {
-    loop {
-        match dir.store.due(REFRESH_EVERY, 25).await {
-            Ok(due) => {
-                if !due.is_empty() {
-                    // A round is a trace, each feed's check a span in it (#213).
-                    let round = tracing::info_span!("refresh", feeds = due.len());
-                    tracing::Instrument::instrument(
-                        async {
-                            // A few at a time: in turn, two publishers timing out held a round
-                            // for a minute and more (#227).
-                            futures_util::StreamExt::for_each_concurrent(futures_util::stream::iter(&due), 5, |d| {
-                                let check = tracing::info_span!("feed", feed = %d.id);
-                                async {
-                                    if let Err(e) = tracing::Instrument::instrument(dir.refresh(d), check).await {
-                                        tracing::warn!(feed = %d.id, error = %format!("{e:#}"), "could not store a feed check");
-                                    }
-                                }
-                            })
-                            .await;
-                            // One line a round, for the dashboard's refresher panel (#210); failures have their own.
-                            tracing::info!(checked = due.len(), "feeds checked");
-                        },
-                        round,
-                    )
-                    .await;
-                    continue;
-                }
-            }
-            Err(e) => tracing::warn!(error = %format!("{e:#}"), "could not read which feeds are due"),
-        }
-        tokio::select! {
-            _ = tokio::time::sleep(Duration::from_secs(60)) => {}
-            _ = dir.wake.notified() => {}
-        }
-    }
-}
-
-/// Reads the config file again whenever it changes, checked each minute. A file that fails to
-/// load leaves the tokens as they were, so a typo locks nobody out.
-pub async fn reloader(dir: Shared, path: std::path::PathBuf) {
-    let mtime = |p: &std::path::Path| std::fs::metadata(p).and_then(|m| m.modified()).ok();
-    let mut last = mtime(&path);
-    loop {
-        tokio::time::sleep(Duration::from_secs(60)).await;
-        let now = mtime(&path);
-        if now == last {
-            continue;
-        }
-        last = now;
-        match config::Config::load(&path).and_then(|c| c.tokens()) {
-            Ok(tokens) => {
-                let n = tokens.len();
-                match dir.set_tokens(tokens).await {
-                    Ok(dropped) => tracing::info!(tokens = n, dropped, "config read again"),
-                    Err(e) => {
-                        tracing::warn!(error = %format!("{e:#}"), "config read again, but old snapshots could not be dropped")
-                    }
-                }
-            }
-            Err(e) => {
-                tracing::warn!(error = %format!("{e:#}"), "config changed but does not load; keeping the tokens as they were")
-            }
-        }
-    }
-}
-
-pub fn router(dir: Shared) -> Router {
-    let v1 = Router::new()
-        .route("/v1/me", get(me))
-        .route("/v1/feeds", get(list))
-        .route("/v1/popular", get(popular))
-        .route("/v1/feeds/{id}", get(feed).delete(remove))
-        .route("/v1/feeds/{id}/refresh", post(refresh))
-        .route("/v1/snapshot", post(snapshot))
-        .route("/v1/blocked", get(blocked))
-        .route("/v1/blocked/{id}", delete(unblock))
-        .route("/v1/instances", get(instances))
-        .route("/v1/instances/{name}", delete(drop_instance))
-        .layer(axum::middleware::from_fn_with_state(dir.clone(), authenticate));
-    Router::new()
-        .route("/healthz", get(healthz))
-        .merge(v1)
-        // Anyone may browse, no token, read only (#215).
-        .merge(site::router())
-        // Brotli, zstd or gzip as the caller takes them: the feed list went to ipx as 603 KB of
-        // JSON, most of a Directory page's time (#253). Inside the access log, which records what
-        // was sent.
-        .layer(tower_http::compression::CompressionLayer::new().br(true).zstd(true).gzip(true))
-        // And takes a request's body gzipped: ipx sends its snapshot so, about 100 KB as JSON.
-        .layer(tower_http::decompression::RequestDecompressionLayer::new().gzip(true))
-        .layer(axum::middleware::from_fn(access_log))
-        .with_state(dir)
-}
-
-struct ApiError(StatusCode, String);
-
-impl IntoResponse for ApiError {
-    fn into_response(self) -> Response {
-        (self.0, Json(serde_json::json!({ "error": self.1 }))).into_response()
-    }
-}
-
-impl From<anyhow::Error> for ApiError {
-    fn from(e: anyhow::Error) -> Self {
-        tracing::error!(error = %format!("{e:#}"), "request failed");
-        ApiError(StatusCode::INTERNAL_SERVER_ERROR, "something went wrong; the directory's log says what".into())
-    }
-}
-
-/// Who made the request, for the access log, set by `authenticate`.
-#[derive(Clone)]
-struct Who(String);
-
-/// One line a request, with the token's name and never the token: what the Grafana panels read.
-async fn access_log(req: Request, next: Next) -> Response {
-    let (method, path) = (req.method().clone(), req.uri().path().to_owned());
-    let start = Instant::now();
-    // A span for each request but the healthcheck's, a child of the caller's when it sent a
-    // traceparent, as ipx does on its calls here (#213).
-    let span = if path == "/healthz" {
-        tracing::Span::none()
-    } else {
-        tracing::info_span!(
-            "http",
-            otel.name = %format!("{method} {path}"),
-            http.request.method = %method,
-            url.path = %path,
-            http.response.status_code = tracing::field::Empty,
-        )
-    };
-    if let Some(parent) = otel::remote_parent(req.headers(), None) {
-        use tracing_opentelemetry::OpenTelemetrySpanExt;
-        let _ = span.set_parent(parent);
-    }
-    let resp = tracing::Instrument::instrument(next.run(req), span.clone()).await;
-    span.record("http.response.status_code", resp.status().as_u16());
-    let _in = span.enter();
-    let who = resp.extensions().get::<Who>().map_or("", |w| w.0.as_str()).to_owned();
-    let status = resp.status().as_u16();
-    if path != "/healthz" {
-        tracing::info!(
-            target: "ipx_directory::http",
-            method = %method, path = %path, status, duration_ms = start.elapsed().as_millis() as u64, token = %who,
-            "{method} {path} -> {status}"
-        );
-    }
-    resp
-}
-
-async fn authenticate(State(dir): State<Shared>, mut req: Request, next: Next) -> Response {
-    let presented = req
-        .headers()
-        .get(header::AUTHORIZATION)
-        .and_then(|v| v.to_str().ok())
-        .and_then(|v| v.strip_prefix("Bearer "))
-        .map(str::trim)
-        .filter(|t| !t.is_empty());
-    let Some(me) = presented.and_then(|t| dir.caller(t)) else {
-        return ApiError(StatusCode::UNAUTHORIZED, "a valid token is needed: Authorization: Bearer <token>".into())
-            .into_response();
-    };
-    let who = Who(me.name.clone());
-    req.extensions_mut().insert(me);
-    let mut resp = next.run(req).await;
-    resp.extensions_mut().insert(who);
-    resp
-}
-
-fn need(me: &Me, access: Access) -> Result<(), ApiError> {
-    if me.access >= access {
-        Ok(())
-    } else {
-        let what = match access {
-            Access::Ro => "browse",
-            Access::Rw => "send snapshots",
-            Access::Rwa => "administer the directory",
-        };
-        Err(ApiError(StatusCode::FORBIDDEN, format!("this token may not {what}")))
-    }
-}
-
-async fn healthz(State(dir): State<Shared>) -> Result<&'static str, ApiError> {
-    dir.store.ping().await?;
-    Ok("ok")
-}
-
-async fn me(Extension(me): Extension<Me>) -> Json<Me> {
-    Json(me)
-}
-
-async fn list(State(dir): State<Shared>) -> Result<Json<Vec<Listing>>, ApiError> {
-    Ok(Json(dir.store.list().await?))
-}
-
-#[derive(Deserialize)]
-struct Limit {
-    limit: Option<u64>,
-}
-
-async fn popular(State(dir): State<Shared>, Query(q): Query<Limit>) -> Result<Json<Vec<Listing>>, ApiError> {
-    Ok(Json(dir.store.popular(q.limit.unwrap_or(10).clamp(1, 100)).await?))
-}
-
-async fn feed(State(dir): State<Shared>, Path(id): Path<String>) -> Result<Json<FeedPage>, ApiError> {
-    dir.store
-        .feed(&id)
-        .await?
-        .map(Json)
-        .ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))
-}
-
-async fn snapshot(
-    State(dir): State<Shared>,
-    Extension(me): Extension<Me>,
-    Json(entries): Json<Vec<SnapshotEntry>>,
-) -> Result<Json<SnapshotResult>, ApiError> {
-    need(&me, Access::Rw)?;
-    if entries.len() > MAX_SNAPSHOT {
-        return Err(ApiError(StatusCode::PAYLOAD_TOO_LARGE, format!("a snapshot lists at most {MAX_SNAPSHOT} feeds")));
-    }
-    let r = dir.store.snapshot(&me.name, &entries).await?;
-    dir.wake.notify_one();
-    tracing::info!(
-        instance = %me.name, accepted = r.accepted, refused_private = r.refused_private,
-        refused_blocked = r.refused_blocked, refused_invalid = r.refused_invalid, "snapshot"
-    );
-    Ok(Json(r))
-}
-
-async fn remove(
-    State(dir): State<Shared>,
-    Extension(me): Extension<Me>,
-    Path(id): Path<String>,
-) -> Result<StatusCode, ApiError> {
-    need(&me, Access::Rwa)?;
-    dir.store.block(&id).await?;
-    tracing::info!(feed = %id, by = %me.name, "feed removed and blocked");
-    Ok(StatusCode::NO_CONTENT)
-}
-
-async fn blocked(State(dir): State<Shared>, Extension(me): Extension<Me>) -> Result<Json<Vec<Blocked>>, ApiError> {
-    need(&me, Access::Rwa)?;
-    Ok(Json(dir.store.blocked().await?))
-}
-
-async fn unblock(
-    State(dir): State<Shared>,
-    Extension(me): Extension<Me>,
-    Path(id): Path<String>,
-) -> Result<StatusCode, ApiError> {
-    need(&me, Access::Rwa)?;
-    if !dir.store.unblock(&id).await? {
-        return Err(ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not blocked")));
-    }
-    tracing::info!(feed = %id, by = %me.name, "feed unblocked");
-    Ok(StatusCode::NO_CONTENT)
-}
-
-async fn instances(State(dir): State<Shared>, Extension(me): Extension<Me>) -> Result<Json<Vec<Instance>>, ApiError> {
-    need(&me, Access::Rwa)?;
-    Ok(Json(dir.store.instances().await?))
-}
-
-async fn drop_instance(
-    State(dir): State<Shared>,
-    Extension(me): Extension<Me>,
-    Path(name): Path<String>,
-) -> Result<StatusCode, ApiError> {
-    need(&me, Access::Rwa)?;
-    let n = dir.store.drop_instance(&name).await?;
-    tracing::info!(instance = %name, feeds = n, by = %me.name, "snapshot dropped");
-    Ok(StatusCode::NO_CONTENT)
-}
-
-async fn refresh(
-    State(dir): State<Shared>,
-    Extension(me): Extension<Me>,
-    Path(id): Path<String>,
-) -> Result<Json<FeedPage>, ApiError> {
-    need(&me, Access::Rwa)?;
-    let due = dir
-        .store
-        .due_one(&id)
-        .await?
-        .ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))?;
-    dir.refresh(&due).await?;
-    feed(State(dir), Path(id)).await
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use axum::body::Body;
-    use tower::ServiceExt;
-
-    async fn app() -> (Router, Shared) {
-        let tokens = ["ro", "rw", "rwa"]
-            .iter()
-            .map(|n| Token {
-                name: format!("{n}-token"),
-                access: serde_json::from_value(serde_json::json!(n)).unwrap(),
-                sha256: config::hash(&format!("secret-{n}")),
-            })
-            .collect();
-        let dir = Directory::new(Store::scratch().await, tokens, Default::default()).unwrap();
-        (router(dir.clone()), dir)
-    }
-
-    async fn call(
-        app: &Router,
-        method: &str,
-        path: &str,
-        token: Option<&str>,
-        body: Option<serde_json::Value>,
-    ) -> (u16, serde_json::Value) {
-        let mut req = axum::http::Request::builder().method(method).uri(path);
-        if let Some(t) = token {
-            req = req.header("authorization", format!("Bearer secret-{t}"));
-        }
-        let req = match body {
-            Some(b) => req.header("content-type", "application/json").body(Body::from(b.to_string())),
-            None => req.body(Body::empty()),
-        }
-        .unwrap();
-        let resp = app.clone().oneshot(req).await.unwrap();
-        let status = resp.status().as_u16();
-        let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap();
-        (status, serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null))
-    }
-
-    #[tokio::test]
-    async fn a_snapshot_may_come_gzipped() {
-        use std::io::Write as _;
-        let (app, _) = app().await;
-        let json = serde_json::json!([{"url": "https://example.com/feed.xml", "subscribers": 2}]).to_string();
-        let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
-        gz.write_all(json.as_bytes()).unwrap();
-        // As ipx sends it (#253).
-        let req = axum::http::Request::builder()
-            .method("POST")
-            .uri("/v1/snapshot")
-            .header("authorization", "Bearer secret-rw")
-            .header("content-type", "application/json")
-            .header("content-encoding", "gzip")
-            .body(Body::from(gz.finish().unwrap()))
-            .unwrap();
-        let resp = app.clone().oneshot(req).await.unwrap();
-        assert_eq!(resp.status(), 200);
-        let got: serde_json::Value =
-            serde_json::from_slice(&axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap()).unwrap();
-        assert_eq!(got["accepted"], 1);
-    }
-
-    #[tokio::test]
-    async fn the_refresher_fetches_nothing_on_this_network() {
-        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
-        let addr = listener.local_addr().unwrap();
-        tokio::spawn(async move { while listener.accept().await.is_ok() {} });
-        let (_, dir) = app().await;
-        let err = dir.client.get(format!("http://{addr}/feed.xml")).send().await.unwrap_err();
-        assert!(format!("{err:?}").contains("internal address"), "{err:?}");
-    }
-
-    #[tokio::test]
-    async fn the_public_pages_read_the_list_once_a_minute() {
-        let (_, dir) = app().await;
-        let first = dir.pages_list().await.unwrap();
-        assert!(Arc::ptr_eq(&first, &dir.pages_list().await.unwrap()), "a second page asks the database nothing");
-        // Once the copy is a minute old, the next page reads the list again.
-        dir.pages_list.lock().await.as_mut().unwrap().0 -= PAGES_LIST_FOR;
-        assert!(!Arc::ptr_eq(&first, &dir.pages_list().await.unwrap()));
-    }
-
-    #[tokio::test]
-    async fn answers_come_compressed_as_the_caller_asks() {
-        let (app, _) = app().await;
-        for (asked, sent) in [("br", Some("br")), ("zstd", Some("zstd")), ("gzip", Some("gzip")), ("identity", None)] {
-            // A page, not an empty feed list: under 32 bytes is sent as it is.
-            let req =
-                axum::http::Request::builder().uri("/").header("accept-encoding", asked).body(Body::empty()).unwrap();
-            let resp = app.clone().oneshot(req).await.unwrap();
-            assert_eq!(resp.status(), 200);
-            let got = resp.headers().get("content-encoding").map(|v| v.to_str().unwrap().to_owned());
-            assert_eq!(got.as_deref(), sent, "asked for {asked} (#253)");
-        }
-    }
-
-    async fn get_page(app: &Router, path: &str) -> (u16, String, Option<String>) {
-        let req = axum::http::Request::builder().uri(path).body(Body::empty()).unwrap();
-        let resp = app.clone().oneshot(req).await.unwrap();
-        let status = resp.status().as_u16();
-        let csp = resp.headers().get("content-security-policy").map(|v| v.to_str().unwrap().to_owned());
-        let bytes = axum::body::to_bytes(resp.into_body(), 1 << 22).await.unwrap();
-        (status, String::from_utf8(bytes.to_vec()).unwrap(), csp)
-    }
-
-    #[tokio::test]
-    async fn anyone_browses_the_pages_and_the_api_still_needs_a_token() {
-        let (app, dir) = app().await;
-        let snap = serde_json::json!([{ "url": "https://x.test/a.xml", "subscribers": 3 }]);
-        call(&app, "POST", "/v1/snapshot", Some("rw"), Some(snap)).await;
-        store::tests::fetch_all(&dir.store).await;
-        let id = dir.store.list().await.unwrap()[0].id.clone();
-
-        let (s, home, csp) = get_page(&app, "/").await;
-        assert_eq!(s, 200);
-        assert!(csp.unwrap().contains("script-src 'self'"));
-        assert!(home.contains("Most subscribed") && home.contains(&format!("/feed/{id}")), "{home}");
-        assert!(get_page(&app, "/all").await.1.contains("3 people subscribe"));
-        let (s, page, _) = get_page(&app, &format!("/feed/{id}")).await;
-        assert_eq!(s, 200);
-        // Its address, to subscribe to anywhere.
-        assert!(page.contains("value=\"https://x.test/a.xml\""), "{page}");
-        assert!(!page.contains("<script>"), "the publisher's HTML is text here");
-        assert_eq!(get_page(&app, "/feed/nope").await.0, 404);
-        assert_eq!(get_page(&app, "/category/Nothing").await.0, 404);
-        assert_eq!(get_page(&app, "/search?q=").await.0, 303);
-        assert_eq!(get_page(&app, "/site.js").await.0, 200);
-        // Reading the pages takes no token; the API still does.
-        assert_eq!(call(&app, "GET", "/v1/feeds", None, None).await.0, 401);
-        assert_eq!(call(&app, "DELETE", &format!("/v1/feeds/{id}"), None, None).await.0, 401);
-        assert_eq!(call(&app, "DELETE", &format!("/feed/{id}"), None, None).await.0, 405);
-    }
-
-    #[tokio::test]
-    async fn each_access_level_does_what_it_may_and_no_more() {
-        let (app, dir) = app().await;
-        let snap = serde_json::json!([{ "url": "https://x.test/a.xml", "subscribers": 2, "category": "Comedy" }]);
-
-        assert_eq!(call(&app, "GET", "/v1/feeds", None, None).await.0, 401);
-        assert_eq!(call(&app, "GET", "/v1/feeds", Some("wrong"), None).await.0, 401);
-        assert_eq!(call(&app, "GET", "/healthz", None, None).await.0, 200);
-
-        let (s, me) = call(&app, "GET", "/v1/me", Some("ro"), None).await;
-        assert_eq!((s, me["name"].as_str(), me["access"].as_str()), (200, Some("ro-token"), Some("ro")));
-        assert_eq!(call(&app, "POST", "/v1/snapshot", Some("ro"), Some(snap.clone())).await.0, 403);
-
-        let (s, r) = call(&app, "POST", "/v1/snapshot", Some("rw"), Some(snap.clone())).await;
-        assert_eq!((s, r["accepted"].as_u64()), (200, Some(1)));
-        store::tests::fetch_all(&dir.store).await;
-        let (_, list) = call(&app, "GET", "/v1/feeds", Some("ro"), None).await;
-        let id = list[0]["id"].as_str().unwrap().to_owned();
-        assert_eq!(list[0]["subscribers"], 2);
-        assert_ne!(list[0]["category"], "Comedy", "the feed's own category wins over the sender's");
-        assert_eq!(call(&app, "GET", &format!("/v1/feeds/{id}"), Some("ro"), None).await.0, 200);
-        assert_eq!(call(&app, "GET", "/v1/feeds/nope", Some("ro"), None).await.0, 404);
-        assert_eq!(call(&app, "GET", "/v1/popular?limit=5", Some("ro"), None).await.1[0]["id"], id.as_str());
-
-        for (m, p) in [
-            ("DELETE", format!("/v1/feeds/{id}")),
-            ("GET", "/v1/blocked".into()),
-            ("DELETE", "/v1/instances/rw-token".into()),
-        ] {
-            assert_eq!(call(&app, m, &p, Some("rw"), None).await.0, 403, "{m} {p}");
-        }
-        assert_eq!(call(&app, "DELETE", &format!("/v1/feeds/{id}"), Some("rwa"), None).await.0, 204);
-        assert_eq!(call(&app, "GET", "/v1/blocked", Some("rwa"), None).await.1[0]["id"], id.as_str());
-        assert_eq!(call(&app, "POST", "/v1/snapshot", Some("rw"), Some(snap.clone())).await.1["refused_blocked"], 1);
-        assert_eq!(call(&app, "DELETE", &format!("/v1/blocked/{id}"), Some("rwa"), None).await.0, 204);
-        assert_eq!(call(&app, "DELETE", &format!("/v1/blocked/{id}"), Some("rwa"), None).await.0, 404);
-        call(&app, "POST", "/v1/snapshot", Some("rw"), Some(snap)).await;
-        assert_eq!(call(&app, "DELETE", "/v1/instances/rw-token", Some("rwa"), None).await.0, 204);
-        assert_eq!(call(&app, "GET", "/v1/feeds", Some("ro"), None).await.1, serde_json::json!([]));
-    }
-
-    #[tokio::test]
-    async fn a_revoked_token_loses_its_access_and_its_counts() {
-        let dir = Directory::new(Store::scratch().await, vec![], Default::default()).unwrap();
-        let t = |n: &str| Token { name: n.into(), access: Access::Rw, sha256: config::hash(&format!("secret-{n}")) };
-        dir.set_tokens(vec![t("a"), t("b")]).await.unwrap();
-        let app = router(dir.clone());
-        let snap = serde_json::json!([{ "url": "https://x.test/a.xml", "subscribers": 1 }]);
-        call(&app, "POST", "/v1/snapshot", Some("a"), Some(snap.clone())).await;
-        call(&app, "POST", "/v1/snapshot", Some("b"), Some(snap)).await;
-        store::tests::fetch_all(&dir.store).await;
-        assert_eq!(call(&app, "GET", "/v1/feeds", Some("a"), None).await.1[0]["subscribers"], 2);
-
-        dir.set_tokens(vec![t("a")]).await.unwrap();
-        assert_eq!(call(&app, "GET", "/v1/feeds", Some("b"), None).await.0, 401);
-        assert_eq!(call(&app, "GET", "/v1/feeds", Some("a"), None).await.1[0]["subscribers"], 1);
-    }
-}
+pub use api::router;
+pub use directory::{Directory, REFRESH_EVERY, Shared, refresher, reloader};
+pub use wire::{Blocked, FeedPage, Instance, Item, Listing, MAX_SNAPSHOT, Me, SnapshotEntry, SnapshotResult};

crates/directory/src/main.rs

@@ -2,7 +2,13 @@
 
 use anyhow::{Context, Result};
 use clap::{Parser, Subcommand};
-use ipx_directory::{Directory, config, store::Store};
+use ipx_directory::store::Store;
+use ipx_directory::{Directory, config};
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use tokio::net::TcpListener;
+use tokio::signal::unix::SignalKind;
+use tracing_subscriber::EnvFilter;
 use tracing_subscriber::prelude::*;
 
 #[derive(Parser)]
@@ -11,7 +17,7 @@ struct Cli {
     /// The config file: where to listen, the database, and the tokens it accepts. Otherwise
     /// DIRECTORY_CONFIG, or else /config/directory.toml.
     #[arg(long)]
-    config: Option<std::path::PathBuf>,
+    config: Option<PathBuf>,
     #[command(subcommand)]
     cmd: Option<Cmd>,
 }
@@ -61,7 +67,7 @@ async fn main() -> Result<()> {
             let port = cfg.bind.rsplit(':').next().unwrap_or("8100");
             let resp = reqwest::Client::new()
                 .get(format!("http://127.0.0.1:{port}/healthz"))
-                .timeout(std::time::Duration::from_secs(4))
+                .timeout(Duration::from_secs(4))
                 .send()
                 .await
                 .context("the directory does not answer")?;
@@ -71,11 +77,10 @@ async fn main() -> Result<()> {
     }
 }
 
-async fn serve(path: &std::path::Path) -> Result<()> {
+async fn serve(path: &Path) -> Result<()> {
     // One JSON object a line for Loki when DIRECTORY_LOG_FORMAT=json, as ipx does; text otherwise.
     let json = std::env::var("DIRECTORY_LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json"));
-    let filter =
-        || tracing_subscriber::EnvFilter::try_from_env("DIRECTORY_LOG").unwrap_or_else(|_| "ipx_directory=info".into());
+    let filter = || EnvFilter::try_from_env("DIRECTORY_LOG").unwrap_or_else(|_| "ipx_directory=info".into());
     // Traces to Tempo when OTEL_EXPORTER_OTLP_ENDPOINT is set, joining ipx's (#213).
     let otel = ipx_directory::otel::provider("ipx-directory")?;
     tracing_subscriber::registry()
@@ -91,7 +96,7 @@ async fn serve(path: &std::path::Path) -> Result<()> {
             use opentelemetry::trace::TracerProvider;
             tracing_opentelemetry::layer()
                 .with_tracer(p.tracer("ipx-directory"))
-                .with_filter(tracing_subscriber::EnvFilter::new("ipx_directory=info,ipx_feed=info"))
+                .with_filter(EnvFilter::new("ipx_directory=info,ipx_feed=info"))
         }))
         .init();
 
@@ -101,15 +106,14 @@ async fn serve(path: &std::path::Path) -> Result<()> {
         .ok()
         .filter(|u| !u.trim().is_empty())
         .with_context(|| format!("{} is not set: the directory needs a database URL", cfg.database_url_env))?;
-    let store = Store::open(&url, true).await.context("opening the database")?;
+    let store = Store::open(&url, /* keep_open */ true).await.context("opening the database")?;
     let dir = Directory::new(store, vec![], ipx_feed::guard::Allowed::parse(&cfg.fetch_internal)?)?;
     let dropped = dir.set_tokens(tokens).await?;
 
-    let listener =
-        tokio::net::TcpListener::bind(&cfg.bind).await.with_context(|| format!("listening on {}", cfg.bind))?;
+    let listener = TcpListener::bind(&cfg.bind).await.with_context(|| format!("listening on {}", cfg.bind))?;
     tracing::info!(version = env!("CARGO_PKG_VERSION"), bind = %cfg.bind, tokens = cfg.tokens.len(), dropped, "directory started");
     if !cfg.metrics_bind.is_empty() {
-        let metrics = tokio::net::TcpListener::bind(&cfg.metrics_bind)
+        let metrics = TcpListener::bind(&cfg.metrics_bind)
             .await
             .with_context(|| format!("listening for metrics on {}", cfg.metrics_bind))?;
         tracing::info!(bind = %cfg.metrics_bind, "serving metrics for Prometheus at /metrics");
@@ -124,8 +128,7 @@ async fn serve(path: &std::path::Path) -> Result<()> {
     tokio::spawn(ipx_directory::reloader(dir.clone(), path.to_owned()));
     axum::serve(listener, ipx_directory::router(dir))
         .with_graceful_shutdown(async {
-            let mut term =
-                tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).expect("SIGTERM handler");
+            let mut term = tokio::signal::unix::signal(SignalKind::terminate()).expect("SIGTERM handler");
             tokio::select! {
                 _ = term.recv() => {}
                 _ = tokio::signal::ctrl_c() => {}

crates/directory/src/metrics.rs

@@ -6,12 +6,17 @@
 //! mimalloc gives freed pages back, so resident memory follows what is in use: a spike that
 //! does not come down again is a leak. Not jemalloc, whose upstream was archived in 2025.
 
+use axum::Router;
 use std::sync::atomic::Ordering::Relaxed;
+use std::time::Instant;
+use tokio::net::TcpListener;
+use tower_http::compression::CompressionLayer;
 
 #[global_allocator]
 static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
 /// The process's resident memory in bytes and its open files, from /proc; None elsewhere.
+#[must_use]
 pub fn process() -> (Option<f64>, Option<f64>) {
     let rss = std::fs::read_to_string("/proc/self/status").ok().and_then(|s| {
         let kb =
@@ -24,7 +29,7 @@ pub fn process() -> (Option<f64>, Option<f64>) {
 
 /// The directory's /metrics, in Prometheus's text format: memory, open files, and the feeds it
 /// has fetched. Nothing from the database, so a scrape costs nothing and answers while it is down.
-fn render(started: std::time::Instant) -> String {
+fn render(started: Instant) -> String {
     let mut out = String::new();
     let mut gauge = |name: &str, kind: &str, help: &str, labels: &str, v: f64| {
         out.push_str(&format!("# HELP {name} {help}\n# TYPE {name} {kind}\n{name}{labels} {v}\n"));
@@ -69,9 +74,9 @@ fn render(started: std::time::Instant) -> String {
 
 /// Serves /metrics on a listener of its own: the public port is behind the tunnel with no
 /// Access in front, and these are for the LAN.
-pub async fn serve(listener: tokio::net::TcpListener) {
-    let started = std::time::Instant::now();
-    let app = axum::Router::new()
+pub async fn serve(listener: TcpListener) {
+    let started = Instant::now();
+    let app = Router::new()
         .route(
             "/metrics",
             axum::routing::get(move || async move {
@@ -79,7 +84,7 @@ pub async fn serve(listener: tokio::net::TcpListener) {
             }),
         )
         // Prometheus asks for gzip (#253).
-        .layer(tower_http::compression::CompressionLayer::new().gzip(true));
+        .layer(CompressionLayer::new().gzip(true));
     if let Err(e) = axum::serve(listener, app).await {
         tracing::error!(error = %e, "metrics stopped");
     }

crates/directory/src/otel.rs

@@ -2,6 +2,8 @@
 //! caller and out to a callee, so one trace follows a request from the browser through ipx-web
 //! and ipx into the directory.
 
+use axum::http::HeaderMap;
+
 /// Traces over OTLP, to Tempo for one, when OTEL_EXPORTER_OTLP_ENDPOINT names a collector
 /// (`http://host:4318`: the exporter speaks OTLP over HTTP and adds `/v1/traces`). The exporter
 /// reads that and the other `OTEL_` variables itself.
@@ -23,7 +25,8 @@ pub fn provider(service: &'static str) -> anyhow::Result<Option<opentelemetry_sd
 /// one action sets off are one trace. None for a missing or malformed one. In the query where no
 /// header can be set: a position saved by sendBeacon, and the playlist Safari's player fetches
 /// itself (#192).
-pub fn remote_parent(headers: &axum::http::HeaderMap, query: Option<&str>) -> Option<opentelemetry::Context> {
+#[must_use]
+pub fn remote_parent(headers: &HeaderMap, query: Option<&str>) -> Option<opentelemetry::Context> {
     use opentelemetry::trace::{SpanContext, SpanId, TraceContextExt, TraceFlags, TraceId, TraceState};
     let v = match headers.get("traceparent") {
         Some(h) => h.to_str().ok()?,
@@ -46,6 +49,7 @@ pub fn remote_parent(headers: &axum::http::HeaderMap, query: Option<&str>) -> Op
 
 /// The current span as a `traceparent`, for a request to another service, so its spans join
 /// this trace. None outside a traced span.
+#[must_use]
 pub fn traceparent() -> Option<String> {
     use opentelemetry::trace::TraceContextExt;
     use tracing_opentelemetry::OpenTelemetrySpanExt;

crates/directory/src/site.rs

@@ -16,6 +16,7 @@ use axum::response::{IntoResponse, Response};
 use axum::routing::get;
 
 use crate::{Listing, Shared};
+use url::form_urlencoded::Serializer;
 
 pub fn router() -> Router<Shared> {
     Router::new()
@@ -140,7 +141,7 @@ async fn listed(dir: &Shared, title: &str, here: &str, f: &Filter, pick: impl Fn
 
     // The query a link keeps, with one of its keys changed.
     let link = |kind: Option<&str>, sub: Option<&str>| {
-        let mut q = url::form_urlencoded::Serializer::new(String::new());
+        let mut q = Serializer::new(String::new());
         if let Some(v) = f.q.as_deref() {
             q.append_pair("q", v);
         }

crates/directory/src/store.rs

@@ -6,6 +6,11 @@ use anyhow::Result;
 use sea_orm::{ConnectionTrait, DatabaseConnection, Statement, TransactionTrait, Value};
 
 use crate::{Blocked, FeedPage, Instance, Item, Listing, SnapshotEntry, SnapshotResult};
+use sea_orm::{ConnectOptions, Database, QueryResult};
+use std::cmp::Reverse;
+use std::collections::{BTreeMap, HashSet};
+use std::time::SystemTime;
+use url::Url;
 
 /// A feed's items as its page shows them, and as many as ipx's own Directory page showed.
 pub const ITEMS: usize = 20;
@@ -16,6 +21,7 @@ const FLAG_AFTER_SECS: i64 = 24 * 3600;
 /// The most subscribers one instance may say a feed has.
 pub const MAX_SUBSCRIBERS: i64 = 1_000_000;
 
+/// The directory's database: feeds, snapshots, blocks and when each sync ran.
 pub struct Store {
     db: DatabaseConnection,
 }
@@ -30,19 +36,25 @@ pub struct Due {
 
 /// What a fetch found, to be stored.
 pub enum Refreshed {
+    /// Unchanged since the last fetch.
     NotModified,
+    /// Read anew, with the validators to send next time.
     Feed { parsed: ipx_feed::ParsedFeed, etag: Option<String>, last_modified: Option<String> },
+    /// The fetch failed, and why, in words.
     Failed(String),
 }
 
+/// Unix seconds now.
+#[must_use]
 pub fn now() -> i64 {
-    std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs() as i64)
+    SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs() as i64)
 }
 
 /// The address as every instance would write it: scheme and host lower-cased, a default port
 /// and a fragment dropped. None for anything that is not http(s).
+#[must_use]
 pub fn normalise(url: &str) -> Option<String> {
-    let mut u = url::Url::parse(url.trim()).ok()?;
+    let mut u = Url::parse(url.trim()).ok()?;
     if !matches!(u.scheme(), "http" | "https") || u.host_str().is_none() {
         return None;
     }
@@ -52,6 +64,7 @@ pub fn normalise(url: &str) -> Option<String> {
 
 /// A feed's id: the first 64 bits of its normalised address's SHA-256, so every instance, and
 /// anyone else, arrives at the same id for the same feed without asking.
+#[must_use]
 pub fn feed_id(normalised: &str) -> String {
     use sha2::Digest;
     sha2::Sha256::digest(normalised.as_bytes())[..8].iter().map(|b| format!("{b:02x}")).collect()
@@ -138,7 +151,7 @@ impl Store {
             let sep = if url.contains('?') { '&' } else { '?' };
             url = format!("{url}{sep}options=-c%20client_min_messages%3Dwarning");
         }
-        let mut opts = sea_orm::ConnectOptions::new(url.clone());
+        let mut opts = ConnectOptions::new(url.clone());
         opts.sqlx_logging(false);
         // An in-memory SQLite database is one per connection: a pool of them would be several.
         if url.contains(":memory:") {
@@ -149,7 +162,7 @@ impl Store {
         if keep_open && url.starts_with("postgres") {
             opts.max_connections(10).min_connections(10);
         }
-        let db = sea_orm::Database::connect(opts).await?;
+        let db = Database::connect(opts).await?;
         for sql in SCHEMA {
             db.execute_unprepared(sql).await?;
         }
@@ -169,7 +182,7 @@ impl Store {
         Ok(self.db.execute_raw(self.stmt(sql, values)).await?.rows_affected())
     }
 
-    async fn rows(&self, sql: &str, values: Vec<Value>) -> Result<Vec<sea_orm::QueryResult>> {
+    async fn rows(&self, sql: &str, values: Vec<Value>) -> Result<Vec<QueryResult>> {
         Ok(self.db.query_all_raw(self.stmt(sql, values)).await?)
     }
 
@@ -181,6 +194,7 @@ impl Store {
         })
     }
 
+    /// Records when the sync `name` (Apple's charts) last ran.
     pub async fn set_synced(&self, name: &str, at: i64) -> Result<()> {
         self.exec(
             "INSERT INTO syncs (name, at) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET at = excluded.at",
@@ -190,6 +204,7 @@ impl Store {
         Ok(())
     }
 
+    /// Whether the database answers, for the healthcheck.
     pub async fn ping(&self) -> Result<()> {
         self.rows("SELECT 1", vec![]).await?;
         Ok(())
@@ -199,14 +214,14 @@ impl Store {
     /// blocked feed is left out and counted; the rest are added to the feeds, new ones to be
     /// fetched by the refresher. Feeds no instance sends any more go.
     pub async fn snapshot(&self, instance: &str, entries: &[SnapshotEntry]) -> Result<SnapshotResult> {
-        let blocked: std::collections::HashSet<String> = self
+        let blocked: HashSet<String> = self
             .rows("SELECT id FROM blocked", vec![])
             .await?
             .iter()
             .map(|r| r.try_get("", "id"))
             .collect::<Result<_, _>>()?;
         let mut result = SnapshotResult::default();
-        let mut keep = std::collections::BTreeMap::new();
+        let mut keep = BTreeMap::new();
         for e in entries {
             let Some(url) = normalise(&e.url) else {
                 result.refused_invalid += 1;
@@ -274,7 +289,8 @@ impl Store {
             .map(|r| r.try_get("", "instance"))
             .collect::<Result<_, _>>()?;
         // Apple's charts are no token's, and stay until the next sync replaces them.
-        for name in present.iter().filter(|n| !names.contains(n) && n.as_str() != crate::apple::INSTANCE) {
+        let gone = present.iter().filter(|n| !names.contains(n) && n.as_str() != crate::apple::INSTANCE);
+        for name in gone {
             dropped += self.drop_instance(name).await?;
         }
         Ok(dropped)
@@ -296,7 +312,7 @@ impl Store {
             f.podcast, CAST(sum(s.subscribers) AS BIGINT) AS subscribers
          FROM feeds f JOIN snapshots s ON s.feed_id = f.id WHERE f.items IS NOT NULL";
 
-    fn listing(r: &sea_orm::QueryResult) -> Result<Listing> {
+    fn listing(r: &QueryResult) -> Result<Listing> {
         Ok(Listing {
             id: r.try_get("", "id")?,
             url: r.try_get("", "url")?,
@@ -373,6 +389,7 @@ impl Store {
         Ok(self.exec("DELETE FROM blocked WHERE id = $1", vec![id.into()]).await? > 0)
     }
 
+    /// Each instance that has sent a snapshot, with how many feeds and people it counted.
     pub async fn instances(&self) -> Result<Vec<Instance>> {
         // Postgres sums a BIGINT as NUMERIC, which does not read as an i64.
         self.rows(
@@ -392,6 +409,7 @@ impl Store {
         .collect()
     }
 
+    /// The feeds an admin has blocked.
     pub async fn blocked(&self) -> Result<Vec<Blocked>> {
         self.rows("SELECT id, url, title, at FROM blocked ORDER BY at DESC, id", vec![])
             .await?
@@ -428,6 +446,7 @@ impl Store {
         .collect()
     }
 
+    /// One feed, due or not, for an admin's Refresh.
     pub async fn due_one(&self, id: &str) -> Result<Option<Due>> {
         self.rows("SELECT id, url, etag, last_modified FROM feeds WHERE id = $1", vec![id.into()])
             .await?
@@ -443,6 +462,7 @@ impl Store {
             .transpose()
     }
 
+    /// Stores what a fetch of feed `id` found, and when.
     pub async fn save(&self, id: &str, got: Refreshed) -> Result<()> {
         let t = now();
         match got {
@@ -468,7 +488,7 @@ impl Store {
                     .flat_map(|e| &e.enclosures)
                     .any(|e| e.mime.as_deref().is_some_and(|m| m.starts_with("audio/") || m.starts_with("video/")));
                 let mut entries = parsed.entries;
-                entries.sort_by_key(|e| std::cmp::Reverse(e.published));
+                entries.sort_by_key(|e| Reverse(e.published));
                 // What an item is, never its files: their addresses are where a key would be.
                 let items: Vec<Item> = entries
                     .into_iter()

crates/directory/src/wire.rs

@@ -0,0 +1,99 @@
+//! What ipx servers and the directory send each other: listings, feed pages, snapshots and their results.
+
+use crate::config::Access;
+use serde::{Deserialize, Serialize};
+
+/// A feed as the directory lists it.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Listing {
+    /// The first 64 bits of the SHA-256 of its normalised address (`store::feed_id`).
+    pub id: String,
+    pub url: String,
+    pub title: Option<String>,
+    pub image: Option<String>,
+    /// Apple's category and the subcategory within it, from the feed's own iTunes category.
+    pub category: Option<String>,
+    pub subcategory: Option<String>,
+    /// Any audio or video in its latest items.
+    pub podcast: bool,
+    /// Summed over every instance's latest snapshot.
+    pub subscribers: i64,
+}
+
+/// One feed's page: what it says it is, and its latest items.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FeedPage {
+    #[serde(flatten)]
+    pub listing: Listing,
+    /// The feed's own text, HTML as often as not, as it was sent: whoever shows it sanitizes it.
+    pub description: Option<String>,
+    /// Why its last check failed, in words, when it did.
+    pub failing: Option<String>,
+    pub items: Vec<Item>,
+}
+
+/// One of a feed's latest items: what it is, never its files.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Item {
+    pub guid: String,
+    pub title: Option<String>,
+    pub link: Option<String>,
+    /// Unix seconds.
+    pub published: Option<i64>,
+    /// Seconds.
+    pub duration: Option<i64>,
+    /// As the feed sent it, unsanitized.
+    pub description: Option<String>,
+}
+
+/// One feed in a snapshot: its address, and how many people on the sending server subscribe.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SnapshotEntry {
+    pub url: String,
+    pub subscribers: i64,
+    /// The category the sending server's admin gave a feed that names none itself: used only
+    /// while the feed names none.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub category: Option<String>,
+}
+
+/// What became of a snapshot's entries.
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
+pub struct SnapshotResult {
+    pub accepted: usize,
+    /// Addresses that look like they carry a subscriber's own access.
+    pub refused_private: usize,
+    /// Feeds an admin removed from the directory.
+    pub refused_blocked: usize,
+    /// Not an http(s) address.
+    pub refused_invalid: usize,
+}
+
+/// Who a token belongs to and what it may do: `/v1/me`'s answer.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Me {
+    pub name: String,
+    pub access: Access,
+}
+
+/// A token whose snapshot the directory holds: how many feeds it gave, and how many people.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Instance {
+    pub name: String,
+    pub feeds: i64,
+    pub subscribers: i64,
+}
+
+/// A feed an admin removed from the directory, which snapshots may not bring back.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Blocked {
+    pub id: String,
+    pub url: Option<String>,
+    pub title: Option<String>,
+    /// When it was blocked, Unix seconds.
+    pub at: i64,
+}
+
+/// The most a snapshot may list: far more than one server's subscriptions, few enough that one
+/// request cannot hold the database for long.
+pub const MAX_SNAPSHOT: usize = 20_000;

crates/feed/Cargo.toml

@@ -4,6 +4,9 @@ version = "0.0.0"
 publish = false
 edition.workspace = true
 
+[lints]
+workspace = true
+
 [dependencies]
 anyhow.workspace = true
 atom_syndication.workspace = true

crates/feed/src/artwork.rs

@@ -0,0 +1,117 @@
+//! Artwork: on https when its host serves it there, and a site icon where a feed has none.
+
+use crate::{discover, fetch};
+use std::collections::HashMap;
+use std::time::Duration;
+
+/// Artwork for a feed that has none: the icon its site's page names, or else the site's
+/// `/favicon.ico`. None if neither is there.
+#[tracing::instrument(skip_all, fields(site = site))]
+pub async fn site_icon(client: &reqwest::Client, site: &str) -> Option<String> {
+    let timeout = Duration::from_secs(20);
+    let resp = client.get(site).timeout(timeout).send().await.ok()?;
+    // Relative to where the page ended up, not where it was asked for: a site that redirects
+    // to /en/ would otherwise have its icon looked for in the wrong place.
+    let base = resp.url().clone();
+    // The icon a page names can be gone: antirez.com names /images/favicon.png, a 404, while its
+    // /favicon.ico is there. Stored unchecked, it was a broken image that was never looked up again.
+    if resp.status().is_success()
+        && let Ok(page) = fetch::read_capped(resp).await
+        && let Some(href) = page_icon(&page)
+        && let Ok(url) = base.join(&href)
+        && is_image(client, url.as_str()).await
+    {
+        return Some(url.into());
+    }
+    let ico = base.join("/favicon.ico").ok()?;
+    is_image(client, ico.as_str()).await.then(|| ico.into())
+}
+
+/// How many hosts one scan of a feed asks whether its artwork is on https.
+pub(crate) const MAX_ART_HOSTS: usize = 8;
+
+/// Artwork's address on https when its host serves it there, else as it was. The page is https
+/// and must not load http; the host is asked once per `known` (one feed's read), and an http
+/// address it does not serve on https stays, for /api/art to fetch (#90). Four of the five
+/// hosts the catalogue had on http served the same image on https; The Secret Cabal's CDN
+/// presents another name's certificate (#110).
+pub async fn prefer_https(client: &reqwest::Client, url: &str, known: &mut HashMap<String, bool>) -> String {
+    let Some(rest) = url.strip_prefix("http://") else { return url.to_owned() };
+    let host = rest.split('/').next().unwrap_or("").to_owned();
+    let secure = format!("https://{rest}");
+    let ok = match known.get(&host) {
+        Some(ok) => *ok,
+        // Each new host is a probe of up to 20 s, one after another on the scan, and the feed's
+        // author decides how many hosts there are: ports and user names make as many as they like
+        // of one server (#339). Real feeds put their artwork on a few.
+        None if known.len() >= MAX_ART_HOSTS => false,
+        None => {
+            let ok = is_image(client, &secure).await;
+            known.insert(host, ok);
+            ok
+        }
+    };
+    if ok { secure } else { url.to_owned() }
+}
+
+/// Whether `url` answers with an image. A site with no favicon often answers 200 with its home
+/// page, which is not an icon.
+pub async fn is_image(client: &reqwest::Client, url: &str) -> bool {
+    let timeout = Duration::from_secs(20);
+    let Ok(resp) = client.get(url).timeout(timeout).send().await else { return false };
+    resp.status().is_success()
+        && resp
+            .headers()
+            .get(reqwest::header::CONTENT_TYPE)
+            .and_then(|v| v.to_str().ok())
+            .is_some_and(|t| t.starts_with("image/"))
+}
+
+/// The icon a web page names in its `<link>` tags, the larger apple-touch-icon first: a plain
+/// `icon` is often 16 pixels, which blurs at the size the list draws artwork.
+pub(crate) fn page_icon(bytes: &[u8]) -> Option<String> {
+    let text = String::from_utf8_lossy(bytes);
+    // ASCII only, so byte offsets in the lowered copy stay valid in the original.
+    let lower = text.to_ascii_lowercase();
+    let (mut touch, mut icon) = (None, None);
+    let mut pos = 0;
+    while let Some(at) = lower[pos..].find("<link") {
+        let start = pos + at;
+        let Some(end) = lower[start..].find('>').map(|e| start + e) else { break };
+        pos = end + 1;
+        let tag = &text[start..end];
+        let Some(rel) = discover::tag_attr(tag, "rel").map(|r| r.to_ascii_lowercase()) else { continue };
+        let rels: Vec<&str> = rel.split_whitespace().collect();
+        if touch.is_none() && rels.iter().any(|r| r.starts_with("apple-touch-icon")) {
+            touch = discover::tag_attr(tag, "href");
+        } else if icon.is_none() && rels.contains(&"icon") {
+            icon = discover::tag_attr(tag, "href");
+        }
+    }
+    touch.or(icon).filter(|h| !h.is_empty())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn artwork_on_more_hosts_than_the_limit_is_not_asked_about() {
+        let mut known: std::collections::HashMap<String, bool> =
+            (0..MAX_ART_HOSTS).map(|n| (format!("h{n}:80"), false)).collect();
+        // No server answers here: were it asked, this would wait on a connection, not return.
+        let url = "http://192.0.2.1:9/a.png";
+        assert_eq!(prefer_https(&reqwest::Client::new(), url, &mut known).await, url);
+        assert_eq!(known.len(), MAX_ART_HOSTS);
+    }
+
+    #[test]
+    fn page_icon_prefers_the_touch_icon() {
+        let page = br#"<head><link rel="stylesheet" href="/a.css">
+            <link rel="shortcut icon" href="/fav.ico">
+            <LINK REL="apple-touch-icon-precomposed" sizes="180x180" href='/touch.png'></head>"#;
+        assert_eq!(page_icon(page).as_deref(), Some("/touch.png"));
+        assert_eq!(page_icon(br#"<link rel="icon" href="i.svg">"#).as_deref(), Some("i.svg"));
+        assert_eq!(page_icon(br#"<link rel="stylesheet" href="/a.css">"#), None);
+    }
+}

crates/feed/src/category.rs

@@ -0,0 +1,125 @@
+//! Apple Podcasts' categories, and where a feed's category sits among them (#118).
+
+/// Apple's podcast categories, each with its subcategories
+/// (https://podcasters.apple.com/support/1691-apple-podcasts-categories): the list a podcast's own
+/// `<itunes:category>` comes from, and a fixed one, so the Directory's categories stay few.
+pub(crate) const CATEGORIES: &[(&str, &[&str])] = &[
+    ("Arts", &["Books", "Design", "Fashion & Beauty", "Food", "Performing Arts", "Visual Arts"]),
+    ("Business", &["Careers", "Entrepreneurship", "Investing", "Management", "Marketing", "Non-Profit"]),
+    ("Comedy", &["Comedy Interviews", "Improv", "Stand-Up"]),
+    ("Education", &["Courses", "How To", "Language Learning", "Self-Improvement"]),
+    ("Fiction", &["Comedy Fiction", "Drama", "Science Fiction"]),
+    ("Government", &[]),
+    ("History", &[]),
+    ("Health & Fitness", &["Alternative Health", "Fitness", "Medicine", "Mental Health", "Nutrition", "Sexuality"]),
+    ("Kids & Family", &["Education for Kids", "Parenting", "Pets & Animals", "Stories for Kids"]),
+    (
+        "Leisure",
+        &["Animation & Manga", "Automotive", "Aviation", "Crafts", "Games", "Hobbies", "Home & Garden", "Video Games"],
+    ),
+    ("Music", &["Music Commentary", "Music History", "Music Interviews"]),
+    (
+        "News",
+        &[
+            "Business News",
+            "Daily News",
+            "Entertainment News",
+            "News Commentary",
+            "Politics",
+            "Sports News",
+            "Tech News",
+        ],
+    ),
+    (
+        "Religion & Spirituality",
+        &["Buddhism", "Christianity", "Hinduism", "Islam", "Judaism", "Religion", "Spirituality"],
+    ),
+    (
+        "Science",
+        &[
+            "Astronomy",
+            "Chemistry",
+            "Earth Sciences",
+            "Life Sciences",
+            "Mathematics",
+            "Natural Sciences",
+            "Nature",
+            "Physics",
+            "Social Sciences",
+        ],
+    ),
+    ("Society & Culture", &["Documentary", "Personal Journals", "Philosophy", "Places & Travel", "Relationships"]),
+    (
+        "Sports",
+        &[
+            "Baseball",
+            "Basketball",
+            "Cricket",
+            "Fantasy Sports",
+            "Football",
+            "Golf",
+            "Hockey",
+            "Rugby",
+            "Running",
+            "Soccer",
+            "Swimming",
+            "Tennis",
+            "Volleyball",
+            "Wilderness",
+            "Wrestling",
+        ],
+    ),
+    ("Technology", &[]),
+    ("True Crime", &[]),
+    ("TV & Film", &["After Shows", "Film History", "Film Interviews", "Film Reviews", "TV Reviews"]),
+];
+
+/// Names that are not in Apple's list but mean one that is (#125). Jev picked from this list when
+/// it spelt Football and Soccer its own way, and stored those answers; Software How-To is Apple's
+/// Technology subcategory from before its 2019 list, which older feeds still name.
+pub(crate) const ALIASES: &[(&str, &str)] =
+    &[("American Football", "Football"), ("Football (Soccer)", "Soccer"), ("Software How-To", "Technology")];
+
+/// Apple's categories as ipx has them, for the directory to check against Apple's own (#252).
+#[must_use]
+pub fn categories() -> &'static [(&'static str, &'static [&'static str])] {
+    CATEGORIES
+}
+
+/// Where a category sits in Apple's list, as (category, subcategory), so the Directory browses a
+/// category and then its subcategories as Apple's does (#118). A podcast's own is stored as its
+/// subcategory, and one an admin typed that is not Apple's stands as a category of its own.
+#[must_use]
+pub fn category_path(name: &str) -> (String, Option<String>) {
+    // A feed's own text, which can end in a stray comma: "Soccer," was a category of its own.
+    let name = name.trim().trim_end_matches([',', ';', '.']).trim_end();
+    let name = ALIASES.iter().find(|(old, _)| old.eq_ignore_ascii_case(name)).map_or(name, |(_, now)| now);
+    for (category, subs) in CATEGORIES {
+        if category.eq_ignore_ascii_case(name) {
+            return (category.to_string(), None);
+        }
+        if let Some(sub) = subs.iter().find(|s| s.eq_ignore_ascii_case(name)) {
+            return (category.to_string(), Some(sub.to_string()));
+        }
+    }
+    (name.to_string(), None)
+}
+
+#[cfg(test)]
+mod tests {
+
+    #[test]
+    fn category_path_finds_a_subcategory_s_category() {
+        use super::category_path;
+        assert_eq!(category_path("Tech News"), ("News".into(), Some("Tech News".into())));
+        assert_eq!(category_path("video games"), ("Leisure".into(), Some("Video Games".into())), "Apple's spelling");
+        assert_eq!(category_path("Technology"), ("Technology".into(), None));
+        assert_eq!(category_path("Homebrew"), ("Homebrew".into(), None), "not Apple's: its own category");
+        // Apple's spelling, an old answer of Jev's, a stray comma and Apple's old list (#125).
+        assert_eq!(category_path("Soccer"), ("Sports".into(), Some("Soccer".into())));
+        assert_eq!(category_path("Football (Soccer)"), ("Sports".into(), Some("Soccer".into())));
+        assert_eq!(category_path("Soccer,"), ("Sports".into(), Some("Soccer".into())));
+        assert_eq!(category_path("Football"), ("Sports".into(), Some("Football".into())));
+        assert_eq!(category_path("Software How-To"), ("Technology".into(), None));
+    }
+}

crates/feed/src/discover.rs

@@ -0,0 +1,107 @@
+//! Finding a feed from what someone typed: a feed address, a site, or a page linking one.
+
+use crate::{fetch, outlines, parse};
+use anyhow::{Context, Result};
+use std::time::Duration;
+
+/// What was typed into Add feed, as a URL. A bare Patreon token is taken as its creator's
+/// feed, since the token alone says whose it is. An address with no scheme is https: 'cnn.com'
+/// was stored as typed and every check failed with "relative URL without a base" (#101).
+#[must_use]
+pub fn expand_input(input: &str) -> String {
+    let s = input.trim();
+    let token = s.len() >= 20 && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_');
+    if token {
+        format!("https://www.patreon.com/rss?auth={s}")
+    } else if let Some(rest) = s.strip_prefix("//") {
+        format!("https://{rest}")
+    } else if !s.contains("://") {
+        format!("https://{s}")
+    } else {
+        s.to_owned()
+    }
+}
+
+/// The feed a web page names as its own via `<link rel="alternate" type="application/rss+xml"
+/// href="...">` (or the Atom equivalent) -- how the new address was found for om.co, ms.now,
+/// Letters of Note, the Daily Dot, Hell Gate, The Frame Lab and Daily Kos.
+pub(crate) fn alternate_feed_link(bytes: &[u8]) -> Option<String> {
+    let text = String::from_utf8_lossy(bytes);
+    // ASCII only: to_lowercase changes some characters' length (U+0130 grows a byte), and the
+    // offsets found in the lowered copy then sliced the original off a char boundary.
+    let lower = text.to_ascii_lowercase();
+    let mut pos = 0;
+    while let Some(rel) = lower[pos..].find("<link") {
+        let start = pos + rel;
+        let Some(end) = lower[start..].find('>').map(|e| start + e) else { break };
+        pos = end + 1;
+        let tag = &text[start..end];
+        let tag_lower = &lower[start..end];
+        let is_alternate = tag_lower.contains("rel=\"alternate\"") || tag_lower.contains("rel='alternate'");
+        let is_feed_type = tag_lower.contains("rss+xml") || tag_lower.contains("atom+xml");
+        if is_alternate
+            && is_feed_type
+            && let Some(href) = tag_attr(tag, "href")
+        {
+            return Some(href);
+        }
+    }
+    None
+}
+
+/// What someone asked to add, as a feed: the address itself when it is a feed or an OPML list,
+/// otherwise the feed its web page links as its own, otherwise an error and nothing is added.
+/// A page that linked no feed used to be added as it was and failed on every check, called a
+/// feed that moved (#102); cnn.com is one.
+pub async fn find_feed(client: &reqwest::Client, url: &str) -> Result<String> {
+    let read = |u: String| async move {
+        let resp = client.get(&u).timeout(Duration::from_secs(30)).send().await.context("connecting")?;
+        anyhow::ensure!(resp.status().is_success(), "HTTP {}", resp.status());
+        let base = resp.url().clone();
+        anyhow::Ok((base, fetch::read_capped(resp).await.context("reading")?))
+    };
+    let is_feed = |b: &[u8]| outlines::is_opml(b) || parse::parse(b).is_ok();
+    let (base, body) = read(url.to_owned()).await.with_context(|| format!("could not read {url}"))?;
+    if is_feed(&body) {
+        return Ok(url.to_owned());
+    }
+    if !parse::looks_like_html(&body) {
+        let why = parse::parse(&body).err().map(|e| format!("{e:#}")).unwrap_or_default();
+        anyhow::bail!("{url} is not a feed: {why}");
+    }
+    let Some(href) = alternate_feed_link(&body) else {
+        anyhow::bail!("{url} is a web page that links no feed, so there is nothing to subscribe to");
+    };
+    let linked =
+        base.join(&href).with_context(|| format!("{url} links {href} as its feed, which is not an address"))?;
+    let (_, feed) = read(linked.to_string()).await.with_context(|| format!("{url} links {linked} as its feed, but"))?;
+    anyhow::ensure!(is_feed(&feed), "{url} links {linked} as its feed, but that is not a feed either");
+    Ok(linked.into())
+}
+
+/// The value of one attribute in an HTML/XML start tag, however it is quoted.
+pub(crate) fn tag_attr(tag: &str, name: &str) -> Option<String> {
+    let key = format!("{name}=");
+    let idx = tag.to_lowercase().find(&key)?;
+    let after = &tag[idx + key.len()..];
+    let quote = after.chars().next()?;
+    if quote != '"' && quote != '\'' {
+        return None;
+    }
+    let rest = &after[1..];
+    let close = rest.find(quote)?;
+    Some(rest[..close].trim().to_owned())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_feed_link_after_non_ascii_text_is_found() {
+        // U+0130 lowercases to three bytes from two, which once shifted every offset after it.
+        let page = "<html><title>\u{130}stanbul \u{130}\u{130}</title>\
+            <link rel=\"alternate\" type=\"application/rss+xml\" href=\"/feed.xml\">";
+        assert_eq!(alternate_feed_link(page.as_bytes()).as_deref(), Some("/feed.xml"));
+    }
+}

crates/feed/src/failure.rs

@@ -0,0 +1,145 @@
+//! A failed read, in words for whoever subscribes, and as a kind for the log.
+
+/// A stored `last_error`, translated into plain words for whoever subscribes: whose problem
+/// it is, and whether there is a new address to switch to.
+pub struct Failure {
+    /// What to tell them, in a sentence.
+    pub reason: &'static str,
+    /// Where the feed went, when the error says.
+    pub new_url: Option<String>,
+}
+
+/// A failure's kind, for the log's `error.type` (#91): the HTTP status where there is one, as
+/// OpenTelemetry names an HTTP error, and otherwise a word for what went wrong. Matches the
+/// same wording as `explain_failure`; a message it does not know is "other", never a wrong kind.
+#[must_use]
+pub fn failure_kind(msg: &str) -> (String, Option<u16>) {
+    let low = msg.to_ascii_lowercase();
+    let code = low.split("http ").skip(1).find_map(|r| r.get(..3)?.parse::<u16>().ok());
+    if let Some(c) = code.filter(|c| (100..600).contains(c)) {
+        return (c.to_string(), Some(c));
+    }
+    let kind = if low.contains("dns error")
+        || low.contains("failed to lookup address")
+        || low.contains("no address associated")
+    {
+        "dns"
+    } else if low.contains("too many redirects") {
+        "redirect_loop"
+    } else if low.contains("timed out") || low.contains("timeout") {
+        "timeout"
+    } else if low.contains("certificate") || low.contains("tls") {
+        "tls"
+    } else if low.contains("got a web page") {
+        "not_a_feed"
+    } else if low.contains("the site sent ") {
+        "site_message"
+    } else if low.contains("connect") {
+        "connect"
+    } else if low.contains("pars") {
+        "parse"
+    } else {
+        "other"
+    };
+    (kind.into(), None)
+}
+
+/// Reads a `last_error` the same way `set_feed_error` received it (`format!("{e:#}")` on the
+/// anyhow chain from `fetch` or `parse`) and says what it means, for the errors worth telling
+/// someone about. Everything else -- a timeout, a 5xx, a 429, a feed that is simply garbled --
+/// comes back `None`: transient by nature, or with nothing more useful to say than the raw
+/// text already shown once a feed is open.
+///
+/// ponytail: matches on the fixed strings this crate itself produces (`anyhow!("HTTP
+/// {status}")`, and `parse`'s "got a web page" and "the site sent") plus the substrings a DNS failure
+/// reliably contains. Fragile if reqwest's own wording changes; the fallback is just showing
+/// nothing extra, so a miss costs a clearer message, not a wrong one.
+pub fn explain_failure(msg: &str) -> Option<Failure> {
+    if let Some(rest) = msg.strip_prefix("got a web page, not a feed") {
+        let new_url = rest.strip_prefix("; it links ").and_then(|r| r.strip_suffix(" as its feed")).map(str::to_owned);
+        return Some(Failure { reason: "The feed moved; this address now shows a web page.", new_url });
+    }
+    if msg.contains("the site sent ") {
+        return Some(Failure {
+            reason: "The site sent a message instead of the feed; the publisher has to fix it.",
+            new_url: None,
+        });
+    }
+    let low = msg.to_ascii_lowercase();
+    if low.contains("http 404") {
+        return Some(Failure { reason: "The publisher took this feed down, or moved it.", new_url: None });
+    }
+    if low.contains("http 401") || low.contains("http 403") {
+        return Some(Failure { reason: "The site refuses ipx's requests.", new_url: None });
+    }
+    if low.contains("answered with nothing") {
+        return Some(Failure { reason: "This address answers with nothing; the site may be gone.", new_url: None });
+    }
+    if low.contains("http 402") {
+        return Some(Failure { reason: "The feed now needs a paid plan.", new_url: None });
+    }
+    // Only a name that does not exist. A lookup that failed for the moment says nothing about the
+    // site, and NPR's and Megaphone's feeds were labelled gone after a DNS outage (#143).
+    if !is_temporary_lookup_failure(msg)
+        && (low.contains("dns error")
+            || low.contains("failed to lookup address")
+            || low.contains("no address associated"))
+    {
+        return Some(Failure { reason: "This address no longer resolves; the site may be gone.", new_url: None });
+    }
+    None
+}
+
+/// A host that could not be looked up for the moment -- the resolver's EAI_AGAIN, glibc's
+/// "Temporary failure in name resolution", musl's "Try again" -- as against a name that does
+/// not exist ("Name or service not known"). It says the network was not answering, not that the
+/// feed is gone (#143).
+#[must_use]
+pub fn is_temporary_lookup_failure(msg: &str) -> bool {
+    let low = msg.to_ascii_lowercase();
+    low.contains("dns error") && (low.contains("temporary failure in name resolution") || low.contains("try again"))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn explain_failure_translates_the_errors_the_ui_should_flag() {
+        assert_eq!(
+            explain_failure("HTTP 404 Not Found").unwrap().reason,
+            "The publisher took this feed down, or moved it."
+        );
+        assert_eq!(explain_failure("HTTP 401 Unauthorized").unwrap().reason, "The site refuses ipx's requests.");
+        assert_eq!(explain_failure("HTTP 403 Forbidden").unwrap().reason, "The site refuses ipx's requests.");
+        assert_eq!(explain_failure("HTTP 402 Payment Required").unwrap().reason, "The feed now needs a paid plan.");
+        assert_eq!(
+            explain_failure("the feed's address answered with nothing at all").unwrap().reason,
+            "This address answers with nothing; the site may be gone."
+        );
+        let dns =
+            explain_failure("connecting: dns error: failed to lookup address information: Name or service not known")
+                .unwrap();
+        assert_eq!(dns.reason, "This address no longer resolves; the site may be gone.");
+        // A lookup that failed for the moment is not a site gone (#143).
+        let blip = "connecting: dns error: failed to lookup address information: Temporary failure in name resolution";
+        assert!(is_temporary_lookup_failure(blip));
+        assert!(explain_failure(blip).is_none());
+        assert!(!is_temporary_lookup_failure(
+            "connecting: dns error: failed to lookup address information: Name or service not known"
+        ));
+        let moved = explain_failure("got a web page, not a feed; it links https://x/feed as its feed").unwrap();
+        assert_eq!(moved.new_url.as_deref(), Some("https://x/feed"));
+        assert!(explain_failure("got a web page, not a feed").unwrap().new_url.is_none());
+        let down = explain_failure("the site sent \"Unable to establish a DB connection\" instead of a feed").unwrap();
+        assert_eq!(down.reason, "The site sent a message instead of the feed; the publisher has to fix it.");
+        for transient in [
+            "HTTP 500 Internal Server Error",
+            "HTTP 429 Too Many Requests",
+            "operation timed out",
+            "not RSS (reached end of input without finding a complete channel) and not Atom (unexpected end of input)",
+        ] {
+            assert!(explain_failure(transient).is_none(), "{transient} must not be flagged");
+        }
+    }
+}

crates/feed/src/fetch.rs

@@ -0,0 +1,190 @@
+//! Reading a feed over HTTP: conditional requests, redirects followed one by one, bounded bodies.
+
+use crate::{privacy, wordpress};
+use anyhow::{Context, Result, anyhow};
+use reqwest::StatusCode;
+use reqwest::header::HeaderName;
+use reqwest::header::{ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED};
+use std::sync::atomic::AtomicU64;
+use std::time::Duration;
+
+/// What `fetch` got.
+pub enum Fetched {
+    /// Server said 304, or returned a body we already have.
+    NotModified,
+    /// A body to read, with the validators to send next time.
+    Body { bytes: Vec<u8>, etag: Option<String>, last_modified: Option<String> },
+}
+
+/// The longest a feed may take, connecting to the last byte: a scan handles feeds in order, and
+/// with no limit one hung server held every scan for as long as it did. Dreamwidth answered 504
+/// after 60-67 s for a day, and each scan took 70-80 s instead of 15 (#108). A feed is small;
+/// downloads, which are not, have no such limit.
+pub const FEED_TIMEOUT: Duration = Duration::from_secs(30);
+
+/// The longest a feed's server may send nothing, for the clients that fetch feeds to set: one
+/// that took the connection and never answered ran the whole 30 s, every scan it was in (#230).
+/// No feed that answered took over 8 s in a day of traces; one that keeps sending, however
+/// slowly, still has until FEED_TIMEOUT.
+pub const FEED_SILENCE: Duration = Duration::from_secs(15);
+
+/// Feeds fetched and the bytes of their bodies, since the process started, for /metrics to set
+/// beside memory (#224): a body is read whole, so a night of large feeds is a night of large
+/// allocations.
+pub static FETCHES: AtomicU64 = AtomicU64::new(0);
+
+/// See `FETCHES`.
+pub static FETCHED_BYTES: AtomicU64 = AtomicU64::new(0);
+
+/// The most of a feed or page read. Without a limit an episode's MP3 given as a feed was read
+/// whole into memory, at every check (#295). Real feeds can be large: a daily show's whole back
+/// catalogue was 62 MB, and 50 MB here stopped it updating (#357).
+pub(crate) const MAX_BODY: usize = 200 << 20;
+
+/// A response's body, up to `MAX_BODY`: past it, an error, and the rest is never read.
+pub(crate) async fn read_capped(mut resp: reqwest::Response) -> Result<Vec<u8>> {
+    if resp.content_length().is_some_and(|n| n > MAX_BODY as u64) {
+        return Err(anyhow!("larger than {} MB, too large to be a feed", MAX_BODY >> 20));
+    }
+    let mut out = Vec::new();
+    while let Some(c) = resp.chunk().await? {
+        out.extend_from_slice(&c);
+        anyhow::ensure!(out.len() <= MAX_BODY, "larger than {} MB, too large to be a feed", MAX_BODY >> 20);
+    }
+    Ok(out)
+}
+
+/// Reads a feed, asking only for what changed since `etag` and `last_modified`. Redirects are
+/// followed here, up to ten, so `auth` goes to the feed's own host alone. Also returns the new
+/// address when every hop said the feed moved for good, for the caller to keep instead.
+#[tracing::instrument(skip_all, fields(url = %privacy::redact(feed_url), http.response.body.size))]
+pub async fn fetch(
+    client: &reqwest::Client,
+    feed_url: &str,
+    auth: Option<(&str, Option<String>)>,
+    etag: Option<&str>,
+    last_modified: Option<&str>,
+) -> Result<(Fetched, Option<String>)> {
+    let start = reqwest::Url::parse(feed_url).context("the feed's address")?;
+    let mut url = start.clone();
+    let mut permanent = true;
+    for _ in 0..10 {
+        let mut req = client.get(url.clone()).timeout(FEED_TIMEOUT);
+        if let Some(tag) = etag {
+            req = req.header(IF_NONE_MATCH, tag);
+        }
+        if let Some(lm) = last_modified {
+            req = req.header(IF_MODIFIED_SINCE, lm);
+        }
+        // The feed's own host only: a redirect elsewhere must not be handed the password.
+        if let Some((user, password)) = &auth
+            && url.host_str() == start.host_str()
+        {
+            req = req.basic_auth(user, password.as_deref());
+        }
+
+        let resp = req.send().await.context("connecting")?;
+        let status = resp.status();
+        if status.is_redirection() && status != StatusCode::NOT_MODIFIED {
+            let to = resp
+                .headers()
+                .get(reqwest::header::LOCATION)
+                .and_then(|v| v.to_str().ok())
+                .ok_or_else(|| anyhow!("HTTP {status} without a Location to go to"))?;
+            url = url.join(to).with_context(|| format!("redirected to {to:?}, which is not an address"))?;
+            permanent &= matches!(status, StatusCode::MOVED_PERMANENTLY | StatusCode::PERMANENT_REDIRECT);
+            continue;
+        }
+        let moved = (permanent && url != start).then(|| url.to_string());
+        FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        if status == StatusCode::NOT_MODIFIED {
+            return Ok((Fetched::NotModified, moved));
+        }
+        if !status.is_success() {
+            // WordPress.com answers some readers' feed requests with a browser check that only
+            // JavaScript passes, for hours at a time and whatever the User-Agent; its API has the
+            // same posts and no check (#317).
+            if status == StatusCode::FORBIDDEN
+                && let Some(host) = url.host_str().map(str::to_owned)
+                && read_capped(resp).await.is_ok_and(|b| wordpress::is_wpcom_challenge(&b))
+            {
+                return Ok((wordpress::wpcom_posts(client, &host).await?, moved));
+            }
+            // The original surfaced 401/407 specially; the code is enough for a UI to switch on.
+            return Err(anyhow!("HTTP {status}"));
+        }
+        let header = |h: HeaderName| resp.headers().get(&h).and_then(|v| v.to_str().ok()).map(str::to_owned);
+        let etag = header(ETAG);
+        let last_modified = header(LAST_MODIFIED);
+        let bytes = read_capped(resp).await.context("reading body")?;
+        tracing::Span::current().record("http.response.body.size", bytes.len());
+        FETCHED_BYTES.fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed);
+        return Ok((Fetched::Body { bytes, etag, last_modified }, moved));
+    }
+    // Worded as reqwest worded it, which failure_kind reads as a redirect loop.
+    Err(anyhow!("error following redirect for url ({url}): too many redirects"))
+}
+
+#[cfg(test)]
+mod tests {
+
+    use crate::failure;
+
+    /// A server answering by path: /old moves for good to /new, /tmp for now, /chain for good
+    /// to /tmp, /new is the feed.
+    async fn redirecting_server() -> String {
+        use tokio::io::{AsyncReadExt, AsyncWriteExt};
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = listener.local_addr().unwrap();
+        tokio::spawn(async move {
+            loop {
+                let Ok((mut sock, _)) = listener.accept().await else { return };
+                tokio::spawn(async move {
+                    let mut buf = [0u8; 2048];
+                    let n = sock.read(&mut buf).await.unwrap_or(0);
+                    let req = String::from_utf8_lossy(&buf[..n]);
+                    let path = req.split_whitespace().nth(1).unwrap_or("/").to_owned();
+                    let body = "<?xml version=\"1.0\"?><rss version=\"2.0\"><channel><title>T</title></channel></rss>";
+                    let resp = match path.as_str() {
+                        "/old" => {
+                            "HTTP/1.1 301 Moved Permanently\r\nLocation: /new\r\nContent-Length: 0\r\n\r\n".to_owned()
+                        }
+                        "/tmp" => "HTTP/1.1 302 Found\r\nLocation: /new\r\nContent-Length: 0\r\n\r\n".to_owned(),
+                        "/chain" => {
+                            "HTTP/1.1 308 Permanent Redirect\r\nLocation: /tmp\r\nContent-Length: 0\r\n\r\n".to_owned()
+                        }
+                        // An episode's file given as a feed: its size said, its body never sent.
+                        "/huge" => "HTTP/1.1 200 OK\r\nContent-Length: 600000000\r\n\r\n".to_owned(),
+                        "/loop" => {
+                            "HTTP/1.1 301 Moved Permanently\r\nLocation: /loop\r\nContent-Length: 0\r\n\r\n".to_owned()
+                        }
+                        _ => format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}", body.len()),
+                    };
+                    let _ = sock.write_all(resp.as_bytes()).await;
+                });
+            }
+        });
+        format!("http://{addr}")
+    }
+
+    #[tokio::test]
+    async fn a_feed_that_moved_for_good_says_where_and_one_moved_for_now_does_not() {
+        let base = redirecting_server().await;
+        let client = reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()).build().unwrap();
+        let get = |path: &str| {
+            let url = format!("{base}{path}");
+            let client = client.clone();
+            async move { super::fetch(&client, &url, None, None, None).await }
+        };
+        let (got, moved) = get("/old").await.unwrap();
+        assert!(matches!(got, super::Fetched::Body { .. }));
+        assert_eq!(moved, Some(format!("{base}/new")));
+        assert_eq!(get("/tmp").await.unwrap().1, None); // 302: for now
+        let huge = get("/huge").await.err().map(|e| format!("{e:#}")).unwrap_or_default();
+        assert!(huge.contains("too large to be a feed"), "{huge}");
+        assert_eq!(get("/chain").await.unwrap().1, None); // 308 then 302: not for good
+        assert_eq!(get("/new").await.unwrap().1, None); // never moved
+        let looped = get("/loop").await.err().unwrap().to_string();
+        assert_eq!(failure::failure_kind(&looped).0, "redirect_loop");
+    }
+}

crates/feed/src/guard.rs

@@ -14,6 +14,7 @@ use std::sync::Arc;
 use std::task::{Context, Poll};
 
 use anyhow::{Context as _, Result};
+use http::Extensions;
 use hyper_util::client::legacy::connect::{Connection, HttpInfo};
 
 type BoxError = Box<dyn std::error::Error + Send + Sync>;
@@ -24,10 +25,13 @@ type BoxError = Box<dyn std::error::Error + Send + Sync>;
 pub struct Allowed(Arc<Vec<(IpAddr, u8)>>);
 
 impl Allowed {
+    /// Reads `fetch_internal`'s list, refusing an entry that is neither an address nor a network.
     pub fn parse(list: &[String]) -> Result<Self> {
         list.iter().map(|s| parse_net(s)).collect::<Result<Vec<_>>>().map(|v| Self(Arc::new(v)))
     }
 
+    /// Whether a connection to `ip` may go ahead: any public address, and the internal ones listed.
+    #[must_use]
     pub fn permits(&self, ip: IpAddr) -> bool {
         let ip = canonical(ip);
         !is_internal(ip) || self.0.iter().any(|&(net, bits)| contains(net, bits, ip))
@@ -73,6 +77,7 @@ fn contains(net: IpAddr, bits: u8, ip: IpAddr) -> bool {
 /// unstable.
 // shortcut: 6to4 (2002::/16) and Teredo addresses that wrap an internal IPv4 one pass; check the
 // embedded address if an IPv6 route to them ever exists here.
+#[must_use]
 pub fn is_internal(ip: IpAddr) -> bool {
     match canonical(ip) {
         IpAddr::V4(a) => {
@@ -107,6 +112,8 @@ pub fn is_internal(ip: IpAddr) -> bool {
 pub struct Guard(Allowed);
 
 impl Guard {
+    /// A guard letting through what `allowed` lists, besides the public internet.
+    #[must_use]
     pub fn new(allowed: Allowed) -> Self {
         Self(allowed)
     }
@@ -120,6 +127,7 @@ impl<S> tower::Layer<S> for Guard {
     }
 }
 
+/// The connector `Guard` wraps, checking each connection it makes.
 #[derive(Clone)]
 pub struct Guarded<S> {
     inner: S,
@@ -146,7 +154,7 @@ where
         let allowed = self.allowed.clone();
         Box::pin(async move {
             let conn = connecting.await.map_err(Into::into)?;
-            let mut extras = http::Extensions::new();
+            let mut extras = Extensions::new();
             conn.connected().get_extras(&mut extras);
             // Refused when the peer is unknown too: a connection this cannot check is one it
             // cannot vouch for.

crates/feed/src/jev.rs

@@ -0,0 +1,67 @@
+//! Asking TypeSafe's Jev which Apple category a feed belongs to.
+
+use crate::parse::ParsedFeed;
+use crate::{category, fetch};
+use anyhow::{Result, anyhow};
+
+/// TypeSafe's System One endpoint, where Jev answers questions about a piece of state.
+pub const JEV_URL: &str = "https://api.typesafe.ai/v1/systemone";
+
+/// A Directory category for a feed that names none of its own, which most blogs do not (#117):
+/// TypeSafe's Jev picks one of Apple's categories or subcategories from the feed's title and
+/// latest items, the subcategory being what a podcast's own category is stored as. The likeliest
+/// is taken however unsure, since an admin can change it and asking again would cost a call a scan.
+pub async fn classify(client: &reqwest::Client, key: &str, feed: &ParsedFeed) -> Result<String> {
+    let items: Vec<&str> = feed.entries.iter().filter_map(|e| e.title.as_deref()).take(15).collect();
+    // A subcategory says which category it is in; without that, "Games" or "Drama" is ambiguous.
+    let mut criteria = serde_json::Map::new();
+    for (category, subs) in category::CATEGORIES {
+        criteria.insert(category.to_string(), serde_json::Value::Null);
+        for sub in *subs {
+            criteria.insert(sub.to_string(), format!("Within {category}").into());
+        }
+    }
+    let body = serde_json::json!({
+        "model": "jev-latest",
+        "state": { "title": feed.title, "recent_items": items },
+        "questions": { "category": {
+            "type": "choice",
+            "instructions": "Which category best describes what this feed publishes?",
+            "criteria": criteria,
+        }},
+    });
+    let answer: serde_json::Value = client
+        .post(JEV_URL)
+        .bearer_auth(key)
+        .json(&body)
+        .timeout(fetch::FEED_TIMEOUT)
+        .send()
+        .await?
+        .error_for_status()?
+        .json()
+        .await?;
+    picked(&answer)
+}
+
+pub(crate) fn picked(answer: &serde_json::Value) -> Result<String> {
+    let choice = answer["answers"]["category"]["choice"].as_str().unwrap_or_default();
+    category::CATEGORIES
+        .iter()
+        .flat_map(|(c, subs)| std::iter::once(c).chain(subs.iter()))
+        .find(|c| **c == choice)
+        .map(|c| c.to_string())
+        .ok_or_else(|| anyhow!("no category in the answer: {answer}"))
+}
+
+#[cfg(test)]
+mod tests {
+
+    #[test]
+    fn picked_takes_only_an_apple_category() {
+        let answer = |c: &str| serde_json::json!({"answers": {"category": {"type": "choice", "choice": c}}});
+        assert_eq!(super::picked(&answer("Technology")).unwrap(), "Technology");
+        assert_eq!(super::picked(&answer("Tech News")).unwrap(), "Tech News", "a subcategory");
+        assert!(super::picked(&answer("Blogs")).is_err(), "not one of Apple's");
+        assert!(super::picked(&serde_json::json!({"error": "rate limited"})).is_err());
+    }
+}

crates/feed/src/outlines.rs

@@ -0,0 +1,111 @@
+//! OPML documents: recognising one, reading it safely, and the feeds it lists.
+
+use anyhow::{Result, anyhow};
+
+/// True when a body is an OPML document rather than a feed.
+///
+/// The original matched on the URL ending in ".opml" (iPXClass.py:34), which misses an
+/// OPML served from a URL without that extension. Sniffing the body catches both.
+#[must_use]
+pub fn is_opml(bytes: &[u8]) -> bool {
+    let head = &bytes[..bytes.len().min(1024)];
+    let text = String::from_utf8_lossy(head).to_lowercase();
+    text.contains("<opml")
+}
+
+/// How deep an OPML document's elements may nest. Real ones are a folder or two deep.
+pub(crate) const MAX_OPML_DEPTH: usize = 100;
+
+/// Parses an OPML document, refusing one nested deeper than `MAX_OPML_DEPTH`. The opml crate's
+/// reader recurses once per level with no limit, and a few MB of `<outline>` overflowed the
+/// stack, which aborts the whole daemon, and again on each restart for a subscribed feed (#322).
+pub fn read_opml(text: &str) -> Result<opml::OPML> {
+    use quick_xml::events::Event;
+    let mut reader = quick_xml::Reader::from_str(text);
+    let mut depth = 0usize;
+    loop {
+        match reader.read_event() {
+            Ok(Event::Start(_)) => {
+                depth += 1;
+                if depth > MAX_OPML_DEPTH {
+                    anyhow::bail!("that OPML nests more than {MAX_OPML_DEPTH} levels deep");
+                }
+            }
+            Ok(Event::End(_)) => depth = depth.saturating_sub(1),
+            // Malformed XML is the opml crate's to describe.
+            Ok(Event::Eof) | Err(_) => break,
+            Ok(_) => {}
+        }
+    }
+    opml::OPML::from_str(text).map_err(|e| anyhow!("{e}"))
+}
+
+/// The feeds listed in an OPML document, as (title, xml_url), walking nested folders.
+pub fn parse_opml(bytes: &[u8]) -> Result<Vec<(String, String)>> {
+    let text = String::from_utf8_lossy(bytes);
+    let doc = read_opml(&text).map_err(|e| anyhow!("that does not parse as OPML: {e}"))?;
+    let mut out = vec![];
+    collect_outlines(&doc.body.outlines, &mut out);
+    Ok(out)
+}
+
+/// OPML nests feeds inside folder outlines, so this walks the whole tree.
+pub fn collect_outlines(outlines: &[opml::Outline], out: &mut Vec<(String, String)>) {
+    for o in outlines {
+        if let Some(url) = &o.xml_url {
+            let title = o.title.clone().unwrap_or_else(|| o.text.clone());
+            out.push((title, url.clone()));
+        }
+        collect_outlines(&o.outlines, out);
+    }
+}
+
+/// The <head><title> of an OPML document.
+#[must_use]
+pub fn opml_title(bytes: &[u8]) -> Option<String> {
+    let text = String::from_utf8_lossy(bytes);
+    let doc = read_opml(&text).ok()?;
+    doc.head.and_then(|h| h.title).map(|t| t.trim().to_owned()).filter(|t| !t.is_empty())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn deeply_nested_opml_is_refused_before_it_is_parsed() {
+        let nest = |n: usize| {
+            format!(
+                r#"<opml version="2.0"><head><title>T</title></head><body>{}<outline text="f" xmlUrl="https://e.com/f"/>{}</body></opml>"#,
+                r#"<outline text="a">"#.repeat(n),
+                "</outline>".repeat(n)
+            )
+        };
+        assert_eq!(parse_opml(nest(5).as_bytes()).unwrap().len(), 1, "a few folders deep is fine");
+        // Deep enough to overflow a 2 MiB stack in the opml crate's reader without the check.
+        let deep = nest(100_000);
+        assert!(parse_opml(deep.as_bytes()).unwrap_err().to_string().contains("levels deep"));
+        assert_eq!(opml_title(deep.as_bytes()), None);
+    }
+
+    #[test]
+    fn opml_is_recognised_and_its_feeds_listed() {
+        let xml = br#"<opml version="2.0"><head><title>My Subscriptions</title></head><body>
+          <outline text="Folder">
+            <outline type="rss" text="Alpha" xmlUrl="https://a.example/rss"/>
+            <outline type="rss" text="Beta"  xmlUrl="https://b.example/rss"/>
+          </outline>
+          <outline text="Not a feed"/>
+        </body></opml>"#;
+        assert!(is_opml(xml));
+        assert_eq!(opml_title(xml).as_deref(), Some("My Subscriptions"));
+
+        let feeds = parse_opml(xml).unwrap();
+        assert_eq!(feeds.len(), 2, "nested folders are walked, non-feed outlines skipped");
+        assert_eq!(feeds[0], ("Alpha".into(), "https://a.example/rss".into()));
+
+        // A feed must never be mistaken for a subscription list.
+        assert!(!is_opml(include_bytes!("../../../tests/data/rss2.xml")));
+        assert!(!is_opml(include_bytes!("../../../tests/data/atom.xml")));
+    }
+}

crates/feed/src/parse.rs

@@ -0,0 +1,980 @@
+//! RSS and Atom, read into `ParsedFeed`: titles as text, notes, enclosures, artwork, dates and durations.
+
+use crate::{discover, wordpress};
+use anyhow::{Result, anyhow};
+use chrono::DateTime;
+use std::collections::HashSet;
+
+/// A feed as `parse` reads it, RSS or Atom alike: the channel's own details and its items.
+#[derive(Debug, Default)]
+pub struct ParsedFeed {
+    pub title: Option<String>,
+    /// RSS's `<ttl>`: how long the publisher asks to be left alone between reads.
+    pub ttl_mins: Option<u64>,
+    /// The show's artwork, `<itunes:image>` before the channel's `<image>`.
+    pub image: Option<String>,
+    /// The site the feed belongs to, where a favicon can stand in for missing artwork.
+    pub site: Option<String>,
+    /// The channel's first `<itunes:category>`, for the Directory's categories.
+    pub category: Option<String>,
+    /// What the feed says it is, for its page in the Directory (#130): RSS's `<description>`,
+    /// or iTunes' summary where that is empty, or Atom's subtitle. HTML as often as not.
+    pub description: Option<String>,
+    pub entries: Vec<Entry>,
+}
+
+/// One item of a feed.
+#[derive(Debug, Default)]
+pub struct Entry {
+    /// What tells this item from the feed's others: its `<guid>`, or failing that its link, its
+    /// enclosure's address or its title (`pick_guid`).
+    pub guid: String,
+    pub title: Option<String>,
+    pub link: Option<String>,
+    /// Unix seconds.
+    pub published: Option<i64>,
+    pub description: Option<String>,
+    /// The item's Markdown, when the feed sends it beside HTML of its own (`<source:markdown>`):
+    /// the page can show either (#285). With no HTML it is the description instead (#284).
+    pub markdown: Option<String>,
+    pub categories: Vec<String>,
+    /// `<itunes:explicit>` said yes.
+    pub explicit: bool,
+    /// Episode artwork; falls back to the feed's in the UI.
+    pub image: Option<String>,
+    /// Seconds.
+    pub duration: Option<i64>,
+    pub episode: Option<i64>,
+    pub season: Option<i64>,
+    pub enclosures: Vec<Enclosure>,
+}
+
+/// A file an item carries, an episode as a rule.
+#[derive(Debug, Default, Clone, PartialEq)]
+pub struct Enclosure {
+    pub url: String,
+    /// The type the feed declares, which may be missing or wrong.
+    pub mime: Option<String>,
+    /// The size the feed declares, in bytes.
+    pub length: Option<i64>,
+}
+
+/// RSS first, then Atom -- the same split the original made on `parsedFeed.version`.
+#[tracing::instrument(skip_all, fields(bytes = bytes.len()))]
+pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> {
+    if bytes.trim_ascii_start().starts_with(b"[")
+        && let Ok(posts) = serde_json::from_slice::<Vec<serde_json::Value>>(bytes)
+    {
+        return Ok(wordpress::from_wp_posts(&posts));
+    }
+    match rss::Channel::read_from(bytes) {
+        Ok(ch) => Ok(from_rss(ch, bytes)),
+        Err(rss_err) => match atom_syndication::Feed::read_from(bytes) {
+            Ok(feed) => Ok(from_atom(feed)),
+            Err(atom_err) => {
+                if let Some(said) = plain_text(bytes) {
+                    return Err(anyhow!("the site sent {said} instead of a feed"));
+                }
+                // Some publishers (kcpw, feedland) write a bare "&" in a URL instead of
+                // "&amp;". Strict XML parsers refuse it; browsers don't. Retry once with
+                // every offending "&" escaped rather than fail outright.
+                let escaped = escape_bare_ampersands(bytes);
+                if escaped != bytes {
+                    if let Ok(ch) = rss::Channel::read_from(escaped.as_slice()) {
+                        return Ok(from_rss(ch, &escaped));
+                    }
+                    if let Ok(feed) = atom_syndication::Feed::read_from(escaped.as_slice()) {
+                        return Ok(from_atom(feed));
+                    }
+                }
+                Err(match discover::alternate_feed_link(bytes) {
+                    Some(href) if looks_like_html(bytes) => {
+                        anyhow!("got a web page, not a feed; it links {href} as its feed")
+                    }
+                    None if looks_like_html(bytes) => anyhow!("got a web page, not a feed"),
+                    _ => anyhow!("not RSS ({rss_err}) and not Atom ({atom_err})"),
+                })
+            }
+        },
+    }
+}
+
+/// Whether a body is a web page rather than a feed: most of the errors traced back to a feed
+/// that moved or a domain that lapsed, with the old URL now serving the site instead (or a
+/// redirect to it). `is_opml` already sniffs the other "not actually a feed" case.
+pub(crate) fn looks_like_html(bytes: &[u8]) -> bool {
+    let head = String::from_utf8_lossy(&bytes[..bytes.len().min(2048)]).to_lowercase();
+    head.contains("<!doctype html") || head.contains("<html")
+}
+
+/// What a site sent when it sent a sentence instead of markup. doghouse's feed answered 200 with
+/// "Unable to establish a DB connection", and the two parsers' errors about end of input buried
+/// it. Anything starting with `<` is markup, however broken, and keeps the parsers' errors.
+pub(crate) fn plain_text(bytes: &[u8]) -> Option<String> {
+    let head = String::from_utf8_lossy(&bytes[..bytes.len().min(512)]);
+    let text = head.trim_start_matches(|c: char| c.is_whitespace() || c == '\u{feff}');
+    if text.starts_with('<') {
+        return None;
+    }
+    let line = text.lines().next().unwrap_or("").trim_end();
+    if line.is_empty() {
+        return Some("an empty reply".into());
+    }
+    let mut said: String = line.chars().take(80).collect();
+    if said.len() < line.len() {
+        said.push('…');
+    }
+    Some(format!("\"{said}\""))
+}
+
+/// Escapes every `&` that does not already start a recognized XML entity
+/// (`&amp;`, `&lt;`, `&gt;`, `&quot;`, `&apos;`, or a numeric reference like `&#39;`).
+pub(crate) fn escape_bare_ampersands(bytes: &[u8]) -> Vec<u8> {
+    fn is_entity_start(rest: &[u8]) -> bool {
+        for named in [&b"amp;"[..], b"lt;", b"gt;", b"quot;", b"apos;"] {
+            if rest.starts_with(named) {
+                return true;
+            }
+        }
+        let digits = if rest.starts_with(b"#x") || rest.starts_with(b"#X") {
+            &rest[2..]
+        } else if rest.starts_with(b"#") {
+            &rest[1..]
+        } else {
+            return false;
+        };
+        let len = digits.iter().take_while(|b| b.is_ascii_alphanumeric()).count();
+        len > 0 && digits.get(len) == Some(&b';')
+    }
+
+    let mut out = Vec::with_capacity(bytes.len());
+    let mut i = 0;
+    while i < bytes.len() {
+        if bytes[i] == b'&' && !is_entity_start(&bytes[i + 1..]) {
+            out.extend_from_slice(b"&amp;");
+        } else {
+            out.push(bytes[i]);
+        }
+        i += 1;
+    }
+    out
+}
+
+/// Every `<enclosure>` of every `<item>`, in document order.
+///
+/// The `rss` crate models an item as having at most one enclosure -- which is what RSS 2.0
+/// says -- and when a feed carries several it keeps only the *last*, silently losing the
+/// rest. Feeds do ship several, so read them from the XML directly.
+pub(crate) fn enclosures_by_item(bytes: &[u8]) -> Vec<Vec<Enclosure>> {
+    use quick_xml::events::Event;
+
+    let mut reader = quick_xml::Reader::from_reader(bytes);
+    reader.config_mut().trim_text(true);
+    let mut buf = Vec::new();
+    let mut out: Vec<Vec<Enclosure>> = Vec::new();
+    let mut current: Option<Vec<Enclosure>> = None;
+
+    let read_enclosure = |e: &quick_xml::events::BytesStart| -> Option<Enclosure> {
+        let (mut url, mut mime, mut length) = (String::new(), None, None);
+        for attr in e.attributes().flatten() {
+            // Values arrive escaped: a feed URL's "&" is "&amp;" in the document.
+            let val = quick_xml::escape::unescape(&attr.value).map(|v| v.trim().to_string()).unwrap_or_default();
+            match attr.key.local_name().as_ref() {
+                "url" => url = val,
+                "type" => mime = Some(val).filter(|v| !v.is_empty()),
+                "length" => length = val.parse().ok(),
+                _ => {}
+            }
+        }
+        (!url.is_empty()).then_some(Enclosure { url, mime, length })
+    };
+
+    loop {
+        match reader.read_event_into(&mut buf) {
+            Ok(Event::Start(e)) => match e.name().local_name().as_ref() {
+                "item" => current = Some(Vec::new()),
+                "enclosure" => {
+                    if let (Some(list), Some(enc)) = (current.as_mut(), read_enclosure(&e)) {
+                        list.push(enc);
+                    }
+                }
+                _ => {}
+            },
+            Ok(Event::Empty(e)) => match e.name().local_name().as_ref() {
+                // <item/> with no children still counts, so the indexes stay aligned.
+                "item" => out.push(Vec::new()),
+                "enclosure" => {
+                    if let (Some(list), Some(enc)) = (current.as_mut(), read_enclosure(&e)) {
+                        list.push(enc);
+                    }
+                }
+                _ => {}
+            },
+            Ok(Event::End(e)) => {
+                if e.name().local_name().as_ref() == "item"
+                    && let Some(list) = current.take()
+                {
+                    out.push(list);
+                }
+            }
+            Ok(Event::Eof) | Err(_) => break,
+            _ => {}
+        }
+        buf.clear();
+    }
+    if let Some(list) = current.take() {
+        out.push(list);
+    }
+    out
+}
+
+pub(crate) fn from_rss(ch: rss::Channel, bytes: &[u8]) -> ParsedFeed {
+    let per_item = enclosures_by_item(bytes);
+    let explicit = ch.itunes_ext().and_then(|it| it.explicit()).is_some_and(is_yes);
+
+    let entries = ch
+        .items()
+        .iter()
+        .enumerate()
+        .filter_map(|(idx, item)| {
+            // Straight from the XML, so an item with several keeps all of them. Falls
+            // back to the parsed one if the scan and the parser disagree on item count.
+            let mut enclosures: Vec<Enclosure> = per_item.get(idx).cloned().unwrap_or_else(|| {
+                item.enclosure()
+                    .into_iter()
+                    .map(|e| Enclosure {
+                        url: e.url().trim().to_owned(),
+                        mime: non_empty(Some(e.mime_type())),
+                        length: e.length().parse().ok(),
+                    })
+                    .filter(|e| !e.url.is_empty())
+                    .collect()
+            });
+            drop_player_repeats(&mut enclosures);
+
+            let guid = pick_guid(
+                item.guid().map(|g| g.value()),
+                item.link(),
+                enclosures.first().map(|e| e.url.as_str()),
+                item.title(),
+            )?;
+
+            let entry_explicit = item.itunes_ext().and_then(|it| it.explicit()).is_some_and(is_yes);
+            let it = item.itunes_ext();
+
+            Some(Entry {
+                guid,
+                title: title_text(item.title()),
+                link: non_empty(item.link()),
+                published: item.pub_date().and_then(parse_date),
+                // Content wins over description, as __getEntries preferred entry.content. The
+                // source namespace's Markdown copy only when there is neither (#284).
+                description: notes(item.content(), item.description(), source_markdown(item)),
+                markdown: non_empty(source_markdown(item))
+                    .filter(|_| non_empty(item.content()).or_else(|| non_empty(item.description())).is_some()),
+                categories: item
+                    .categories()
+                    .iter()
+                    .map(|c| c.name().to_owned())
+                    .filter(|c| !c.is_empty() && !c.starts_with("http"))
+                    .collect(),
+                explicit: explicit || entry_explicit,
+                image: item_image(item, &enclosures),
+                duration: it.and_then(|i| i.duration()).and_then(parse_duration),
+                episode: it.and_then(|i| i.episode()).and_then(|e| e.trim().parse().ok()),
+                season: it.and_then(|i| i.season()).and_then(|e| e.trim().parse().ok()),
+                enclosures,
+            })
+        })
+        .collect();
+
+    ParsedFeed {
+        title: title_text(Some(ch.title())),
+        ttl_mins: ch.ttl().and_then(|t| t.trim().parse().ok()),
+        // itunes:image is the square artwork; <image><url> is the older, often smaller one.
+        image: ch
+            .itunes_ext()
+            .and_then(|i| i.image())
+            .map(str::to_owned)
+            .or_else(|| ch.image().map(|i| i.url().to_owned())),
+        site: non_empty(Some(ch.link().trim())),
+        // Only the iTunes one: Apple's list is fixed, while a plain <category> is freeform and
+        // would fill the Directory with one-off tags. The subcategory where there is one: Apple
+        // files every tabletop and gaming show under Leisure, which says little; Games says it.
+        description: non_empty(Some(ch.description())).or_else(|| non_empty(ch.itunes_ext().and_then(|i| i.summary()))),
+        category: ch
+            .itunes_ext()
+            .and_then(|i| i.categories().first())
+            .map(|c| c.subcategory().filter(|s| !s.text().trim().is_empty()).unwrap_or(c))
+            .and_then(|c| non_empty(Some(c.text().trim()))),
+        entries,
+    }
+}
+
+pub(crate) fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed {
+    let entries = feed
+        .entries()
+        .iter()
+        .filter_map(|e| {
+            // Atom carries enclosures as <link rel="enclosure">.
+            let mut enclosures: Vec<Enclosure> = e
+                .links()
+                .iter()
+                .filter(|l| l.rel() == "enclosure")
+                .map(|l| Enclosure {
+                    url: l.href().trim().to_owned(),
+                    mime: non_empty(l.mime_type()),
+                    length: l.length().and_then(|s| s.parse().ok()),
+                })
+                .filter(|e| !e.url.is_empty())
+                .collect();
+            drop_player_repeats(&mut enclosures);
+
+            let alt = e.links().iter().find(|l| l.rel() == "alternate" || l.rel().is_empty()).map(|l| l.href());
+
+            let guid =
+                pick_guid(Some(e.id()), alt, enclosures.first().map(|x| x.url.as_str()), Some(e.title().as_str()))?;
+
+            Some(Entry {
+                guid,
+                title: title_text(Some(e.title().as_str())),
+                link: alt.map(str::to_owned),
+                published: e.published().or(Some(e.updated())).map(|d| d.timestamp()),
+                description: notes(e.content().and_then(|c| c.value()), e.summary().map(|s| s.as_str()), None),
+                markdown: None,
+                categories: e.categories().iter().map(|c| c.term().to_owned()).collect(),
+                explicit: false,
+                image: None,
+                duration: None,
+                episode: None,
+                season: None,
+                enclosures,
+            })
+        })
+        .collect();
+
+    ParsedFeed {
+        title: title_text(Some(feed.title().as_str())),
+        ttl_mins: None,
+        image: feed.logo().or_else(|| feed.icon()).map(str::to_owned),
+        site: feed
+            .links()
+            .iter()
+            .find(|l| l.rel() == "alternate")
+            .map(|l| l.href().trim().to_owned())
+            .filter(|h| !h.is_empty()),
+        category: None,
+        description: non_empty(feed.subtitle().map(|t| t.as_str())),
+        entries,
+    }
+}
+
+/// The original fell back to hashing the title or description. A guid, permalink or
+/// enclosure URL is a stable identifier already, so no hashing is needed; an entry with
+/// none of them has nothing to download and is dropped.
+pub(crate) fn pick_guid(
+    guid: Option<&str>,
+    link: Option<&str>,
+    enclosure: Option<&str>,
+    title: Option<&str>,
+) -> Option<String> {
+    [guid, link, enclosure, title].into_iter().flatten().map(str::trim).find(|s| !s.is_empty()).map(str::to_owned)
+}
+
+pub(crate) fn is_yes(s: &str) -> bool {
+    matches!(s.trim().to_ascii_lowercase().as_str(), "yes" | "true" | "explicit")
+}
+
+pub(crate) fn non_empty(s: Option<&str>) -> Option<String> {
+    s.map(str::trim).filter(|s| !s.is_empty()).map(str::to_owned)
+}
+
+/// WordPress numbers each audio player on a page by adding `?_=N` to its file's URL, so a post
+/// that embeds the file it encloses lists the same file twice: Rands in Repose's "The Promotion
+/// Paradox" was downloaded twice and offered two play buttons for one mp3. The first stays.
+pub(crate) fn drop_player_repeats(encs: &mut Vec<Enclosure>) {
+    let mut seen = HashSet::new();
+    encs.retain(|e| seen.insert(same_file_key(&e.url)));
+}
+
+/// An enclosure URL without WordPress's player number, for telling repeats of one file apart
+/// from different files.
+#[must_use]
+pub fn same_file_key(url: &str) -> String {
+    let Ok(mut u) = url::Url::parse(url) else { return url.to_owned() };
+    let kept: Vec<(String, String)> = u
+        .query_pairs()
+        .filter(|(k, v)| !(k == "_" && !v.is_empty() && v.bytes().all(|b| b.is_ascii_digit())))
+        .map(|(k, v)| (k.into_owned(), v.into_owned()))
+        .collect();
+    if kept.is_empty() {
+        u.set_query(None);
+    } else {
+        u.query_pairs_mut().clear().extend_pairs(kept);
+    }
+    u.to_string()
+}
+
+/// A title as plain text. An Atom title of `type="html"`, or an RSS one in CDATA, comes through
+/// the XML parser with its HTML entities intact: The Verge's "Meta&#8217;s" reached the page as
+/// typed. Decoded one entity at a time, so an `&` that starts none, as in "Q&A", stays as it is
+/// instead of failing the whole title.
+pub(crate) fn title_text(s: Option<&str>) -> Option<String> {
+    let s = non_empty(s)?;
+    let mut out = String::with_capacity(s.len());
+    let mut rest = s.as_str();
+    while let Some(at) = rest.find('&') {
+        out.push_str(&rest[..at]);
+        rest = &rest[at..];
+        let len = 1 + rest[1..].find(|c: char| !(c.is_ascii_alphanumeric() || c == '#')).unwrap_or(rest.len() - 1);
+        let decoded = rest[len..]
+            .starts_with(';')
+            .then(|| quick_xml::escape::unescape_with(&rest[..=len], quick_xml::escape::resolve_html5_entity).ok())
+            .flatten();
+        match decoded {
+            Some(v) => {
+                out.push_str(&v);
+                rest = &rest[len + 1..];
+            }
+            None => {
+                out.push('&');
+                rest = &rest[1..];
+            }
+        }
+    }
+    out.push_str(rest);
+    non_empty(Some(&out))
+}
+
+/// An item's notes as HTML: `body`, with Markdown made HTML, whether the feed wrote its text in
+/// Markdown or sent only a Markdown copy (#284). The page cleans it as it does any feed's HTML.
+pub(crate) fn notes(content: Option<&str>, description: Option<&str>, markdown: Option<&str>) -> Option<String> {
+    match body(content, description) {
+        Some(b) if is_markdown(&b) => Some(markdown_html(&b)),
+        Some(b) => Some(b),
+        None => non_empty(markdown).map(|m| markdown_html(&m)),
+    }
+}
+
+/// `<source:markdown>`, from https://source.scripting.com/: the item's text in Markdown.
+///
+/// shortcut: found by the `source` prefix the namespace's own docs and feeds use; a feed binding
+/// it to another prefix is not read, until one turns up.
+pub(crate) fn source_markdown(item: &rss::Item) -> Option<&str> {
+    item.extensions().get("source")?.get("markdown")?.first()?.value()
+}
+
+/// Text with no HTML tag and some Markdown: a link, a heading, a list, a quote, code or bold.
+/// Plain text without any is left as it was, since rendering it would change nothing but
+/// could still read a line starting "1984." as a list.
+pub(crate) fn is_markdown(s: &str) -> bool {
+    let tag = s
+        .match_indices('<')
+        .any(|(i, _)| s[i + 1..].starts_with(|c: char| c.is_ascii_alphabetic() || c == '/' || c == '!'));
+    let block = |l: &str| ["# ", "## ", "### ", "- ", "* ", "> ", "```"].iter().any(|m| l.trim_start().starts_with(m));
+    !tag && (s.contains("](") || s.contains("**") || s.lines().any(block))
+}
+
+/// Markdown made HTML, uncleaned: whoever shows it cleans it as any feed's HTML.
+#[must_use]
+pub fn markdown_html(md: &str) -> String {
+    use pulldown_cmark::{Options, Parser, html};
+    let mut out = String::new();
+    html::push_html(&mut out, Parser::new_ext(md, Options::ENABLE_TABLES | Options::ENABLE_STRIKETHROUGH));
+    out
+}
+
+/// An item's show notes: its full body when that is whole, else its description.
+///
+/// libsyn served Daily Meditation Podcast's `content:encoded` cut at the `>` inside a class name
+/// pasted from a web app (`[&:has([data-writing-block])>*]:pointer-events-auto`), so the body
+/// began halfway through a tag and the page showed the rest of the tag as text. The same item's
+/// `description` was whole. With no description to fall back on, a damaged body beats none.
+pub(crate) fn body(content: Option<&str>, description: Option<&str>) -> Option<String> {
+    match non_empty(content).filter(|c| !starts_mid_tag(c)) {
+        Some(c) => Some(match subtitle(&c, description) {
+            Some(s) => format!("<p><em>{}</em></p>{c}", quick_xml::escape::escape(s.as_str())),
+            None => c,
+        }),
+        None => non_empty(description).or_else(|| non_empty(content)),
+    }
+}
+
+/// A description that is a subtitle rather than a second copy of the notes: Substack puts the
+/// post's subtitle there and leaves it out of `content:encoded`, so taking the body alone lost it.
+/// Podcast feeds mostly repeat their notes in both, whole or cut short with an ellipsis, and a
+/// description found in the body is not shown twice.
+///
+/// ponytail: short plain text not found in the body. A summary a podcast writes apart from its
+/// notes passes too and shows above them, which reads fine; a real subtitle field would need an
+/// `entries` column.
+pub(crate) fn subtitle(body: &str, description: Option<&str>) -> Option<String> {
+    let d = title_text(description)?;
+    if d.contains('<') || d.chars().count() > 300 {
+        return None;
+    }
+    // Words alone: a tag taken out leaves "tape ," where the description has "tape,", and a cut
+    // description ends in "…" or "[...]".
+    let words = |s: &str| {
+        s.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).collect::<Vec<_>>().join(" ").to_lowercase()
+    };
+    let want = words(&d);
+    let text = title_text(Some(&text_of(body))).unwrap_or_default();
+    (!want.is_empty() && !words(&text).contains(&want)).then_some(d)
+}
+
+/// HTML with its tags taken out, each replaced by a space so words either side stay apart.
+pub(crate) fn text_of(html: &str) -> String {
+    let mut out = String::with_capacity(html.len());
+    let mut in_tag = false;
+    for c in html.chars() {
+        match c {
+            '<' => in_tag = true,
+            '>' if in_tag => {
+                in_tag = false;
+                out.push(' ');
+            }
+            _ if !in_tag => out.push(c),
+            _ => {}
+        }
+    }
+    out
+}
+
+/// Text that closes an attribute list (`">`) before any tag has opened is the tail of a tag whose
+/// start was cut off.
+pub(crate) fn starts_mid_tag(html: &str) -> bool {
+    html[..html.find('<').unwrap_or(html.len())].contains("\">")
+}
+
+/// The picture to show beside an item, in order of how deliberate it is:
+/// `itunes:image`, then Media RSS `media:thumbnail`, then a `media:content` that is an
+/// image, and finally an image enclosure -- which is how a blog's article picture arrives
+/// (Substack puts it there), so those entries get artwork rather than a blank square.
+pub(crate) fn item_image(item: &rss::Item, enclosures: &[Enclosure]) -> Option<String> {
+    if let Some(url) = item.itunes_ext().and_then(|i| i.image()) {
+        return non_empty(Some(url));
+    }
+
+    let media = item.extensions().get("media");
+    let attr = |name: &str, want_image: bool| -> Option<String> {
+        media?.get(name)?.iter().find_map(|e| {
+            if want_image {
+                // media:content carries anything; only take it when it says it is a picture.
+                let is_image = e.attrs.get("type").is_some_and(|t| t.starts_with("image/"))
+                    || e.attrs.get("medium").is_some_and(|m| m == "image");
+                if !is_image {
+                    return None;
+                }
+            }
+            non_empty(e.attrs.get("url").map(String::as_str))
+        })
+    };
+    attr("thumbnail", false).or_else(|| attr("content", true)).or_else(|| {
+        enclosures.iter().find(|e| e.mime.as_deref().is_some_and(|m| m.starts_with("image/"))).map(|e| e.url.clone())
+    })
+}
+
+/// itunes:duration is either plain seconds ("5649") or a clock ("1:34:09", "23:45").
+pub(crate) fn parse_duration(s: &str) -> Option<i64> {
+    let s = s.trim();
+    if s.is_empty() {
+        return None;
+    }
+    if !s.contains(':') {
+        return s.parse().ok().filter(|n| *n > 0);
+    }
+    let mut total: i64 = 0;
+    for part in s.split(':') {
+        total = total * 60 + part.trim().parse::<i64>().ok()?;
+    }
+    Some(total).filter(|n| *n > 0)
+}
+
+/// RSS pubDate is RFC 2822; some feeds ship RFC 3339 instead.
+pub(crate) fn parse_date(s: &str) -> Option<i64> {
+    let s = s.trim();
+    DateTime::parse_from_rfc2822(s).or_else(|_| DateTime::parse_from_rfc3339(s)).ok().map(|d| d.timestamp())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::failure;
+
+    #[test]
+    fn parses_rss_with_itunes_extensions() {
+        let bytes = include_bytes!("../../../tests/data/rss2.xml");
+        let feed = parse(bytes).unwrap();
+
+        assert_eq!(feed.title.as_deref(), Some("Test Cast"));
+        assert_eq!(feed.ttl_mins, Some(45));
+        assert_eq!(feed.category.as_deref(), Some("Podcasting"), "the first, by its subcategory");
+        assert_eq!(feed.description.as_deref(), Some("A synthetic feed used by the parser tests."));
+        assert_eq!(feed.entries.len(), 3);
+
+        let ep = &feed.entries[0];
+        assert_eq!(ep.guid, "https://example.com/ep/1");
+        assert_eq!(ep.title.as_deref(), Some("Episode One"));
+        assert_eq!(ep.published, Some(1_078_016_400));
+        assert_eq!(ep.categories, vec!["Tech"]);
+        assert!(!ep.explicit);
+        assert_eq!(
+            ep.enclosures,
+            vec![Enclosure {
+                url: "https://example.com/ep1.mp3".into(),
+                mime: Some("audio/mpeg".into()),
+                length: Some(12_345_678),
+            }]
+        );
+
+        assert!(feed.entries[1].explicit, "entry-level itunes:explicit=yes");
+        assert_eq!(feed.entries[2].enclosures[0].mime.as_deref(), Some("application/x-bittorrent"));
+    }
+
+    #[test]
+    fn a_feed_says_what_it_is_in_itunes_summary_or_atom_subtitle() {
+        // An empty <description> gives way to iTunes' summary (#130).
+        let rss = br#"<?xml version="1.0"?><rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
+        <channel><title>R</title><description> </description><itunes:summary>Said here.</itunes:summary></channel></rss>"#;
+        assert_eq!(parse(rss).unwrap().description.as_deref(), Some("Said here."));
+        let atom = br#"<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom"><title>A</title>
+        <subtitle>Or here.</subtitle><id>urn:a</id><updated>2026-01-01T00:00:00Z</updated></feed>"#;
+        assert_eq!(parse(atom).unwrap().description.as_deref(), Some("Or here."));
+    }
+
+    #[test]
+    fn markdown_notes_are_made_html_and_html_is_left_alone() {
+        let feed = |items: &str| {
+            let xml = format!(
+                r#"<?xml version="1.0"?><rss version="2.0" xmlns:source="https://source.scripting.com/"><channel><title>R</title>{items}</channel></rss>"#
+            );
+            parse(xml.as_bytes()).unwrap().entries.into_iter().map(|e| e.description).collect::<Vec<_>>()
+        };
+        let got = feed(
+            r#"<item><guid>a</guid><description># Hi
+
+Some **bold** and [a link](https://x.example/).</description></item>
+            <item><guid>b</guid><source:markdown>Only *Markdown* here.</source:markdown></item>
+            <item><guid>c</guid><description>&lt;p&gt;HTML **as sent**&lt;/p&gt;</description><source:markdown>ignored</source:markdown></item>
+            <item><guid>d</guid><description>1984. Plain text, left as it is.</description></item>"#,
+        );
+        assert_eq!(
+            got[0].as_deref(),
+            Some("<h1>Hi</h1>\n<p>Some <strong>bold</strong> and <a href=\"https://x.example/\">a link</a>.</p>\n")
+        );
+        assert_eq!(got[1].as_deref(), Some("<p>Only <em>Markdown</em> here.</p>\n"), "the Markdown copy, alone");
+        assert_eq!(got[2].as_deref(), Some("<p>HTML **as sent**</p>"), "HTML wins, untouched");
+        assert_eq!(got[3].as_deref(), Some("1984. Plain text, left as it is."));
+    }
+
+    #[test]
+    fn markdown_sent_beside_html_is_kept_for_the_page_to_switch_to() {
+        let xml = br#"<?xml version="1.0"?><rss version="2.0" xmlns:source="https://source.scripting.com/"><channel><title>R</title>
+        <item><guid>both</guid><description>&lt;p&gt;Hi&lt;/p&gt;</description><source:markdown>*Hi*</source:markdown></item>
+        <item><guid>alone</guid><source:markdown>*Hi*</source:markdown></item></channel></rss>"#;
+        let got: Vec<_> = parse(xml).unwrap().entries.into_iter().map(|e| e.markdown).collect();
+        assert_eq!(got, [Some("*Hi*".to_owned()), None], "only beside HTML; alone, it is the description (#284)");
+    }
+
+    #[test]
+    fn a_file_wordpress_lists_twice_is_one_enclosure() {
+        let xml = br#"<?xml version="1.0"?><rss version="2.0"><channel><title>R</title>
+        <item><title>The Promotion Paradox</title><guid>p</guid>
+        <enclosure url="https://x/ep.mp3" length="1" type="audio/mpeg"/>
+        <enclosure url="https://x/ep.mp3?_=2" length="1" type="audio/mpeg"/>
+        <enclosure url="https://x/other.mp3?_=3&amp;key=k" length="1" type="audio/mpeg"/>
+        </item></channel></rss>"#;
+        let urls: Vec<String> = parse(xml).unwrap().entries[0].enclosures.iter().map(|e| e.url.clone()).collect();
+        assert_eq!(
+            urls,
+            ["https://x/ep.mp3", "https://x/other.mp3?_=3&key=k"],
+            "the repeat goes, a different file stays"
+        );
+        assert_eq!(same_file_key("https://x/a.mp3?key=k&_=2"), same_file_key("https://x/a.mp3?key=k"));
+        assert_ne!(same_file_key("https://x/a.mp3?_=x"), same_file_key("https://x/a.mp3"), "only a number");
+    }
+
+    #[test]
+    fn titles_are_read_as_text_not_html() {
+        // The Verge: an Atom title of type="html", its entity inside CDATA.
+        let xml = br#"<?xml version="1.0"?>
+        <feed xmlns="http://www.w3.org/2005/Atom"><title type="text">V</title><id>v</id>
+        <updated>2026-09-15T00:00:00Z</updated>
+        <entry><title type="html"><![CDATA[Meta&#8217;s new One]]></title><id>e1</id>
+        <updated>2026-09-15T00:00:00Z</updated></entry></feed>"#;
+        assert_eq!(parse(xml).unwrap().entries[0].title.as_deref(), Some("Meta\u{2019}s new One"));
+        // HTML names as well as numbers; a bare `&` and an unknown name are left as they are.
+        assert_eq!(
+            title_text(Some("Pe&ntilde;a &amp; &#x201C;Q&A&#8221; &bogus; AT&T;")).as_deref(),
+            Some("Pe\u{f1}a & \u{201c}Q&A\u{201d} &bogus; AT&T;")
+        );
+    }
+
+    #[test]
+    fn a_body_cut_off_mid_tag_gives_way_to_the_description() {
+        // How libsyn served Daily Meditation Podcast #3477: content:encoded began inside a tag.
+        let cut = r#"*]:pointer-events-auto R6Vx5W_threadScrollVars" dir="auto" data-turn="assistant"> <p>What if</p>"#;
+        let whole = r#"<div class="[&:has([data-writing-block])>*]:pointer-events-auto"><p>What if</p></div>"#;
+        assert_eq!(body(Some(cut), Some(whole)).as_deref(), Some(whole));
+        assert_eq!(
+            body(Some("<p>Notes</p>"), Some("<p>Notes</p>")).as_deref(),
+            Some("<p>Notes</p>"),
+            "a whole body wins"
+        );
+        assert_eq!(
+            body(Some("Plain notes, no tags."), Some("Plain notes, no tags.")).as_deref(),
+            Some("Plain notes, no tags.")
+        );
+        assert_eq!(body(Some(cut), None).as_deref(), Some(cut), "a damaged body beats none");
+        assert_eq!(body(None, Some("Summary")).as_deref(), Some("Summary"));
+    }
+
+    #[test]
+    fn a_subtitle_missing_from_the_body_is_kept_above_it() {
+        // Substack: the subtitle is the description, and content:encoded does not repeat it.
+        assert_eq!(
+            body(Some("<p>The post.</p>"), Some("Why the <b> tag & I fell out")).as_deref(),
+            Some("<p>The post.</p>"),
+            "a description with markup in it is notes, not a subtitle",
+        );
+        assert_eq!(
+            body(Some("<p>The post.</p>"), Some("Why Q&amp;A threads go wrong")).as_deref(),
+            Some("<p><em>Why Q&amp;A threads go wrong</em></p><p>The post.</p>"),
+        );
+        // A podcast repeating its notes, whole, cut short, or differently spaced: shown once.
+        let notes = "<p>This week we talk about <a href=\"x\">tape</a>, drums and a very long list.</p>";
+        for d in [
+            "This week we talk about tape, drums and a very long list.",
+            "This week we talk about tape, drums…",
+            "This week we  talk about\ntape [...]",
+        ] {
+            assert_eq!(body(Some(notes), Some(d)).as_deref(), Some(notes), "{d:?} is already in the body");
+        }
+        let long = "word ".repeat(80);
+        assert_eq!(
+            body(Some("<p>The post.</p>"), Some(&long)).as_deref(),
+            Some("<p>The post.</p>"),
+            "a long description is notes, not a subtitle"
+        );
+    }
+
+    #[test]
+    fn feed_level_explicit_overrides_entries() {
+        let xml = br#"<?xml version="1.0"?>
+        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
+        <channel><title>X</title><link>https://x</link><description>d</description>
+        <itunes:explicit>yes</itunes:explicit>
+        <item><title>a</title><guid>g1</guid>
+        <enclosure url="https://x/a.mp3" length="1" type="audio/mpeg"/></item>
+        </channel></rss>"#;
+        let feed = parse(xml).unwrap();
+        assert!(feed.entries[0].explicit, "the entry says nothing; the feed-level flag must still mark it explicit");
+    }
+
+    #[test]
+    fn failures_are_named_by_kind_for_the_log() {
+        let k = |m: &str| failure::failure_kind(m);
+        assert_eq!(k("HTTP 404 Not Found"), ("404".into(), Some(404)));
+        assert_eq!(k("HTTP 503 Service Unavailable"), ("503".into(), Some(503)));
+        assert_eq!(
+            k("connecting: error following redirect for url (https://www.toddstashwick.com/): too many redirects").0,
+            "redirect_loop"
+        );
+        assert_eq!(k("connecting: dns error: failed to lookup address information").0, "dns");
+        assert_eq!(k("operation timed out").0, "timeout");
+        assert_eq!(k("something new").0, "other");
+    }
+
+    #[test]
+    fn a_web_page_says_so_and_names_the_feed_it_links() {
+        let html = br#"<!doctype html><html><head>
+            <link rel="alternate" type="application/rss+xml" href="https://x.example/feed">
+            </head><body>not a feed</body></html>"#;
+        let err = parse(html).unwrap_err().to_string();
+        assert_eq!(err, "got a web page, not a feed; it links https://x.example/feed as its feed");
+    }
+
+    #[test]
+    fn a_web_page_with_no_feed_link_still_says_so() {
+        let html = b"<!doctype html><html><body>moved</body></html>";
+        assert_eq!(parse(html).unwrap_err().to_string(), "got a web page, not a feed");
+    }
+
+    #[test]
+    fn malformed_xml_gets_the_original_parser_errors() {
+        let err = parse(b"<rss><channel><title>cut off").unwrap_err().to_string();
+        assert!(err.starts_with("not RSS ("), "{err}");
+    }
+
+    #[test]
+    fn a_body_with_no_markup_says_what_the_site_sent() {
+        let err = parse(b"\xef\xbb\xbf\r\n  Unable to establish a DB connection\nmore").unwrap_err().to_string();
+        assert_eq!(err, "the site sent \"Unable to establish a DB connection\" instead of a feed");
+        let long = parse("x".repeat(200).as_bytes()).unwrap_err().to_string();
+        assert_eq!(long, format!("the site sent \"{}…\" instead of a feed", "x".repeat(80)));
+        assert_eq!(parse(b" \n").unwrap_err().to_string(), "the site sent an empty reply instead of a feed");
+    }
+
+    #[test]
+    fn parses_atom_enclosure_links() {
+        let bytes = include_bytes!("../../../tests/data/atom.xml");
+        let feed = parse(bytes).unwrap();
+
+        assert_eq!(feed.title.as_deref(), Some("Atom Cast"));
+        assert_eq!(feed.entries.len(), 1);
+        let ep = &feed.entries[0];
+        assert_eq!(ep.guid, "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a");
+        assert_eq!(ep.link.as_deref(), Some("https://example.org/ep/1"));
+        assert_eq!(ep.published, Some(1_078_016_400));
+        assert_eq!(
+            ep.enclosures,
+            vec![Enclosure {
+                url: "https://example.org/ep1.m4a".into(),
+                mime: Some("audio/mp4".into()),
+                length: Some(9_876_543),
+            }],
+            "rel=enclosure only; the alternate link must not become an enclosure"
+        );
+    }
+
+    #[test]
+    fn a_bare_ampersand_in_a_link_is_repaired_and_parsed() {
+        // kcpw.org: <link>https://kcpw.org/?post_type=post&p=125715</link> -- a bare "&"
+        // that strict XML rejects but browsers accept.
+        let xml = br#"<?xml version="1.0"?>
+        <rss version="2.0"><channel><title>X</title><link>https://x</link><description>d</description>
+        <item><title>a</title><guid>g1</guid>
+        <link>https://kcpw.org/?post_type=post&p=125715</link>
+        <enclosure url="https://x/a.mp3?a=1&b=2" length="1" type="audio/mpeg"/></item>
+        </channel></rss>"#;
+        let feed = parse(xml).unwrap();
+        assert_eq!(feed.entries[0].link.as_deref(), Some("https://kcpw.org/?post_type=post&p=125715"));
+        assert_eq!(feed.entries[0].enclosures[0].url, "https://x/a.mp3?a=1&b=2");
+    }
+
+    #[test]
+    fn escape_bare_ampersands_leaves_real_entities_alone() {
+        let out = escape_bare_ampersands(b"a&amp;b &lt;x&gt; &#39; &#x2F; c&d");
+        assert_eq!(out, b"a&amp;b &lt;x&gt; &#39; &#x2F; c&amp;d");
+    }
+
+    #[test]
+    fn the_rss_title_always_wins_and_episode_numbers_stay_metadata() {
+        // Some feeds set a different itunes:title. The displayed title is always the RSS
+        // <title>, verbatim -- separators and all -- and season/episode are stored
+        // alongside it rather than folded into it.
+        let xml = br#"<?xml version="1.0"?>
+        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
+        <channel><title>Show</title><link>https://x</link><description>d</description>
+        <item>
+          <title>Music from a Darkened Room | Session Zero</title>
+          <itunes:title>Session Zero</itunes:title>
+          <guid>sz</guid>
+          <itunes:season>8</itunes:season>
+          <itunes:duration>6720</itunes:duration>
+          <enclosure url="https://x/sz.mp3" length="1" type="audio/mpeg"/>
+        </item>
+        <item>
+          <title>Music from a Darkened Room Part 1 | Murphy's Drawer</title>
+          <guid>p1</guid>
+          <itunes:season>8</itunes:season><itunes:episode>1</itunes:episode>
+          <enclosure url="https://x/p1.mp3" length="1" type="audio/mpeg"/>
+        </item>
+        </channel></rss>"#;
+        let feed = parse(xml).unwrap();
+
+        let sz = &feed.entries[0];
+        assert_eq!(
+            sz.title.as_deref(),
+            Some("Music from a Darkened Room | Session Zero"),
+            "itunes:title must not override the RSS title"
+        );
+        assert_eq!(sz.season, Some(8));
+        assert_eq!(sz.episode, None, "a missing episode number stays missing");
+        assert_eq!(sz.duration, Some(6720));
+
+        let p1 = &feed.entries[1];
+        assert_eq!(p1.title.as_deref(), Some("Music from a Darkened Room Part 1 | Murphy's Drawer"));
+        assert_eq!((p1.season, p1.episode), (Some(8), Some(1)));
+    }
+
+    #[test]
+    fn an_item_may_carry_several_enclosures() {
+        // The rss crate keeps only one per item -- the last -- so these come from the XML.
+        let xml = br#"<?xml version="1.0"?>
+        <rss version="2.0"><channel><title>M</title><link>https://x</link><description>d</description>
+        <item><title>Two files</title><guid>m1</guid>
+          <enclosure url="https://x/a.mp3?v=1&amp;t=2" length="111" type="audio/mpeg"/>
+          <enclosure url="https://x/b.mp4" length="222" type="video/mp4"/>
+        </item>
+        <item><title>One file</title><guid>m2</guid>
+          <enclosure url="https://x/c.mp3" length="333" type="audio/mpeg"/></item>
+        <item><title>None</title><guid>m3</guid></item>
+        </channel></rss>"#;
+        let f = parse(xml).unwrap();
+        assert_eq!(f.entries.len(), 3);
+
+        let two = &f.entries[0].enclosures;
+        assert_eq!(two.len(), 2, "both enclosures survive");
+        assert_eq!(two[0].url, "https://x/a.mp3?v=1&t=2", "document order, and the escaped ampersand is decoded");
+        assert_eq!(two[0].length, Some(111));
+        assert_eq!(two[1].url, "https://x/b.mp4");
+        assert_eq!(two[1].mime.as_deref(), Some("video/mp4"));
+
+        assert_eq!(f.entries[1].enclosures.len(), 1);
+        assert_eq!(f.entries[2].enclosures.len(), 0, "an item may have none");
+    }
+
+    #[test]
+    fn an_items_picture_comes_from_the_most_deliberate_source() {
+        let xml = br#"<?xml version="1.0"?>
+        <rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"
+             xmlns:media="http://search.yahoo.com/mrss/">
+        <channel><title>P</title><link>https://x</link><description>d</description>
+        <item><title>Has itunes</title><guid>a</guid>
+          <itunes:image href="https://x/itunes.jpg"/>
+          <media:thumbnail url="https://x/thumb.jpg"/>
+          <enclosure url="https://x/a.jpg" length="1" type="image/jpeg"/></item>
+        <item><title>Has thumbnail</title><guid>b</guid>
+          <media:thumbnail url="https://x/thumb.jpg"/>
+          <enclosure url="https://x/b.jpg" length="1" type="image/jpeg"/></item>
+        <item><title>Has media content</title><guid>c</guid>
+          <media:content url="https://x/pic.jpg" type="image/jpeg"/>
+          <media:content url="https://x/clip.mp4" type="video/mp4"/></item>
+        <item><title>Only an image enclosure</title><guid>d</guid>
+          <enclosure url="https://x/d.jpg" length="1" type="image/jpeg"/></item>
+        <item><title>Audio only</title><guid>e</guid>
+          <enclosure url="https://x/e.mp3" length="1" type="audio/mpeg"/></item>
+        </channel></rss>"#;
+        let f = parse(xml).unwrap();
+        let img = |i: usize| f.entries[i].image.as_deref();
+        assert_eq!(img(0), Some("https://x/itunes.jpg"), "itunes:image wins");
+        assert_eq!(img(1), Some("https://x/thumb.jpg"), "then media:thumbnail");
+        assert_eq!(img(2), Some("https://x/pic.jpg"), "media:content, and only the image one");
+        assert_eq!(img(3), Some("https://x/d.jpg"), "a blog's article picture arrives as an enclosure");
+        assert_eq!(img(4), None, "audio is not a picture");
+    }
+
+    #[test]
+    fn durations_parse_from_seconds_or_a_clock() {
+        assert_eq!(parse_duration("5649"), Some(5649));
+        assert_eq!(parse_duration("23:45"), Some(1425));
+        assert_eq!(parse_duration("1:34:09"), Some(5649));
+        assert_eq!(parse_duration("0"), None, "zero is not a duration");
+        assert_eq!(parse_duration(""), None);
+        assert_eq!(parse_duration("garbage"), None);
+    }
+
+    #[test]
+    fn rejects_html_masquerading_as_a_feed() {
+        assert!(parse(b"<html><body>nope</body></html>").is_err());
+    }
+
+    #[test]
+    fn guid_falls_back_through_link_then_enclosure() {
+        assert_eq!(pick_guid(Some(" "), Some("l"), Some("e"), None).as_deref(), Some("l"));
+        assert_eq!(pick_guid(None, None, Some("e"), Some("t")).as_deref(), Some("e"));
+        assert_eq!(pick_guid(None, None, None, None), None);
+    }
+}

crates/feed/src/patreon.rs

@@ -0,0 +1,147 @@
+//! Patreon: a creator's feed, its shows, and when two Patreon addresses are the same feed.
+
+use crate::fetch;
+use anyhow::{Context, Result, anyhow};
+
+/// The token and show of a Patreon feed link, or None for any other URL.
+///
+/// Patreon gives each patron one token per creator. With no show it stands for the creator,
+/// whose feed carries every show at once.
+pub(crate) fn patreon_parts(url: &str) -> Option<(String, Option<String>)> {
+    let u = url::Url::parse(url).ok()?;
+    if !matches!(u.host_str()?, "patreon.com" | "www.patreon.com") || !u.path().starts_with("/rss") {
+        return None;
+    }
+    let param = |name: &str| u.query_pairs().find(|(k, _)| k == name).map(|(_, v)| v.into_owned());
+    Some((param("auth")?, param("show")))
+}
+
+/// A Patreon link naming a creator but no show.
+#[must_use]
+pub fn is_patreon_creator(url: &str) -> bool {
+    matches!(patreon_parts(url), Some((_, None)))
+}
+
+/// Whether two URLs are the same feed. One Patreon show has several spellings -- by the
+/// creator's name, by number, or with no creator at all -- and the token and show are what
+/// identify it.
+#[must_use]
+pub fn same_feed(a: &str, b: &str) -> bool {
+    // Parsing both is the slow part, and only Patreon's addresses can differ and still match.
+    a == b
+        || (a.contains("patreon.com")
+            && b.contains("patreon.com")
+            && patreon_parts(a).is_some_and(|p| Some(p) == patreon_parts(b)))
+}
+
+/// A Patreon creator's name and shows, each show as (title, feed URL).
+///
+/// ponytail: Patreon's own web API, undocumented, asked without signing in. If it changes,
+/// finding shows stops and the show feeds already found keep working. The documented API
+/// needs an OAuth client per install and does not list shows.
+pub async fn patreon_shows(client: &reqwest::Client, url: &str) -> Result<(Option<String>, Vec<(String, String)>)> {
+    // The creator feed names its campaign by number in its self link, a few hundred bytes in.
+    // The whole feed runs to megabytes and Patreon ignores Range, so read until it turns up.
+    let mut resp = client.get(url).timeout(fetch::FEED_TIMEOUT).send().await.context("connecting")?;
+    if !resp.status().is_success() {
+        return Err(anyhow!("Patreon refused the feed: HTTP {}", resp.status()));
+    }
+    let mut head = Vec::new();
+    while patreon_campaign(&head).is_none() && head.len() < 64 * 1024 {
+        let Some(chunk) = resp.chunk().await.context("reading the feed")? else { break };
+        head.extend_from_slice(&chunk);
+    }
+    let campaign = patreon_campaign(&head).ok_or_else(|| anyhow!("the Patreon feed does not say whose it is"))?;
+
+    let api = format!(
+        "https://www.patreon.com/api/campaigns/{campaign}\
+         ?include=shows&fields%5Bcampaign%5D=name&fields%5Bcollection%5D=title"
+    );
+    let resp = client.get(api).send().await.context("asking Patreon for the shows")?;
+    if !resp.status().is_success() {
+        return Err(anyhow!("Patreon would not list the shows: HTTP {}", resp.status()));
+    }
+    let (name, shows) = parse_patreon_shows(&resp.bytes().await.context("reading the shows")?)?;
+    Ok((name, shows.into_iter().map(|(id, title)| (title, format!("{url}&show={id}"))).collect()))
+}
+
+/// The campaign number in the start of a Patreon feed.
+pub(crate) fn patreon_campaign(head: &[u8]) -> Option<String> {
+    let text = String::from_utf8_lossy(head);
+    text.match_indices("patreon.com/rss/").find_map(|(i, m)| {
+        let id: String = text[i + m.len()..].chars().take_while(char::is_ascii_digit).collect();
+        (!id.is_empty()).then_some(id)
+    })
+}
+
+/// A campaign's name, and its shows as (id, title).
+pub(crate) type PatreonShows = (Option<String>, Vec<(String, String)>);
+
+/// A campaign's name and its shows as (id, title), from Patreon's JSON:API answer.
+pub(crate) fn parse_patreon_shows(json: &[u8]) -> Result<PatreonShows> {
+    let v: serde_json::Value = serde_json::from_slice(json).context("Patreon's answer is not JSON")?;
+    // Missing is not the same as none. Read as no shows, the creator feed would be scanned as
+    // a plain feed, claim every show's files, and leave the shows empty once the list returned.
+    let ids = v["data"]["relationships"]["shows"]["data"]
+        .as_array()
+        .ok_or_else(|| anyhow!("Patreon's answer does not list the shows"))?;
+    let title = |id: &str| -> Option<String> {
+        let show = v["included"].as_array()?.iter().find(|x| x["type"] == "collection" && x["id"] == id)?;
+        show["attributes"]["title"].as_str().map(|t| t.trim().to_owned())
+    };
+    let shows = ids
+        .iter()
+        .filter_map(|s| s["id"].as_str())
+        .map(|id| (id.to_owned(), title(id).unwrap_or_else(|| format!("Show {id}"))))
+        .collect();
+    Ok((v["data"]["attributes"]["name"].as_str().map(str::to_owned), shows))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::discover;
+
+    #[test]
+    fn a_patreon_creator_is_a_list_of_its_shows() {
+        let tok = "AbCdEfGhIjKlMnOpQrStUvWxYz012_-9";
+        assert_eq!(discover::expand_input(&format!(" {tok} ")), format!("https://www.patreon.com/rss?auth={tok}"));
+        assert_eq!(discover::expand_input("https://example.com/rss"), "https://example.com/rss");
+        assert_eq!(discover::expand_input("http://example.com/rss"), "http://example.com/rss");
+        assert_eq!(discover::expand_input(" cnn.com "), "https://cnn.com");
+        assert_eq!(discover::expand_input("example.com/feed.xml"), "https://example.com/feed.xml");
+        assert_eq!(discover::expand_input("//example.com/rss"), "https://example.com/rss");
+
+        assert!(is_patreon_creator(&format!("https://www.patreon.com/rss/glasscannon?auth={tok}")));
+        assert!(is_patreon_creator(&format!("https://www.patreon.com/rss?auth={tok}")));
+        assert!(!is_patreon_creator(&format!("https://www.patreon.com/rss/x?auth={tok}&show=1")), "one show is a feed");
+        assert!(!is_patreon_creator(&format!("https://example.com/rss?auth={tok}")));
+
+        // The show you already have by name is the one a bare token would add by number.
+        assert!(same_feed(
+            &format!("https://www.patreon.com/rss/glasscannon?auth={tok}&show=2073588"),
+            &format!("https://www.patreon.com/rss?auth={tok}&show=2073588"),
+        ));
+        assert!(!same_feed(
+            &format!("https://www.patreon.com/rss?auth={tok}&show=1"),
+            &format!("https://www.patreon.com/rss?auth={tok}&show=2"),
+        ));
+
+        // The self link carries the campaign by number, whichever spelling was asked for.
+        let head = br#"<rss><channel><link>https://www.patreon.com/glasscannon</link>
+            <atom:link href="https://www.patreon.com/rss/369921?auth=t" rel="self"/>"#;
+        assert_eq!(patreon_campaign(head).as_deref(), Some("369921"));
+        assert_eq!(patreon_campaign(b"<rss><channel><title>T"), None);
+
+        let json = br#"{"data":{"id":"369921","type":"campaign","attributes":{"name":"The Glass Cannon Network"},
+            "relationships":{"shows":{"data":[{"id":"2073588","type":"collection"},{"id":"2073636","type":"collection"}]}}},
+            "included":[{"id":"2073588","type":"collection","attributes":{"title":"Get in the Trunk "}},
+                        {"id":"2073636","type":"collection","attributes":{"title":"Shadowdark"}}]}"#;
+        let (name, shows) = parse_patreon_shows(json).unwrap();
+        assert_eq!(name.as_deref(), Some("The Glass Cannon Network"));
+        assert_eq!(shows, [("2073588".into(), "Get in the Trunk".into()), ("2073636".into(), "Shadowdark".into())]);
+
+        // An answer that stops naming the shows is an error, never "this creator has none".
+        assert!(parse_patreon_shows(br#"{"data":{"attributes":{"name":"X"}}}"#).is_err());
+    }
+}

crates/feed/src/privacy.rs

@@ -0,0 +1,105 @@
+//! Addresses that carry someone's access: taking it out of what is logged, and keeping such feeds private.
+
+/// Conditional GET, following redirects itself: `client` must follow none (`feed_client`), so
+/// each hop is seen. The second value is where the feed now is when every hop said so for good
+/// (301 or 308): a publisher that moved its feed, which the catalogue should follow rather
+/// than be redirected on every read. A temporary redirect (302, 307) moves nothing.
+/// `system-proxy` reads http_proxy/https_proxy.
+/// `auth` is a username and password, sent only to the feed's own host.
+/// An address as a trace may keep it: without its query or user and password, where a private
+/// feed carries its credential (Patreon's `?auth=`), which Tempo kept for anyone reading it (#225).
+/// shortcut: a token in the path (`/feed/<token>.xml`) stays; strip paths too if a feed does that.
+#[must_use]
+pub fn redact(url: &str) -> String {
+    match url::Url::parse(url) {
+        Ok(mut u) => {
+            u.set_query(None);
+            u.set_fragment(None);
+            let _ = u.set_username("");
+            let _ = u.set_password(None);
+            u.to_string()
+        }
+        // Not an address ipx could fetch; keep what comes before anything that could be a query.
+        Err(_) => url.split(['?', '#', '@']).next().unwrap_or_default().to_owned(),
+    }
+}
+
+/// Text with each address in it redacted, for what is logged and sent to every open page: an
+/// error from reqwest names the whole address it fetched, a private feed's key with it (#238).
+#[must_use]
+pub fn redact_text(text: &str) -> String {
+    let mut out = String::with_capacity(text.len());
+    let mut rest = text;
+    while let Some(i) = rest.find("http://").into_iter().chain(rest.find("https://")).min() {
+        out.push_str(&rest[..i]);
+        let end = rest[i..].find(|c: char| c.is_whitespace() || "()<>\"'".contains(c)).map_or(rest.len(), |n| i + n);
+        out.push_str(&redact(&rest[i..end]));
+        rest = &rest[end..];
+    }
+    out.push_str(rest);
+    out
+}
+
+/// An address that carries a subscriber's own access: a paid-feed host, credentials in the URL,
+/// or a key in the query. ipx leaves such a feed out of anything others see, and the directory
+/// refuses one sent to it (#202).
+///
+/// ponytail: a heuristic. A key hidden in the path of a host not in `PAID_HOSTS` gets
+/// through; a per-feed private flag is the upgrade (#205).
+#[must_use]
+pub fn url_looks_private(feed_url: &str) -> bool {
+    let Ok(u) = url::Url::parse(feed_url) else { return true };
+    // Paid-feed services put the subscriber's key in the path as often as in the query, and a
+    // long path segment alone proves nothing (acast's public show ids look the same). So a
+    // feed from one of them is private whatever its URL looks like. Supercast is why this
+    // exists: `feeds.supercast.com/feeds/<key>` reached the directory before it did.
+    let host = u.host_str().unwrap_or("");
+    PAID_HOSTS.iter().any(|h| host == *h || host.ends_with(&format!(".{h}")))
+        || !u.username().is_empty()
+        || u.password().is_some()
+        || u.query_pairs().any(|(k, _)| {
+            let k = k.to_ascii_lowercase();
+            ["auth", "token", "key", "secret", "pass", "sig", "session", "user", "uid"].iter().any(|w| k.contains(w))
+        })
+}
+
+/// Services whose feeds are always one subscriber's own.
+pub(crate) const PAID_HOSTS: &[&str] =
+    &["patreon.com", "supercast.com", "supportingcast.fm", "glow.fm", "memberful.com"];
+
+#[cfg(test)]
+mod tests {
+
+    #[test]
+    fn redact_drops_what_carries_a_credential() {
+        assert_eq!(super::redact("https://www.patreon.com/rss/x?auth=SECRET&show=1"), "https://www.patreon.com/rss/x");
+        assert_eq!(super::redact("https://me:pw@example.com/feed.xml#top"), "https://example.com/feed.xml");
+        assert_eq!(super::redact("https://example.com/feed.xml"), "https://example.com/feed.xml");
+        assert_eq!(super::redact("not a url?token=SECRET"), "not a url");
+        // As reqwest words a failure, and with two addresses and none.
+        assert_eq!(
+            super::redact_text(
+                "connecting: error sending request for url (https://www.patreon.com/rss/x?auth=SECRET&show=1): operation timed out"
+            ),
+            "connecting: error sending request for url (https://www.patreon.com/rss/x): operation timed out"
+        );
+        assert_eq!(super::redact_text("from http://a/f?k=1 to https://u:p@b/g"), "from http://a/f to https://b/g");
+        assert_eq!(super::redact_text("HTTP 404 Not Found"), "HTTP 404 Not Found");
+    }
+
+    #[test]
+    fn an_address_carrying_someones_access_looks_private() {
+        let private = super::url_looks_private;
+        assert!(!private("https://feeds.twit.tv/twit.xml"));
+        // Patreon's shape: the key is a query parameter.
+        assert!(private("https://www.patreon.com/rss/x?auth=abc123&show=2073588"));
+        assert!(private("https://example.com/rss?api_key=abc"));
+        // Paid-feed services put the key in the path; the host gives them away.
+        assert!(private("https://feeds.supercast.com/feeds/abcdefghijklmnopqrstuvwx"));
+        assert!(private("https://someshow.supportingcast.fm/content/abc123.rss"));
+        // A long path segment alone is not a key: acast's public show ids look the same.
+        assert!(!private("https://feeds.acast.com/public/shows/0123456789abcdef01234567"));
+        assert!(private("https://ray:hunter2@example.com/rss"));
+        assert!(private("not a url"), "unparseable is not safe to list");
+    }
+}

crates/feed/src/wordpress.rs

@@ -0,0 +1,86 @@
+//! WordPress.com sites that answer their feed with a browser check, read from WordPress.com's API instead (#317).
+
+use crate::fetch::Fetched;
+use crate::parse::{Entry, ParsedFeed};
+use crate::{fetch, parse};
+use anyhow::{Context, Result};
+use chrono::NaiveDateTime;
+
+/// WordPress.com's browser check, in place of the feed it was asked for.
+pub(crate) fn is_wpcom_challenge(body: &[u8]) -> bool {
+    String::from_utf8_lossy(&body[..body.len().min(4096)]).contains("/__challenge/")
+}
+
+/// A WordPress.com site's latest posts, from WordPress.com's API, which `parse` reads as a feed.
+/// shortcut: the API's default page of ten posts, as many as WordPress puts in a feed; a feed
+/// set to carry more gets ten here.
+pub(crate) async fn wpcom_posts(client: &reqwest::Client, host: &str) -> Result<Fetched> {
+    let api = format!("https://public-api.wordpress.com/wp/v2/sites/{host}/posts");
+    let resp = client
+        .get(&api)
+        .timeout(fetch::FEED_TIMEOUT)
+        .send()
+        .await
+        .context("HTTP 403 Forbidden, a browser check; connecting to WordPress.com's API in its place")?;
+    anyhow::ensure!(
+        resp.status().is_success(),
+        "HTTP 403 Forbidden, a browser check, and WordPress.com's API in its place answered HTTP {}",
+        resp.status()
+    );
+    let bytes = fetch::read_capped(resp).await.context("reading WordPress.com's API")?;
+    fetch::FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+    fetch::FETCHED_BYTES.fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed);
+    Ok(Fetched::Body { bytes, etag: None, last_modified: None })
+}
+
+/// WordPress's posts as its REST API sends them (`wpcom_posts`). The guid is the one the site's
+/// own feed gives each post, so they are the items already stored. No title for the feed: the
+/// stored one stays.
+pub(crate) fn from_wp_posts(posts: &[serde_json::Value]) -> ParsedFeed {
+    let entries = posts
+        .iter()
+        .filter_map(|p| {
+            Some(Entry {
+                guid: parse::non_empty(p["guid"]["rendered"].as_str())
+                    .or_else(|| parse::non_empty(p["link"].as_str()))?,
+                title: parse::title_text(p["title"]["rendered"].as_str()),
+                link: parse::non_empty(p["link"].as_str()),
+                published: p["date_gmt"]
+                    .as_str()
+                    .and_then(|d| NaiveDateTime::parse_from_str(d, "%Y-%m-%dT%H:%M:%S").ok())
+                    .map(|d| d.and_utc().timestamp()),
+                description: parse::non_empty(p["content"]["rendered"].as_str()),
+                image: parse::non_empty(p["jetpack_featured_media_url"].as_str()),
+                ..Default::default()
+            })
+        })
+        .collect();
+    ParsedFeed { entries, ..Default::default() }
+}
+
+#[cfg(test)]
+mod tests {
+
+    #[test]
+    fn reads_wordpress_posts_as_a_feed() {
+        // As WordPress.com's API sends them, trimmed (#317).
+        let json = br#"[{"id":90248260,"date_gmt":"2026-10-09T13:34:00",
+        "guid":{"rendered":"https:\/\/pelgranepress.com\/?p=90248260"},
+        "link":"https:\/\/pelgranepress.com\/2026\/10\/09\/whisky\/",
+        "title":{"rendered":"Baby&#8217;s First Whisky"},"content":{"rendered":"<p>Notes<\/p>"},
+        "jetpack_featured_media_url":""}]"#;
+        let feed = super::parse(json).expect("WordPress posts parse as a feed");
+        assert_eq!(feed.title, None);
+        let e = &feed.entries[0];
+        assert_eq!(e.guid, "https://pelgranepress.com/?p=90248260");
+        assert_eq!(e.title.as_deref(), Some("Baby\u{2019}s First Whisky"));
+        assert_eq!(e.link.as_deref(), Some("https://pelgranepress.com/2026/10/09/whisky/"));
+        assert_eq!(e.published, Some(1_791_552_840));
+        assert_eq!(e.description.as_deref(), Some("<p>Notes</p>"));
+        assert_eq!(e.image, None);
+        assert!(super::is_wpcom_challenge(
+            b"<!DOCTYPE html><title>Checking your browser...</title><script src=\"/__challenge/probes.js\">"
+        ));
+        assert!(!super::is_wpcom_challenge(b"<html>Forbidden</html>"));
+    }
+}

crates/ipx/CHANGELOG.md

@@ -12,6 +12,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.5.1] - 2026-10-24
+
+### Changed
+
+- Log lines about starting, stopping, accounts and the configuration name the part of iPX that wrote them in `target` (`ipx::daemon`, `ipx::cli`, `ipx::catalogue`, ...), where they all said `ipx` (#356).
+
+### Fixed
+
+- A feed larger than 50 MB is read again, up to 200 MB: a podcast with a long back catalogue had stopped updating (#357).
+
 ## [1.5.0] - 2026-10-23
 
 ### Changed
@@ -924,6 +934,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `ipx import` and `ipx export` for OPML, and systemd units in `contrib/`.
 
 [unreleased]: https://git.sdf1.net/rays/ipx/compare/v1.5.0...main
+[1.5.1]: https://git.sdf1.net/rays/ipx/compare/v1.5.0...v1.5.1
 [1.5.0]: https://git.sdf1.net/rays/ipx/compare/v1.4.0...v1.5.0
 [1.4.0]: https://git.sdf1.net/rays/ipx/compare/v1.3.0...v1.4.0
 [1.3.0]: https://git.sdf1.net/rays/ipx/compare/v1.2.0...v1.3.0

crates/ipx/Cargo.toml

@@ -1,9 +1,12 @@
 [package]
 name = "ipx"
-version = "1.5.0"
+version = "1.5.1"
 edition.workspace = true
 publish = false
 
+[lints]
+workspace = true
+
 [dependencies]
 ipx-feed.workspace = true
 ipx-directory.workspace = true
@@ -18,6 +21,7 @@ futures-util.workspace = true
 jsonwebtoken.workspace = true
 librqbit.workspace = true
 opentelemetry.workspace = true
+opentelemetry_sdk.workspace = true
 opml.workspace = true
 percent-encoding.workspace = true
 reqwest.workspace = true

crates/ipx/src/access.rs

@@ -11,7 +11,7 @@ use std::time::{Duration, Instant};
 
 use anyhow::{Context, Result};
 use jsonwebtoken::jwk::JwkSet;
-use jsonwebtoken::{Algorithm, DecodingKey, Validation, decode, decode_header};
+use jsonwebtoken::{Algorithm, DecodingKey, Validation};
 
 /// Cloudflare rotates its keys every six weeks or so, publishing the new one before using it.
 /// A token naming a key not seen yet refetches, but no more often than this, so a stream of
@@ -55,7 +55,7 @@ impl Keys {
         F: FnOnce() -> Fut,
         Fut: Future<Output = Result<JwkSet>>,
     {
-        let kid = decode_header(token).ok()?.kid?;
+        let kid = jsonwebtoken::decode_header(token).ok()?.kid?;
         let key = match self.key(&kid) {
             Some(k) => k,
             None => {
@@ -78,7 +78,7 @@ impl Keys {
         v.set_audience(&[aud]);
         v.set_issuer(&[format!("https://{team}")]);
         v.validate_nbf = true;
-        match decode::<Claims>(token, &key, &v) {
+        match jsonwebtoken::decode::<Claims>(token, &key, &v) {
             Ok(data) => crate::auth::name_from_header(&data.claims.email?),
             Err(e) => {
                 tracing::warn!(error = %e, "refused a Cloudflare Access token");
@@ -88,12 +88,12 @@ impl Keys {
     }
 
     fn key(&self, kid: &str) -> Option<DecodingKey> {
-        self.cache.lock().unwrap().keys.get(kid).cloned()
+        self.cache.lock().expect("a panic elsewhere poisoned this lock").keys.get(kid).cloned()
     }
 
     /// Takes the slot as it answers, so two requests at once do not both fetch.
     fn may_refetch(&self) -> bool {
-        let mut c = self.cache.lock().unwrap();
+        let mut c = self.cache.lock().expect("a panic elsewhere poisoned this lock");
         if c.fetched.is_some_and(|t| t.elapsed() < REFETCH_EVERY) {
             return false;
         }
@@ -105,7 +105,7 @@ impl Keys {
     fn store(&self, set: JwkSet) {
         let keys =
             set.keys.iter().filter_map(|k| Some((k.common.key_id.clone()?, DecodingKey::from_jwk(k).ok()?))).collect();
-        let mut c = self.cache.lock().unwrap();
+        let mut c = self.cache.lock().expect("a panic elsewhere poisoned this lock");
         c.keys = keys;
         c.fetched = Some(Instant::now());
     }
@@ -128,7 +128,7 @@ async fn fetch(client: &reqwest::Client, team: &str) -> Result<JwkSet> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use jsonwebtoken::{EncodingKey, Header, encode, get_current_timestamp};
+    use jsonwebtoken::{EncodingKey, Header};
 
     const TEAM: &str = "team.cloudflareaccess.com";
     const AUD: &str = "aud-tag";
@@ -141,11 +141,11 @@ mod tests {
         let mut h = Header::new(alg);
         h.kid = Some("k1".into());
         let k = if alg == Algorithm::RS256 { EncodingKey::from_rsa_der(key) } else { EncodingKey::from_secret(key) };
-        encode(&h, &claims, &k).unwrap()
+        jsonwebtoken::encode(&h, &claims, &k).unwrap()
     }
 
     fn claims(aud: &str, exp_in: i64) -> serde_json::Value {
-        let now = get_current_timestamp() as i64;
+        let now = jsonwebtoken::get_current_timestamp() as i64;
         serde_json::json!({
             "aud": [aud], "iss": format!("https://{TEAM}"), "email": "Rays@SDF1.net",
             "iat": now, "nbf": now, "exp": now + exp_in, "type": "app",
@@ -201,7 +201,7 @@ mod tests {
         // A made-up key id straight after: not fetched again.
         let mut h = Header::new(Algorithm::RS256);
         h.kid = Some("nobody".into());
-        let stray = encode(&h, &claims(AUD, 300), &EncodingKey::from_rsa_der(SIGNER)).unwrap();
+        let stray = jsonwebtoken::encode(&h, &claims(AUD, 300), &EncodingKey::from_rsa_der(SIGNER)).unwrap();
         let count = || {
             fetches.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
             async { Ok(jwks()) }

crates/ipx/src/art.rs

@@ -2,8 +2,11 @@
 //! of each publisher's server for every visit, and still there when a publisher is not.
 
 use std::collections::HashMap;
+use std::collections::hash_map::DefaultHasher;
+use std::fs::File;
 use std::path::{Path, PathBuf};
 use std::sync::{LazyLock, Mutex};
+use std::time::SystemTime;
 use std::time::{Duration, Instant};
 
 pub fn dir() -> PathBuf {
@@ -16,7 +19,7 @@ pub fn dir() -> PathBuf {
 // one in 10^12. Move to a SHA if either ever matters.
 pub fn path(url: &str) -> PathBuf {
     use std::hash::{Hash, Hasher};
-    let mut h = std::collections::hash_map::DefaultHasher::new();
+    let mut h = DefaultHasher::new();
     url.hash(&mut h);
     dir().join(format!("{:016x}", h.finish()))
 }
@@ -25,8 +28,8 @@ pub fn path(url: &str) -> PathBuf {
 pub fn read(file: &Path) -> Option<(String, Vec<u8>)> {
     let kind = std::fs::read_to_string(file.with_extension("type")).ok()?;
     let body = std::fs::read(file).ok()?;
-    if let Ok(f) = std::fs::File::options().write(true).open(file) {
-        let _ = f.set_modified(std::time::SystemTime::now());
+    if let Ok(f) = File::options().write(true).open(file) {
+        let _ = f.set_modified(SystemTime::now());
     }
     Some((kind, body))
 }
@@ -75,7 +78,7 @@ pub fn fail(url: &str, why: &str) {
 /// Returns how many went.
 pub fn trim(dir: &Path, limit: u64) -> usize {
     let Ok(entries) = std::fs::read_dir(dir) else { return 0 };
-    let mut kept: Vec<(std::time::SystemTime, u64, PathBuf)> = entries
+    let mut kept: Vec<(SystemTime, u64, PathBuf)> = entries
         .flatten()
         .map(|e| e.path())
         .filter(|p| p.extension().is_none())

crates/ipx/src/auth.rs

@@ -2,9 +2,14 @@
 //! this -- Cloudflare Zero Trust on `ipodderx.sdf1.net`, which puts the authenticated
 //! address in `Cf-Access-Authenticated-User-Email`.
 
-use anyhow::{Result, bail};
+use anyhow::{Context, Result, bail};
 use argon2::Argon2;
-use argon2::password_hash::{PasswordHasher, PasswordVerifier, phc::PasswordHash};
+use argon2::password_hash::phc::PasswordHash;
+use argon2::password_hash::{PasswordHasher, PasswordVerifier};
+use std::fs::{File, OpenOptions};
+use std::path::Path;
+use std::sync::{Arc, LazyLock};
+use tokio::sync::Semaphore;
 
 /// Argon2id with the crate's defaults, which are the OWASP-recommended parameters. The
 /// salt is generated per password by the hasher itself.
@@ -31,17 +36,15 @@ pub fn verify_password(password: &str, stored: &str) -> bool {
 /// How many password checks run at once: each is tens of milliseconds of CPU, and forty wrong
 /// passwords at a time, run on the async workers, made every other request wait 4s (#137).
 /// Half the cores, so a flood of sign-ins waits on itself and the rest of the server has the rest.
-static CHECKS: std::sync::LazyLock<std::sync::Arc<tokio::sync::Semaphore>> = std::sync::LazyLock::new(|| {
-    std::sync::Arc::new(tokio::sync::Semaphore::new(
-        std::thread::available_parallelism().map_or(1, |n| (n.get() / 2).max(1)),
-    ))
+static CHECKS: LazyLock<Arc<Semaphore>> = LazyLock::new(|| {
+    Arc::new(Semaphore::new(std::thread::available_parallelism().map_or(1, |n| (n.get() / 2).max(1))))
 });
 
 /// What a name that is not an account is checked against, so that refusing it takes as long as
 /// refusing a wrong password does. Refused without a check, it came back 31ms sooner, and the
 /// time told anyone which names are accounts here (#138).
-static DECOY: std::sync::LazyLock<String> =
-    std::sync::LazyLock::new(|| hash_password("no account here has this password").expect("hashing a fixed password"));
+static DECOY: LazyLock<String> =
+    LazyLock::new(|| hash_password("no account here has this password").expect("hashing a fixed password"));
 
 /// A sign-in's password against the account's hash, or against `DECOY` when there is no account
 /// or it has no password: false either way, in the same time. Off the async workers, and a few at
@@ -90,7 +93,7 @@ pub fn api_token_hash(token: &str) -> String {
 
 fn getrandom(buf: &mut [u8]) -> std::io::Result<()> {
     use std::io::Read;
-    std::fs::File::open("/dev/urandom")?.read_exact(buf)
+    File::open("/dev/urandom")?.read_exact(buf)
 }
 
 /// A username taken from a proxy header. Cloudflare sends an email address; the local part
@@ -107,6 +110,36 @@ pub fn name_from_header(raw: &str) -> Option<String> {
     Some(name.to_ascii_lowercase())
 }
 
+/// The first admin's password: `IPX_ADMIN_PASSWORD` when set, or a random one written to
+/// `admin-password` beside config.toml, readable by the daemon's user alone. Every install began
+/// as admin / ipodderx, published in the README, and nothing made anyone change it (#321).
+/// Returns the password and where it can be found, for the log, which never shows it.
+pub(crate) fn first_password(config_dir: &Path, from_env: Option<String>) -> Result<(String, String)> {
+    if let Some(p) = from_env.filter(|p| !p.is_empty()) {
+        return Ok((p, "with the password in IPX_ADMIN_PASSWORD".into()));
+    }
+    let password = crate::auth::new_session_token()[..24].to_owned();
+    let file = config_dir.join("admin-password");
+    write_private(&file, &password)?;
+    Ok((password, format!("with the password in {}", file.display())))
+}
+
+/// Writes a secret only its owner may read: created 0600, never readable for a moment between.
+pub(crate) fn write_private(file: &Path, text: &str) -> Result<()> {
+    use std::io::Write;
+    use std::os::unix::fs::OpenOptionsExt;
+    std::fs::create_dir_all(file.parent().unwrap_or(Path::new(".")))?;
+    let _ = std::fs::remove_file(file);
+    let mut f = OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .mode(0o600)
+        .open(file)
+        .with_context(|| format!("writing {}", file.display()))?;
+    f.write_all(text.as_bytes())?;
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -162,4 +195,28 @@ mod tests {
         assert_eq!(name_from_header("ray\nadmin"), None);
         assert_eq!(name_from_header("ray;admin"), None);
     }
+
+    #[test]
+    fn the_first_admin_gets_a_password_nobody_else_knows() {
+        use std::os::unix::fs::PermissionsExt;
+        let dir = std::env::temp_dir().join(format!("ipx-first-password-{}", std::process::id()));
+        let _ = std::fs::remove_dir_all(&dir);
+
+        let (a, told) = first_password(&dir, None).unwrap();
+        let file = dir.join("admin-password");
+        assert_eq!(std::fs::read_to_string(&file).unwrap(), a);
+        assert_eq!(std::fs::metadata(&file).unwrap().permissions().mode() & 0o777, 0o600);
+        assert!(told.contains("admin-password") && !told.contains(&a), "the log says where, not what: {told}");
+        assert_eq!(a.len(), 24);
+        assert_ne!(a, "ipodderx");
+        assert_ne!(first_password(&dir, None).unwrap().0, a, "each new database gets its own");
+
+        std::fs::remove_dir_all(&dir).unwrap();
+        let (b, told) = first_password(&dir, Some("chosen".into())).unwrap();
+        assert_eq!(b, "chosen");
+        assert!(!told.contains("chosen"));
+        assert!(!file.exists(), "a password set in the environment is not written anywhere");
+        assert_eq!(first_password(&dir, Some(String::new())).unwrap().0.len(), 24, "empty means unset");
+        std::fs::remove_dir_all(&dir).unwrap();
+    }
 }

crates/ipx/src/catalogue.rs

@@ -0,0 +1,616 @@
+//! The feeds the server knows and who reads them: adding, removing, leaving, and what an
+//! OPML or Patreon list leaves behind when it goes.
+
+use crate::ctx::Ctx;
+use crate::ipc::Event;
+use crate::{cli, config, db, feed, lists};
+use anyhow::{Context, Result};
+use std::collections::BTreeMap;
+use std::path::Path;
+use url::Url;
+
+/// Subscribes to one feed, naming it from its own title.
+pub(crate) async fn add(
+    ctx: &Ctx,
+    url: &str,
+    folder: Option<String>,
+    keywords: Vec<String>,
+    category: Option<String>,
+    list: bool,
+) -> Result<()> {
+    let url = feed::find_feed(&ctx.client, &feed::expand_input(url)).await?;
+    let (id, existed) = add_to_catalogue(ctx, &url, folder, keywords).await?;
+    // Listing a feed the catalogue already has, or giving it a category, is the point of asking
+    // again: it changes those, and nothing else. A feed an OPML lists has no entry to change.
+    if list || category.is_some() {
+        let mut cfg = ctx.edit_cfg().await;
+        if let Some(f) = cfg.feeds.get_mut(&id) {
+            f.listed |= list;
+            if category.is_some() {
+                f.category = category;
+            }
+            ctx.store_cfg(cfg).await?;
+            if existed {
+                println!("{id} is already in the catalogue; updated its Directory listing");
+                return Ok(());
+            }
+        }
+    }
+    if existed {
+        anyhow::bail!("already subscribed as {id:?}");
+    }
+    println!("added {id}{}", if list { ", listed in the Directory" } else { "" });
+    Ok(())
+}
+
+/// Adds the feed at `url` to the catalogue unless it is there already, a feed an OPML lists
+/// included, so one show is never added twice: its id, and whether it was there. For `ipx add`
+/// and the page's Add feed alike, which were the same steps written twice (#161); each finds the
+/// feed first, to say its own way why an address is not one.
+pub async fn add_to_catalogue(
+    ctx: &Ctx,
+    url: &str,
+    folder: Option<String>,
+    keywords: Vec<String>,
+) -> Result<(String, bool)> {
+    // Taken before the duplicate check, so two people adding the same feed at once add it once.
+    let mut cfg = ctx.edit_cfg().await;
+    if let Some(existing) = subscriptions(ctx).await?.into_iter().find(|s| feed::same_feed(&s.cfg.url, url)) {
+        return Ok((existing.id, true));
+    }
+    let id = add_one(ctx, &mut cfg, url, folder, keywords).await?;
+    ctx.store_cfg(cfg).await?;
+    Ok((id, false))
+}
+
+/// Returns the new feed id. Adding checks the address is a feed first (`feed::find_feed`); this
+/// still names one it cannot read from its URL rather than failing.
+pub(crate) async fn add_one(
+    ctx: &Ctx,
+    cfg: &mut config::Config,
+    url: &str,
+    folder: Option<String>,
+    keywords: Vec<String>,
+) -> Result<String> {
+    let probe = config::Feed {
+        url: url.to_owned(),
+        folder: folder.clone(),
+        group: None,
+        media_types: None,
+        schedule: None,
+        keywords: keywords.clone(),
+        allow_explicit: false,
+        auto_download: true,
+        max_new_per_check: None,
+        username: None,
+        password: None,
+        password_env: None,
+        category: None,
+        listed: false,
+        private: false,
+    };
+
+    let title = match feed::fetch(&ctx.feed_client, &probe.url, None, None, None).await.map(|(got, _)| got) {
+        // An OPML subscription is named from its own <head><title>, not by trying to
+        // parse it as a feed and falling back to the hostname.
+        Ok(feed::Fetched::Body { bytes, .. }) if feed::is_opml(&bytes) => {
+            feed::opml_title(&bytes).unwrap_or_else(|| url_stem(url))
+        }
+        Ok(feed::Fetched::Body { bytes, .. }) => {
+            feed::parse(&bytes).ok().and_then(|f| f.title).unwrap_or_else(|| url_stem(url))
+        }
+        _ => {
+            tracing::warn!(url, "could not read the feed; naming it from its URL");
+            url_stem(url)
+        }
+    };
+
+    // Slugs must be unique across derived feeds too, or a new feed can collide with one
+    // an OPML already introduced.
+    let mut taken: BTreeMap<String, config::Feed> =
+        subscriptions(ctx).await?.into_iter().map(|s| (s.id, s.cfg)).collect();
+    // A removed feed keeps its rows, so its id is only free again for the same feed: re-adding
+    // it gets its history back, and a different feed does not inherit someone else's.
+    for (id, other) in ctx.db.feed_urls().await? {
+        if !feed::same_feed(&other, url) {
+            taken.entry(id).or_insert_with(|| probe.clone());
+        }
+    }
+    let id = config::unique_slug(&title, |s| taken.contains_key(s));
+    cfg.feeds.insert(id.clone(), probe);
+    Ok(id)
+}
+
+/// Host plus last path segment, for naming a feed we could not read.
+pub(crate) fn url_stem(url: &str) -> String {
+    Url::parse(url).ok().and_then(|u| u.host_str().map(str::to_owned)).unwrap_or_else(|| url.to_owned())
+}
+
+pub(crate) async fn rm(ctx: &Ctx, feed: &str) -> Result<()> {
+    let mut cfg = ctx.edit_cfg().await;
+    let Some(parent) = cfg.feeds.remove(feed) else {
+        // Derived from an OPML: drop it here, though the subscription will list it again
+        // on the next read unless the OPML itself goes.
+        ctx.db.drop_managed(feed).await?;
+        println!("removed {feed}; it came from an OPML subscription and may return on the next read");
+        return Ok(());
+    };
+    ctx.store_cfg(cfg).await?;
+    // State and files stay: re-adding the feed should not re-download its back catalogue.
+    println!("removed {feed}; downloads and history kept");
+    retire_group(ctx, feed, Some(&parent)).await?;
+    Ok(())
+}
+
+pub(crate) async fn import(ctx: &Ctx, file: &Path) -> Result<()> {
+    let text = std::fs::read_to_string(file).with_context(|| format!("reading {}", file.display()))?;
+    // The CLI speaks for the operator, as the shared web token does.
+    let admin = ctx
+        .db
+        .users()
+        .await?
+        .into_iter()
+        .find(|u| u.is_admin)
+        .ok_or_else(|| anyhow::anyhow!("no admin account to subscribe: ipx user add <name> --admin"))?;
+    let doc = feed::read_opml(&text).map_err(|e| anyhow::anyhow!("{} is not OPML: {e}", file.display()))?;
+    let (added, had) = lists::subscribe_opml(ctx, lists::opml_feeds(&doc)?, admin.id).await?;
+    println!("subscribed {} to {added} feed(s); {had} already there", admin.name);
+    Ok(())
+}
+
+/// The configuration ipx runs with: config.toml for where things are and who may sign in, the
+/// database for the feeds and the server settings (issue #18). The first time a database holds
+/// neither, it takes them from config.toml, which is then cut down to the rest, the original kept
+/// beside it as config.toml.pre-database.
+pub(crate) async fn assemble_config(db: &db::Db, mut cfg: config::Config, path: &Path) -> Result<config::Config> {
+    for _ in 0..2 {
+        if let Some((stored, feeds)) = db.stored_config().await? {
+            if config::Config::file_holds_stored(path) {
+                tracing::warn!(
+                    "config.toml still lists feeds or server settings; they are ignored, since the \
+                     database holds them now. Change them in the web UI, or with ipx add and rm."
+                );
+            }
+            stored.apply(&mut cfg);
+            cfg.feeds = feeds;
+            return Ok(cfg);
+        }
+        if db.import_config(&cfg).await? {
+            if path.exists() {
+                let original = path.with_extension("toml.pre-database");
+                if !original.exists() {
+                    std::fs::copy(path, &original)
+                        .with_context(|| format!("keeping the original as {}", original.display()))?;
+                }
+                cfg.save_bootstrap(path)?;
+            }
+            tracing::info!(
+                feeds = cfg.feeds.len(),
+                "moved the feeds and server settings from config.toml into the database"
+            );
+            return Ok(cfg);
+        }
+        // Another ipx imported between our look and our insert; the next pass reads its copy.
+    }
+    anyhow::bail!("the database says it holds the configuration and then that it does not")
+}
+
+pub(crate) async fn export(ctx: &Ctx, file: &std::path::Path) -> Result<()> {
+    let mut doc = opml::OPML {
+        head: Some(opml::Head { title: Some("ipx subscriptions".into()), ..Default::default() }),
+        ..Default::default()
+    };
+    for (id, feed) in &ctx.cfg().feeds {
+        let title = ctx.db.feed_summary(id).await.ok().and_then(|s| s.title).unwrap_or_else(|| id.clone());
+        doc.add_feed(&title, &feed.url);
+    }
+    let xml = doc.to_string().map_err(|e| anyhow::anyhow!("writing OPML: {e}"))?;
+    std::fs::write(file, xml).with_context(|| format!("writing {}", file.display()))?;
+    println!("exported {} feed(s) to {}", ctx.cfg().feeds.len(), file.display());
+    Ok(())
+}
+
+pub(crate) async fn list(ctx: &Ctx) -> Result<()> {
+    let cfg = ctx.cfg();
+    if cfg.feeds.is_empty() {
+        println!("No feeds configured. Add one with `ipx add <url>`, or in the web UI.");
+        return Ok(());
+    }
+    for (id, feed) in &cfg.feeds {
+        let s = ctx.db.feed_summary(id).await?;
+        // The id on a line of its own, labelled: first on the title's line, antirez.com's id
+        // "feed" read as a heading and 'ipx fetch antirez' was tried instead (issue #82).
+        println!("{}", s.title.as_deref().filter(|t| !t.is_empty()).unwrap_or(id));
+        println!("    id           {id}");
+        println!("    url          {}", feed.url);
+        println!("    last checked {}", cli::ago(s.last_checked));
+        println!("    entries      {} ({} downloaded)", s.entries, s.downloaded);
+        if let Some(err) = &s.last_error {
+            println!("    last error   {err}");
+        }
+    }
+    Ok(())
+}
+
+/// Removes feeds nobody subscribes to that are dead (failing for 30 days) or quiet (nothing new
+/// in a year), from the catalogue and the database, so the Directory lists feeds worth taking.
+/// A feed listed in it with no subscribers stays for as long as it works and publishes.
+pub(crate) async fn clean_directory(ctx: &Ctx) -> Result<()> {
+    const DEAD: i64 = 30 * 86_400;
+    const QUIET: i64 = 365 * 86_400;
+    let now = db::now();
+    // Once a day, for feeds dead a month or quiet a year: before every scan, its query took 1 to
+    // 12 s every few minutes, and the scans and downloads waited behind it (#193).
+    use std::sync::atomic::Ordering::Relaxed;
+    if now - ctx.directory_cleaned.load(Relaxed) < 86_400 {
+        return Ok(());
+    }
+    ctx.directory_cleaned.store(now, Relaxed);
+    let stale = ctx.db.stale_unsubscribed(now - DEAD, now - QUIET).await?;
+    if stale.is_empty() {
+        return Ok(());
+    }
+    let mut cfg = ctx.edit_cfg().await;
+    // Every one removed, so not `any`, which would stop at the first.
+    let mut removed = std::collections::HashMap::new();
+    for id in &stale {
+        if let Some(f) = cfg.feeds.remove(id) {
+            removed.insert(id.clone(), f);
+        }
+    }
+    if !removed.is_empty() {
+        ctx.store_cfg(cfg).await?;
+    }
+    for id in &stale {
+        retire_group(ctx, id, removed.get(id)).await?;
+        ctx.db.forget_feed(id).await?;
+        tracing::info!(
+            feed = id,
+            "removed a feed nobody subscribes to that is dead or has published nothing in a year"
+        );
+    }
+    Ok(())
+}
+
+/// One feed to scan: either an entry you wrote in config.toml, or one derived from an
+/// OPML subscription and held only in the database.
+pub struct Sub {
+    pub id: String,
+    pub cfg: config::Feed,
+    /// True when it came from an OPML and has no config entry of its own.
+    pub managed: bool,
+}
+
+/// Everything to scan: your config entries, plus whatever the OPML subscriptions listed.
+///
+/// A derived feed borrows its parent's settings wholesale. That is why it needs no config
+/// entry -- there is nothing to store but its URL and where it came from.
+pub async fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
+    let cfg = ctx.cfg();
+    let mut out: Vec<Sub> =
+        cfg.feeds.iter().map(|(id, f)| Sub { id: id.clone(), cfg: f.clone(), managed: false }).collect();
+
+    // A group's title, once a group: asked for each feed in it, three queries a feed, it was about
+    // 96 for The Glass Cannon Network's 32 shows on every feed list, live update and scan (#159).
+    let mut group_titles = std::collections::HashMap::<String, String>::new();
+    for m in ctx.db.managed_feeds().await? {
+        if cfg.feeds.contains_key(&m.id) {
+            continue; // promoted to config at some point; that entry wins
+        }
+        let parent = cfg.feeds.get(&m.group_id);
+        if parent.is_none() {
+            // The OPML or Patreon feed this was derived from is no longer in config --
+            // removing it should have retired these rows too (see `retire_group`), but
+            // skip them here regardless so a row that slips through is never scanned.
+            continue;
+        }
+        let base = match parent.and_then(|p| p.folder.clone()) {
+            Some(folder) => folder,
+            None => {
+                if !group_titles.contains_key(&m.group_id) {
+                    let title = ctx.db.feed_summary(&m.group_id).await.ok().and_then(|s| s.title);
+                    group_titles.insert(m.group_id.clone(), title.unwrap_or_else(|| m.group_id.clone()));
+                }
+                group_titles[&m.group_id].clone()
+            }
+        };
+        let title = m.title.clone().unwrap_or_else(|| m.id.clone());
+        let (username, password, password_env) = parent.map(|p| p.credentials_for(&m.url)).unwrap_or_default();
+        out.push(Sub {
+            id: m.id.clone(),
+            cfg: config::Feed {
+                url: m.url.clone(),
+                folder: Some(format!("{base}/{title}")),
+                group: Some(m.group_id.clone()),
+                media_types: parent.and_then(|p| p.media_types.clone()),
+                schedule: parent.and_then(|p| p.schedule.clone()),
+                keywords: parent.map(|p| p.keywords.clone()).unwrap_or_default(),
+                allow_explicit: parent.is_some_and(|p| p.allow_explicit),
+                auto_download: parent.is_none_or(|p| p.auto_download),
+                max_new_per_check: parent.and_then(|p| p.max_new_per_check),
+                username,
+                password,
+                password_env,
+                category: None,
+                listed: false,
+                private: false,
+            },
+            managed: true,
+        });
+    }
+    out.sort_by(|a, b| a.id.cmp(&b.id));
+    Ok(out)
+}
+
+/// Takes a feed off one person's list, and the feeds in it if it is a group. Everyone else's
+/// lists are left alone.
+pub async fn leave(ctx: &Ctx, user_id: i64, id: &str) -> Result<()> {
+    use tracing::Instrument;
+    ctx.db.unsubscribe(user_id, id).instrument(db::step("unsubscribe")).await?;
+    // What they kept stays on their list, a feed of its own once nobody reads the list (#280).
+    let kept = ctx.db.kept_feeds(user_id).instrument(db::step("kept")).await?;
+    let all = subscriptions(ctx).instrument(db::step("subscriptions")).await?;
+    let unkept = all.iter().filter(|s| s.cfg.group.as_deref() == Some(id) && !kept.contains(&s.id));
+    for child in unkept {
+        ctx.db.unsubscribe(user_id, &child.id).instrument(db::step("unsubscribe")).await?;
+    }
+    if ctx.db.subscriber_counts().instrument(db::step("subscriber_counts")).await?.contains_key(id) {
+        return Ok(());
+    }
+
+    // Nobody is left: the feed stops being scanned. Its files and history stay, so if
+    // someone subscribes again they do not pull the back catalogue a second time. A feed an
+    // admin listed stays in the Directory, for the next person.
+    let mut cfg = ctx.edit_cfg().await;
+    if cfg.feeds.get(id).is_some_and(|f| f.listed) {
+        return Ok(());
+    }
+    let Some(parent) = cfg.feeds.remove(id) else {
+        // A derived feed: forget it here, though the OPML will list it again on the next
+        // read unless you unsubscribe from the OPML itself.
+        return ctx.db.drop_managed(id).await;
+    };
+    ctx.store_cfg(cfg).await?;
+    retire_group(ctx, id, Some(&parent)).await
+}
+
+/// Removes an account, leaving each of its feeds first as unsubscribing does. The foreign keys
+/// took the subscriptions with the account, but not the feed: one only they read stayed in the
+/// catalogue with nobody subscribed, downloading every new episode for no one (#157).
+pub async fn remove_user(ctx: &Ctx, user_id: i64) -> Result<()> {
+    let catalogue = subscriptions(ctx).await?;
+    let mine: std::collections::HashSet<String> =
+        ctx.db.subscriptions_for(user_id).await?.into_iter().map(|s| s.feed_id).collect();
+    for id in &mine {
+        // A feed in a group of theirs goes with the group.
+        let in_theirs = catalogue.iter().any(|s| &s.id == id && s.cfg.group.as_ref().is_some_and(|g| mine.contains(g)));
+        if !in_theirs {
+            leave(ctx, user_id, id).await?;
+        }
+    }
+    ctx.db.delete_user(user_id).await
+}
+
+/// Retires every feed derived from `parent_id`, now that nothing subscribes to the OPML or
+/// Patreon feed that listed them: the same rule `sync_group` applies to one the list drops --
+/// removed if nothing was downloaded, orphaned and kept otherwise. Called once the parent
+/// itself is removed, since `subscriptions()` would otherwise keep scanning them under a
+/// fallback policy meant for a feed with no parent at all. A feed promoted to config is not
+/// derived any more, so it is only unmanaged. `parent` is the list's settings as they were before
+/// it was removed, None when that is no longer known.
+pub async fn retire_group(ctx: &Ctx, parent_id: &str, parent: Option<&config::Feed>) -> Result<()> {
+    let cfg = ctx.cfg();
+    let managed = ctx.db.managed_feeds().await?;
+    let derived = managed.into_iter().filter(|m| m.group_id == parent_id);
+    for m in derived {
+        if cfg.feeds.contains_key(&m.id) {
+            // Scanned from its config entry and still read. Dropped as derived, its stored
+            // entries would go with it: davewiner's 11 were promoted without being unmanaged.
+            ctx.db.unmanage(&m.id).await?;
+        } else if promote_kept(ctx, &m, parent).await? {
+            tracing::info!(feed = %m.id, "its list is gone but someone kept it; a feed of its own now");
+        } else if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 {
+            ctx.db.set_orphaned(&m.id, true).await?;
+        } else {
+            ctx.db.drop_managed(&m.id).await?;
+        }
+    }
+    Ok(())
+}
+
+/// Makes a feed from an OPML or Patreon list a feed of its own, for those who kept it (#280): a
+/// catalogue entry in no group, which the list no longer adds or takes away. Whoever had it only
+/// through the list, without keeping it, loses it, as they would have when the list dropped it.
+/// False, changing nothing, when nobody kept it. `base` is the list's settings: most callers have
+/// removed the list from config by now, so they pass them.
+pub(crate) async fn promote_kept(ctx: &Ctx, m: &db::Managed, base: Option<&config::Feed>) -> Result<bool> {
+    let subs = ctx.db.subscribers_of(&m.id).await?;
+    if !subs.iter().any(|(_, kept)| *kept) {
+        return Ok(false);
+    }
+    let mut cfg = ctx.edit_cfg().await;
+    // The list's own settings carry over; its folder and group do not.
+    let (username, password, password_env) = base.map(|p| p.credentials_for(&m.url)).unwrap_or_default();
+    cfg.feeds.entry(m.id.clone()).or_insert_with(|| config::Feed {
+        url: m.url.clone(),
+        folder: None,
+        group: None,
+        media_types: base.and_then(|p| p.media_types.clone()),
+        schedule: base.and_then(|p| p.schedule.clone()),
+        keywords: base.map(|p| p.keywords.clone()).unwrap_or_default(),
+        allow_explicit: base.is_some_and(|p| p.allow_explicit),
+        auto_download: base.is_none_or(|p| p.auto_download),
+        max_new_per_check: base.and_then(|p| p.max_new_per_check),
+        username,
+        password,
+        password_env,
+        category: None,
+        listed: false,
+        // Looked up after the list was removed, private was always false, and a private OPML's
+        // kept show went to the directory (#332). A list no longer known may have been private.
+        private: base.is_none_or(|p| p.private),
+    });
+    ctx.store_cfg(cfg).await?;
+    ctx.db.unmanage(&m.id).await?;
+    ctx.db.set_orphaned(&m.id, false).await?;
+    for (user, kept) in subs {
+        if !kept {
+            ctx.db.unsubscribe(user, &m.id).await?;
+        }
+    }
+    Ok(true)
+}
+
+/// Retires every group whose parent is gone from config, and returns how many derived rows that
+/// dropped or unmanaged. An OPML removed before `retire_group` existed left its feeds behind:
+/// davewiner's 922 were skipped by every scan and never cleared, and their stale errors were
+/// most of the ones stored.
+pub(crate) async fn retire_stranded(ctx: &Ctx) -> Result<usize> {
+    let cfg = ctx.cfg();
+    let before = ctx.db.managed_feeds().await?;
+    let stranded: std::collections::BTreeSet<&str> =
+        before.iter().map(|m| m.group_id.as_str()).filter(|g| !cfg.feeds.contains_key(*g)).collect();
+    for group in stranded {
+        retire_group(ctx, group, None).await?;
+    }
+    Ok(before.len() - ctx.db.managed_feeds().await?.len())
+}
+
+/// A feed that answered from a new address after permanent redirects moves there in the
+/// catalogue, so it is read from there and no longer redirected every time. One an OPML lists
+/// is left alone, since the OPML would only put the old address back; so is a move onto an
+/// address another feed already has.
+pub(crate) async fn follow_move(ctx: &Ctx, id: &str, to: &str) -> Result<()> {
+    // Said in the event below, not logged here: the event is the log line.
+    if subscriptions(ctx).await?.iter().any(|s| s.id != id && feed::same_feed(&s.cfg.url, to)) {
+        tracing::info!(feed = id, to, "the feed moved to an address another feed already has; leaving it");
+        return Ok(());
+    }
+    let mut cfg = ctx.edit_cfg().await;
+    let Some(f) = cfg.feeds.get_mut(id) else { return Ok(()) };
+    let from = std::mem::replace(&mut f.url, to.to_owned());
+    ctx.store_cfg(cfg).await?;
+    ctx.out.emit(Event::FeedMoved { feed: id.to_owned(), from, to: to.to_owned() });
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_util::{feed, test_ctx};
+
+    #[tokio::test]
+    async fn the_first_start_moves_the_configuration_in_and_trims_the_file() {
+        let dir = std::env::temp_dir().join(format!("ipx-assemble-{}", std::process::id()));
+        std::fs::create_dir_all(&dir).unwrap();
+        let path = dir.join("config.toml");
+        std::fs::write(
+            &path,
+            "[general]\nschedule = \"every 2h\"\n[web]\ntoken = \"t\"\n[feeds.show]\nurl = \"http://x/show.xml\"\n",
+        )
+        .unwrap();
+        let db = db::Db::memory().await.unwrap();
+
+        let first = assemble_config(&db, config::Config::load(&path).unwrap(), &path).await.unwrap();
+        assert_eq!(first.feeds.keys().collect::<Vec<_>>(), ["show"]);
+        assert_eq!(first.general.schedule, "every 2h");
+        assert!(dir.join("config.toml.pre-database").exists(), "the original is kept");
+        assert!(!config::Config::file_holds_stored(&path), "and the file no longer lists them");
+
+        // The next start reads them from the database, the trimmed file notwithstanding.
+        let next = assemble_config(&db, config::Config::load(&path).unwrap(), &path).await.unwrap();
+        assert_eq!(next.feeds.keys().collect::<Vec<_>>(), ["show"]);
+        assert_eq!(next.general.schedule, "every 2h");
+        assert_eq!(next.web.token, "t");
+        std::fs::remove_dir_all(&dir).unwrap();
+    }
+
+    #[tokio::test]
+    async fn dead_feeds_leave_the_directory_once_a_day_not_before_every_scan() {
+        let ctx = test_ctx(config::Config::default()).await;
+        let dead = |id: &str| {
+            format!("INSERT INTO feeds (id, url, error_since, managed) VALUES ('{id}', 'u-{id}', 100, false)")
+        };
+        let listed = async || ctx.db.feed_urls().await.unwrap().into_iter().map(|(id, _)| id).collect::<Vec<_>>();
+        ctx.db.sql(&dead("a")).await.unwrap();
+        clean_directory(&ctx).await.unwrap();
+        assert!(listed().await.is_empty(), "the first pass since starting looks");
+        ctx.db.sql(&dead("b")).await.unwrap();
+        clean_directory(&ctx).await.unwrap();
+        assert_eq!(listed().await, ["b"], "not again the same day (#193)");
+        ctx.directory_cleaned.store(db::now() - 86_401, std::sync::atomic::Ordering::Relaxed);
+        clean_directory(&ctx).await.unwrap();
+        assert!(listed().await.is_empty(), "a day on, it looks again");
+    }
+
+    #[tokio::test]
+    async fn a_removed_account_takes_the_feeds_only_it_read_with_it() {
+        let ctx = test_ctx(config::Config::default()).await;
+        let mut cfg = ctx.edit_cfg().await;
+        for id in ["only-theirs", "shared", "listed"] {
+            cfg.feeds.insert(id.into(), config::Feed { listed: id == "listed", ..feed() });
+        }
+        ctx.store_cfg(cfg).await.unwrap();
+        let gone = ctx.db.create_user("gone", None, false).await.unwrap();
+        let stays = ctx.db.create_user("stays", None, true).await.unwrap();
+        for id in ["only-theirs", "shared", "listed"] {
+            ctx.db.subscribe(gone, id).await.unwrap();
+        }
+        ctx.db.subscribe(stays, "shared").await.unwrap();
+
+        remove_user(&ctx, gone).await.unwrap();
+        assert!(ctx.db.user_by_id(gone).await.unwrap().is_none());
+        // Nobody reads it now: out of the catalogue, not downloading for no one (#157).
+        assert_eq!(ctx.cfg().feeds.keys().collect::<Vec<_>>(), ["listed", "shared"]);
+        assert!(ctx.db.subscription(stays, "shared").await.unwrap().is_some());
+    }
+
+    #[tokio::test]
+    async fn a_derived_feed_is_not_scanned_once_its_opml_leaves_config() {
+        // davewiner: the OPML subscription left config.toml, but its 922 derived rows
+        // stayed in the database and kept being scanned under the no-parent fallback.
+        let ctx = test_ctx(config::Config::default()).await;
+        ctx.db.upsert_managed("child", "http://x/child.xml", "Child", "gone-opml").await.unwrap();
+        assert!(
+            subscriptions(&ctx).await.unwrap().iter().all(|s| s.id != "child"),
+            "a derived feed whose parent is gone from config must not be scanned"
+        );
+    }
+
+    #[tokio::test]
+    async fn retiring_a_group_drops_what_was_never_downloaded_and_orphans_the_rest() {
+        let ctx = test_ctx(config::Config::default()).await;
+        ctx.db.upsert_managed("empty", "http://x/empty.xml", "Empty", "parent").await.unwrap();
+        ctx.db.upsert_managed("has-file", "http://x/has-file.xml", "Has File", "parent").await.unwrap();
+        let enc = feed::Enclosure { url: "http://x/ep.mp3".into(), mime: None, length: None };
+        ctx.db.record_enclosure("has-file", "g1", &enc).await.unwrap();
+        ctx.db.mark_downloaded(&enc.url, std::path::Path::new("/downloads/ep.mp3"), 1).await.unwrap();
+
+        retire_group(&ctx, "parent", None).await.unwrap();
+
+        let managed = ctx.db.managed_feeds().await.unwrap();
+        assert!(!managed.iter().any(|m| m.id == "empty"), "nothing downloaded, so it is forgotten");
+        assert!(managed.iter().any(|m| m.id == "has-file"), "has a file on disk, so it is kept");
+        assert!(ctx.db.feed_summary("has-file").await.unwrap().orphaned, "and flagged as orphaned");
+    }
+
+    #[tokio::test]
+    async fn a_stranded_group_is_retired_but_a_promoted_feed_keeps_its_entries() {
+        // davewiner: the OPML left config before retire_group existed, and 11 of its feeds
+        // promoted to config since still said managed = 1.
+        let mut cfg = config::Config::default();
+        cfg.feeds.insert("promoted".into(), feed());
+        cfg.feeds.insert("live-opml".into(), feed());
+        let ctx = test_ctx(cfg).await;
+        ctx.db.upsert_managed("promoted", "http://x/p.xml", "Promoted", "gone-opml").await.unwrap();
+        ctx.db.upsert_managed("empty", "http://x/e.xml", "Empty", "gone-opml").await.unwrap();
+        ctx.db.upsert_managed("listed", "http://x/l.xml", "Listed", "live-opml").await.unwrap();
+        ctx.db.record_entry("promoted", &feed::Entry { guid: "g1".into(), ..Default::default() }).await.unwrap();
+
+        assert_eq!(retire_stranded(&ctx).await.unwrap(), 2, "empty dropped, promoted unmanaged");
+
+        let managed: Vec<String> = ctx.db.managed_feeds().await.unwrap().into_iter().map(|m| m.id).collect();
+        assert_eq!(managed, ["listed"], "a group still in config is left alone");
+        assert_eq!(ctx.db.feed_summary("promoted").await.unwrap().entries, 1, "its entries survive");
+    }
+}

crates/ipx/src/cli.rs

@@ -0,0 +1,234 @@
+//! The command line: its commands, and what each does when no daemon takes it.
+
+use crate::ctx::Ctx;
+use crate::ipc::Command as Cmd;
+use crate::{catalogue, daemon, db, metrics, retention, scan};
+use anyhow::Result;
+use chrono::DateTime;
+use clap::{Parser, Subcommand};
+use std::path::PathBuf;
+use std::sync::Arc;
+
+#[derive(Parser)]
+#[command(name = "ipx", version, about = "A headless podcatcher")]
+pub(crate) struct Cli {
+    /// Config file (default: $XDG_CONFIG_HOME/ipx/config.toml)
+    #[arg(long, global = true)]
+    pub(crate) config: Option<PathBuf>,
+
+    /// Do the work here even if a daemon is running
+    #[arg(long, global = true)]
+    pub(crate) local: bool,
+
+    #[command(subcommand)]
+    pub(crate) command: Command,
+}
+
+#[derive(Subcommand)]
+pub(crate) enum Command {
+    /// Show configured feeds and their state
+    List,
+    /// Scan feeds for new entries
+    Fetch {
+        /// Only this feed id
+        feed: Option<String>,
+        /// Read the feed in full now, even when it is not due and has not changed
+        #[arg(long)]
+        force: bool,
+    },
+    /// Delete old or over-quota downloads
+    Reap {
+        /// Show what would go, delete nothing
+        #[arg(long)]
+        dry_run: bool,
+    },
+    /// Counts of feeds, pending and downloaded enclosures
+    Status,
+    /// Subscribe to a feed
+    Add {
+        url: String,
+        /// Download folder name (default: the feed title)
+        #[arg(long)]
+        folder: Option<String>,
+        /// Only take enclosures matching these keywords
+        #[arg(long, value_delimiter = ',')]
+        keywords: Vec<String>,
+        /// The Directory's category, for a feed that names none of its own (News, Technology)
+        #[arg(long)]
+        category: Option<String>,
+        /// Put it in the Directory for anyone to subscribe to, and keep it there when nobody does
+        #[arg(long)]
+        list: bool,
+    },
+    /// Unsubscribe. Downloads and history are left alone.
+    Rm { feed: String },
+    /// Add every feed in an OPML file
+    Import { file: PathBuf },
+    /// Write subscriptions out as OPML
+    Export { file: PathBuf },
+    /// Add, list or remove the accounts that can sign in to the web UI
+    User {
+        #[command(subcommand)]
+        cmd: UserCmd,
+    },
+    /// Run the scheduler and serve the control socket
+    Daemon {
+        /// Serve the web UI on this address, overriding [web] in the config
+        #[arg(long, value_name = "ADDR")]
+        web: Option<String>,
+    },
+}
+
+#[derive(Subcommand)]
+pub(crate) enum UserCmd {
+    /// Create an account. The password is read from stdin: `echo -n hunter2 | ipx user add ray`
+    Add {
+        name: String,
+        /// May manage other accounts, and is who the shared web token signs in as
+        #[arg(long)]
+        admin: bool,
+        /// Sign-in comes from the proxy instead, so there is no password to set
+        #[arg(long)]
+        no_password: bool,
+    },
+    /// Show the accounts and how each one signs in
+    List,
+    /// Replace a password, read from stdin
+    Passwd { name: String },
+    /// Delete an account and everything it knows: its subscriptions and read state
+    Rm { name: String },
+    /// Rename an account, keeping its feeds, read state and admin rights. This is how an
+    /// account made before the proxy takes the name the proxy signs it in as
+    Rename { name: String, new_name: String },
+}
+
+/// Accounts. Passwords come in on stdin so they never reach a shell history or a `ps`
+/// listing.
+pub(crate) async fn user_cmd(ctx: &Arc<Ctx>, cmd: UserCmd) -> Result<()> {
+    let read_password = || -> Result<String> {
+        use std::io::Read;
+        let mut buf = String::new();
+        std::io::stdin().read_to_string(&mut buf)?;
+        let pw = buf.trim_end_matches(['\n', '\r']).to_string();
+        if pw.is_empty() {
+            anyhow::bail!("no password on stdin: try `echo -n secret | ipx user ...`");
+        }
+        Ok(pw)
+    };
+
+    match cmd {
+        UserCmd::Add { name, admin, no_password } => {
+            let name = name.trim().to_ascii_lowercase();
+            if name.is_empty() {
+                anyhow::bail!("a name is required");
+            }
+            if ctx.db.user_by_name(&name).await?.is_some() {
+                anyhow::bail!("{name} already exists");
+            }
+            let hash = if no_password { None } else { Some(crate::auth::hash_password(&read_password()?)?) };
+            // The first account runs the place; there is nobody else to grant it.
+            let first = ctx.db.users().await?.is_empty();
+            ctx.db.create_user(&name, hash.as_deref(), admin || first).await?;
+            println!(
+                "added {name}{}{}",
+                if admin || first { " (admin)" } else { "" },
+                if no_password { ", signs in through the proxy" } else { "" }
+            );
+            Ok(())
+        }
+        UserCmd::List => {
+            let users = ctx.db.users().await?;
+            if users.is_empty() {
+                println!("no accounts yet: ipx user add <name>");
+            }
+            for u in users {
+                let added = u
+                    .created
+                    .and_then(|t| DateTime::from_timestamp(t, 0))
+                    .map_or("?".into(), |d| d.format("%Y-%m-%d").to_string());
+                let seen = u.last_login.map_or("never signed in".into(), |t| format!("signed in {}", ago(Some(t))));
+                println!(
+                    "{:<20} {:<6} {:<11} added {added}  {seen}",
+                    u.name,
+                    if u.is_admin { "admin" } else { "" },
+                    if u.pass_hash.is_some() { "password" } else { "proxy only" },
+                );
+            }
+            Ok(())
+        }
+        UserCmd::Passwd { name } => {
+            let name = name.trim().to_ascii_lowercase();
+            let user = ctx.db.user_by_name(&name).await?.ok_or_else(|| anyhow::anyhow!("no such account: {name}"))?;
+            ctx.db.set_password(user.id, &crate::auth::hash_password(&read_password()?)?).await?;
+            println!("password changed for {name}, and every browser signed in as {name} signed out");
+            let tokens = ctx.db.api_tokens(user.id).await?.len();
+            if tokens > 0 {
+                println!(
+                    "{tokens} API token(s) of {name}'s still work: remove any that may be someone else's in Settings"
+                );
+            }
+            Ok(())
+        }
+        UserCmd::Rename { name, new_name } => {
+            let name = name.trim().to_ascii_lowercase();
+            // The same rules as a name the proxy vouches for, or the proxy would never find it.
+            let new_name = crate::auth::name_from_header(&new_name)
+                .ok_or_else(|| anyhow::anyhow!("not a usable name: no commas, semicolons or line breaks"))?;
+            let user = ctx.db.user_by_name(&name).await?.ok_or_else(|| anyhow::anyhow!("no such account: {name}"))?;
+            if ctx.db.user_by_name(&new_name).await?.is_some() {
+                anyhow::bail!("{new_name} already exists");
+            }
+            ctx.db.rename_user(user.id, &new_name).await?;
+            println!("renamed {name} to {new_name}");
+            Ok(())
+        }
+        UserCmd::Rm { name } => {
+            let name = name.trim().to_ascii_lowercase();
+            let user = ctx.db.user_by_name(&name).await?.ok_or_else(|| anyhow::anyhow!("no such account: {name}"))?;
+            catalogue::remove_user(ctx, user.id).await?;
+            println!("removed {name}");
+            Ok(())
+        }
+    }
+}
+
+pub(crate) async fn run(ctx: &Arc<Ctx>, cmd: Cmd) -> Result<()> {
+    // What the worker is on, for /metrics: the healthcheck answers while it is stuck (#195).
+    let _job = metrics::STATS.job(match &cmd {
+        Cmd::Fetch { .. } => "scan",
+        Cmd::Reap { .. } => "reap",
+        Cmd::Download { .. } => "download",
+        Cmd::Status => "status",
+        Cmd::Reload => "reload",
+    });
+    match cmd {
+        Cmd::Fetch { feed, force, feeds } => {
+            // Make room before pulling more down, as the original did per download.
+            retention::reap(ctx, /* dry_run */ false, /* standalone */ false).await?;
+            scan::fetch(ctx, feed.as_deref(), force, &feeds).await?;
+            metrics::STATS.last_scan_done.store(db::now(), std::sync::atomic::Ordering::Relaxed);
+            Ok(())
+        }
+        Cmd::Reap { dry_run } => retention::reap(ctx, dry_run, /* standalone */ true).await,
+        Cmd::Download { enclosure } => scan::download_one(ctx, enclosure).await,
+        Cmd::Status => {
+            ctx.out.emit(daemon::status(ctx).await);
+            Ok(())
+        }
+        Cmd::Reload => daemon::reload(ctx).await,
+    }
+}
+
+pub(crate) fn ago(t: Option<i64>) -> String {
+    let Some(t) = t else { return "never".into() };
+    format!("{} ago", duration((db::now() - t).max(0) as u64))
+}
+
+pub(crate) fn duration(secs: u64) -> String {
+    match secs {
+        s if s < 90 => format!("{s}s"),
+        s if s < 5400 => format!("{}m", s / 60),
+        s if s < 172_800 => format!("{}h", s / 3600),
+        s => format!("{}d", s / 86_400),
+    }
+}

crates/ipx/src/config.rs

@@ -3,7 +3,9 @@
 use anyhow::{Context, Result};
 use serde::{Deserialize, Serialize};
 use std::collections::BTreeMap;
+use std::fs::Permissions;
 use std::path::{Path, PathBuf};
+use url::Url;
 
 #[derive(Debug, Default, Clone, Deserialize, Serialize)]
 pub struct Config {
@@ -319,7 +321,7 @@ impl Feed {
 }
 
 fn same_host(a: &str, b: &str) -> bool {
-    let host = |u: &str| url::Url::parse(u).ok().and_then(|u| u.host_str().map(str::to_ascii_lowercase));
+    let host = |u: &str| Url::parse(u).ok().and_then(|u| u.host_str().map(str::to_ascii_lowercase));
     host(a).is_some_and(|h| host(b).as_ref() == Some(&h))
 }
 
@@ -456,7 +458,7 @@ fn write_private(path: &Path, text: &str) -> Result<()> {
     #[cfg(unix)]
     {
         use std::os::unix::fs::PermissionsExt;
-        std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?;
+        std::fs::set_permissions(path, Permissions::from_mode(0o600))?;
     }
     Ok(())
 }
@@ -527,7 +529,7 @@ pub fn unique_slug(text: &str, taken: impl Fn(&str) -> bool) -> String {
     if !taken(&base) {
         return base;
     }
-    (2..).map(|n| format!("{base}-{n}")).find(|s| !taken(s)).unwrap()
+    (2..).map(|n| format!("{base}-{n}")).find(|s| !taken(s)).expect("an unending count finds a free name")
 }
 
 /// `$var`, or `~/fallback` when it is unset or empty, as the XDG base directory spec says.
@@ -756,8 +758,9 @@ mod tests {
         };
         assert_eq!(f.password().as_deref(), Some("literal"));
 
-        unsafe { std::env::set_var("IPX_TEST_PASS", "from-env") };
-        f.password_env = Some("IPX_TEST_PASS".into());
-        assert_eq!(f.password().as_deref(), Some("from-env"));
+        // A variable every test run already has: setting one is unsafe while other tests'
+        // threads read the environment.
+        f.password_env = Some("PATH".into());
+        assert_eq!(f.password(), std::env::var("PATH").ok(), "password_env wins over password");
     }
 }

crates/ipx/src/ctx.rs

@@ -0,0 +1,261 @@
+//! What every command shares: the configuration, the database, the clients, and the state the
+//! daemon's tasks hand each other.
+
+use crate::ipc::{Emitter, Event};
+use crate::{config, db, directory, feed, scan, torrent};
+use anyhow::Result;
+use std::collections::{HashMap, HashSet};
+use std::ops::{Deref, DerefMut};
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicI64};
+use std::time::Duration;
+use tokio::sync::broadcast;
+use tokio::sync::{MutexGuard, Notify, OnceCell, Semaphore};
+
+/// Everything a command needs. One per process.
+pub struct Ctx {
+    /// Swapped wholesale when the web UI rewrites config.toml, so a running daemon picks
+    /// up feed changes without a restart. Callers take a snapshot; no guard is ever held
+    /// across an await.
+    pub cfg: std::sync::RwLock<Arc<config::Config>>,
+    pub db: db::Db,
+    /// For anything a feed or a person names: refuses internal addresses (#320).
+    pub client: reqwest::Client,
+    /// For the services this server is set up to call, which are on its own network by design:
+    /// the directory, as `http://ipx-directory:8100`. Never for an address a feed or a person names.
+    pub local_client: reqwest::Client,
+    /// For feeds only, refusing internal addresses as `client` does: follows no redirects, so `feed::fetch` sees each hop and can tell a feed
+    /// that moved for good from one sent elsewhere for now.
+    pub feed_client: reqwest::Client,
+    pub out: Emitter,
+    /// Started on first use: a BitTorrent session binds ports and starts a DHT, which is
+    /// rude to do for a config that has never seen a torrent.
+    pub torrents: OnceCell<torrent::Torrents>,
+    /// Caps how many torrents run at once when they are detached.
+    pub torrent_slots: Arc<Semaphore>,
+    /// Needed because a subscribed OPML rewrites the feed list as it syncs.
+    pub config_path: PathBuf,
+    /// Run torrents off the command worker. A torrent takes minutes to fetch metadata and
+    /// then seeds for up to an hour, and the worker is sequential -- inline, one torrent
+    /// stops every feed scan, every HTTP download and every status command behind it.
+    /// A one-shot CLI run keeps them inline, or the process would exit mid-download.
+    pub detach_torrents: bool,
+    /// Feeds whose host could not be looked up for the moment, and when to try them again: a
+    /// failure that says nothing about the feed, kept off it and out of the database (#143).
+    /// Forgotten on a restart, which only makes the retry sooner.
+    pub retry: std::sync::Mutex<HashMap<String, i64>>,
+    /// Held from taking a copy of the config to storing it (`edit_cfg`). Each change stores the
+    /// whole catalogue from its copy, and two at once stored over each other: a feed added while
+    /// another add spent a minute fetching was gone once that one stored (#153).
+    // ponytail: this process only. `ipx add`, `rm` and `import` write from a process of their
+    // own, and a change in flight here can still store over theirs; take the copy from the
+    // database rather than memory if that ever happens.
+    pub cfg_write: tokio::sync::Mutex<()>,
+    /// Wakes DeadAir's maker when a show is turned on, so its copies start at once instead of
+    /// at the next pass (#171).
+    pub deadair: Notify,
+    /// Wakes the Signals judge when there are new items or a Signal is made or changed (#254).
+    pub signals: Notify,
+    /// People whose TypeSafe key was refused, passed over until they save another.
+    pub signals_refused: std::sync::Mutex<HashSet<i64>>,
+    /// When the Directory was last cleared of dead feeds (`clean_directory`), 0 for not since
+    /// starting.
+    pub directory_cleaned: AtomicI64,
+    /// Sends the snapshot to the directory now, as when its settings change (#205).
+    pub directory_wake: Notify,
+    /// When the snapshot last went and what came of it, for the admin page.
+    pub directory_shared: std::sync::Mutex<Option<directory::Shared>>,
+    /// Send the snapshot even if it has not changed: after an unblock or a dropped snapshot the
+    /// directory needs it again, though nothing here differs (#206).
+    pub directory_resend: AtomicBool,
+}
+
+/// A copy of the config to change, holding off every other change until `store_cfg` keeps it.
+pub struct CfgEdit<'a> {
+    pub(crate) cfg: config::Config,
+    pub(crate) _held: MutexGuard<'a, ()>,
+}
+
+impl Deref for CfgEdit<'_> {
+    type Target = config::Config;
+    fn deref(&self) -> &config::Config {
+        &self.cfg
+    }
+}
+
+impl DerefMut for CfgEdit<'_> {
+    fn deref_mut(&mut self) -> &mut config::Config {
+        &mut self.cfg
+    }
+}
+
+/// How soon a feed whose host could not be looked up is tried again (#143).
+pub(crate) const RETRY_LOOKUP_SECS: i64 = 600;
+
+impl Ctx {
+    /// Everything a command shares, built once at start: the clients refuse internal addresses
+    /// (#320) but `local_client`, and a daemon sends its events to the socket and the page.
+    pub(crate) fn new(
+        cfg: config::Config,
+        db: db::Db,
+        config_path: PathBuf,
+        is_daemon: bool,
+        events: &broadcast::Sender<Event>,
+    ) -> Result<Self> {
+        let guard = feed::guard::Guard::new(feed::guard::Allowed::parse(&cfg.general.fetch_internal)?);
+        Ok(Ctx {
+            cfg: std::sync::RwLock::new(Arc::new(cfg)),
+            db,
+            client: reqwest::Client::builder()
+                .user_agent(concat!("ipx/", env!("CARGO_PKG_VERSION")))
+                // Connecting only, so it bounds a download's start, not a long download (#108).
+                .connect_timeout(Duration::from_secs(10))
+                // Two minutes without a byte is a stalled download, however long it is: unbounded, one
+                // held the worker, and every scan behind it, for good, while the daemon read healthy (#152).
+                .read_timeout(Duration::from_secs(120))
+                .connector_layer(guard.clone())
+                .build()?,
+            local_client: reqwest::Client::builder()
+                .user_agent(concat!("ipx/", env!("CARGO_PKG_VERSION")))
+                .connect_timeout(Duration::from_secs(10))
+                .build()?,
+            feed_client: reqwest::Client::builder()
+                .user_agent(concat!("ipx/", env!("CARGO_PKG_VERSION")))
+                .connect_timeout(Duration::from_secs(10))
+                .read_timeout(feed::FEED_SILENCE)
+                .redirect(reqwest::redirect::Policy::none())
+                .connector_layer(guard)
+                .build()?,
+            out: if is_daemon {
+                Emitter::socket(events.clone(), /* print */ false)
+            } else {
+                Emitter::terminal()
+            },
+            torrents: OnceCell::new(),
+            torrent_slots: Arc::new(Semaphore::new(2)),
+            config_path,
+            detach_torrents: is_daemon,
+            retry: Default::default(),
+            cfg_write: Default::default(),
+            directory_cleaned: Default::default(),
+            deadair: Default::default(),
+            signals: Default::default(),
+            signals_refused: Default::default(),
+            directory_wake: Default::default(),
+            directory_shared: Default::default(),
+            directory_resend: Default::default(),
+        })
+    }
+
+    /// When a feed is next due, taking a lookup retry into account: not before it.
+    pub(crate) fn due(
+        &self,
+        cfg: &config::Config,
+        feed: &config::Feed,
+        id: &str,
+        state: &db::HttpState,
+        subscribed: bool,
+    ) -> Option<i64> {
+        let at = scan::due_at(cfg, feed, state, subscribed);
+        match self.retry.lock().expect("a panic elsewhere poisoned this lock").get(id) {
+            Some(&retry) => Some(at.unwrap_or(0).max(retry)),
+            None => at,
+        }
+    }
+
+    /// A scan's failure, recorded on the feed, unless it was only a moment's failure to look its
+    /// host up: on 2026-10-05 a DNS outage on Tower failed 204 feeds in a minute with
+    /// "Temporary failure in name resolution", and each, read once a day and backed off as a
+    /// failing feed, sat a day failing, unread and labelled gone (#143). Tried again in minutes
+    /// instead, with nothing on the feed to say it failed.
+    pub(crate) async fn feed_failed(&self, id: &str, url: &str, msg: &str) -> Result<()> {
+        if feed::is_temporary_lookup_failure(msg) {
+            self.retry
+                .lock()
+                .expect("a panic elsewhere poisoned this lock")
+                .insert(id.to_owned(), db::now() + RETRY_LOOKUP_SECS);
+            return Ok(());
+        }
+        self.db.set_feed_error(id, url, msg).await
+    }
+
+    pub fn cfg(&self) -> Arc<config::Config> {
+        self.cfg.read().expect("a panic elsewhere poisoned this lock").clone()
+    }
+
+    /// The config as it stands, to change and then keep with `store_cfg`; any other change waits
+    /// until then.
+    pub async fn edit_cfg(&self) -> CfgEdit<'_> {
+        // A step of its own: waiting here is waiting on another change to the catalogue.
+        let held = tracing::Instrument::instrument(self.cfg_write.lock(), db::step("cfg_lock")).await;
+        CfgEdit { cfg: (*self.cfg()).clone(), _held: held }
+    }
+
+    /// Keeps a changed catalogue or server settings: in the database, and for everything running
+    /// here from now on. What config.toml's save and reload did, before the database held them.
+    pub async fn store_cfg(&self, edit: CfgEdit<'_>) -> Result<()> {
+        tracing::Instrument::instrument(self.db.store_config(&edit.cfg), db::step("store_config")).await?;
+        self.set_cfg(edit.cfg);
+        Ok(())
+    }
+
+    pub(crate) fn set_cfg(&self, cfg: config::Config) {
+        *self.cfg.write().expect("a panic elsewhere poisoned this lock") = Arc::new(cfg);
+        tracing::info!("config reloaded");
+        // A feed added, removed, made private or given a category, or the directory's own
+        // settings changed: the directory hears of it now, not at the next half hour (#205).
+        self.directory_wake.notify_one();
+    }
+
+    pub(crate) async fn torrents(&self) -> Result<&torrent::Torrents> {
+        let cfg = self.cfg();
+        self.torrents.get_or_try_init(|| torrent::Torrents::new(&cfg)).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_util::{feed, test_ctx};
+
+    #[tokio::test]
+    async fn two_changes_to_the_catalogue_at_once_both_keep() {
+        let ctx = Arc::new(test_ctx(config::Config::default()).await);
+        // A slow change, as adding a feed is while it fetches, and a quick one made meanwhile.
+        let slow = {
+            let ctx = ctx.clone();
+            tokio::spawn(async move {
+                let mut cfg = ctx.edit_cfg().await;
+                tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+                cfg.feeds.insert("a".into(), feed());
+                ctx.store_cfg(cfg).await.unwrap();
+            })
+        };
+        tokio::time::sleep(std::time::Duration::from_millis(10)).await;
+        let mut cfg = ctx.edit_cfg().await;
+        cfg.feeds.insert("b".into(), feed());
+        ctx.store_cfg(cfg).await.unwrap();
+        slow.await.unwrap();
+        assert_eq!(ctx.cfg().feeds.keys().collect::<Vec<_>>(), ["a", "b"], "neither stored over the other (#153)");
+        let (_, stored) = ctx.db.stored_config().await.unwrap().unwrap();
+        assert_eq!(stored.keys().collect::<Vec<_>>(), ["a", "b"]);
+    }
+
+    #[tokio::test]
+    async fn a_moments_lookup_failure_is_tried_again_soon_and_not_held_against_the_feed() {
+        let ctx = test_ctx(config::Config::default()).await;
+        let blip = "connecting: dns error: failed to lookup address information: Temporary failure in name resolution";
+        ctx.feed_failed("npr", "https://feeds.npr.org/x", blip).await.unwrap();
+        // Nothing on the feed: no error, no run of failures to back off from or call it gone by.
+        let s = ctx.db.feed_summary("npr").await.unwrap();
+        assert!(s.last_error.is_none() && s.error_since.is_none(), "{:?}", s.last_error);
+        // Due again in minutes, though a feed nobody subscribes to is otherwise read once a day.
+        let state = db::HttpState { last_checked: Some(db::now() - 2 * 86_400), ..Default::default() };
+        let due = ctx.due(&config::Config::default(), &feed(), "npr", &state, false).unwrap();
+        assert!((due - db::now() - RETRY_LOOKUP_SECS).abs() <= 2, "{due}");
+        // A real failure is the feed's.
+        ctx.feed_failed("gone", "https://x/feed", "HTTP 404 Not Found").await.unwrap();
+        assert_eq!(ctx.db.feed_summary("gone").await.unwrap().last_error.as_deref(), Some("HTTP 404 Not Found"));
+    }
+}

crates/ipx/src/daemon.rs

@@ -0,0 +1,356 @@
+//! The daemon: its worker, its web server, and how it stops.
+
+use crate::ctx::Ctx;
+use crate::ipc::{Command as Cmd, Event};
+use crate::{access, auth, catalogue, cli, config, db, deadair, directory, ipc, metrics, signals, web};
+use anyhow::Result;
+use std::future::Future;
+use std::path::Path;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tokio::runtime::Handle;
+use tokio::sync::{broadcast, mpsc, watch};
+use tokio::task::JoinHandle;
+
+/// The catalogue and server settings as the database now has them, in place of the copy in
+/// memory, for a daemon told another ipx changed them (#120); config.toml, read again, for where
+/// things are and who gets in.
+pub(crate) async fn reload(ctx: &Ctx) -> Result<()> {
+    let _held = ctx.cfg_write.lock().await;
+    let cfg = catalogue::assemble_config(&ctx.db, config::Config::load(&ctx.config_path)?, &ctx.config_path).await?;
+    ctx.set_cfg(cfg);
+    Ok(())
+}
+
+/// The counts `ipx status` prints, and /api/status serves. A running daemon's socket answers with
+/// this directly rather than through the job queue.
+pub(crate) async fn status(ctx: &Ctx) -> Event {
+    match tracing::Instrument::instrument(ctx.db.counts(), db::step("counts")).await {
+        Ok((pending, downloaded)) => {
+            let feeds = tracing::Instrument::instrument(catalogue::subscriptions(ctx), db::step("subscriptions"))
+                .await
+                .map(|s| s.len())
+                .unwrap_or(0);
+            // Kept for /metrics, which asks the database nothing (#195); the daemon counts every 30 s.
+            use std::sync::atomic::Ordering::Relaxed;
+            let st = &metrics::STATS;
+            st.feeds.store(feeds as i64, Relaxed);
+            st.pending.store(pending, Relaxed);
+            st.downloaded.store(downloaded, Relaxed);
+            Event::Status { feeds, pending, downloaded }
+        }
+        Err(e) => Event::Error { msg: format!("{e:#}") },
+    }
+}
+
+pub(crate) async fn daemon(
+    ctx: Arc<Ctx>,
+    config_path: PathBuf,
+    web_addr: Option<String>,
+    events: broadcast::Sender<Event>,
+) -> Result<()> {
+    let socket = ctx.cfg().general.socket.clone();
+    if ipc::daemon_is_live(&socket).await {
+        anyhow::bail!("a daemon is already listening on {}", socket.display());
+    }
+
+    // A database with nobody in it cannot be signed into.
+    if ctx.db.users().await?.is_empty() {
+        let dir = ctx.config_path.parent().unwrap_or(Path::new("."));
+        let (password, told) = auth::first_password(dir, std::env::var("IPX_ADMIN_PASSWORD").ok())?;
+        ctx.db.create_user("admin", Some(&crate::auth::hash_password(&password)?), /* admin */ true).await?;
+        tracing::warn!(
+            "no accounts yet: created 'admin' {told}. Change it with `echo -n <password> | ipx user passwd admin`"
+        );
+    }
+
+    // Unread lists start from here, and a reap may be a day off (#347). Slower without, never wrong.
+    if let Err(e) = ctx.db.advance_unread_from().await {
+        tracing::warn!(error = %format!("{e:#}"), "could not work out where unread items start");
+    }
+
+    if let Some(admin) = ctx.db.users().await?.into_iter().find(|u| u.is_admin) {
+        let catalogue: Vec<String> = ctx.cfg().feeds.keys().cloned().collect();
+        match ctx.db.adopt_catalogue(admin.id, &catalogue).await {
+            Ok(0) => {}
+            Ok(n) => tracing::info!(user = %admin.name, feeds = n, "subscribed the first admin to the catalogue"),
+            Err(e) => tracing::error!(error = %e, "could not subscribe the first admin to the catalogue"),
+        }
+    }
+
+    match ctx.db.requeue_interrupted().await {
+        Ok(n) if n > 0 => tracing::info!(count = n, "requeued downloads interrupted by a restart"),
+        Ok(_) => {}
+        Err(e) => tracing::warn!(error = %format!("{e:#}"), "could not requeue interrupted downloads"),
+    }
+
+    match catalogue::retire_stranded(&ctx).await {
+        Ok(0) => {}
+        Ok(n) => tracing::info!(feeds = n, "retired feeds whose OPML is no longer in config"),
+        Err(e) => tracing::warn!(error = %format!("{e:#}"), "could not retire feeds whose OPML is no longer in config"),
+    }
+
+    // Room for any one request's commands: a web request waits inside itself for a free place,
+    // and Download latest sends up to 100. With 64 the rest waited on downloads an hour long, and
+    // every other request that queues work waited behind them (#154).
+    let (tx_cmd, mut rx_cmd) = mpsc::channel::<Cmd>(1024);
+
+    let web = start_web(&ctx, &config_path, web_addr, &tx_cmd, &events).await?;
+    // status and reload are answered by the socket itself; everything else waits its turn in the
+    // queue.
+    let answer: ipc::StatusFn = {
+        let ctx = ctx.clone();
+        Arc::new(move |cmd| {
+            let ctx = ctx.clone();
+            Box::pin(async move {
+                if matches!(cmd, Cmd::Reload)
+                    && let Err(e) = reload(&ctx).await
+                {
+                    return Event::Error { msg: format!("reading the catalogue again: {e:#}") };
+                }
+                status(&ctx).await
+            })
+        })
+    };
+    let server = tokio::spawn(ipc::serve(socket.clone(), events.clone(), tx_cmd, answer));
+
+    // One command at a time: the queue is what keeps two scans from overlapping.
+    tracing::info!(feeds = catalogue::subscriptions(&ctx).await.map(|s| s.len()).unwrap_or(0), "daemon started");
+
+    // A signal has to be able to interrupt work in progress, not just the wait between
+    // jobs. Racing `shutdown()` in the outer select only cancels branch selection: once
+    // inside a long download the daemon stopped listening and had to be SIGKILLed.
+    let (tx_stop, rx_stop) = tokio::sync::watch::channel(false);
+    tokio::spawn(async move {
+        shutdown().await;
+        let _ = tx_stop.send(true);
+    });
+    // Off the worker: the first pass over a library is hours of ffmpeg, and scans and downloads
+    // must not wait behind it.
+    tokio::spawn(deadair::run(ctx.clone(), rx_stop.clone()));
+    tokio::spawn(signals::run(ctx.clone(), rx_stop.clone()));
+    tokio::spawn(directory::run(ctx.clone(), rx_stop.clone()));
+    // The library's counts for /metrics. The Docker healthcheck asked for them every 30 s until
+    // it was removed: Docker runs it with exec, which failed on Tower whenever root was logged
+    // out (#218).
+    {
+        let (ctx, mut stop) = (ctx.clone(), rx_stop.clone());
+        tokio::spawn(async move {
+            let mut every = tokio::time::interval(Duration::from_secs(30));
+            loop {
+                tokio::select! {
+                    _ = every.tick() => { status(&ctx).await; }
+                    _ = stop.changed() => break,
+                }
+            }
+        });
+    }
+
+    // Runs one job, abandoning it if a signal arrives. Returns false to end the loop.
+    async fn until_stopped(ctx: &Ctx, rx: &watch::Receiver<bool>, job: impl Future<Output = Result<()>>) -> bool {
+        let mut stop = rx.clone();
+        tokio::select! {
+            _ = stop.changed() => {
+                tracing::info!("signal received; abandoning the job in progress");
+                false
+            }
+            result = job => {
+                if let Err(e) = result {
+                    ctx.out.emit(Event::Error { msg: format!("{e:#}") });
+                }
+                true
+            }
+        }
+    }
+
+    let mut stop = rx_stop.clone();
+    // The first pass at once, as the minute's tick did: what came due while it was down.
+    let mut first = true;
+    loop {
+        if *stop.borrow() {
+            break;
+        }
+        let wait = if std::mem::take(&mut first) { Duration::ZERO } else { until_next_scan(&ctx).await };
+        tokio::select! {
+            biased;
+            _ = stop.changed() => break,
+            Some(cmd) = rx_cmd.recv() => {
+                // Both halves of the protocol are logged under one target so the UI can
+                // show the conversation on its own: this is everything arriving, whatever
+                // the source -- a socket client, the CLI proxying, or the web UI.
+                tracing::info!(
+                    target: "ipx::io",
+                    "-> {}",
+                    serde_json::to_string(&cmd).unwrap_or_else(|_| format!("{cmd:?}"))
+                );
+                if !until_stopped(&ctx, &rx_stop, cli::run(&ctx, cmd)).await {
+                    break;
+                }
+            }
+            _ = tokio::time::sleep(wait) => {
+                // Per-feed schedule and TTL decide what actually gets polled.
+                let job = cli::run(&ctx, Cmd::Fetch { feed: None, force: false, feeds: vec![] });
+                if !until_stopped(&ctx, &rx_stop, job).await {
+                    break;
+                }
+            }
+        }
+    }
+
+    server.abort();
+    if let Some(w) = web {
+        w.abort();
+    }
+    let _ = std::fs::remove_file(&socket);
+    tracing::info!("daemon stopped");
+    Ok(())
+}
+
+/// Starts the web UI when it is switched on, minting and saving a token if there is none.
+pub(crate) async fn start_web(
+    ctx: &Arc<Ctx>,
+    config_path: &Path,
+    web_addr: Option<String>,
+    cmds: &mpsc::Sender<Cmd>,
+    events: &broadcast::Sender<Event>,
+) -> Result<Option<JoinHandle<()>>> {
+    let cfg = ctx.cfg();
+    let enabled = cfg.web.enabled || web_addr.is_some();
+    if !enabled {
+        return Ok(None);
+    }
+    let bind = web_addr.unwrap_or_else(|| cfg.web.bind.clone());
+
+    if cfg.web.token.is_empty() {
+        let mut fresh = (*cfg).clone();
+        fresh.web.enabled = true;
+        fresh.web.bind = bind.clone();
+        fresh.web.token = crate::auth::new_session_token();
+        // The token is config.toml's, not the database's: it decides who gets in.
+        fresh.save_bootstrap(config_path)?;
+        ctx.set_cfg(fresh.clone());
+        // The token signs in as the admin, and whatever reads this process's output (docker logs,
+        // for one) is wider than who reads config.toml. So say where it is, never what it is.
+        // Logged, not printed, so a JSON log stays one object a line (#91).
+        tracing::info!(
+            "web ui token generated and saved to {} as [web] token. Open http://{bind}/?token=<that token>",
+            config_path.display()
+        );
+    } else {
+        tracing::info!("web ui at http://{bind}/ (the sign-in token is [web] token in {})", config_path.display());
+    }
+
+    // Bound beyond localhost, as a container has to be for its port to be published. Worth a
+    // warning only when nothing but a password stands in front of it: it said "the token is all
+    // that guards it" on every start of production, behind Cloudflare Access, and was the only
+    // warning in a healthy log (#106).
+    if ctx.cfg().web.binds_publicly() {
+        if ctx.cfg().web.access().is_some() {
+            tracing::info!(
+                bind,
+                "web ui is reachable off this machine; signing in takes an account's password or the admin token, or Cloudflare Access through a trusted proxy"
+            );
+        } else {
+            tracing::warn!(
+                bind,
+                "web ui is reachable off this machine; an account's password or the admin token is all that guards it"
+            );
+        }
+    }
+
+    let access = Arc::new(access::Keys::default());
+    if let Some((team, _)) = ctx.cfg().web.access() {
+        let (access, ctx, team) = (access.clone(), ctx.clone(), team.to_owned());
+        tokio::spawn(async move { access.prefetch(&ctx.client, &team).await });
+    }
+    let playing = deadair::Playing::default();
+    if !cfg.web.metrics_bind.is_empty() {
+        let sources = metrics::Sources {
+            ctx: ctx.clone(),
+            cmds: cmds.clone(),
+            events: events.clone(),
+            playing: playing.clone(),
+            runtime: Handle::current(),
+            started: Instant::now(),
+        };
+        let at = metrics::spawn(&cfg.web.metrics_bind, sources)?;
+        tracing::info!(%at, "serving metrics for Prometheus at /metrics");
+    }
+    let state = web::WebState { ctx: ctx.clone(), cmds: cmds.clone(), events: events.clone(), access, playing };
+    Ok(Some(tokio::spawn(async move {
+        if let Err(e) = web::serve(state, &bind).await {
+            tracing::error!(error = %format!("{e:#}"), "web ui stopped");
+        }
+    })))
+}
+
+pub(crate) async fn shutdown() {
+    use tokio::signal::unix;
+    use tokio::signal::unix::SignalKind;
+    let mut term = match unix::signal(SignalKind::terminate()) {
+        Ok(s) => s,
+        Err(_) => return std::future::pending().await,
+    };
+    tokio::select! {
+        _ = tokio::signal::ctrl_c() => {}
+        _ = term.recv() => {}
+    }
+}
+
+/// How long the daemon may sleep before a feed is due: until the earliest one, at least 30 s and
+/// at most 10 minutes. It ticked every minute and ran a scan pass each time, 80% of them finding
+/// nothing due (#114). The floor keeps a feed that never gets a check time from spinning it; the
+/// ceiling picks up within ten minutes what no command announces, such as `ipx add` or a
+/// shorter schedule. A command, a refresh or a feed added on the page, wakes it at once anyway.
+pub(crate) async fn until_next_scan(ctx: &Ctx) -> std::time::Duration {
+    let next = async {
+        let cfg = ctx.cfg();
+        let subscribed = ctx.db.subscriber_counts().await?;
+        let states = ctx.db.http_states().await?;
+        anyhow::Ok(
+            catalogue::subscriptions(ctx)
+                .await?
+                .iter()
+                // A feed with no row yet has never been checked: due now.
+                .map(|s| {
+                    let never = db::HttpState::default();
+                    let state = states.get(&s.id).unwrap_or(&never);
+                    ctx.due(&cfg, &s.cfg, &s.id, state, subscribed.contains_key(&s.id)).unwrap_or(0)
+                })
+                .min(),
+        )
+    };
+    let wait = match next.await {
+        Ok(Some(at)) => (at - db::now()).max(0) as u64,
+        Ok(None) => u64::MAX,
+        Err(e) => {
+            tracing::warn!(error = %format!("{e:#}"), "could not work out when the next feed is due");
+            0
+        }
+    };
+    std::time::Duration::from_secs(wait.clamp(30, 600))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_util::{feed, test_ctx};
+
+    #[tokio::test]
+    async fn a_reload_takes_the_catalogue_another_ipx_wrote() {
+        let with = |id: &str| {
+            let mut cfg = config::Config::default();
+            cfg.feeds.insert(id.into(), config::Feed { url: format!("http://x/{id}.xml"), ..feed() });
+            cfg
+        };
+        let held = with("kept-in-memory");
+        let ctx = test_ctx(held.clone()).await;
+        ctx.db.store_config(&held).await.unwrap();
+        // `ipx add` and `ipx rm`, from a process of their own (#120).
+        ctx.db.store_config(&with("added-by-the-cli")).await.unwrap();
+        reload(&ctx).await.unwrap();
+        let urls: Vec<String> = ctx.cfg().feeds.values().map(|f| f.url.clone()).collect();
+        assert_eq!(urls, vec!["http://x/added-by-the-cli.xml".to_string()]);
+    }
+}

crates/ipx/src/db.rs

@@ -7,15 +7,19 @@ use crate::entity::{
 };
 use anyhow::{Context, Result};
 use sea_orm::sea_query::{Expr, Func};
+use sea_orm::sqlx::sqlite::SqliteSynchronous;
 use sea_orm::{
     ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, Set,
     Statement,
 };
+use sea_orm::{ConnectOptions, Database, DatabaseConnection};
 use std::path::Path;
+use std::time::SystemTime;
+use tracing::Instrument;
 
 /// A pool of connections, where there was one connection behind a global lock.
 pub struct Db {
-    orm: sea_orm::DatabaseConnection,
+    orm: DatabaseConnection,
     /// A test's database file, removed when the test is done with it.
     #[cfg(test)]
     tmp: Option<std::path::PathBuf>,
@@ -46,7 +50,7 @@ impl Drop for Db {
 /// here, as the old migrate() did for SQLite, or sea-orm-migration once there are several. A
 /// database from before 0.7 took its last columns from that migrate(), so it upgrades through
 /// a 0.7 release first.
-async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> {
+async fn create_missing(orm: &DatabaseConnection) -> Result<()> {
     use crate::entity::*;
     use sea_orm::{ConnectionTrait, Schema};
     let schema = Schema::new(orm.get_database_backend());
@@ -241,15 +245,15 @@ fn is_postgres(location: &str) -> bool {
 
 /// sqlx's defaults for SQLite are what ipx wants, foreign keys on and a five-second wait for a
 /// lock, which is what rusqlite was set to, but for one.
-async fn connect(location: &str, keep_open: bool) -> Result<sea_orm::DatabaseConnection> {
-    let mut opts = sea_orm::ConnectOptions::new(url_for(location));
+async fn connect(location: &str, keep_open: bool) -> Result<DatabaseConnection> {
+    let mut opts = ConnectOptions::new(url_for(location));
     opts.sqlx_logging(false);
     // NORMAL, SQLite's own advice for WAL: synced at checkpoints, not after every commit. FULL,
     // the default, waited for the disk after nearly every statement, and a feed's items are
     // written a row at a time: 1.3s a feed on Tower's /tmp, where one synchronous write takes
     // 52ms, and 39s to subscribe an admin to 1,500 feeds (#135). A crash of ipx loses nothing;
     // only a power cut can lose the last transactions, and it never corrupts the file.
-    opts.map_sqlx_sqlite_opts(|o| o.synchronous(sea_orm::sqlx::sqlite::SqliteSynchronous::Normal));
+    opts.map_sqlx_sqlite_opts(|o| o.synchronous(SqliteSynchronous::Normal));
     // The same on Postgres, for ipx's own sessions: a commit waited for the WAL to reach the
     // disk, 12.8ms on the databases project's server, and at about fifty commits a feed a scan
     // took most of a second a feed (#140). A crash of the Postgres server can lose the last
@@ -271,7 +275,7 @@ async fn connect(location: &str, keep_open: bool) -> Result<sea_orm::DatabaseCon
     if keep_open && is_postgres(location) {
         opts.min_connections(20);
     }
-    sea_orm::Database::connect(opts).await.with_context(|| format!("opening {}", redact(location)))
+    Database::connect(opts).await.with_context(|| format!("opening {}", redact(location)))
 }
 
 /// The URL `connect` hands sqlx for a location: a SQLite file, created if missing, or a Postgres
@@ -475,7 +479,6 @@ impl Db {
         Ok(Self { orm, tmp: Some(path) })
     }
 
-    #[cfg(test)]
     /// The first column of every row, for a test to check what a query left behind.
     #[cfg(test)]
     pub async fn i64s_for_test(&self, sql: &str) -> Vec<i64> {
@@ -494,6 +497,7 @@ impl Db {
     }
 
     /// What a feed says it is, for its page in the Directory.
+    #[cfg(test)]
     pub async fn feed_description(&self, feed_id: &str) -> Result<Option<String>> {
         Ok(feeds::Entity::find_by_id(feed_id.to_owned()).one(&self.orm).await?.and_then(|f| f.description))
     }
@@ -533,7 +537,7 @@ impl Db {
         let one = || sea_orm::Value::from(only.map(str::to_owned));
         // A step a query: the feed list's trace was 290 ms with nothing under it.
         let counts = |of: &'static str, sql: &'static str, args: Vec<sea_orm::Value>| async move {
-            tracing::Instrument::instrument(self.rows(sql, args), step(of))
+            Instrument::instrument(self.rows(sql, args), step(of))
                 .await?
                 .iter()
                 .map(|r| Ok((r.try_get::<String>("", "feed_id")?, r.try_get::<i64>("", "n")?)))
@@ -569,7 +573,7 @@ impl Db {
             ),
             vec![user_id.into(), one()],
         );
-        let blocked = tracing::Instrument::instrument(
+        let blocked = Instrument::instrument(
             blocklists::Entity::find().filter(blocklists::Column::UserId.eq(user_id)).all(&self.orm),
             step("blocked"),
         );
@@ -577,7 +581,7 @@ impl Db {
         if let Some(id) = only {
             rows = rows.filter(feeds::Column::Id.eq(id));
         }
-        let rows = tracing::Instrument::instrument(rows.all(&self.orm), step("feeds"));
+        let rows = Instrument::instrument(rows.all(&self.orm), step("feeds"));
         let (entries, downloaded, unread, blocked, rows) = tokio::try_join!(
             entries,
             downloaded,
@@ -820,7 +824,7 @@ impl Db {
 
     /// Returns true when this enclosure URL is new. False means we have downloaded it
     /// before, or deliberately reaped it -- either way it is not fetched again.
-    pub async fn mark_downloaded(&self, url: &str, path: &std::path::Path, bytes: u64) -> Result<()> {
+    pub async fn mark_downloaded(&self, url: &str, path: &Path, bytes: u64) -> Result<()> {
         self.exec(
             "UPDATE enclosures SET state = 'done', path = $2, bytes_done = $3,
                 downloaded_at = $4, last_error = NULL WHERE url = $1",
@@ -1003,7 +1007,7 @@ impl Db {
             .await?
             .into_iter()
             .filter_map(|e| Some((e.id, e.path?)))
-            .filter(|(_, p)| !std::path::Path::new(p).exists())
+            .filter(|(_, p)| !Path::new(p).exists())
             .collect())
     }
 
@@ -1137,6 +1141,7 @@ impl Db {
 }
 
 impl Db {
+    #[cfg(test)]
     pub async fn unread_count(&self, user_id: i64, feed_id: &str) -> Result<i64> {
         let r = self
             .rows(
@@ -1397,6 +1402,7 @@ impl Db {
     /// person subscribes to when `feed_id` is None (All Subscriptions). `search` matches title
     /// and description, case-insensitively.
     #[allow(clippy::too_many_arguments)]
+    #[cfg(test)]
     pub async fn entries_in(
         &self,
         user_id: i64,
@@ -1554,6 +1560,7 @@ impl Db {
     }
 
     /// How many entries `entries_in` would page through, so the UI knows whether there is more.
+    #[cfg(test)]
     pub async fn count_in(
         &self,
         user_id: i64,
@@ -1792,16 +1799,6 @@ impl Db {
             .collect()
     }
 
-    /// Feeds with any audio or video enclosure: the Directory's Podcasts, with the rest Blogs.
-    /// Reaped files keep their rows, so a show whose files have all been purged still counts.
-    pub async fn media_feeds(&self) -> Result<std::collections::HashSet<String>> {
-        self.rows("SELECT DISTINCT feed_id FROM enclosures WHERE mime LIKE 'audio/%' OR mime LIKE 'video/%'", vec![])
-            .await?
-            .iter()
-            .map(|r| Ok(r.try_get("", "feed_id")?))
-            .collect()
-    }
-
     /// Who else would miss this file: subscribers other than `user_id` who have starred
     /// the item or have not read it yet. Deleting is deleting their copy too.
     pub async fn others_wanting(&self, enclosure_id: i64, user_id: i64) -> Result<(i64, i64)> {
@@ -3087,7 +3084,7 @@ impl Db {
 }
 
 pub fn now() -> i64 {
-    std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0)
+    SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0)
 }
 
 #[cfg(test)]

crates/ipx/src/deadair.rs

@@ -20,12 +20,18 @@ use std::sync::Arc;
 
 use tracing::Instrument;
 
-use crate::db::step;
+use crate::db;
 
 use anyhow::{Context, Result, bail};
 use serde::{Deserialize, Serialize};
 
-use crate::Ctx;
+use crate::ctx::Ctx;
+use axum::body::Bytes;
+use std::collections::{HashSet, VecDeque};
+use std::io::SeekFrom;
+use std::process::Stdio;
+use std::time::{Duration, Instant};
+use tokio::fs::File;
 
 /// Quieter than this for longer than `MIN_SILENCE` is a silence: what the measurements on the
 /// issue used. A podcast's room tone sits well below it and speech well above.
@@ -44,7 +50,7 @@ pub const DEFAULT_PAUSE: i64 = 300;
 const MAX_PIECE: f64 = 10.0;
 /// How long the maker waits when there is nothing to do: a download or a feed turned on is
 /// picked up this long after, at most.
-const IDLE: std::time::Duration = std::time::Duration::from_secs(120);
+const IDLE: Duration = Duration::from_secs(120);
 
 /// A span of the original, in milliseconds, that DeadAir leaves out.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -81,28 +87,28 @@ pub fn id3_len(head: &[u8]) -> u64 {
 pub async fn kept(
     path: &Path,
     plan: &Plan,
-) -> std::io::Result<(u64, impl futures_util::Stream<Item = std::io::Result<axum::body::Bytes>> + use<>)> {
+) -> std::io::Result<(u64, impl futures_util::Stream<Item = std::io::Result<Bytes>> + use<>)> {
     use tokio::io::{AsyncReadExt, AsyncSeekExt};
-    let mut file = tokio::fs::File::open(path).await?;
+    let mut file = File::open(path).await?;
     let mut head = [0u8; 10];
     let n = file.read(&mut head).await?;
     // Only up to the first frame kept: past the tag can be the Xing frame, which gives the
     // original's length.
     let first = plan.pieces.first().map_or(0, |p| p.0);
     let tag = id3_len(&head[..n]).min(first);
-    let ranges: std::collections::VecDeque<(u64, u64)> =
+    let ranges: VecDeque<(u64, u64)> =
         std::iter::once((0, tag)).chain(plan.pieces.iter().map(|&(o, n, _)| (o, n))).filter(|r| r.1 > 0).collect();
     let len = ranges.iter().map(|r| r.1).sum();
     let body = futures_util::stream::try_unfold((file, ranges), |(mut f, mut left)| async move {
         let Some((off, n)) = left.pop_front() else { return Ok(None) };
         let mut buf = vec![0; n.min(1 << 16) as usize];
-        f.seek(std::io::SeekFrom::Start(off)).await?;
+        f.seek(SeekFrom::Start(off)).await?;
         f.read_exact(&mut buf).await?;
         let got = buf.len() as u64;
         if got < n {
             left.push_front((off + got, n - got));
         }
-        Ok(Some((axum::body::Bytes::from(buf), (f, left))))
+        Ok(Some((Bytes::from(buf), (f, left))))
     });
     Ok((len, body))
 }
@@ -341,7 +347,11 @@ pub async fn on_save(
     secs: i64,
     duration: Option<i64>,
 ) -> Result<(i64, Option<i64>)> {
-    let plan = playing.lock().unwrap().get(&(user, feed.to_owned(), guid.to_owned())).cloned();
+    let plan = playing
+        .lock()
+        .expect("a panic elsewhere poisoned this lock")
+        .get(&(user, feed.to_owned(), guid.to_owned()))
+        .cloned();
     let Some(p) = plan else { return Ok((secs, duration)) };
     let original = |s: i64| to_original(&p.cuts, s * 1000) / 1000;
     let (secs, duration) = (original(secs), duration.map(original));
@@ -367,7 +377,7 @@ pub async fn run(ctx: Arc<Ctx>, mut stop: tokio::sync::watch::Receiver<bool>) {
     }
     // Files ffmpeg failed on, passed over until the next start. Not stored: a failure recorded
     // as the file having nothing to cut was never tried again, even once a fix was deployed (#174).
-    let mut failed = std::collections::HashSet::new();
+    let mut failed = HashSet::new();
     loop {
         // Dropping the pass on a signal kills the ffmpeg it is running (`kill_on_drop`).
         let done = tokio::select! {
@@ -390,7 +400,7 @@ pub async fn run(ctx: Arc<Ctx>, mut stop: tokio::sync::watch::Receiver<bool>) {
 async fn have_ffmpeg() -> bool {
     tokio::process::Command::new("ffmpeg")
         .arg("-version")
-        .stdout(std::process::Stdio::null())
+        .stdout(Stdio::null())
         .status()
         .await
         .is_ok_and(|s| s.success())
@@ -400,7 +410,7 @@ async fn have_ffmpeg() -> bool {
 /// shortened has already heard through gets none: finished episodes are rarely played again.
 // shortcut: those stay on the list and are looked at again each pass, one query each; record
 // them as passed over if a large library makes the pass slow.
-async fn next(ctx: &Ctx, failed: &mut std::collections::HashSet<i64>) -> Result<bool> {
+async fn next(ctx: &Ctx, failed: &mut HashSet<i64>) -> Result<bool> {
     // Nobody with DeadAir on for anything, which is most passes: nothing more to ask the database.
     use std::sync::atomic::Ordering::Relaxed;
     let st = &crate::metrics::STATS;
@@ -429,7 +439,7 @@ async fn next(ctx: &Ctx, failed: &mut std::collections::HashSet<i64>) -> Result<
             continue;
         }
         let src = crate::download::playable(Path::new(&t.path));
-        let started = std::time::Instant::now();
+        let started = Instant::now();
         // A trace of its own for each file looked at, not for the passes that find nothing to do,
         // which are most of them.
         let span = tracing::info_span!("deadair", feed = t.feed_id, enclosure = t.id, cuts = tracing::field::Empty);
@@ -446,7 +456,7 @@ async fn next(ctx: &Ctx, failed: &mut std::collections::HashSet<i64>) -> Result<
         };
         ctx.db
             .store_plans(t.id, &serde_json::to_string(&plans)?)
-            .instrument(span.in_scope(|| step("store_plans")))
+            .instrument(span.in_scope(|| db::step("store_plans")))
             .await?;
         let usual = plans.get(&DEFAULT_PAUSE);
         span.record("cuts", usual.map_or(0, |p| p.cuts.len()));
@@ -474,7 +484,7 @@ async fn next(ctx: &Ctx, failed: &mut std::collections::HashSet<i64>) -> Result<
 /// anything but MP3 or AAC on its own (ADTS; AAC in MP4 has no byte ranges a playlist can
 /// use), or one with no silence long enough to cut.
 pub async fn analyse(src: &Path) -> Result<Option<BTreeMap<i64, Plan>>> {
-    let Some((first, frame)) = probe(src).instrument(step("probe")).await? else { return Ok(None) };
+    let Some((first, frame)) = probe(src).instrument(db::step("probe")).await? else { return Ok(None) };
     let path = src.display().to_string();
     let log = ffmpeg(&[
         "-hide_banner",
@@ -489,11 +499,11 @@ pub async fn analyse(src: &Path) -> Result<Option<BTreeMap<i64, Plan>>> {
         "null",
         "-",
     ])
-    .instrument(step("silencedetect"))
+    .instrument(db::step("silencedetect"))
     .await
     .context("finding the silences")?;
     let silences = parse_silences(&log);
-    let frames = frame_bytes(&path).instrument(step("frame_bytes")).await?;
+    let frames = frame_bytes(&path).instrument(db::step("frame_bytes")).await?;
     let plans: BTreeMap<i64, Plan> = PAUSES
         .iter()
         .map(|&pause| {
@@ -590,7 +600,7 @@ async fn run_out(prog: &str, args: &[&str]) -> Result<String> {
 
 fn command(prog: &str) -> tokio::process::Command {
     let mut c = tokio::process::Command::new(prog);
-    c.kill_on_drop(true).stdin(std::process::Stdio::null());
+    c.kill_on_drop(true).stdin(Stdio::null());
     c
 }
 
@@ -795,6 +805,133 @@ mod tests {
         }
         std::fs::remove_dir_all(&dir).unwrap();
     }
+
+    use crate::config;
+    use crate::test_util::test_ctx;
+
+    #[tokio::test]
+    async fn deadair_hands_out_each_persons_clock_and_saves_in_the_originals() {
+        let ctx = test_ctx(config::Config::default()).await;
+        let db = &ctx.db;
+        let u = db.create_user("dee", None, false).await.unwrap();
+        db.subscribe(u, "f").await.unwrap();
+        for sql in [
+            "INSERT INTO entries (feed_id, guid, first_seen, duration) VALUES ('f', 'g', 1, 600)",
+            "INSERT INTO enclosures (id, feed_id, guid, url, path, state, downloaded_at)
+             VALUES (7, 'f', 'g', 'http://x/a.mp3', '/x/a.mp3', 'done', 100)",
+        ] {
+            db.sql(sql).await.unwrap();
+        }
+        // Two minutes cut at the usual pause, both before the place saved; 40 s at the longest.
+        let plans = r#"{"300": {"cuts": [[60000,120000],[200000,260000]], "pieces": []},
+                        "500": {"cuts": [[60000,100000]], "pieces": []}}"#;
+        db.store_plans(7, plans).await.unwrap();
+        db.set_position(u, "f", "g", 300, None).await.unwrap();
+        let order = db::order_sql("published", "desc");
+        let page = || {
+            let (order, ctx) = (order.clone(), &ctx);
+            async move {
+                let mut rows = ctx.db.entries_in(u, Some("f"), db::Filter::All, None, 0, 10, &order).await.unwrap();
+                for_reader(ctx, u, &mut rows).await.unwrap();
+                rows.remove(0)
+            }
+        };
+        let off = page().await;
+        assert_eq!((off.position, off.enclosures[0].deadair), (300, None), "off until turned on for the feed");
+        assert!(!off.enclosures[0].deadair_pending);
+
+        db.set_deadair(u, "f", Some(true)).await.unwrap();
+        db.sql("DELETE FROM deadair_plans").await.unwrap();
+        let waiting = page().await;
+        assert!(waiting.enclosures[0].deadair_pending, "on, and its silences not found yet");
+        db.store_plans(7, plans).await.unwrap();
+        let on = page().await;
+        assert!(!on.enclosures[0].deadair_pending);
+        assert_eq!((on.position, on.duration, on.enclosures[0].deadair), (180, Some(480), Some(120)));
+        // A longer pause of their own: their clock, and the playlist, are that plan's (#182).
+        db.set_deadair_pause(u, "f", 500).await.unwrap();
+        let long = page().await;
+        assert_eq!((long.position, long.duration, long.enclosures[0].deadair), (260, Some(560), Some(40)));
+        assert_eq!(plan_for(&ctx, u, "f", 7).await.unwrap().unwrap().saved_ms(), 40_000);
+        db.set_deadair_pause(u, "f", 300).await.unwrap();
+
+        // Saved from the playlist at 3:20 of its 8:00: 5:20 of the original's 10:00, short of 75%.
+        let playing = Playing::default();
+        let plan = plan_for(&ctx, u, "f", 7).await.unwrap().unwrap();
+        playing.lock().unwrap().insert((u, "f".into(), "g".into()), Arc::new(plan));
+        let saved = on_save(&ctx, &playing, u, "f", "g", 200, Some(480)).await.unwrap();
+        assert_eq!(saved, (320, Some(600)));
+        assert_eq!(db.deadair_total(u).await.unwrap(), 0);
+        // Past 75% (6:00 of the playlist is 8:00 of the original), counted once.
+        for _ in 0..2 {
+            on_save(&ctx, &playing, u, "f", "g", 360, None).await.unwrap();
+        }
+        assert_eq!(db.deadair_total(u).await.unwrap(), 120_000);
+        // Playing the original again: saved as it is.
+        playing.lock().unwrap().clear();
+        assert_eq!(on_save(&ctx, &playing, u, "f", "g", 200, Some(600)).await.unwrap(), (200, Some(600)));
+        // Turned off again: the original's clock, and no playlist.
+        db.set_deadair(u, "f", Some(false)).await.unwrap();
+        assert_eq!(page().await.enclosures[0].deadair, None);
+        assert!(plan_for(&ctx, u, "f", 7).await.unwrap().is_none());
+    }
+
+    #[tokio::test]
+    async fn two_people_play_one_file_each_in_their_own_version_and_place() {
+        let ctx = test_ctx(config::Config::default()).await;
+        let db = &ctx.db;
+        for sql in [
+            "INSERT INTO entries (feed_id, guid, first_seen, duration) VALUES ('f', 'g', 1, 600)",
+            "INSERT INTO enclosures (id, feed_id, guid, url, path, state, downloaded_at)
+             VALUES (7, 'f', 'g', 'http://x/a.mp3', '/x/a.mp3', 'done', 100)",
+        ] {
+            db.sql(sql).await.unwrap();
+        }
+        db.store_plans(
+            7,
+            r#"{"300": {"cuts": [[60000,120000],[200000,260000]], "pieces": []},
+                              "500": {"cuts": [[60000,100000]], "pieces": []}}"#,
+        )
+        .await
+        .unwrap();
+        // One file on disk, three listeners at once: DeadAir at the usual pause, at a longer
+        // one, and the original.
+        let (usual, long, plain) = (
+            db.create_user("usual", None, false).await.unwrap(),
+            db.create_user("long", None, false).await.unwrap(),
+            db.create_user("plain", None, false).await.unwrap(),
+        );
+        for u in [usual, long, plain] {
+            db.subscribe(u, "f").await.unwrap();
+        }
+        for u in [usual, long] {
+            db.set_deadair(u, "f", Some(true)).await.unwrap();
+        }
+        db.set_deadair_pause(long, "f", 500).await.unwrap();
+        // Each is handed their own playlist, as /media/7/deadair.m3u8 does.
+        let playing = Playing::default();
+        for u in [usual, long] {
+            let plan = plan_for(&ctx, u, "f", 7).await.unwrap().unwrap();
+            playing.lock().unwrap().insert((u, "f".into(), "g".into()), Arc::new(plan));
+        }
+        assert!(plan_for(&ctx, plain, "f", 7).await.unwrap().is_none());
+        // All three save 3:20 of what they hear, as the position route does: each is read in
+        // that person's own clock and kept as a place in the original, theirs alone.
+        let mut kept = vec![];
+        for u in [usual, long, plain] {
+            let (secs, len) = on_save(&ctx, &playing, u, "f", "g", 200, None).await.unwrap();
+            db.set_position(u, "f", "g", secs, len).await.unwrap();
+            kept.push(secs);
+        }
+        assert_eq!(kept, [320, 240, 200], "two minutes cut before 3:20, forty seconds, none");
+        // Each comes back to 3:20 of their own version, whatever the others saved.
+        let order = db::order_sql("published", "desc");
+        for (u, len) in [(usual, 480), (long, 560), (plain, 600)] {
+            let mut rows = db.entries_in(u, Some("f"), db::Filter::All, None, 0, 10, &order).await.unwrap();
+            for_reader(&ctx, u, &mut rows).await.unwrap();
+            assert_eq!((rows[0].position, rows[0].duration), (200, Some(len)), "user {u}");
+        }
+    }
 }
 
 /// A real episode, for trying the cut on something other than a test tone:

crates/ipx/src/directory.rs

@@ -7,10 +7,16 @@ use std::collections::{HashMap, HashSet};
 use ipx_directory::{FeedPage, Listing, Me, SnapshotEntry, SnapshotResult};
 use serde::de::DeserializeOwned;
 
-use crate::{Ctx, Sub, config, feed};
+use crate::catalogue::Sub;
+use crate::ctx::Ctx;
+use crate::{config, feed};
+use std::fmt::{Display, Formatter};
+use std::sync::{Arc, LazyLock};
+use std::time::{Duration, Instant};
+use tracing::Instrument;
 
 /// How often the snapshot goes when nothing has woken the sender sooner.
-const SHARE_EVERY: std::time::Duration = std::time::Duration::from_secs(30 * 60);
+const SHARE_EVERY: Duration = Duration::from_secs(30 * 60);
 
 /// What went wrong asking the directory, in words for an admin.
 #[derive(Debug)]
@@ -23,8 +29,8 @@ pub enum DirError {
     Unreachable(String),
 }
 
-impl std::fmt::Display for DirError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl Display for DirError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
             DirError::NotConfigured => {
                 write!(f, "No directory is set up on this server. An admin sets one in Admin, Directory.")
@@ -69,11 +75,8 @@ async fn call<T: DeserializeOwned>(
     body: Option<&[SnapshotEntry]>,
 ) -> Result<T, DirError> {
     let (base, token) = target(&ctx.cfg()).ok_or(DirError::NotConfigured)?;
-    let mut req = ctx
-        .local_client
-        .request(method, format!("{base}{path}"))
-        .bearer_auth(token)
-        .timeout(std::time::Duration::from_secs(20));
+    let mut req =
+        ctx.local_client.request(method, format!("{base}{path}")).bearer_auth(token).timeout(Duration::from_secs(20));
     // The directory's spans join this request's trace (#213).
     if let Some(tp) = ipx_directory::otel::traceparent() {
         req = req.header("traceparent", tp);
@@ -139,7 +142,7 @@ pub async fn page(ctx: &Ctx, id: &str) -> Result<FeedPage, DirError> {
 ///
 /// shortcut: in memory, one address a listed feed, forgotten on a restart until the Directory is
 /// next read, which the page does when it loads; a table if that ever shows as initials.
-static LISTED_ART: std::sync::LazyLock<std::sync::Mutex<HashSet<String>>> = std::sync::LazyLock::new(Default::default);
+static LISTED_ART: LazyLock<std::sync::Mutex<HashSet<String>>> = LazyLock::new(Default::default);
 
 fn note_art<'a>(listings: impl Iterator<Item = &'a Listing>) {
     let mut art = LISTED_ART.lock().unwrap_or_else(|e| e.into_inner());
@@ -225,7 +228,7 @@ pub fn snapshot_entries(catalogue: &[Sub], counts: &HashMap<String, i64>) -> Vec
 
 /// Sends the snapshot now.
 async fn snapshot(ctx: &Ctx) -> Result<Vec<SnapshotEntry>, DirError> {
-    let catalogue = crate::subscriptions(ctx).await.map_err(|e| DirError::Unreachable(format!("{e:#}")))?;
+    let catalogue = crate::catalogue::subscriptions(ctx).await.map_err(|e| DirError::Unreachable(format!("{e:#}")))?;
     let counts = ctx.db.subscriber_counts().await.map_err(|e| DirError::Unreachable(format!("{e:#}")))?;
     Ok(snapshot_entries(&catalogue, &counts))
 }
@@ -240,10 +243,10 @@ pub async fn share(ctx: &Ctx, entries: &[SnapshotEntry]) -> Result<SnapshotResul
 }
 
 /// Sends the snapshot every half hour while sharing is on, and at once when the settings change.
-pub async fn run(ctx: std::sync::Arc<Ctx>, mut stop: tokio::sync::watch::Receiver<bool>) {
+pub async fn run(ctx: Arc<Ctx>, mut stop: tokio::sync::watch::Receiver<bool>) {
     // What went last, where, and when: a scan wakes this every half minute or so, and the same
     // 1,500 feeds went each time, the directory rewriting its rows for nothing (#211).
-    let mut last: Option<(Vec<u8>, std::time::Instant)> = None;
+    let mut last: Option<(Vec<u8>, Instant)> = None;
     loop {
         let cfg = ctx.cfg();
         if cfg.general.directory_share && configured(&cfg) {
@@ -254,9 +257,9 @@ pub async fn run(ctx: std::sync::Arc<Ctx>, mut stop: tokio::sync::watch::Receive
                     if forced || resend(last.as_ref(), &key) {
                         let span =
                             tracing::info_span!(target: "ipx::directory", "directory_share", feeds = entries.len());
-                        let got = tracing::Instrument::instrument(share(&ctx, &entries), span).await;
+                        let got = Instrument::instrument(share(&ctx, &entries), span).await;
                         if got.is_ok() {
-                            last = Some((key, std::time::Instant::now()));
+                            last = Some((key, Instant::now()));
                         }
                         Some(got)
                     } else {
@@ -279,7 +282,8 @@ pub async fn run(ctx: std::sync::Arc<Ctx>, mut stop: tokio::sync::watch::Receive
                     Ok(r) => (Some(r), None),
                     Err(e) => (None, Some(e.to_string())),
                 };
-                *ctx.directory_shared.lock().unwrap() = Some(Shared { at: crate::db::now(), result, error });
+                *ctx.directory_shared.lock().expect("a panic elsewhere poisoned this lock") =
+                    Some(Shared { at: crate::db::now(), result, error });
             }
         }
         tokio::select! {
@@ -293,7 +297,7 @@ pub async fn run(ctx: std::sync::Arc<Ctx>, mut stop: tokio::sync::watch::Receive
 /// Whether to send the snapshot `key` stands for: one that changed, or went to another directory
 /// or with another token, goes now; the same again only once SHARE_EVERY has passed, so a
 /// directory that lost it, or dropped this server's counts, has it back within the half hour.
-fn resend(last: Option<&(Vec<u8>, std::time::Instant)>, key: &[u8]) -> bool {
+fn resend(last: Option<&(Vec<u8>, Instant)>, key: &[u8]) -> bool {
     last.is_none_or(|(k, at)| k != key || at.elapsed() >= SHARE_EVERY)
 }
 

crates/ipx/src/download.rs

@@ -6,6 +6,9 @@ use std::path::{Path, PathBuf};
 use tokio::io::AsyncWriteExt;
 
 use crate::config::{Config, Feed as FeedCfg, Organize};
+use chrono::Local;
+use tokio::fs::File;
+use url::Url;
 
 /// Forbidden characters that were separating words: they become "-" so the words stay
 /// apart. The original's stringCleaning() deleted them, turning "Show | Series" into
@@ -83,7 +86,7 @@ pub fn filename_for(url: &str, content_disposition: Option<&str>) -> String {
     if let Some(name) = content_disposition.and_then(disposition_filename) {
         return sanitize(&name);
     }
-    let from_url = url::Url::parse(url)
+    let from_url = Url::parse(url)
         .ok()
         .and_then(|u| u.path_segments().and_then(|mut s| s.rfind(|p| !p.is_empty())).map(str::to_owned))
         .map(|seg| percent_encoding::percent_decode_str(&seg).decode_utf8_lossy().into_owned())
@@ -151,7 +154,7 @@ pub async fn download(
     tokio::fs::create_dir_all(&tmp_dir).await.with_context(|| format!("creating {}", tmp_dir.display()))?;
     let tmp = tmp_dir.join(&name);
 
-    let mut file = tokio::fs::File::create(&tmp).await.with_context(|| format!("creating {}", tmp.display()))?;
+    let mut file = File::create(&tmp).await.with_context(|| format!("creating {}", tmp.display()))?;
     let mut done: u64 = 0;
     let mut stream = resp.bytes_stream();
     let _active = crate::metrics::busy(&crate::metrics::STATS.downloads_active);
@@ -246,7 +249,7 @@ async fn sniff(path: &Path) -> Result<Sniffed> {
 
 async fn read_head(path: &Path, n: usize) -> Result<Vec<u8>> {
     use tokio::io::AsyncReadExt;
-    let mut f = tokio::fs::File::open(path).await?;
+    let mut f = File::open(path).await?;
     let mut buf = vec![0u8; n];
     let read = f.read(&mut buf).await?;
     buf.truncate(read);
@@ -277,7 +280,7 @@ fn unique_path(dir: &Path, name: &str) -> PathBuf {
 /// letting the sanitizer eat the separator.
 pub fn folder_for(cfg: &Config, id: &str, feed_cfg: &FeedCfg, title: Option<&str>) -> String {
     match cfg.general.organize {
-        Organize::Date => chrono::Local::now().format("%m-%d-%Y").to_string(),
+        Organize::Date => Local::now().format("%m-%d-%Y").to_string(),
         Organize::Feed => {
             let raw = feed_cfg.folder.as_deref().or(title).filter(|s| !s.trim().is_empty()).unwrap_or(id);
             raw.split('/')
@@ -312,7 +315,8 @@ pub fn blocked(words: &[String], haystacks: &[&str]) -> bool {
     // at either end: " new york " is in " the new york times ", " york " is not in " yorkshire ".
     let norm = |s: &str| {
         let mut out = String::from(" ");
-        for w in s.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()) {
+        let words = s.split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty());
+        for w in words {
             out.push_str(&w.to_lowercase());
             out.push(' ');
         }

crates/ipx/src/ipc.rs

@@ -2,7 +2,12 @@
 
 use anyhow::{Context, Result};
 use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use std::future::Future;
 use std::path::{Path, PathBuf};
+use std::pin::Pin;
+use std::sync::Arc;
+use std::time::Duration;
 use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
 use tokio::net::{UnixListener, UnixStream};
 use tokio::sync::{broadcast, mpsc};
@@ -124,7 +129,7 @@ impl Event {
     /// Whether a page signed in as this person is sent it. Everyone had every event, so a key in
     /// a paid feed's path, which `redacted` keeps, reached people who do not read the feed (#325).
     /// An admin has them all, for the log.
-    pub fn is_for(&self, user: &crate::db::User, mine: &std::collections::HashSet<String>) -> bool {
+    pub fn is_for(&self, user: &crate::db::User, mine: &HashSet<String>) -> bool {
         if user.is_admin {
             return true;
         }
@@ -262,21 +267,20 @@ impl Event {
     /// terminal. Addresses lose their query and password, where a private feed keeps its key, and
     /// so do the ones in an error's words (#238).
     fn redacted(mut self) -> Self {
-        use crate::feed::{redact, redact_text};
         match &mut self {
             Event::DownloadError { url, msg, .. } => {
-                *url = redact(url);
-                *msg = redact_text(msg);
+                *url = crate::feed::redact(url);
+                *msg = crate::feed::redact_text(msg);
             }
             // Progress and DownloadDone too: a Patreon file's address carries its signature (#292).
             Event::TorrentDeferred { url, .. } | Event::Progress { url, .. } | Event::DownloadDone { url, .. } => {
-                *url = redact(url)
+                *url = crate::feed::redact(url)
             }
             Event::FeedMoved { from, to, .. } => {
-                *from = redact(from);
-                *to = redact(to);
+                *from = crate::feed::redact(from);
+                *to = crate::feed::redact(to);
             }
-            Event::FeedError { msg, .. } | Event::Error { msg } => *msg = redact_text(msg),
+            Event::FeedError { msg, .. } | Event::Error { msg } => *msg = crate::feed::redact_text(msg),
             _ => {}
         }
         self
@@ -351,8 +355,7 @@ pub async fn daemon_is_live(path: &Path) -> bool {
 /// the meantime. The answer goes to the client that asked and no one else: broadcast, it ended
 /// any `ipx fetch` that was watching a scan, since `status` is a terminal event.
 /// A future, since reading the counts is a database query.
-pub type StatusFn =
-    std::sync::Arc<dyn Fn(Command) -> std::pin::Pin<Box<dyn std::future::Future<Output = Event> + Send>> + Send + Sync>;
+pub type StatusFn = Arc<dyn Fn(Command) -> Pin<Box<dyn Future<Output = Event> + Send>> + Send + Sync>;
 
 /// Accepts connections, feeding commands to `cmds` and events from `events` back out.
 pub async fn serve(
@@ -424,7 +427,7 @@ async fn handle(
             Ok(cmd @ (Command::Status | Command::Reload)) => {
                 tracing::info!(target: "ipx::io", "-> {line}");
                 let ev = status(cmd).await;
-                log_event(&ev, true);
+                log_event(&ev, /* wire */ true);
                 let _ = reply.send(ev).await;
             }
             Ok(cmd) => {
@@ -456,9 +459,7 @@ pub async fn reload(path: &Path) -> Result<()> {
         }
         anyhow::bail!("the daemon closed the connection without answering")
     };
-    tokio::time::timeout(std::time::Duration::from_secs(10), answer)
-        .await
-        .context("the daemon did not answer in 10s")?
+    tokio::time::timeout(Duration::from_secs(10), answer).await.context("the daemon did not answer in 10s")?
 }
 
 /// Sends one command to a running daemon and prints the events it produces.

crates/ipx/src/lists.rs

@@ -0,0 +1,270 @@
+//! OPML: importing a file's feeds, and keeping a subscribed list's feeds in step with it.
+
+use crate::ctx::Ctx;
+use crate::scan::Outcome;
+use crate::{catalogue, config, feed};
+use anyhow::Result;
+use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
+
+/// The feeds an OPML document lists, up to as many as a subscribed list may bring in: each becomes
+/// a feed in the catalogue, scanned for everyone, and one upload of tens of thousands held the
+/// catalogue for minutes and stayed in it (#338).
+pub fn opml_feeds(doc: &opml::OPML) -> Result<Vec<(String, String)>> {
+    let mut found = vec![];
+    feed::collect_outlines(&doc.body.outlines, &mut found);
+    anyhow::ensure!(
+        found.len() <= MAX_LISTED,
+        "it lists {} feeds, more than the {MAX_LISTED} one may have",
+        found.len()
+    );
+    Ok(found)
+}
+
+/// Subscribes one person to every feed in an OPML document, for the CLI and the web alike.
+/// A feed already in the catalogue costs nothing; an unknown one is added under the OPML's
+/// title rather than refetching each. Returns (newly subscribed, already subscribed).
+///
+/// Before accounts, importing only added unknown URLs to config.toml. Once subscriptions
+/// decided what each person sees, that imported nothing at all for a feed someone else
+/// already had, and a new one had no subscriber, so it was never scanned.
+///
+/// The caller parses the document and lists its feeds (`opml_feeds`), so each refuses a file in
+/// its own terms, before anything is touched: a 400 from the web, a message from the CLI.
+pub async fn subscribe_opml(ctx: &Ctx, found: Vec<(String, String)>, user_id: i64) -> Result<(usize, usize)> {
+    let mut cfg = ctx.edit_cfg().await;
+    // By address, once: each feed looked through every subscription and the whole catalogue, and
+    // the catalogue's lock is held throughout (#338).
+    let mut known = HashMap::new();
+    for s in catalogue::subscriptions(ctx).await? {
+        // The first with an address wins, as a search in order found it.
+        known.entry(s.cfg.url).or_insert(s.id);
+    }
+    let mut ids = vec![];
+    let mut grew = false;
+    for (title, url) in found {
+        // Known already, or the same URL listed twice in one file.
+        let existing = known.get(&url).cloned();
+        let id = match existing {
+            Some(id) => id,
+            None => {
+                let id = config::unique_slug(&title, |s| cfg.feeds.contains_key(s));
+                known.insert(url.clone(), id.clone());
+                cfg.feeds.insert(
+                    id.clone(),
+                    config::Feed {
+                        url,
+                        folder: None,
+                        group: None,
+                        media_types: None,
+                        schedule: None,
+                        keywords: vec![],
+                        allow_explicit: false,
+                        auto_download: true,
+                        max_new_per_check: None,
+                        username: None,
+                        password: None,
+                        password_env: None,
+                        category: None,
+                        listed: false,
+                        private: false,
+                    },
+                );
+                grew = true;
+                id
+            }
+        };
+        ids.push(id);
+    }
+    if grew {
+        ctx.store_cfg(cfg).await?;
+    }
+
+    let (mut added, mut had) = (0, 0);
+    for id in ids {
+        if ctx.db.subscription(user_id, &id).await?.is_some() {
+            had += 1;
+        } else {
+            ctx.db.subscribe(user_id, &id).await?;
+            added += 1;
+        }
+    }
+    Ok((added, had))
+}
+
+pub(crate) async fn sync_opml(ctx: &Arc<Ctx>, parent_id: &str, bytes: &[u8]) -> Result<Outcome> {
+    let listed = feed::parse_opml(bytes)?;
+    if let Some(title) = feed::opml_title(bytes) {
+        ctx.db.set_title(parent_id, &title).await?;
+    }
+    sync_group(ctx, parent_id, &listed).await
+}
+
+/// How many feeds an OPML or Patreon list may bring in.
+pub(crate) const MAX_LISTED: usize = 2000;
+
+/// Brings the feed list in step with a list of feeds: a subscribed OPML, or a Patreon
+/// creator's shows.
+///
+/// New entries are added under the list's group and folder. An entry that has gone from
+/// the list is unsubscribed *only if nothing was ever downloaded for it* -- otherwise it
+/// is kept and flagged, because dropping it would orphan files on disk with nothing in
+/// the UI to explain them.
+pub(crate) async fn sync_group(ctx: &Arc<Ctx>, parent_id: &str, listed: &[(String, String)]) -> Result<Outcome> {
+    // Each is a feed scanned for everyone subscribed to the list, and the list is its author's to
+    // write: one of tens of thousands held the scan for hours (#337). davewiner's had 922.
+    anyhow::ensure!(
+        listed.len() <= MAX_LISTED,
+        "it lists {} feeds, more than the {MAX_LISTED} a list may have",
+        listed.len()
+    );
+    let cfg = ctx.cfg();
+    let existing = ctx.db.managed_feeds().await?;
+    let mut added = vec![];
+    // Read once, not once a new feed: each read is the whole feeds table.
+    let known = ctx.db.feed_urls().await?;
+    let fixed: HashSet<&str> = cfg.feeds.keys().chain(existing.iter().map(|m| &m.id)).map(String::as_str).collect();
+    let known_ids: HashSet<&str> = known.iter().map(|(id, _)| id.as_str()).collect();
+
+    for (title, url) in listed {
+        // Already known, whether derived or promoted into the config.
+        if let Some(m) = existing.iter().find(|m| &m.url == url) {
+            ctx.db.upsert_managed(&m.id, url, title, parent_id).await?;
+            continue;
+        }
+        // A Patreon show you added by hand may be spelled differently from the one listed.
+        if cfg.feeds.values().any(|f| feed::same_feed(&f.url, url)) {
+            continue;
+        }
+        // A removed feed keeps its rows, so its id is only free again for the same feed.
+        let its_own: Vec<&str> =
+            known.iter().filter(|(_, u)| feed::same_feed(u, url)).map(|(id, _)| id.as_str()).collect();
+        let id = config::unique_slug(title, |s| {
+            fixed.contains(s) || added.iter().any(|a| a == s) || (known_ids.contains(s) && !its_own.contains(&s))
+        });
+        ctx.db.upsert_managed(&id, url, title, parent_id).await?;
+        added.push(id);
+    }
+
+    // Whoever subscribes to the OPML subscribes to what it lists: that is what taking a
+    // subscription means. Their own feeds are untouched.
+    let managed = ctx.db.managed_feeds().await?;
+    let list_and_feeds = managed
+        .iter()
+        .filter(|m| m.group_id == parent_id)
+        .map(|m| m.id.clone())
+        .chain(std::iter::once(parent_id.to_string()));
+    for id in list_and_feeds {
+        for user in ctx.db.users().await? {
+            if ctx.db.subscription(user.id, parent_id).await?.is_some() {
+                ctx.db.subscribe(user.id, &id).await?;
+            }
+        }
+    }
+
+    // Anything in this group the OPML no longer lists.
+    let mut removed = 0;
+    let mut kept = 0;
+    let parent = ctx.cfg().feeds.get(parent_id).cloned();
+    let in_group = existing.iter().filter(|m| m.group_id == parent_id);
+    for m in in_group {
+        if listed.iter().any(|(_, u)| u == &m.url) {
+            continue;
+        }
+        if catalogue::promote_kept(ctx, m, parent.as_ref()).await? {
+            kept += 1;
+            tracing::info!(feed = %m.id, "dropped from the OPML but someone kept it; a feed of its own now");
+            continue;
+        }
+        if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 {
+            // Never orphan a downloaded file: keep the feed and say why in the UI.
+            ctx.db.set_orphaned(&m.id, true).await?;
+            kept += 1;
+            tracing::info!(feed = %m.id, "dropped from the OPML but has downloads; keeping it");
+        } else {
+            ctx.db.drop_managed(&m.id).await?;
+            removed += 1;
+            tracing::info!(feed = %m.id, "dropped from the OPML with nothing downloaded; removed");
+        }
+    }
+
+    Ok(Outcome::Opml { added, removed, kept, total: listed.len() })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_util::{feed, test_ctx};
+
+    #[tokio::test]
+    async fn an_opml_upload_is_capped_and_takes_a_repeated_address_once() {
+        let opml = |n: usize| {
+            let outlines: String =
+                (0..n).map(|i| format!(r#"<outline text="Same" xmlUrl="http://x/{}.xml"/>"#, i % 3)).collect();
+            feed::read_opml(&format!(r#"<opml version="2.0"><head/><body>{outlines}</body></opml>"#)).unwrap()
+        };
+        assert!(opml_feeds(&opml(MAX_LISTED + 1)).unwrap_err().to_string().contains("more than the 2000"));
+        let ctx = test_ctx(config::Config::default()).await;
+        let me = ctx.db.create_user("me", None, false).await.unwrap();
+        // Six outlines, three addresses, one title: three feeds, named apart.
+        let (added, had) = subscribe_opml(&ctx, opml_feeds(&opml(6)).unwrap(), me).await.unwrap();
+        let ids: std::collections::BTreeSet<String> = ctx.cfg().feeds.keys().cloned().collect();
+        assert_eq!(ids, ["same", "same-2", "same-3"].map(String::from).into());
+        assert_eq!((added, had), (3, 3));
+    }
+
+    #[tokio::test]
+    async fn a_list_of_more_feeds_than_the_limit_is_refused() {
+        let ctx = Arc::new(test_ctx(config::Config::default()).await);
+        let listed: Vec<(String, String)> =
+            (0..=MAX_LISTED).map(|n| (format!("Show {n}"), format!("http://x/{n}.xml"))).collect();
+        let err = sync_group(&ctx, "opml", &listed).await.err().expect("refused");
+        assert!(err.to_string().contains("more than the 2000"), "{err:#}");
+        assert!(ctx.db.managed_feeds().await.unwrap().is_empty(), "nothing taken from it");
+        // At the limit, all of them, each its own id.
+        sync_group(&ctx, "opml", &listed[..MAX_LISTED]).await.unwrap();
+        let ids: std::collections::HashSet<String> =
+            ctx.db.managed_feeds().await.unwrap().into_iter().map(|m| m.id).collect();
+        assert_eq!(ids.len(), MAX_LISTED);
+    }
+
+    #[tokio::test]
+    async fn a_kept_feed_outlives_its_opml_for_whoever_kept_it() {
+        let mut cfg = config::Config::default();
+        cfg.feeds.insert("opml".into(), config::Feed { private: true, ..feed() });
+        let ctx = Arc::new(test_ctx(cfg).await);
+        let (me, you) = (
+            ctx.db.create_user("me", None, false).await.unwrap(),
+            ctx.db.create_user("you", None, false).await.unwrap(),
+        );
+        for id in ["opml", "a", "b"] {
+            if id != "opml" {
+                ctx.db.upsert_managed(id, &format!("http://x/{id}.xml"), id, "opml").await.unwrap();
+            }
+            for u in [me, you] {
+                ctx.db.subscribe(u, id).await.unwrap();
+            }
+        }
+        assert!(ctx.db.set_kept(me, "a", true).await.unwrap());
+        assert!(ctx.db.set_kept(me, "b", true).await.unwrap());
+
+        // The OPML drops a: mine to keep, a feed of its own; yours, unkept, goes.
+        sync_group(&ctx, "opml", &[("b".into(), "http://x/b.xml".into())]).await.unwrap();
+        let a = ctx.cfg().feeds.get("a").cloned().expect("a is in the catalogue");
+        assert_eq!((a.url.as_str(), a.group, a.private), ("http://x/a.xml", None, true));
+        assert!(!ctx.db.managed_feeds().await.unwrap().iter().any(|m| m.id == "a"));
+        assert!(ctx.db.subscription(me, "a").await.unwrap().is_some());
+        assert!(ctx.db.subscription(you, "a").await.unwrap().is_none());
+
+        // Leaving the OPML takes what I did not keep and leaves b, still the OPML's while you read it.
+        catalogue::leave(&ctx, me, "opml").await.unwrap();
+        assert!(ctx.db.subscription(me, "b").await.unwrap().is_some());
+        assert!(ctx.db.managed_feeds().await.unwrap().iter().any(|m| m.id == "b"));
+        // Once nobody reads it, b is mine alone, and yours goes with the OPML.
+        catalogue::leave(&ctx, you, "opml").await.unwrap();
+        // Still private: the OPML is gone from config by the time b is promoted (#332).
+        assert!(ctx.cfg().feeds.get("b").is_some_and(|f| f.group.is_none() && f.private));
+        assert!(ctx.db.subscription(me, "b").await.unwrap().is_some());
+        assert!(ctx.db.subscription(you, "b").await.unwrap().is_none());
+    }
+}

crates/ipx/src/logbuf.rs

@@ -4,7 +4,9 @@
 //! to read. Capturing inside the tracing pipeline works the same either way.
 
 use std::collections::VecDeque;
+use std::fmt::Debug;
 use std::sync::{LazyLock, Mutex};
+use tracing::Subscriber;
 use tracing::field::{Field, Visit};
 use tracing_subscriber::Layer;
 use tracing_subscriber::layer::Context;
@@ -67,7 +69,7 @@ pub fn since(after: u64, limit: usize) -> (Vec<LogLine>, u64) {
 /// A tracing layer that mirrors every event into the ring.
 pub struct RingLayer;
 
-impl<S: tracing::Subscriber> Layer<S> for RingLayer {
+impl<S: Subscriber> Layer<S> for RingLayer {
     fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) {
         let mut v = Collect::default();
         event.record(&mut v);
@@ -102,7 +104,7 @@ impl Collect {
 }
 
 impl Visit for Collect {
-    fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
+    fn record_debug(&mut self, field: &Field, value: &dyn Debug) {
         self.add(field, format!("{value:?}"));
     }
     // Numbers and bools reach record_debug through the trait's defaults, which prints them the

crates/ipx/src/logging.rs

@@ -0,0 +1,89 @@
+//! Log lines as JSON for Loki, carrying the trace and span they belong to.
+
+use crate::logbuf;
+use std::io::IsTerminal;
+use tracing::Subscriber;
+use tracing_subscriber::EnvFilter;
+use tracing_subscriber::fmt::format::JsonFields;
+use tracing_subscriber::fmt::{FmtContext, FormatEvent, FormatFields};
+use tracing_subscriber::registry::LookupSpan;
+
+/// The JSON log line with the trace and span it belongs to (#91), so a line in Loki leads to its
+/// trace in Tempo. The JSON formatter cannot take a field of its own, so the ids go on the end of
+/// the object it writes. A line outside any traced span, or with no trace exporter, is unchanged.
+pub(crate) struct WithTrace<F>(F);
+
+impl<S, N, F> FormatEvent<S, N> for WithTrace<F>
+where
+    F: FormatEvent<S, N>,
+    S: Subscriber + for<'a> LookupSpan<'a>,
+    N: for<'w> FormatFields<'w> + 'static,
+{
+    fn format_event(
+        &self,
+        ctx: &FmtContext<'_, S, N>,
+        mut w: tracing_subscriber::fmt::format::Writer<'_>,
+        ev: &tracing::Event<'_>,
+    ) -> std::fmt::Result {
+        use opentelemetry::trace::TraceContextExt;
+        use tracing_opentelemetry::OpenTelemetrySpanExt;
+        let current = tracing::Span::current();
+        let sc = if current.is_none() { None } else { Some(current.context().span().span_context().clone()) };
+        let Some(sc) = sc.filter(|c| c.is_valid()) else {
+            return self.0.format_event(ctx, w, ev);
+        };
+        let mut line = String::new();
+        self.0.format_event(ctx, tracing_subscriber::fmt::format::Writer::new(&mut line), ev)?;
+        match line.trim_end().strip_suffix('}') {
+            Some(body) => writeln!(w, r#"{body},"trace_id":"{}","span_id":"{}"}}"#, sc.trace_id(), sc.span_id()),
+            None => w.write_str(&line),
+        }
+    }
+}
+
+/// Sets up logging: to stderr, as text or as JSON (`IPX_LOG_FORMAT`), into the ring the page's log
+/// reads, and to the trace exporter when there is one.
+pub(crate) fn init(otel: Option<&opentelemetry_sdk::trace::SdkTracerProvider>) {
+    // Everything goes to stderr as before, and is mirrored into a ring the UI can read.
+    use tracing_subscriber::Layer;
+    use tracing_subscriber::layer::SubscriberExt;
+    use tracing_subscriber::util::SubscriberInitExt;
+    // Two filters, deliberately different. stderr follows IPX_LOG; the in-app buffer
+    // keeps debug as well, so the log view can show protocol traffic and routine
+    // skips that would be noise on a terminal. IPX_UI_LOG overrides it.
+    let stderr_filter = || EnvFilter::try_from_env("IPX_LOG").unwrap_or_else(|_| "ipx=info".into());
+    // One JSON object a line for Loki (#91), with each event's fields as its own; text
+    // otherwise, for someone reading a terminal.
+    let json = std::env::var("IPX_LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json"));
+    let ui_filter = EnvFilter::try_from_env("IPX_UI_LOG").unwrap_or_else(|_| "ipx=debug".into());
+    tracing_subscriber::registry()
+        .with((!json).then(|| {
+            tracing_subscriber::fmt::layer()
+                .with_writer(std::io::stderr)
+                // Colour for a terminal only: in docker logs and Loki the escapes are noise
+                // every query has to strip (#88).
+                .with_ansi(IsTerminal::is_terminal(&std::io::stderr()))
+                .with_filter(stderr_filter())
+        }))
+        .with(json.then(|| {
+            tracing_subscriber::fmt::layer()
+                .fmt_fields(JsonFields::new())
+                .event_format(WithTrace(
+                    tracing_subscriber::fmt::format()
+                        .json()
+                        .flatten_event(true)
+                        .with_current_span(true)
+                        .with_span_list(false),
+                ))
+                .with_writer(std::io::stderr)
+                .with_filter(stderr_filter())
+        }))
+        .with(logbuf::RingLayer.with_filter(ui_filter))
+        .with(otel.as_ref().map(|p| {
+            use opentelemetry::trace::TracerProvider;
+            tracing_opentelemetry::layer()
+                .with_tracer(p.tracer("ipx"))
+                .with_filter(EnvFilter::new("ipx=info,ipx_feed=info"))
+        }))
+        .init();
+}

crates/ipx/src/metrics.rs

@@ -6,9 +6,16 @@
 //! Prometheus exporters have none: checking a token is a database lookup. Written out by hand:
 //! the text format is a few lines, and a client library would be a dependency for them.
 
+use axum::Router;
+use std::collections::BTreeMap;
 use std::fmt::Write as _;
+use std::net::SocketAddr;
 use std::sync::Arc;
-use std::sync::atomic::{AtomicI64, AtomicU64, Ordering::Relaxed};
+use std::sync::atomic::Ordering::Relaxed;
+use std::sync::atomic::{AtomicI64, AtomicU64};
+use std::time::Instant;
+use tokio::runtime::Handle;
+use tower_http::compression::CompressionLayer;
 
 /// What the daemon counts as it works, for /metrics to read without asking anything (#195).
 /// Global: the web layer, the worker, downloads, artwork and DeadAir each add to it, and a handle
@@ -50,11 +57,11 @@ pub struct Stats {
     /// What each person's Signals, icons and key checks used, and whose key paid (#278): items
     /// judged, Signals they fit, and TypeSafe's input and output tokens. Floats: a call shared by
     /// several people on the server's key is split between them.
-    people: std::sync::Mutex<std::collections::BTreeMap<(String, &'static str), [f64; 4]>>,
+    people: std::sync::Mutex<BTreeMap<(String, &'static str), [f64; 4]>>,
     /// The worker's job now, and since when.
-    job: std::sync::Mutex<Option<(&'static str, std::time::Instant)>>,
+    job: std::sync::Mutex<Option<(&'static str, Instant)>>,
     /// Jobs finished, by kind: how many, and their seconds all told.
-    jobs: std::sync::Mutex<std::collections::BTreeMap<&'static str, (u64, f64)>>,
+    jobs: std::sync::Mutex<BTreeMap<&'static str, (u64, f64)>>,
 }
 
 impl Stats {
@@ -79,16 +86,16 @@ impl Stats {
             jev_calls: [const { AtomicU64::new(0) }; 5],
             jev_micros: AtomicU64::new(0),
             jev_paced_micros: AtomicU64::new(0),
-            people: std::sync::Mutex::new(std::collections::BTreeMap::new()),
+            people: std::sync::Mutex::new(BTreeMap::new()),
             job: std::sync::Mutex::new(None),
-            jobs: std::sync::Mutex::new(std::collections::BTreeMap::new()),
+            jobs: std::sync::Mutex::new(BTreeMap::new()),
         }
     }
 
     /// Adds to what `who` used on `key` ("server" or "own"): items judged, fits, input and output
     /// tokens.
     pub fn bill(&self, who: &str, key: &'static str, used: [f64; 4]) {
-        let mut people = self.people.lock().unwrap();
+        let mut people = self.people.lock().expect("a panic elsewhere poisoned this lock");
         let row = people.entry((who.to_owned(), key)).or_default();
         for (r, u) in row.iter_mut().zip(used) {
             *r += u;
@@ -98,7 +105,7 @@ impl Stats {
     /// Marks the worker as on a job of `kind` until what it returns is dropped, however the job
     /// ends: finished, failed or stopped.
     pub fn job(&'static self, kind: &'static str) -> Job {
-        *self.job.lock().unwrap() = Some((kind, std::time::Instant::now()));
+        *self.job.lock().expect("a panic elsewhere poisoned this lock") = Some((kind, Instant::now()));
         Job(self)
     }
 }
@@ -107,8 +114,8 @@ pub struct Job(&'static Stats);
 
 impl Drop for Job {
     fn drop(&mut self) {
-        if let Some((kind, since)) = self.0.job.lock().unwrap().take() {
-            let mut jobs = self.0.jobs.lock().unwrap();
+        if let Some((kind, since)) = self.0.job.lock().expect("a panic elsewhere poisoned this lock").take() {
+            let mut jobs = self.0.jobs.lock().expect("a panic elsewhere poisoned this lock");
             let e = jobs.entry(kind).or_default();
             e.0 += 1;
             e.1 += since.elapsed().as_secs_f64();
@@ -137,17 +144,17 @@ impl Drop for Busy {
 
 /// What the metrics are read from.
 pub struct Sources {
-    pub ctx: Arc<crate::Ctx>,
+    pub ctx: Arc<crate::ctx::Ctx>,
     pub cmds: tokio::sync::mpsc::Sender<crate::ipc::Command>,
     pub events: tokio::sync::broadcast::Sender<crate::ipc::Event>,
     pub playing: crate::deadair::Playing,
     /// The daemon's runtime, not the metrics thread's own.
-    pub runtime: tokio::runtime::Handle,
-    pub started: std::time::Instant,
+    pub runtime: Handle,
+    pub started: Instant,
 }
 
 /// Starts serving /metrics on `bind`, on a thread of its own, and says where.
-pub fn spawn(bind: &str, s: Sources) -> anyhow::Result<std::net::SocketAddr> {
+pub fn spawn(bind: &str, s: Sources) -> anyhow::Result<SocketAddr> {
     use anyhow::Context as _;
     // Bound here, so a port already taken is the daemon's error at start, not a thread's later.
     let listener = std::net::TcpListener::bind(bind).with_context(|| format!("binding metrics to {bind}"))?;
@@ -160,7 +167,7 @@ pub fn spawn(bind: &str, s: Sources) -> anyhow::Result<std::net::SocketAddr> {
             Err(e) => return tracing::error!(error = %e, "metrics could not start"),
         };
         rt.block_on(async move {
-            let app = axum::Router::new()
+            let app = Router::new()
                 .route(
                     "/metrics",
                     axum::routing::get(move || {
@@ -169,7 +176,7 @@ pub fn spawn(bind: &str, s: Sources) -> anyhow::Result<std::net::SocketAddr> {
                     }),
                 )
                 // Prometheus asks for gzip, every 15 s (#253).
-                .layer(tower_http::compression::CompressionLayer::new().gzip(true));
+                .layer(CompressionLayer::new().gzip(true));
             match tokio::net::TcpListener::from_std(listener) {
                 Ok(l) => {
                     if let Err(e) = axum::serve(l, app).await {
@@ -273,7 +280,7 @@ pub fn render(s: &Sources) -> String {
 /// The work in progress and done, from `STATS`: jobs, queues and what is in flight (#195).
 fn work(out: &mut String, st: &Stats, event_queue: usize, retries: usize, art_limit: u64) {
     let now = crate::db::now();
-    let job = *st.job.lock().unwrap();
+    let job = *st.job.lock().expect("a panic elsewhere poisoned this lock");
     gauge(out, "ipx_worker_busy", "1 while the worker is on a job.", &[("", job.is_some() as u8 as f64)]);
     if let Some((kind, since)) = job {
         let label = format!("kind=\"{kind}\"");
@@ -284,7 +291,7 @@ fn work(out: &mut String, st: &Stats, event_queue: usize, retries: usize, art_li
             &[(&label, since.elapsed().as_secs_f64())],
         );
     }
-    let jobs = st.jobs.lock().unwrap().clone();
+    let jobs = st.jobs.lock().expect("a panic elsewhere poisoned this lock").clone();
     let labels: Vec<String> = jobs.keys().map(|k| format!("kind=\"{k}\"")).collect();
     let pairs = |f: fn(&(u64, f64)) -> f64| {
         labels.iter().zip(jobs.values()).map(|(l, v)| (l.as_str(), f(v))).collect::<Vec<_>>()
@@ -359,7 +366,7 @@ fn work(out: &mut String, st: &Stats, event_queue: usize, retries: usize, art_li
     );
     // By person and whose key paid (#278). An account's name is often an email address: /metrics
     // is on the LAN only, as Prometheus is.
-    let people = st.people.lock().unwrap().clone();
+    let people = st.people.lock().expect("a panic elsewhere poisoned this lock").clone();
     let labels: Vec<String> =
         people.keys().map(|(who, key)| format!("user=\"{}\",key=\"{key}\"", label(who))).collect();
     let by = |n: usize| labels.iter().zip(people.values()).map(|(l, v)| (l.as_str(), v[n])).collect::<Vec<_>>();
@@ -512,4 +519,51 @@ mod tests {
         let (rss, fds) = ipx_directory::metrics::process();
         assert!(rss.unwrap() > 0.0 && fds.unwrap() > 0.0);
     }
+
+    use crate::config;
+    use crate::test_util::test_ctx;
+    use tokio::sync::{broadcast, mpsc};
+
+    #[tokio::test]
+    async fn metrics_answer_from_a_thread_of_their_own() {
+        let ctx = Arc::new(test_ctx(config::Config::default()).await);
+        let (cmds, _rx) = mpsc::channel(8);
+        let (events, _) = broadcast::channel(8);
+        let sources = Sources {
+            ctx: ctx.clone(),
+            cmds,
+            events,
+            playing: Default::default(),
+            runtime: tokio::runtime::Handle::current(),
+            started: std::time::Instant::now(),
+        };
+        let at = spawn("127.0.0.1:0", sources).unwrap();
+        let text = ctx.client.get(format!("http://{at}/metrics")).send().await.unwrap().text().await.unwrap();
+        // Every sample a name, optional labels and a number, as Prometheus parses them (#194).
+        let samples = text.lines().filter(|l| !l.starts_with('#'));
+        for line in samples {
+            let (name, value) = line.rsplit_once(' ').unwrap();
+            assert!(name.starts_with("ipx_") && value.parse::<f64>().is_ok(), "{line}");
+        }
+        for name in
+            ["ipx_db_connections{state=\"in_use\"}", "ipx_db_connections_max", "ipx_worker_queue", "ipx_runtime_tasks"]
+        {
+            assert!(text.contains(name), "{name}");
+        }
+        assert!(
+            spawn(
+                &at.to_string(),
+                Sources {
+                    ctx: ctx.clone(),
+                    cmds: mpsc::channel(1).0,
+                    events: broadcast::channel(1).0,
+                    playing: Default::default(),
+                    runtime: tokio::runtime::Handle::current(),
+                    started: std::time::Instant::now(),
+                }
+            )
+            .is_err(),
+            "a port already taken is an error at start"
+        );
+    }
 }

crates/ipx/src/retention.rs

@@ -4,7 +4,13 @@ use anyhow::Result;
 use tracing::Instrument;
 
 use crate::config::Config;
-use crate::db::{Candidate, Db, now, step};
+use crate::ctx::Ctx;
+use crate::db;
+use crate::db::{Candidate, Db};
+use crate::ipc::Event;
+use crate::{art, catalogue};
+use std::io::ErrorKind;
+use std::path::Path;
 
 /// Headroom kept below the quota, as getToQuota()'s `1048576 * 50` did.
 pub const PAD_BYTES: u64 = 50 * 1024 * 1024;
@@ -50,7 +56,7 @@ pub async fn run(cfg: &Config, db: &Db, dry_run: bool) -> Result<Report> {
     // and the whole library be forgotten at once (#293).
     // Each step is a span of the reap's trace, so a slow one says which part was slow.
     let missing = if cfg.general.download_dir.is_dir() {
-        db.missing_files().instrument(step("missing_files")).await?
+        db.missing_files().instrument(db::step("missing_files")).await?
     } else {
         tracing::warn!(dir = %cfg.general.download_dir.display(), "the downloads folder is not there; left every file as it was");
         vec![]
@@ -63,20 +69,20 @@ pub async fn run(cfg: &Config, db: &Db, dry_run: bool) -> Result<Report> {
         report.reconciled += 1;
     }
 
-    for id in db.prune_abandoned_failures().instrument(step("prune_abandoned_failures")).await? {
+    for id in db.prune_abandoned_failures().instrument(db::step("prune_abandoned_failures")).await? {
         tracing::info!(feed = id, "forgot a failing feed nobody subscribes to");
     }
 
-    let candidates = db.reap_candidates().instrument(step("reap_candidates")).await?;
+    let candidates = db.reap_candidates().instrument(db::step("reap_candidates")).await?;
 
     if cfg.general.max_age_days > 0 {
-        let cutoff = now() - (cfg.general.max_age_days * 86_400) as i64;
+        let cutoff = db::now() - (cfg.general.max_age_days * 86_400) as i64;
         report.aged_out = aged(&candidates, cutoff);
         for c in &report.aged_out {
-            report.bytes_freed += remove(db, c, dry_run).instrument(step("remove_aged")).await?;
+            report.bytes_freed += remove(db, c, dry_run).instrument(db::step("remove_aged")).await?;
         }
         if !dry_run {
-            report.entries_pruned = db.prune_entries(cutoff).instrument(step("prune_entries")).await?;
+            report.entries_pruned = db.prune_entries(cutoff).instrument(db::step("prune_entries")).await?;
         }
     }
 
@@ -88,7 +94,7 @@ pub async fn run(cfg: &Config, db: &Db, dry_run: bool) -> Result<Report> {
         let total: u64 = remaining.iter().map(|c| c.bytes.max(0) as u64).sum();
         report.over_quota = pick(&remaining, total, limit);
         for c in &report.over_quota {
-            report.bytes_freed += remove(db, c, dry_run).instrument(step("remove_over_quota")).await?;
+            report.bytes_freed += remove(db, c, dry_run).instrument(db::step("remove_over_quota")).await?;
         }
     }
 
@@ -102,8 +108,8 @@ async fn remove(db: &Db, c: &Candidate, dry_run: bool) -> Result<u64> {
     // A torrent's folder measures as a directory entry, not what is in it: the recorded size then.
     let size = std::fs::metadata(&c.path).ok().filter(|m| m.is_file()).map_or(c.bytes.max(0) as u64, |m| m.len());
     // Missing is fine: the point is that it is gone.
-    if let Err(e) = crate::download::remove(std::path::Path::new(&c.path))
-        && e.kind() != std::io::ErrorKind::NotFound
+    if let Err(e) = crate::download::remove(Path::new(&c.path))
+        && e.kind() != ErrorKind::NotFound
     {
         tracing::warn!(path = c.path, error = %e, "could not delete");
         return Ok(0);
@@ -112,6 +118,38 @@ async fn remove(db: &Db, c: &Candidate, dry_run: bool) -> Result<u64> {
     Ok(size)
 }
 
+/// `standalone` false means this is the sweep that runs before a scan: it reports what it
+/// deleted, but must not emit the terminal ReapDone, or a client waiting on its `fetch`
+/// would stop reading before the scan had even started.
+#[tracing::instrument(name = "reap", skip_all, fields(dry_run = dry_run, files = tracing::field::Empty, bytes = tracing::field::Empty, entries = tracing::field::Empty))]
+pub(crate) async fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> {
+    let r = run(&ctx.cfg(), &ctx.db, dry_run).await?;
+    let span = tracing::Span::current();
+    span.record("files", r.aged_out.len() + r.over_quota.len());
+    span.record("bytes", r.bytes_freed);
+    span.record("entries", r.entries_pruned);
+    if !dry_run {
+        tracing::Instrument::instrument(catalogue::clean_directory(ctx), db::step("clean_directory")).await?;
+        let moved =
+            tracing::Instrument::instrument(ctx.db.advance_unread_from(), db::step("advance_unread_from")).await?;
+        tracing::debug!(moved, "unread lists start from each person's oldest unread item");
+        // Off the async threads: it walks every file kept, on FUSE (#345).
+        let (span, limit) = (db::step("trim_artwork"), ctx.cfg().general.art_cache_mb * 1_048_576);
+        let gone = tokio::task::spawn_blocking(move || span.in_scope(|| art::trim(&art::dir(), limit))).await?;
+        if gone > 0 {
+            tracing::debug!(gone, "trimmed the artwork kept on disk");
+        }
+    }
+    let removed = r.aged_out.iter().chain(r.over_quota.iter());
+    for c in removed {
+        ctx.out.emit(Event::Reaped { path: c.path.clone(), bytes: c.bytes.max(0) as u64 });
+    }
+    if standalone {
+        ctx.out.emit(Event::ReapDone { files: r.aged_out.len() + r.over_quota.len(), bytes: r.bytes_freed });
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/ipx/src/signals.rs

@@ -14,16 +14,21 @@ use tracing::Instrument;
 
 use std::sync::atomic::Ordering::Relaxed;
 
-use crate::Ctx;
-use crate::db::{Ask, Unjudged, step};
+use crate::ctx::Ctx;
+use crate::db;
+use crate::db::{Ask, Unjudged};
 use crate::feed;
 use crate::metrics::STATS;
+use std::collections::BTreeMap;
+use std::fmt::{Display, Formatter};
+use std::future::Future;
+use std::time::Duration;
 
 /// Items put to Jev a pass. A pass ends early once there are none left, so this only says how
 /// long the queue is held before people who got a key later are looked at.
 const BATCH: i64 = 50;
 /// Between passes when nothing wakes it: a scan or a new Signal does, so this is a backstop.
-const IDLE: std::time::Duration = std::time::Duration::from_secs(300);
+const IDLE: Duration = Duration::from_secs(300);
 /// Calls to Jev in flight at once (#275). One at a time, about 150 ms each, a new Signal's month
 /// of items took twenty minutes. Enough for `MAX_RATE` at that pace; the pace is `pace`'s to keep.
 const IN_FLIGHT: usize = 10;
@@ -79,8 +84,11 @@ pub fn typesafe_key() -> Option<String> {
 async fn pass(ctx: &Ctx) -> Result<bool> {
     let server = server_key(ctx);
     // People's refused keys are not what is asked with when the server's is.
-    let skip: Vec<i64> =
-        if server.is_some() { Vec::new() } else { ctx.signals_refused.lock().unwrap().iter().copied().collect() };
+    let skip: Vec<i64> = if server.is_some() {
+        Vec::new()
+    } else {
+        ctx.signals_refused.lock().expect("a panic elsewhere poisoned this lock").iter().copied().collect()
+    };
     let days = ctx.cfg().general.signal_days;
     let items = ctx.db.unjudged(&skip, BATCH, server.is_some(), days).await?;
     if items.is_empty() {
@@ -99,7 +107,7 @@ async fn pass(ctx: &Ctx) -> Result<bool> {
         tracing::Span::current().record("calls", groups.len());
         let refused = std::sync::Mutex::new(HashSet::new());
         let stopped = AtomicBool::new(false);
-        let mut hits_for = std::collections::BTreeMap::<i64, usize>::new();
+        let mut hits_for = BTreeMap::<i64, usize>::new();
         let mut judged = 0;
         // Each call starts only when a place is free, so one that sees a key refused or the pass
         // stopped is not sent; those already in flight are let finish, and their answers kept.
@@ -110,7 +118,7 @@ async fn pass(ctx: &Ctx) -> Result<bool> {
                 let (group, refused, stopped, server) = (&groups[n], &refused, &stopped, &server);
                 async move {
                     let first = group[0];
-                    if stopped.load(Relaxed) || refused.lock().unwrap().contains(&first.user_id) {
+                    if stopped.load(Relaxed) || refused.lock().expect("a panic elsewhere poisoned this lock").contains(&first.user_id) {
                         return None;
                     }
                     let key = server.as_deref().or(first.key.as_deref())?;
@@ -131,7 +139,7 @@ async fn pass(ctx: &Ctx) -> Result<bool> {
                             *hits_for.entry(item.user_id).or_default() += hits;
                             tracing::info!(ev = "signal", user = item.user_id, feed = %item.feed_id, new = hits, "an item fits a Signal");
                         }
-                        ctx.db.record_verdicts(&item.feed_id, &item.guid, &verdicts).instrument(step("record_verdicts")).await?;
+                        ctx.db.record_verdicts(&item.feed_id, &item.guid, &verdicts).instrument(db::step("record_verdicts")).await?;
                     }
                 }
                 // The server's key refused is everyone's: the queue waits for the next pass.
@@ -143,7 +151,7 @@ async fn pass(ctx: &Ctx) -> Result<bool> {
                 // A key TypeSafe will not take is not sent the rest of the queue. It is tried
                 // again once its owner saves one (`web::patch_me`) or the daemon restarts.
                 Err(e) if refuses_key(&e) => {
-                    if refused.lock().unwrap().insert(first.user_id) {
+                    if refused.lock().expect("a panic elsewhere poisoned this lock").insert(first.user_id) {
                         tracing::warn!(user = first.user_id, error = %format!("{e:#}"), "TypeSafe refused a Signal key");
                     }
                 }
@@ -173,9 +181,9 @@ async fn pass(ctx: &Ctx) -> Result<bool> {
         for (user, hits) in hits_for {
             ctx.out.emit(crate::ipc::Event::SignalHits { user, hits });
         }
-        let refused = refused.into_inner().unwrap();
+        let refused = refused.into_inner().expect("a panic elsewhere poisoned this lock");
         let some_refused = !refused.is_empty();
-        ctx.signals_refused.lock().unwrap().extend(refused);
+        ctx.signals_refused.lock().expect("a panic elsewhere poisoned this lock").extend(refused);
         // A pass that only failed would otherwise go straight round again on the same items, and
         // one TypeSafe refused as malformed would ask again at once, however full.
         anyhow::Ok(!stopped.into_inner() && (full(&items) || some_refused))
@@ -199,8 +207,8 @@ struct Refused {
     body: String,
 }
 
-impl std::fmt::Display for Refused {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+impl Display for Refused {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
         write!(f, "TypeSafe answered {}: {}", self.status, self.body)
     }
 }
@@ -234,7 +242,7 @@ fn slot(
     tokens: f64,
 ) -> (tokio::time::Instant, tokio::time::Instant) {
     let start = next.max(now);
-    (start, start + std::time::Duration::from_secs_f64((1.0 / MAX_RATE).max(tokens / MAX_TOKENS)))
+    (start, start + Duration::from_secs_f64((1.0 / MAX_RATE).max(tokens / MAX_TOKENS)))
 }
 
 /// Waits for this call's turn. Its tokens are guessed from the request, a token to three bytes
@@ -257,7 +265,7 @@ async fn pace(body: &serde_json::Value) {
 
 /// TypeSafe said too many: nobody calls again until it says, or a second.
 async fn back_off(retry_after: Option<u64>) {
-    let until = tokio::time::Instant::now() + std::time::Duration::from_secs(retry_after.unwrap_or(1).min(60));
+    let until = tokio::time::Instant::now() + Duration::from_secs(retry_after.unwrap_or(1).min(60));
     let mut next = NEXT_CALL.lock().await;
     *next = Some(next.map_or(until, |n| n.max(until)));
 }
@@ -431,7 +439,7 @@ fn usage(answer: &serde_json::Value) -> [f64; 2] {
 
 /// Runs a call to Jev in `span`, counting it in /metrics by how it ended, and marking the span
 /// failed, which a field named `error` alone does not: Tempo's failed traces left Jev's out.
-async fn called<T>(span: tracing::Span, call: impl std::future::Future<Output = Result<T>>) -> Result<T> {
+async fn called<T>(span: tracing::Span, call: impl Future<Output = Result<T>>) -> Result<T> {
     let started = std::time::Instant::now();
     let r = call.instrument(span.clone()).await;
     STATS.jev_micros.fetch_add(started.elapsed().as_micros() as u64, Relaxed);

crates/ipx/src/test_util.rs

@@ -0,0 +1,104 @@
+//! What several modules' tests share: a context on a scratch database, feeds to put in it, and a
+//! server answering with feeds.
+
+use crate::ctx::Ctx;
+use crate::ipc::Emitter;
+use crate::{config, db};
+use std::path::PathBuf;
+use std::sync::Arc;
+
+pub(crate) fn feed() -> config::Feed {
+    // Whatever `ipx add` would write, which is the shape every code path sees.
+    let mut cfg = config::Config::default();
+
+    add_one_cfg(&mut cfg, "http://x/f.xml", None, vec![])
+}
+
+/// The feed entry `add` builds, without the network round trip it does for a title.
+pub(crate) fn add_one_cfg(
+    _cfg: &mut config::Config,
+    url: &str,
+    folder: Option<String>,
+    keywords: Vec<String>,
+) -> config::Feed {
+    config::Feed {
+        url: url.into(),
+        folder,
+        keywords,
+        allow_explicit: false,
+        auto_download: true,
+        group: None,
+        media_types: None,
+        schedule: None,
+        max_new_per_check: None,
+        username: None,
+        password: None,
+        password_env: None,
+        category: None,
+        listed: false,
+        private: false,
+    }
+}
+
+pub(crate) fn sub(kw: Option<&[&str]>, auto: Option<bool>, max: Option<i64>) -> db::Sub {
+    db::Sub {
+        feed_id: "f".into(),
+        keywords: kw.map(|k| k.iter().map(|s| s.to_string()).collect()),
+        auto_download: auto,
+        allow_explicit: None,
+        max_new_per_check: max,
+        blocked: vec![],
+    }
+}
+
+pub(crate) async fn test_ctx(cfg: config::Config) -> Ctx {
+    Ctx {
+        cfg: std::sync::RwLock::new(Arc::new(cfg)),
+        db: db::Db::memory().await.unwrap(),
+        client: reqwest::Client::new(),
+        local_client: reqwest::Client::new(),
+        feed_client: reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()).build().unwrap(),
+        out: Emitter::terminal(),
+        torrents: tokio::sync::OnceCell::new(),
+        torrent_slots: Arc::new(tokio::sync::Semaphore::new(2)),
+        config_path: PathBuf::new(),
+        detach_torrents: false,
+        retry: Default::default(),
+        cfg_write: Default::default(),
+        deadair: Default::default(),
+        signals: Default::default(),
+        signals_refused: Default::default(),
+        directory_wake: Default::default(),
+        directory_shared: Default::default(),
+        directory_resend: Default::default(),
+        directory_cleaned: Default::default(),
+    }
+}
+
+/// Answers /empty with a 200 and nothing, /fixed with a feed of one item as it should read,
+/// anything else with the same item's text broken.
+pub(crate) async fn empty_or_feed_server() -> String {
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let addr = listener.local_addr().unwrap();
+    tokio::spawn(async move {
+        loop {
+            let Ok((mut sock, _)) = listener.accept().await else { return };
+            tokio::spawn(async move {
+                let mut buf = [0u8; 2048];
+                let n = sock.read(&mut buf).await.unwrap_or(0);
+                let req = String::from_utf8_lossy(&buf[..n]).into_owned();
+                let body = if req.starts_with("GET /empty") {
+                    "\n"
+                } else if req.starts_with("GET /fixed") {
+                    "<?xml version=\"1.0\"?><rss version=\"2.0\"><channel><title>T</title><item><title>One</title><guid>g1</guid><description>As it should read.</description></item></channel></rss>"
+                } else {
+                    "<?xml version=\"1.0\"?><rss version=\"2.0\"><channel><title>T</title><item><title>One</title><guid>g1</guid><description>Somebody else's text.</description></item></channel></rss>"
+                };
+                let resp = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}", body.len());
+                let _ = sock.write_all(resp.as_bytes()).await;
+            });
+        }
+    });
+    format!("http://{addr}")
+}

crates/ipx/src/torrent.rs

@@ -9,6 +9,7 @@ use std::sync::Arc;
 use std::time::{Duration, Instant};
 
 use crate::config::Config;
+use librqbit::{ListenerOptions, ManagedTorrent};
 
 pub struct Torrents {
     session: Arc<Session>,
@@ -26,7 +27,7 @@ impl Torrents {
         let listen_addr: SocketAddr = format!("0.0.0.0:{lo}").parse()?;
 
         let opts = SessionOptions {
-            listen: Some(librqbit::ListenerOptions { listen_addr, ..Default::default() }),
+            listen: Some(ListenerOptions { listen_addr, ..Default::default() }),
             client_name_and_version: Some(concat!("ipx ", env!("CARGO_PKG_VERSION")).into()),
             ..Default::default()
         };
@@ -127,7 +128,7 @@ impl Torrents {
     }
 
     /// Seeds until the ratio or the time limit is reached, whichever comes first.
-    async fn seed(&self, cfg: &Config, handle: &Arc<librqbit::ManagedTorrent>) {
+    async fn seed(&self, cfg: &Config, handle: &Arc<ManagedTorrent>) {
         if cfg.torrent.seed_ratio <= 0.0 || cfg.torrent.seed_time_mins == 0 {
             return;
         }

crates/ipx/src/web.rs

@@ -20,10 +20,19 @@ use tokio::sync::{broadcast, mpsc};
 use tower::ServiceExt;
 use tower_http::services::ServeFile;
 
-use crate::Ctx;
-use crate::db::step;
+use crate::ctx::Ctx;
+use crate::db;
 use crate::ipc::{Command, Event};
+use axum::Extension;
+use axum::extract::{ConnectInfo, FromRequestParts};
+use axum::http::request::Parts;
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::net::SocketAddr;
+use std::sync::LazyLock;
+use std::time::{Duration, Instant};
+use tokio::net::TcpListener;
 use tracing::Instrument;
+use url::Url;
 
 const COOKIE: &str = "ipx_token";
 
@@ -98,14 +107,13 @@ pub async fn serve(state: WebState, bind: &str) -> Result<()> {
     use axum::serve::ListenerExt;
     // Nagle off, as axum 0.7 had it: under load a response goes out in parts, and with Nagle
     // on its last small part waits for the browser's delayed ACK, up to 40 ms (#170).
-    let listener =
-        tokio::net::TcpListener::bind(bind).await.with_context(|| format!("binding {bind}"))?.tap_io(|tcp| {
-            if let Err(e) = tcp.set_nodelay(true) {
-                tracing::debug!(error = %e, "could not set TCP_NODELAY");
-            }
-        });
+    let listener = TcpListener::bind(bind).await.with_context(|| format!("binding {bind}"))?.tap_io(|tcp| {
+        if let Err(e) = tcp.set_nodelay(true) {
+            tracing::debug!(error = %e, "could not set TCP_NODELAY");
+        }
+    });
     tracing::info!(bind, "web ui listening");
-    axum::serve(listener, router(state).into_make_service_with_connect_info::<std::net::SocketAddr>())
+    axum::serve(listener, router(state).into_make_service_with_connect_info::<SocketAddr>())
         .await
         .context("serving the web ui")
 }
@@ -240,7 +248,7 @@ async fn vouched_name(
     state: &WebState,
     cfg: &crate::config::Config,
     peer: String,
-    headers: &axum::http::HeaderMap,
+    headers: &HeaderMap,
 ) -> Option<String> {
     if cfg.web.trusted_header.is_empty() || !cfg.web.trusted_proxies.iter().any(|p| p == &peer) {
         return None;
@@ -257,21 +265,15 @@ async fn vouched_name(
 }
 
 fn peer(req: &Request) -> String {
-    req.extensions()
-        .get::<axum::extract::ConnectInfo<std::net::SocketAddr>>()
-        .map(|c| c.0.ip().to_string())
-        .unwrap_or_default()
+    req.extensions().get::<ConnectInfo<SocketAddr>>().map(|c| c.0.ip().to_string()).unwrap_or_default()
 }
 
 /// Handlers take `User` to say they need one; the auth layer put it there, and nothing
 /// reaches a handler without passing through it.
-impl<S: Send + Sync> axum::extract::FromRequestParts<S> for crate::db::User {
+impl<S: Send + Sync> FromRequestParts<S> for crate::db::User {
     type Rejection = (StatusCode, &'static str);
 
-    async fn from_request_parts(
-        parts: &mut axum::http::request::Parts,
-        _state: &S,
-    ) -> std::result::Result<Self, Self::Rejection> {
+    async fn from_request_parts(parts: &mut Parts, _state: &S) -> std::result::Result<Self, Self::Rejection> {
         parts.extensions.get::<crate::db::User>().cloned().ok_or((StatusCode::UNAUTHORIZED, "sign in"))
     }
 }
@@ -346,7 +348,7 @@ async fn logout(State(state): State<WebState>, req: Request) -> Response {
 async fn me(
     State(state): State<WebState>,
     user: crate::db::User,
-    axum::Extension(Proxied(by_proxy)): axum::Extension<Proxied>,
+    Extension(Proxied(by_proxy)): Extension<Proxied>,
 ) -> Json<serde_json::Value> {
     let url = state.ctx.cfg().web.sign_out_url.clone();
     let sign_out = (by_proxy && !url.is_empty()).then_some(url);
@@ -403,7 +405,7 @@ async fn patch_me(
             })?;
         }
         state.ctx.db.set_typesafe_key(user.id, key.as_deref()).await?;
-        state.ctx.signals_refused.lock().unwrap().remove(&user.id);
+        state.ctx.signals_refused.lock().expect("a panic elsewhere poisoned this lock").remove(&user.id);
         state.ctx.signals.notify_one();
     }
     Ok(StatusCode::NO_CONTENT)
@@ -502,17 +504,12 @@ struct IconAsk {
 /// the route is not tied to a Signal, so a loop on it ran up the server's bill and held back
 /// everyone's Signals behind the shared pace (#340). A Signal is named a few times at most.
 const ICON_PICKS_AN_HOUR: u32 = 20;
-static ICON_PICKS: std::sync::LazyLock<std::sync::Mutex<std::collections::HashMap<i64, (std::time::Instant, u32)>>> =
-    std::sync::LazyLock::new(Default::default);
+static ICON_PICKS: LazyLock<std::sync::Mutex<HashMap<i64, (Instant, u32)>>> = LazyLock::new(Default::default);
 
 /// Counts a pick against the person's hour, which starts at their first pick in it.
-fn icon_pick_allowed(
-    picks: &mut std::collections::HashMap<i64, (std::time::Instant, u32)>,
-    user: i64,
-    now: std::time::Instant,
-) -> bool {
+fn icon_pick_allowed(picks: &mut HashMap<i64, (Instant, u32)>, user: i64, now: Instant) -> bool {
     let (start, used) = picks.entry(user).or_insert((now, 0));
-    if now.duration_since(*start) >= std::time::Duration::from_secs(3600) {
+    if now.duration_since(*start) >= Duration::from_secs(3600) {
         (*start, *used) = (now, 0);
     }
     if *used >= ICON_PICKS_AN_HOUR {
@@ -532,7 +529,11 @@ async fn pick_signal_icon(
     let Some(key) = crate::signals::typesafe_key() else {
         return Err(ApiError::not_found("picking an icon is not set up on this server"));
     };
-    if !icon_pick_allowed(&mut ICON_PICKS.lock().unwrap(), user.id, std::time::Instant::now()) {
+    if !icon_pick_allowed(
+        &mut ICON_PICKS.lock().expect("a panic elsewhere poisoned this lock"),
+        user.id,
+        Instant::now(),
+    ) {
         return Err(ApiError::too_many("enough icons picked for one hour; choose one by hand"));
     }
     let name = signal_name(&body.name)?;
@@ -767,7 +768,7 @@ async fn remove_user(
             target.name
         )));
     }
-    crate::remove_user(&state.ctx, id).await?;
+    crate::catalogue::remove_user(&state.ctx, id).await?;
     tracing::info!(by = %user.name, user = %target.name, "account removed");
     Ok(StatusCode::NO_CONTENT)
 }
@@ -894,27 +895,27 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
     let cfg = state.ctx.cfg();
     // Config entries plus the feeds derived from OPML subscriptions -- the catalogue.
     // What comes back is only the part of it this person subscribes to.
-    let subs = crate::subscriptions(&state.ctx).instrument(step("catalogue")).await?;
-    let mine: std::collections::HashMap<String, crate::db::Sub> = state
+    let subs = crate::catalogue::subscriptions(&state.ctx).instrument(db::step("catalogue")).await?;
+    let mine: HashMap<String, crate::db::Sub> = state
         .ctx
         .db
         .subscriptions_for(user.id)
-        .instrument(step("subscriptions"))
+        .instrument(db::step("subscriptions"))
         .await?
         .into_iter()
         .map(|s| (s.feed_id.clone(), s))
         .collect();
-    let counts = state.ctx.db.subscriber_counts().instrument(step("subscriber_counts")).await?;
-    let pinned = state.ctx.db.pinned_feeds(user.id).instrument(step("pinned")).await?;
-    let kept = state.ctx.db.kept_feeds(user.id).instrument(step("kept")).await?;
-    let (hidden, quiet) = state.ctx.db.hidden_from(user.id).instrument(step("hidden_from")).await?;
+    let counts = state.ctx.db.subscriber_counts().instrument(db::step("subscriber_counts")).await?;
+    let pinned = state.ctx.db.pinned_feeds(user.id).instrument(db::step("pinned")).await?;
+    let kept = state.ctx.db.kept_feeds(user.id).instrument(db::step("kept")).await?;
+    let (hidden, quiet) = state.ctx.db.hidden_from(user.id).instrument(db::step("hidden_from")).await?;
     let prefs = state.ctx.db.deadair_prefs(Some(user.id)).await?;
-    let deadair: std::collections::HashMap<&str, bool> = prefs.iter().map(|(_, f, w, _)| (f.as_str(), *w)).collect();
-    let pauses: std::collections::HashMap<&str, i64> =
-        prefs.iter().filter_map(|(_, f, _, p)| Some((f.as_str(), (*p)?))).collect();
-    let mut listed = state.ctx.db.feed_list(user.id, only).instrument(step("feed_list")).await?;
+    let deadair: HashMap<&str, bool> = prefs.iter().map(|(_, f, w, _)| (f.as_str(), *w)).collect();
+    let pauses: HashMap<&str, i64> = prefs.iter().filter_map(|(_, f, _, p)| Some((f.as_str(), (*p)?))).collect();
+    let mut listed = state.ctx.db.feed_list(user.id, only).instrument(db::step("feed_list")).await?;
     let mut out = Vec::with_capacity(mine.len());
-    for sub in subs.iter().filter(|s| only.is_none_or(|o| o == s.id)) {
+    let asked = subs.iter().filter(|s| only.is_none_or(|o| o == s.id));
+    for sub in asked {
         let (id, feed) = (&sub.id, &sub.cfg);
         // In a group, what you have not set on the feed comes from your settings on the group,
         // the same fallback the scanner uses (`Db::subscribers`).
@@ -964,11 +965,11 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
             managed: sub.managed,
             schedule: feed.schedule.clone(),
             schedule_mins: feed.schedule.as_deref().and_then(crate::config::parse_interval),
-            every_mins: crate::due_after(&cfg, feed, ttl_mins, None, None, 0) / 60,
+            every_mins: crate::scan::due_after(&cfg, feed, ttl_mins, None, None, 0) / 60,
             last_checked: s.last_checked,
             next_check: s
                 .last_checked
-                .map(|t| t + crate::due_after(&cfg, feed, ttl_mins, s.error_since, s.same_since, t) as i64),
+                .map(|t| t + crate::scan::due_after(&cfg, feed, ttl_mins, s.error_since, s.same_since, t) as i64),
             failing: s
                 .error_since
                 .filter(|since| crate::db::now() - since >= FLAG_AFTER_SECS)
@@ -1029,23 +1030,23 @@ fn dir_error(e: crate::directory::DirError) -> ApiError {
 /// address, since the directory and this server name a feed differently.
 async fn rows_for(state: &WebState, user_id: i64, listed: Vec<ipx_directory::Listing>) -> Result<Vec<PopularRow>> {
     // A step each, as the feed list has: a Directory page of 983 ms showed 19 under spans (#250).
-    let mine: std::collections::HashSet<String> = state
+    let mine: HashSet<String> = state
         .ctx
         .db
         .subscriptions_for(user_id)
-        .instrument(step("subscriptions"))
+        .instrument(db::step("subscriptions"))
         .await?
         .into_iter()
         .map(|s| s.feed_id)
         .collect();
-    let here: std::collections::HashMap<String, String> = crate::subscriptions(&state.ctx)
-        .instrument(step("catalogue"))
+    let here: HashMap<String, String> = crate::catalogue::subscriptions(&state.ctx)
+        .instrument(db::step("catalogue"))
         .await?
         .into_iter()
         .filter(|s| mine.contains(&s.id))
         .filter_map(|s| ipx_directory::store::normalise(&s.cfg.url).map(|u| (u, s.id)))
         .collect();
-    let _matching = step("match").entered();
+    let _matching = db::step("match").entered();
     Ok(listed
         .into_iter()
         .map(|l| {
@@ -1215,11 +1216,11 @@ async fn subscribe_popular(
         crate::directory::DirError::Refused(404, _) => ApiError::bad_request(format!("{id:?} is not in the directory")),
         e => dir_error(e),
     })?;
-    let (feed_id, existed) = crate::add_to_catalogue(&state.ctx, &page.listing.url, None, vec![]).await?;
+    let (feed_id, existed) = crate::catalogue::add_to_catalogue(&state.ctx, &page.listing.url, None, vec![]).await?;
     let already = existed && state.ctx.db.subscription(user.id, &feed_id).await?.is_some();
     state.ctx.db.subscribe(user.id, &feed_id).await?;
     if !already {
-        explicit_on_add(&state, user.id, &feed_id, false).await?;
+        explicit_on_add(&state, user.id, &feed_id, /* allow */ false).await?;
     }
     scan_soon(&state, Some(feed_id.clone())).await;
     Ok(Json(serde_json::json!({ "id": feed_id })))
@@ -1227,16 +1228,12 @@ async fn subscribe_popular(
 
 /// Validates a replacement feed URL: present, parseable, and not already subscribed under
 /// a different id. Returns the trimmed URL.
-fn check_url(
-    url: &str,
-    id: &str,
-    feeds: &std::collections::BTreeMap<String, crate::config::Feed>,
-) -> Result<String, String> {
+fn check_url(url: &str, id: &str, feeds: &BTreeMap<String, crate::config::Feed>) -> Result<String, String> {
     let url = url.trim();
     if url.is_empty() {
         return Err("the feed URL cannot be empty".into());
     }
-    match url::Url::parse(url) {
+    match Url::parse(url) {
         Ok(u) if u.scheme() == "http" || u.scheme() == "https" => {}
         Ok(u) => return Err(format!("{:?} is not an http(s) URL", u.scheme())),
         Err(e) => return Err(format!("that is not a valid URL: {e}")),
@@ -1606,20 +1603,20 @@ async fn entry_page(
     };
     let count = async {
         match signal {
-            Some(id) => db.signal_count(user_id, id, filter, search).instrument(step("count_in")).await,
-            None => db.count_of(user_id, feeds, filter, search).instrument(step("count_in")).await,
+            Some(id) => db.signal_count(user_id, id, filter, search).instrument(db::step("count_in")).await,
+            None => db.count_of(user_id, feeds, filter, search).instrument(db::step("count_in")).await,
         }
     };
     // The count beside the list, not after it: each waited on the other, about 40 and 24 ms on the
     // busiest route (#346).
     let (mut rows, total) = tokio::try_join!(list, count)?;
-    step("sanitize").in_scope(|| {
+    db::step("sanitize").in_scope(|| {
         let mut sanitizer = feed_sanitizer();
         for row in &mut rows {
             clean_row(&mut sanitizer, row);
         }
     });
-    crate::deadair::for_reader(&state.ctx, user_id, &mut rows).instrument(step("deadair")).await?;
+    crate::deadair::for_reader(&state.ctx, user_id, &mut rows).instrument(db::step("deadair")).await?;
     Ok(Json(EntryPage { total, entries: rows }))
 }
 
@@ -1709,7 +1706,7 @@ async fn add_feed(
         .map_err(|e| ApiError::bad_request(format!("{e:#}")))?;
     // Someone else may already have it. Then adding costs nothing: no second fetch, no
     // second copy on disk, just another name against the same feed.
-    let (id, existed) = crate::add_to_catalogue(&state.ctx, &url, body.folder, body.keywords).await?;
+    let (id, existed) = crate::catalogue::add_to_catalogue(&state.ctx, &url, body.folder, body.keywords).await?;
     let already = existed && state.ctx.db.subscription(user.id, &id).await?.is_some();
     state.ctx.db.subscribe(user.id, &id).await?;
     if !already {
@@ -1771,7 +1768,8 @@ struct FeedPatch {
 /// looked for in every item of every feed the list covers.
 fn clean_words(words: Vec<String>) -> Result<Vec<String>, ApiError> {
     let mut out: Vec<String> = Vec::new();
-    for w in words.iter().map(|w| w.trim()).filter(|w| !w.is_empty()) {
+    let trimmed = words.iter().map(|w| w.trim()).filter(|w| !w.is_empty());
+    for w in trimmed {
         if !out.iter().any(|o| o.eq_ignore_ascii_case(w)) {
             out.push(w.to_owned());
         }
@@ -1798,22 +1796,22 @@ async fn patch_feed(
 ) -> Result<StatusCode, ApiError> {
     // Pinning is yours alone too, and means nothing for a feed you do not subscribe to.
     if let Some(on) = body.pinned
-        && !state.ctx.db.set_pinned(user.id, &id, on).instrument(step("set_pinned")).await?
+        && !state.ctx.db.set_pinned(user.id, &id, on).instrument(db::step("set_pinned")).await?
     {
         return Err(ApiError::not_found("you do not subscribe to that feed"));
     }
     if let Some(on) = body.kept
-        && !state.ctx.db.set_kept(user.id, &id, on).instrument(step("set_kept")).await?
+        && !state.ctx.db.set_kept(user.id, &id, on).instrument(db::step("set_kept")).await?
     {
         return Err(ApiError::not_found("you do not subscribe to that feed"));
     }
     if let Some(on) = body.hide_from_all
-        && !state.ctx.db.set_hide_from_all(user.id, &id, on).instrument(step("set_hide_from_all")).await?
+        && !state.ctx.db.set_hide_from_all(user.id, &id, on).instrument(db::step("set_hide_from_all")).await?
     {
         return Err(ApiError::not_found("you do not subscribe to that feed"));
     }
     if let Some(on) = body.hide_from_signals
-        && !state.ctx.db.set_hide_from_signals(user.id, &id, on).instrument(step("set_hide_from_signals")).await?
+        && !state.ctx.db.set_hide_from_signals(user.id, &id, on).instrument(db::step("set_hide_from_signals")).await?
     {
         return Err(ApiError::not_found("you do not subscribe to that feed"));
     }
@@ -1823,15 +1821,15 @@ async fn patch_feed(
         return Err(ApiError::bad_request("DeadAir's pause is 200, 300, 400 or 500 ms"));
     }
     if let Some(wanted) = body.deadair {
-        state.ctx.db.set_deadair(user.id, &id, wanted).instrument(step("set_deadair")).await?;
+        state.ctx.db.set_deadair(user.id, &id, wanted).instrument(db::step("set_deadair")).await?;
         state.ctx.deadair.notify_one();
     }
     if let Some(p) = body.deadair_pause {
-        state.ctx.db.set_deadair_pause(user.id, &id, p).instrument(step("set_deadair_pause")).await?;
+        state.ctx.db.set_deadair_pause(user.id, &id, p).instrument(db::step("set_deadair_pause")).await?;
     }
     // What one person wants -- which items, whether to fetch them, how many at a time --
     // is theirs. It goes on their subscription and nobody else sees the change.
-    if let Some(mut mine) = state.ctx.db.subscription(user.id, &id).instrument(step("subscription")).await? {
+    if let Some(mut mine) = state.ctx.db.subscription(user.id, &id).instrument(db::step("subscription")).await? {
         let mut touched = false;
         if let Some(v) = body.keywords.clone() {
             mine.keywords = Some(v.into_iter().filter(|k| !k.trim().is_empty()).collect());
@@ -1850,10 +1848,10 @@ async fn patch_feed(
             touched = true;
         }
         if touched {
-            state.ctx.db.set_subscription(user.id, &mine).instrument(step("set_subscription")).await?;
+            state.ctx.db.set_subscription(user.id, &mine).instrument(db::step("set_subscription")).await?;
         }
         if let Some(v) = body.blocked.clone() {
-            state.ctx.db.set_blocklist(user.id, &id, &clean_words(v)?).instrument(step("set_blocklist")).await?;
+            state.ctx.db.set_blocklist(user.id, &id, &clean_words(v)?).instrument(db::step("set_blocklist")).await?;
         }
     }
 
@@ -1874,7 +1872,7 @@ async fn patch_feed(
     }
     // Marking a feed private is for those who read it: anyone could keep anyone's feed, or a
     // whole OPML's, out of the directory, and only an admin could undo it (#333).
-    if !user.is_admin && state.ctx.db.subscription(user.id, &id).instrument(step("subscription")).await?.is_none() {
+    if !user.is_admin && state.ctx.db.subscription(user.id, &id).instrument(db::step("subscription")).await?.is_none() {
         return Err(ApiError::not_found("you do not subscribe to that feed"));
     }
     let mut cfg = state.ctx.edit_cfg().await;
@@ -1882,7 +1880,7 @@ async fn patch_feed(
     // Derived feeds have no config entry. Editing one is the moment it earns a real
     // entry: promote it, so the config holds your decisions and nothing else.
     if !cfg.feeds.contains_key(&id) {
-        let subs = crate::subscriptions(&state.ctx).instrument(step("subscriptions")).await?;
+        let subs = crate::catalogue::subscriptions(&state.ctx).instrument(db::step("subscriptions")).await?;
         let found =
             subs.iter().find(|s| s.id == id).ok_or_else(|| ApiError::bad_request(format!("no feed with id {id:?}")))?;
         cfg.feeds.insert(id.clone(), found.cfg.clone());
@@ -1926,7 +1924,7 @@ async fn patch_feed(
     if url_changed {
         // Refreshing a rotated auth token is the common case; entries and download history
         // are keyed by feed id, so they survive the change.
-        state.ctx.db.clear_validators(&id).instrument(step("clear_validators")).await?;
+        state.ctx.db.clear_validators(&id).instrument(db::step("clear_validators")).await?;
     }
     Ok(StatusCode::NO_CONTENT)
 }
@@ -1936,7 +1934,7 @@ async fn remove_feed(
     Path(id): Path<String>,
     user: crate::db::User,
 ) -> Result<StatusCode, ApiError> {
-    crate::leave(&state.ctx, user.id, &id).await?;
+    crate::catalogue::leave(&state.ctx, user.id, &id).await?;
     Ok(StatusCode::NO_CONTENT)
 }
 
@@ -2137,7 +2135,7 @@ fn changed_feed(ev: &Event) -> Option<&str> {
 
 /// Audio, served by ServeFile so Range requests work and the player can seek.
 async fn media(State(state): State<WebState>, Path(id): Path<i64>, user: crate::db::User, req: Request) -> Response {
-    serve_media(state, id, user, req, false).await
+    serve_media(state, id, user, req, /* piece */ false).await
 }
 
 /// DeadAir's playlist of a file (#181): the original's frames kept by this person's pause
@@ -2150,7 +2148,8 @@ async fn media_deadair(State(state): State<WebState>, Path(id): Path<i64>, user:
     if let Err(e) = subscribed(&state, &user, &enc.feed_id).await {
         return e.into_response();
     }
-    let plan = match crate::deadair::plan_for(&state.ctx, user.id, &enc.feed_id, id).instrument(step("plan")).await {
+    let plan = match crate::deadair::plan_for(&state.ctx, user.id, &enc.feed_id, id).instrument(db::step("plan")).await
+    {
         Ok(Some(p)) => p,
         Ok(None) => return (StatusCode::NOT_FOUND, "DeadAir has nothing to cut in this file for you").into_response(),
         Err(e) => return (StatusCode::INTERNAL_SERVER_ERROR, format!("{e:#}")).into_response(),
@@ -2178,7 +2177,8 @@ async fn media_save(State(state): State<WebState>, Path(id): Path<i64>, user: cr
     let Some(path) = enc.path else {
         return (StatusCode::NOT_FOUND, "not downloaded").into_response();
     };
-    let plan = match crate::deadair::plan_for(&state.ctx, user.id, &enc.feed_id, id).instrument(step("plan")).await {
+    let plan = match crate::deadair::plan_for(&state.ctx, user.id, &enc.feed_id, id).instrument(db::step("plan")).await
+    {
         Ok(Some(p)) => p,
         Ok(None) => return (StatusCode::NOT_FOUND, "DeadAir has nothing to cut in this file for you").into_response(),
         Err(e) => return (StatusCode::INTERNAL_SERVER_ERROR, format!("{e:#}")).into_response(),
@@ -2212,7 +2212,7 @@ async fn media_piece(
     user: crate::db::User,
     req: Request,
 ) -> Response {
-    serve_media(state, id, user, req, true).await
+    serve_media(state, id, user, req, /* piece */ true).await
 }
 
 async fn serve_media(state: WebState, id: i64, user: crate::db::User, req: Request, piece: bool) -> Response {
@@ -2294,7 +2294,7 @@ async fn art(State(state): State<WebState>, Query(q): Query<ArtQuery>) -> Respon
     // Only what a feed here or the directory names: anything else would make ipx fetch any
     // address it is given. The Directory shows podcasts nobody here subscribes to (#273).
     if !crate::directory::listed_art(&q.u)
-        && !state.ctx.db.names_image(&q.u).instrument(step("names_image")).await.unwrap_or(false)
+        && !state.ctx.db.names_image(&q.u).instrument(db::step("names_image")).await.unwrap_or(false)
     {
         return (StatusCode::NOT_FOUND, "no feed names that artwork").into_response();
     }
@@ -2323,10 +2323,10 @@ async fn art(State(state): State<WebState>, Query(q): Query<ArtQuery>) -> Respon
         let kind = image_kind(&said, &body).ok_or_else(|| anyhow::anyhow!("not an image: {said}"))?;
         anyhow::Ok((kind, body))
     };
-    match got.instrument(step("art_fetch")).await {
+    match got.instrument(db::step("art_fetch")).await {
         Ok((kind, body)) => {
             if keep {
-                let (span, file, kind, body) = (step("art_keep"), file.clone(), kind.clone(), body.clone());
+                let (span, file, kind, body) = (db::step("art_keep"), file.clone(), kind.clone(), body.clone());
                 match tokio::task::spawn_blocking(move || span.in_scope(|| crate::art::write(&file, &kind, &body)))
                     .await
                 {
@@ -2368,7 +2368,7 @@ impl Drop for ArtFetchDone<'_> {
 /// Artwork kept on disk, read off the async threads: the cache is on FUSE, and four calls a
 /// hit, fifty images a page, held a worker each, 1 s at p90 for a hit (#345).
 async fn kept_art(file: &std::path::Path) -> Option<(String, Vec<u8>)> {
-    let (span, file) = (step("art_cached"), file.to_owned());
+    let (span, file) = (db::step("art_cached"), file.to_owned());
     tokio::task::spawn_blocking(move || span.in_scope(|| crate::art::read(&file))).await.ok().flatten()
 }
 
@@ -2442,7 +2442,7 @@ async fn read_all(
     // A subscription's own row has no entries, so marking it read means everything under it.
     let mut ids = vec![id.clone()];
     ids.extend(
-        crate::subscriptions(&state.ctx)
+        crate::catalogue::subscriptions(&state.ctx)
             .await?
             .into_iter()
             .filter(|s| s.cfg.group.as_deref() == Some(id.as_str()))
@@ -2504,7 +2504,7 @@ async fn export_opml(State(state): State<WebState>, user: crate::db::User) -> Re
         head: Some(opml::Head { title: Some("ipx subscriptions".into()), ..Default::default() }),
         ..Default::default()
     };
-    for s in crate::subscriptions(&state.ctx).await? {
+    for s in crate::catalogue::subscriptions(&state.ctx).await? {
         // A feed from an OPML subscription comes back with the OPML itself.
         if s.managed || !mine.contains(&s.id) {
             continue;
@@ -2538,8 +2538,8 @@ async fn import_opml(
     // file arrives as text, is read here, and is gone when the request ends.
     let doc = crate::feed::read_opml(&body.xml)
         .map_err(|e| ApiError::bad_request(format!("that is not an OPML file: {e}")))?;
-    let found = crate::opml_feeds(&doc).map_err(|e| ApiError::bad_request(format!("{e:#}")))?;
-    let (added, already) = crate::subscribe_opml(&state.ctx, found, user.id).await?;
+    let found = crate::lists::opml_feeds(&doc).map_err(|e| ApiError::bad_request(format!("{e:#}")))?;
+    let (added, already) = crate::lists::subscribe_opml(&state.ctx, found, user.id).await?;
     if added > 0 {
         scan_soon(&state, None).await;
     }
@@ -2549,7 +2549,7 @@ async fn import_opml(
 /// The numbers `ipx status` prints, and the version, for a dashboard such as Homepage's
 /// customapi widget. Behind sign-in like the rest of /api; Homepage sends the shared token.
 async fn status(State(state): State<WebState>) -> Response {
-    match crate::status(&state.ctx).await {
+    match crate::daemon::status(&state.ctx).await {
         Event::Status { feeds, pending, downloaded } => Json(serde_json::json!({
             "feeds": feeds,
             "pending": pending,

docs/architecture.md

@@ -27,11 +27,30 @@ id, and Caddy's request lines theirs. ipx also serves Prometheus metrics (`[web]
 
 | File | Responsibility | What it replaced in the Python |
 |---|---|---|
-| `crates/ipx/src/main.rs` | CLI, dispatch, scan loop, download policy | `iPXAgent.py` |
+| `crates/ipx/src/main.rs` | The module list and `main`: reads the command line and hands it on | `iPXAgent.py` |
+| `crates/ipx/src/cli.rs` | The commands, and what each does when no daemon takes it | `iPXAgent.py` |
+| `crates/ipx/src/ctx.rs` | `Ctx`, what every command shares: configuration, database, clients | — |
+| `crates/ipx/src/daemon.rs` | The daemon: worker loop, web server, shutdown, `status` | `iPXAgent.py` |
+| `crates/ipx/src/scan.rs` | When a feed is due, reading it, download policy, downloads and torrents | `iPXAgent.py` |
+| `crates/ipx/src/catalogue.rs` | Adding, removing and leaving feeds; subscriptions; retiring a list's feeds | — |
+| `crates/ipx/src/lists.rs` | OPML import, and keeping an OPML's or a Patreon creator's feeds in step | — |
+| `crates/ipx/src/logging.rs` | Logging setup, and JSON lines carrying their trace | — |
 | `crates/ipx/src/config.rs` | TOML load/save, `General`/`Feed`/`Web`, intervals, slugs | `iPXSettings.py`, `feeds.plist` |
 | `crates/ipx/src/db.rs` | Every query, through SeaORM; creates missing tables | `.ipxd` plists, `history.dat`, `qmcache.dat` |
 | `crates/ipx/src/entity.rs` | The tables, as SeaORM entities: the schema | — |
-| `crates/feed/src/lib.rs` | Conditional GET, RSS/Atom/OPML parsing, `url_looks_private` | `FeedData.__getFeed/__getEntries` |
+| `crates/feed/src/lib.rs` | ipx-feed's modules and what it exports | — |
+| `crates/feed/src/fetch.rs` | Conditional GET, redirects followed one by one, bounded bodies | `FeedData.__getFeed` |
+| `crates/feed/src/parse.rs` | RSS and Atom into `ParsedFeed`: titles, notes, enclosures, artwork, dates | `FeedData.__getEntries` |
+| `crates/feed/src/outlines.rs` | OPML: recognising, reading safely, listing its feeds | — |
+| `crates/feed/src/patreon.rs` | A Patreon creator's shows | — |
+| `crates/feed/src/discover.rs` | Finding a feed from an address, a site or a page | — |
+| `crates/feed/src/wordpress.rs` | WordPress.com's API for a site behind a browser check | — |
+| `crates/feed/src/failure.rs` | A failed read in words, and its kind for the log | — |
+| `crates/feed/src/privacy.rs` | `redact`, and `url_looks_private` | — |
+| `crates/feed/src/artwork.rs` | Artwork on https, and site icons | — |
+| `crates/feed/src/category.rs` | Apple's categories | — |
+| `crates/feed/src/jev.rs` | Jev's pick of a category | — |
+| `crates/feed/src/guard.rs` | The connection guard against internal addresses (#320) | — |
 | `crates/ipx/src/download.rs` | Streaming download, naming, type sniffing, placement | `iPXDownloader.getFile` |
 | `crates/ipx/src/torrent.rs` | librqbit session, seeding limits, stall abort | vendored BitTorrent 4.2.1 |
 | `crates/ipx/src/retention.rs` | Quota and age sweeps | `iPXQuotaManager.py` |
@@ -42,7 +61,10 @@ id, and Caddy's request lines theirs. ipx also serves Prometheus metrics (`[web]
 | `crates/ipx/src/directory.rs` | Asking the directory service, and sharing this server's public feeds with it | — |
 | `crates/ipx/src/metrics.rs` | `/metrics` for Prometheus | — |
 | `crates/ipx/src/signals.rs` | Signals: asks Jev, on each person's TypeSafe key, which of their items fit their Signals | — |
-| `crates/directory/src/lib.rs` | The directory's HTTP API, its token check, the refresher | — |
+| `crates/directory/src/lib.rs` | The directory's modules and what it exports | — |
+| `crates/directory/src/api.rs` | The directory's HTTP API and its token check | — |
+| `crates/directory/src/directory.rs` | `Directory`: its store and tokens, the refresher, the config reloader | — |
+| `crates/directory/src/wire.rs` | What servers and the directory send each other | — |
 | `crates/directory/src/store.rs` | The directory's database: feeds, snapshots, blocked | — |
 | `crates/directory/src/config.rs` | The directory's config file: its tokens and their access | — |
 | `crates/directory/src/otel.rs` | Traces for both Rust services: the exporter, `traceparent` in and out | — |

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "ipx-web",
-  "version": "1.3.0",
+  "version": "1.3.1-dev",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "ipx-web",
-      "version": "1.3.0",
+      "version": "1.3.1-dev",
       "dependencies": {
         "@fortawesome/fontawesome-free": "7.3.1",
         "@swc/core": "^1.16.2",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ipx-web",
-  "version": "1.3.0",
+  "version": "1.3.1-dev",
   "private": true,
   "description": "Builds the ipx web pages from web/src (web/build.mjs, run by build.rs) and tests them. The Rust tests cover the server.",
   "scripts": {