From 1698cf8d1e642731e7b1ddaacd949f5737b3b761 Mon Sep 17 00:00:00 2001 From: rays Date: Sat, 12 Sep 2026 14:19:00 +0000 Subject: [PATCH] Answer status on the socket instead of queuing it behind the worker The worker runs one job at a time, and status was one of its jobs, so the Docker healthcheck waited behind the startup scan (54 seconds of it after the last deploy) and timed out at 5. Any scan or download longer than three probes would have had a working daemon marked unhealthy. The socket now answers status straight away; everything else still queues. A test fills the queue and checks status comes back anyway. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TAC7sLVqfKmY6rsTLXzNgk --- CHANGELOG.md | 2 ++ CLAUDE.md | 10 +++++++--- docs/architecture.md | 3 ++- docs/history.md | 16 ++++++++++++++++ src/ipc.rs | 39 ++++++++++++++++++++++++++++++++++++++- src/main.rs | 23 +++++++++++++++++++---- 6 files changed, 84 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 719449a..4fb95b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ The long form, with what was wrong before and how it was found, is in - Show notes that the podcast's host cut off in the middle of a tag no longer open with a scrap of HTML: the item's other copy of its notes is shown instead. Daily Meditation Podcast had 57. +- Docker no longer shows ipodderx as starting, or calls it unhealthy, while it scans or downloads: + `ipx status` answers at once instead of waiting for the job in progress to finish. - Signing out after signing in through Cloudflare Access no longer lands on ipodderx's own password page. With the new `sign_out_url` set, Sign out ends the Access session, and the password page sends anyone the proxy signs in straight to their feeds. diff --git a/CLAUDE.md b/CLAUDE.md index 64359e9..3a48440 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,7 +46,10 @@ docker tag mirror.gcr.io/library/rust:1-slim-bookworm rust:1-slim-bookworm Run those again now and then, or the local copies go stale. The healthcheck runs `ipx status` against the control socket, so `(healthy)` in `docker ps` means -the worker is alive, not just the web port. The container restarts on its own after a reboot. +the daemon answers there and can read its database, not just that the web port is up. The socket +answers `status` itself instead of queuing it behind the worker's current job, so a long scan or +download does not fail the check; it also means a worker stuck on one job would still pass. The +container restarts on its own after a reboot. Before the container, ipx ran by hand in code-server, with its files in `/config/.config/ipx/` and `/config/.local/share/ipx/`. Those are still there and the container does not read them. If you run @@ -114,8 +117,9 @@ Non-trivial logic leaves one runnable check behind. Pure functions (`merge_polic watch the shutdown channel itself; the daemon ignored SIGTERM for exactly this reason. * Only one daemon per socket. Removing the socket file defeats the guard and you get two daemons fighting over the database, with the stale one still holding the port. -* `/api/settings` answering `200` does **not** mean the worker is alive — it is a different task. - Probe the control socket (`ipx status`) to check that. +* `/api/settings` answering `200` does **not** mean the daemon is well — the web server is a + different task. `ipx status` checks the control socket and the database; to see the worker + getting through its jobs, watch for `scan complete` in the log. * **Every `ipx` command runs `migrate()` when it opens the database**, the healthcheck's `ipx status` included. A migration that rewrites a big table (`DROP COLUMN`) takes seconds on production, and a command run meanwhile fails with `migrating schema`. It changes nothing; wait diff --git a/docs/architecture.md b/docs/architecture.md index 48d8966..c06bc28 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -87,7 +87,8 @@ printf '{"cmd":"fetch","force":true}\n' | socat - UNIX-CONNECT:$XDG_RUNTIME_DIR/ **Events** — `feed_start`, `feed_skip`, `feed_done`, `feed_error`, `progress`, `download_done`, `download_error`, `torrent_deferred`, `reaped`, `reap_done`, `scan_done`, `status`, `error`. `scan_done`, `reap_done` and `status` are terminal: a client that asked for work stops reading -there. +there. Commands run one at a time, in the order they arrive, except `status`: the socket answers it +straight away, so the Docker healthcheck is never left waiting behind a scan or a download. Progress carries the enclosure id, without which a UI cannot tell one download from another and ends up animating every pending row. It is throttled to whole percents. The stream is a broadcast, diff --git a/docs/history.md b/docs/history.md index 9d3fbac..61d97a7 100644 --- a/docs/history.md +++ b/docs/history.md @@ -6,6 +6,22 @@ reasoning lives. New write-ups go at the top. See [README.md](../README.md) for what the thing is. +## 2026-09-12 — Healthy while busy + +After a deploy the container sat at "starting" for a minute, and Docker's health log showed two +`ipx status` probes exceeding their 5-second timeout. The daemon's own log explained it. The first +scan after the start fetched 23 feeds, from 14:10:41 to 14:11:35, and both probes' `status` +commands waited in the job queue behind it; they were answered together at 14:11:35, straight after +`scan_done`. The worker runs one job at a time and `status` was one of its jobs, so any scan or +download longer than about a minute and a half, three 30-second probes, would have had Docker call +a working daemon unhealthy. + +The socket now answers `status` itself, from two short queries, and only real work goes through the +queue. The trade is that healthy now means the daemon answers on its socket and can read its +database; a worker stuck on one job would still pass. Asking a daemon that downloads hour-long +podcasts to be idle within five seconds was never a fair test of whether it was alive. A test holds +the queue full and checks `status` still comes back. + ## 2026-09-12 — Signing in through Authentik, for real Ray could not get Authentik's sign-in to reach ipx, following `docs/sso.md`, which had been written diff --git a/src/ipc.rs b/src/ipc.rs index d703bf7..a55977b 100644 --- a/src/ipc.rs +++ b/src/ipc.rs @@ -172,11 +172,17 @@ pub async fn daemon_is_live(path: &Path) -> bool { UnixStream::connect(path).await.is_ok() } +/// Answers `status` for the socket, without the worker. The worker runs one job at a time, and a +/// healthcheck left waiting behind a scan or a long download timed out and called a busy daemon +/// dead. +pub type StatusFn = std::sync::Arc; + /// Accepts connections, feeding commands to `cmds` and events from `events` back out. pub async fn serve( path: PathBuf, events: broadcast::Sender, cmds: mpsc::Sender, + status: StatusFn, ) -> Result<()> { // A socket file left by a crashed daemon would block the bind; a live one was already // rejected by the caller's daemon_is_live() check. @@ -195,8 +201,9 @@ pub async fn serve( let (stream, _) = listener.accept().await?; let rx = events.subscribe(); let cmds = cmds.clone(); + let status = status.clone(); tokio::spawn(async move { - if let Err(e) = handle(stream, rx, cmds).await { + if let Err(e) = handle(stream, rx, cmds, status).await { tracing::debug!(error = %e, "client gone"); } }); @@ -207,6 +214,7 @@ async fn handle( stream: UnixStream, mut rx: broadcast::Receiver, cmds: mpsc::Sender, + status: StatusFn, ) -> Result<()> { let (read, mut write) = stream.into_split(); @@ -229,6 +237,11 @@ async fn handle( continue; } match serde_json::from_str::(line) { + // Answered here, not queued behind whatever the worker is on: see StatusFn. + Ok(Command::Status) => { + tracing::info!(target: "ipx::io", "-> {line}"); + status(); + } Ok(cmd) => { if cmds.send(cmd).await.is_err() { break; // Worker is gone; so are we. @@ -329,4 +342,28 @@ mod tests { } .is_terminal()); } + + #[tokio::test] + async fn status_is_answered_while_the_worker_is_busy() { + // The queue is full and nobody drains it, as when the worker is deep in a long download: + // anything sent to it would wait for ever. + let (cmds, _worker) = mpsc::channel::(1); + cmds.send(Command::Reap { dry_run: true }).await.unwrap(); + let (events, _) = broadcast::channel::(8); + let tx = events.clone(); + let status: StatusFn = std::sync::Arc::new(move || { + let _ = tx.send(Event::Status { feeds: 1, pending: 2, downloaded: 3 }); + }); + let (client, server) = UnixStream::pair().unwrap(); + tokio::spawn(handle(server, events.subscribe(), cmds, status)); + + let (read, mut write) = client.into_split(); + write.write_all(b"{\"cmd\":\"status\"}\n").await.unwrap(); + let line = tokio::time::timeout(std::time::Duration::from_secs(2), BufReader::new(read).lines().next_line()) + .await + .expect("status waited behind the worker") + .unwrap() + .unwrap(); + assert!(line.contains(r#""ev":"status""#), "{line}"); + } } diff --git a/src/main.rs b/src/main.rs index 5608f1e..7e64d20 100644 --- a/src/main.rs +++ b/src/main.rs @@ -339,14 +339,24 @@ async fn run(ctx: &Arc, cmd: Cmd) -> Result<()> { Cmd::Reap { dry_run } => reap(ctx, dry_run, true), Cmd::Download { enclosure } => download_one(ctx, enclosure).await, Cmd::Status => { - let (pending, downloaded) = ctx.db.counts()?; - let feeds = subscriptions(ctx).map(|s| s.len()).unwrap_or(0); - ctx.out.emit(Event::Status { feeds, pending, downloaded }); + ctx.out.emit(status(ctx)); Ok(()) } } } +/// The counts `ipx status` prints. A running daemon's socket answers with this directly rather +/// than through the job queue. +fn status(ctx: &Ctx) -> Event { + match ctx.db.counts() { + Ok((pending, downloaded)) => { + let feeds = subscriptions(ctx).map(|s| s.len()).unwrap_or(0); + Event::Status { feeds, pending, downloaded } + } + Err(e) => Event::Error { msg: format!("{e:#}") }, + } +} + async fn daemon( ctx: Arc, config_path: PathBuf, @@ -385,7 +395,12 @@ async fn daemon( let (tx_cmd, mut rx_cmd) = mpsc::channel::(64); let web = start_web(&ctx, &config_path, web_addr, &tx_cmd, &events).await?; - let server = tokio::spawn(ipc::serve(socket.clone(), events.clone(), tx_cmd)); + // status is answered by the socket itself; everything else waits its turn in the queue. + let answer: ipc::StatusFn = { + let ctx = ctx.clone(); + Arc::new(move || ctx.out.emit(status(&ctx))) + }; + let server = tokio::spawn(ipc::serve(socket.clone(), events.clone(), tx_cmd, answer)); // One command at a time: the queue is what keeps two scans from overlapping. let mut ticker = tokio::time::interval(std::time::Duration::from_secs(60));