rays/ipx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 18 Wiki Activity

No Docker healthchecks: the daemon keeps /metrics' counts itself (#218)

Browse Source 
Docker runs a healthcheck with exec, and containerd puts each exec's state in XDG_RUNTIME_DIR,
/run/user/0 on Tower, which elogind removes when root's last session ends. Every check then
failed and every container showed unhealthy while all of them answered. The checks are gone from
the sample compose file and production's; ipx's library counts for /metrics, which only the
healthcheck's status call refreshed, are counted every 30 s by the daemon.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-10-12 21:08:22 +00:00
parent 02bfa9571b
commit cc155274e9
11 changed files with 45 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/directory/CHANGELOG.md
View File

@@ -10,6 +10,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Changed
- The example in docs/directory.md has no healthcheck; `ipx-directory health` is still there to run by hand (#218).
## [1.0.0] - 2026-10-12
### Added

3
crates/directory/src/main.rs
View File

@@ -25,8 +25,7 @@ enum Cmd {
#[command(subcommand)]
cmd: TokenCmd,
},
/// Check that the directory on this machine answers and can read its database: the
/// container's healthcheck.
/// Check that the directory on this machine answers and can read its database.
Health,
}

4
crates/ipx/CHANGELOG.md
View File

@@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Changed
- `docker-compose.yml` has no healthchecks: Docker runs them with exec, which can fail on the host while ipx is well. The library's counts in `/metrics` are kept every 30 s by the daemon itself (#218).
## [1.0.0] - 2026-10-12
### Added

2
crates/ipx/src/ipc.rs
View File

@@ -198,7 +198,7 @@ fn log_event(e: &Event, wire: bool) {
)
};
}
// The healthcheck's answer, every 30s: a reply on the socket rather than work done, so it
// `ipx status`'s answer: a reply on the socket rather than work done, so it
// stays with the rest of the conversation, and the Scans tab stays about scans.
if matches!(e, Event::Status { .. }) {
return line!(info, "ipx::io", format!("<- {json}"));

21
crates/ipx/src/main.rs
View File

@@ -254,8 +254,8 @@ impl Ctx {
#[tokio::main]
async fn main() -> Result<()> {
let cli = Cli::parse();
// The daemon only: `ipx status` runs every half minute as the healthcheck, and a trace
// for each would bury the ones worth looking at.
// The daemon only: a trace for every `ipx status` a script runs would bury the ones worth
// looking at.
let otel = match cli.command {
Command::Daemon { .. } => ipx_directory::otel::provider("ipx")?,
_ => None,
@@ -568,7 +568,7 @@ pub(crate) async fn status(ctx: &Ctx) -> Event {
match ctx.db.counts().await {
Ok((pending, downloaded)) => {
let feeds = subscriptions(ctx).await.map(|s| s.len()).unwrap_or(0);
// Kept for /metrics, which asks the database nothing (#195): the healthcheck asks this every 30 s.
// Kept for /metrics, which asks the database nothing (#195); the daemon counts every 30 s.
use std::sync::atomic::Ordering::Relaxed;
let st = &metrics::STATS;
st.feeds.store(feeds as i64, Relaxed);
@@ -663,6 +663,21 @@ async fn daemon(
// must not wait behind it.
tokio::spawn(deadair::run(ctx.clone(), rx_stop.clone()));
tokio::spawn(directory::run(ctx.clone(), rx_stop.clone()));
// The library's counts for /metrics. The Docker healthcheck asked for them every 30 s until
// it was removed: Docker runs it with exec, which failed on Tower whenever root was logged
// out (#218).
{
let (ctx, mut stop) = (ctx.clone(), rx_stop.clone());
tokio::spawn(async move {
let mut every = tokio::time::interval(std::time::Duration::from_secs(30));
loop {
tokio::select! {
_ = every.tick() => { status(&ctx).await; }
_ = stop.changed() => break,
}
}
});
}
// Runs one job, abandoning it if a signal arrives. Returns false to end the loop.
async fn until_stopped(

10
crates/ipx/src/metrics.rs
View File

@@ -36,7 +36,7 @@ pub struct Stats {
/// Artwork served from disk, and fetched from the publisher because it was not there.
pub art_hits: AtomicU64,
pub art_misses: AtomicU64,
/// The library as the healthcheck last counted it, every 30 s; -1 until it has.
/// The library as the daemon last counted it, every 30 s; -1 until it has.
pub feeds: AtomicI64,
pub pending: AtomicI64,
pub downloaded: AtomicI64,
@@ -218,9 +218,9 @@ fn work(out: &mut String, st: &Stats, event_queue: usize, retries: usize, art_li
counter(out, "ipx_art_requests_total", "Artwork asked for: served from disk, or fetched from the publisher.",
&[("result=\"hit\"", st.art_hits.load(Relaxed) as f64), ("result=\"miss\"", st.art_misses.load(Relaxed) as f64)]);
for (name, help, v) in [
("ipx_feeds", "Feeds in the catalogue, as the healthcheck last counted.", &st.feeds),
("ipx_downloads_pending", "Files a scan will download on its own, as the healthcheck last counted.", &st.pending),
("ipx_files_downloaded", "Files on disk, as the healthcheck last counted.", &st.downloaded),
("ipx_feeds", "Feeds in the catalogue, as last counted.", &st.feeds),
("ipx_downloads_pending", "Files a scan will download on its own, as last counted.", &st.pending),
("ipx_files_downloaded", "Files on disk, as last counted.", &st.downloaded),
] {
let v = v.load(Relaxed);
if v >= 0 {
@@ -293,7 +293,7 @@ mod tests {
assert!(out.contains("ipx_worker_busy 0\n") && !out.contains("ipx_worker_job_seconds"), "{out}");
assert!(out.contains("ipx_jobs_total{kind=\"scan\"} 1\n") && out.contains("# TYPE ipx_jobs_total counter"), "{out}");
assert!(out.contains("ipx_event_queue 3\n") && out.contains("ipx_feed_retries 1\n"));
// The library's counts are left out until the healthcheck has counted it.
// The library's counts are left out until the daemon has counted it.
assert!(!out.contains("ipx_feeds "));
}