ipx-play: metrics for Prometheus, a dashboard row and a down alert (#458)

ipx-play sent traces and logged JSON but served no /metrics, so nothing showed whether
shared links were opening or failing. It now serves them on metrics_bind (9189 in
production, off the tunnel's port), with the directory's histogram and /proc reading:
requests by method, route and status (by route, so each shared episode is not a row of its
own, and an unknown address is route=""), episodes shared, memory, open files, uptime and
version.

The iPX dashboard gets an ipx-play row, its warnings join the other containers' log panel,
and alerts.py fires when Prometheus cannot reach it. The prod-check skill and docs/play.md
name it, with the scrape job and the OTEL_RESOURCE_ATTRIBUTES tag the trace panels need.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/play/CHANGELOG.md

@@ -14,5 +14,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - A page for each shared episode, at `/e/<id>`: its show, title, date, length and notes, a player that plays the publisher's own file from the time it was shared at (`?t=754` or `?t=12:34`), buttons to subscribe in Apple Podcasts, Overcast, Pocket Casts or Castro, the feed's address, and a preview where the link is pasted (#447).
 - `POST /v1/shares`, for an ipx server with a token from `ipx-play token <name>`: an episode in, its short link out, the same link however often and from whichever server it is shared. An address that looks like a private or paid feed's is refused (#447).
+- Metrics for Prometheus on a port of its own, `metrics_bind`: requests by route and status and their time, episodes shared, memory (#458).
 
 [unreleased]: https://git.sdf1.net/rays/ipx/commits/main/crates/play

crates/play/src/config.rs

@@ -15,6 +15,10 @@ pub struct Config {
     /// The environment variable holding the database URL, so the file holds no secret.
     #[serde(default = "default_database_url_env")]
     pub database_url_env: String,
+    /// Where to serve /metrics for Prometheus (#458), apart from `bind`, which the tunnel
+    /// publishes: `0.0.0.0:9189`. Empty serves none.
+    #[serde(default)]
+    pub metrics_bind: String,
     /// The address people open, which the links ipx is handed begin with.
     #[serde(default = "default_public_url")]
     pub public_url: String,

crates/play/src/lib.rs

@@ -9,10 +9,11 @@
 //! its file is, nothing more.
 
 pub mod config;
+pub mod metrics;
 mod page;
 pub mod store;
 
-use axum::extract::{Path, Query, Request, State};
+use axum::extract::{MatchedPath, Path, Query, Request, State};
 use axum::http::{StatusCode, header};
 use axum::middleware::Next;
 use axum::response::{IntoResponse, Response};
@@ -160,6 +161,7 @@ async fn share(State(st): State<St>, Extension(who): Extension<Who>, Json(mut e)
     }
     match st.store.put(&who.0, &e).await {
         Ok(id) => {
+            metrics::SHARES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
             tracing::info!(server = %who.0, id = %id, "an episode was shared");
             Json(Shared { url: format!("{}/e/{id}", st.base), id }).into_response()
         }
@@ -198,6 +200,9 @@ async fn healthz(State(st): State<St>) -> Response {
 /// calls here; and a line, with the server's name and never its token.
 async fn access_log(req: Request, next: Next) -> Response {
     let (method, path) = (req.method().clone(), req.uri().path().to_owned());
+    // By route, not path: by path each shared episode would be a row of its own. A path no route
+    // has is "", which someone trying addresses cannot multiply.
+    let route = req.extensions().get::<MatchedPath>().map_or("", |r| r.as_str()).to_owned();
     let start = Instant::now();
     let span = if path == "/healthz" {
         tracing::Span::none()
@@ -218,6 +223,14 @@ async fn access_log(req: Request, next: Next) -> Response {
     let status = resp.status().as_u16();
     span.record("http.response.status_code", status);
     if path != "/healthz" {
+        metrics::REQUESTS.observe(
+            &format!(
+                "method=\"{}\",route=\"{}\",status=\"{status}\"",
+                ipx_directory::metrics::method(method.as_str()),
+                ipx_directory::metrics::label(&route)
+            ),
+            start.elapsed().as_secs_f64(),
+        );
         let _in = span.enter();
         let who = resp.extensions().get::<Who>().map_or("", |w| w.0.as_str()).to_owned();
         tracing::info!(
@@ -301,6 +314,18 @@ mod tests {
         assert!(!html.contains("javascript:"), "a link that is not a web address is dropped");
 
         assert_eq!(get(&app, "/e/nothere").await.0, StatusCode::NOT_FOUND);
+        get(&app, "/no/such/page").await;
+
+        let mut out = String::new();
+        metrics::REQUESTS.render(&mut out, "r", "");
+        for row in [
+            r#"r_count{method="GET",route="/e/{id}",status="200"}"#,
+            r#"r_count{method="POST",route="/v1/shares",status="200"}"#,
+            r#"r_count{method="GET",route="",status="404"}"#,
+        ] {
+            assert!(out.contains(row), "requests are counted by route, not path: {row} in {out}");
+        }
+        assert!(!out.contains("nothere") && !out.contains("/no/such"), "{out}");
     }
 
     #[tokio::test]

crates/play/src/main.rs

@@ -98,6 +98,13 @@ async fn serve(path: &Path) -> Result<()> {
         .with_context(|| format!("{} is not set: ipx-play needs a database URL", cfg.database_url_env))?;
     let store = Store::open(&url, true).await.context("opening the database")?;
     let listener = TcpListener::bind(&cfg.bind).await.with_context(|| format!("listening on {}", cfg.bind))?;
+    if !cfg.metrics_bind.is_empty() {
+        let metrics = TcpListener::bind(&cfg.metrics_bind)
+            .await
+            .with_context(|| format!("listening for metrics on {}", cfg.metrics_bind))?;
+        tracing::info!(bind = %cfg.metrics_bind, "serving metrics for Prometheus at /metrics");
+        tokio::spawn(ipx_play::metrics::serve(metrics));
+    }
     tracing::info!(version = env!("CARGO_PKG_VERSION"), bind = %cfg.bind, tokens = tokens.len(), "ipx-play started");
     let app = ipx_play::router(Play { store, tokens, base: cfg.base().to_owned() });
     axum::serve(listener, app)

crates/play/src/metrics.rs

@@ -0,0 +1,99 @@
+//! ipx-play's /metrics (#458), in Prometheus's text format: memory, open files, the requests it
+//! answered and the episodes it was given to share. The histogram and /proc reading are the
+//! directory's, so the two read alike on the dashboard.
+
+use axum::Router;
+use ipx_directory::metrics::{Histograms, SECONDS, process};
+use std::sync::atomic::AtomicU64;
+use std::sync::atomic::Ordering::Relaxed;
+use std::time::Instant;
+use tokio::net::TcpListener;
+
+/// Requests answered, by method, route and status: a page opened is `route="/e/{id}"`.
+pub static REQUESTS: Histograms = Histograms::new(SECONDS);
+/// Episodes an ipx server sent to be shared, the same one again included.
+pub static SHARES: AtomicU64 = AtomicU64::new(0);
+
+/// Nothing from the database, so a scrape costs nothing and answers while it is down.
+fn render(started: Instant) -> String {
+    let mut out = String::new();
+    let mut gauge = |name: &str, kind: &str, help: &str, labels: &str, v: f64| {
+        out.push_str(&format!("# HELP {name} {help}\n# TYPE {name} {kind}\n{name}{labels} {v}\n"));
+    };
+    let (rss, fds) = process();
+    if let Some(v) = rss {
+        gauge("play_process_resident_bytes", "gauge", "Memory the process holds.", "", v);
+    }
+    if let Some(v) = fds {
+        gauge("play_process_open_fds", "gauge", "Files and sockets the process has open.", "", v);
+    }
+    gauge("play_uptime_seconds", "gauge", "Seconds since ipx-play started.", "", started.elapsed().as_secs() as f64);
+    gauge(
+        "play_build_info",
+        "gauge",
+        "The version running, as a label.",
+        &format!("{{version=\"{}\"}}", env!("CARGO_PKG_VERSION")),
+        1.0,
+    );
+    gauge("play_shares_total", "counter", "Episodes ipx servers sent to be shared.", "", SHARES.load(Relaxed) as f64);
+    REQUESTS.render(
+        &mut out,
+        "play_http_request_duration_seconds",
+        "Requests answered and the seconds each took, by method, route and status.",
+    );
+    out
+}
+
+/// Serves /metrics on a listener of its own: the public port is behind the tunnel with no Access
+/// in front, and these are for the LAN.
+// ponytail: no gzip, unlike the directory's (#253): a few dozen lines a scrape. Add tower-http's
+// CompressionLayer if the routes grow.
+pub async fn serve(listener: TcpListener) {
+    let started = Instant::now();
+    let app = Router::new().route(
+        "/metrics",
+        axum::routing::get(move || async move {
+            ([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(started))
+        }),
+    );
+    if let Err(e) = axum::serve(listener, app).await {
+        tracing::error!(error = %e, "metrics stopped");
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn every_play_metric_grafana_queries_is_served() {
+        // A row for each name, so a histogram's lines are there to find.
+        REQUESTS.observe("method=\"GET\",route=\"/e/{id}\",status=\"200\"", 0.01);
+        let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let at = l.local_addr().unwrap();
+        tokio::spawn(serve(l));
+        let text = reqwest::get(format!("http://{at}/metrics")).await.unwrap().text().await.unwrap();
+        let src = [include_str!("../../../grafana/dashboard.py"), include_str!("../../../grafana/alerts.py")].concat();
+        let names: std::collections::BTreeSet<&str> = src
+            .match_indices("play_")
+            .filter(|(i, _)| !src[..*i].ends_with(|c: char| c.is_ascii_alphanumeric() || c == '_'))
+            .filter_map(|(i, _)| {
+                let rest = &src[i..];
+                let end = rest.find(|c: char| !(c.is_ascii_lowercase() || c == '_')).unwrap_or(rest.len());
+                // A query names one with its labels, or the dashboard keeps it in a string; ipx's
+                // events, such as play_start in a panel's description, are neither.
+                rest[end..].starts_with(['{', '"']).then(|| &rest[..end])
+            })
+            // A histogram is queried by its series' names and served under its own.
+            .map(|n| n.trim_end_matches("_bucket").trim_end_matches("_count").trim_end_matches("_sum"))
+            .collect();
+        assert!(names.len() >= 3, "{names:?}");
+        for n in names {
+            // Memory comes from /proc, which a Mac has not.
+            if n.starts_with("play_process_") && process().0.is_none() {
+                continue;
+            }
+            assert!(text.contains(&format!("# TYPE {n} ")), "{n} is queried by grafana/ but not served: {text}");
+        }
+    }
+}