ipx-play: metrics for Prometheus, a dashboard row and a down alert (#458)

ipx-play sent traces and logged JSON but served no /metrics, so nothing showed whether
shared links were opening or failing. It now serves them on metrics_bind (9189 in
production, off the tunnel's port), with the directory's histogram and /proc reading:
requests by method, route and status (by route, so each shared episode is not a row of its
own, and an unknown address is route=""), episodes shared, memory, open files, uptime and
version.

The iPX dashboard gets an ipx-play row, its warnings join the other containers' log panel,
and alerts.py fires when Prometheus cannot reach it. The prod-check skill and docs/play.md
name it, with the scrape job and the OTEL_RESOURCE_ATTRIBUTES tag the trace panels need.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/play/src/metrics.rs

@@ -0,0 +1,99 @@
+//! ipx-play's /metrics (#458), in Prometheus's text format: memory, open files, the requests it
+//! answered and the episodes it was given to share. The histogram and /proc reading are the
+//! directory's, so the two read alike on the dashboard.
+
+use axum::Router;
+use ipx_directory::metrics::{Histograms, SECONDS, process};
+use std::sync::atomic::AtomicU64;
+use std::sync::atomic::Ordering::Relaxed;
+use std::time::Instant;
+use tokio::net::TcpListener;
+
+/// Requests answered, by method, route and status: a page opened is `route="/e/{id}"`.
+pub static REQUESTS: Histograms = Histograms::new(SECONDS);
+/// Episodes an ipx server sent to be shared, the same one again included.
+pub static SHARES: AtomicU64 = AtomicU64::new(0);
+
+/// Nothing from the database, so a scrape costs nothing and answers while it is down.
+fn render(started: Instant) -> String {
+    let mut out = String::new();
+    let mut gauge = |name: &str, kind: &str, help: &str, labels: &str, v: f64| {
+        out.push_str(&format!("# HELP {name} {help}\n# TYPE {name} {kind}\n{name}{labels} {v}\n"));
+    };
+    let (rss, fds) = process();
+    if let Some(v) = rss {
+        gauge("play_process_resident_bytes", "gauge", "Memory the process holds.", "", v);
+    }
+    if let Some(v) = fds {
+        gauge("play_process_open_fds", "gauge", "Files and sockets the process has open.", "", v);
+    }
+    gauge("play_uptime_seconds", "gauge", "Seconds since ipx-play started.", "", started.elapsed().as_secs() as f64);
+    gauge(
+        "play_build_info",
+        "gauge",
+        "The version running, as a label.",
+        &format!("{{version=\"{}\"}}", env!("CARGO_PKG_VERSION")),
+        1.0,
+    );
+    gauge("play_shares_total", "counter", "Episodes ipx servers sent to be shared.", "", SHARES.load(Relaxed) as f64);
+    REQUESTS.render(
+        &mut out,
+        "play_http_request_duration_seconds",
+        "Requests answered and the seconds each took, by method, route and status.",
+    );
+    out
+}
+
+/// Serves /metrics on a listener of its own: the public port is behind the tunnel with no Access
+/// in front, and these are for the LAN.
+// ponytail: no gzip, unlike the directory's (#253): a few dozen lines a scrape. Add tower-http's
+// CompressionLayer if the routes grow.
+pub async fn serve(listener: TcpListener) {
+    let started = Instant::now();
+    let app = Router::new().route(
+        "/metrics",
+        axum::routing::get(move || async move {
+            ([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(started))
+        }),
+    );
+    if let Err(e) = axum::serve(listener, app).await {
+        tracing::error!(error = %e, "metrics stopped");
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn every_play_metric_grafana_queries_is_served() {
+        // A row for each name, so a histogram's lines are there to find.
+        REQUESTS.observe("method=\"GET\",route=\"/e/{id}\",status=\"200\"", 0.01);
+        let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let at = l.local_addr().unwrap();
+        tokio::spawn(serve(l));
+        let text = reqwest::get(format!("http://{at}/metrics")).await.unwrap().text().await.unwrap();
+        let src = [include_str!("../../../grafana/dashboard.py"), include_str!("../../../grafana/alerts.py")].concat();
+        let names: std::collections::BTreeSet<&str> = src
+            .match_indices("play_")
+            .filter(|(i, _)| !src[..*i].ends_with(|c: char| c.is_ascii_alphanumeric() || c == '_'))
+            .filter_map(|(i, _)| {
+                let rest = &src[i..];
+                let end = rest.find(|c: char| !(c.is_ascii_lowercase() || c == '_')).unwrap_or(rest.len());
+                // A query names one with its labels, or the dashboard keeps it in a string; ipx's
+                // events, such as play_start in a panel's description, are neither.
+                rest[end..].starts_with(['{', '"']).then(|| &rest[..end])
+            })
+            // A histogram is queried by its series' names and served under its own.
+            .map(|n| n.trim_end_matches("_bucket").trim_end_matches("_count").trim_end_matches("_sum"))
+            .collect();
+        assert!(names.len() >= 3, "{names:?}");
+        for n in names {
+            // Memory comes from /proc, which a Mac has not.
+            if n.starts_with("play_process_") && process().0.is_none() {
+                continue;
+            }
+            assert!(text.contains(&format!("# TYPE {n} ")), "{n} is queried by grafana/ but not served: {text}");
+        }
+    }
+}