ipx/crates/play/src/metrics.rs

//! ipx-play's /metrics (#458), in Prometheus's text format: memory, open files, the requests it
//! answered and the episodes it was given to share. The histogram and /proc reading are the
//! directory's, so the two read alike on the dashboard.

use axum::Router;
use ipx_directory::metrics::{Histograms, SECONDS, process};
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering::Relaxed;
use std::time::Instant;
use tokio::net::TcpListener;

/// Requests answered, by method, route and status: a page opened is `route="/e/{id}"`.
pub static REQUESTS: Histograms = Histograms::new(SECONDS);
/// Episodes an ipx server sent to be shared, the same one again included.
pub static SHARES: AtomicU64 = AtomicU64::new(0);

/// Nothing from the database, so a scrape costs nothing and answers while it is down.
fn render(started: Instant) -> String {
    let mut out = String::new();
    let mut gauge = |name: &str, kind: &str, help: &str, labels: &str, v: f64| {
        out.push_str(&format!("# HELP {name} {help}\n# TYPE {name} {kind}\n{name}{labels} {v}\n"));
    };
    let (rss, fds) = process();
    if let Some(v) = rss {
        gauge("play_process_resident_bytes", "gauge", "Memory the process holds.", "", v);
    }
    if let Some(v) = fds {
        gauge("play_process_open_fds", "gauge", "Files and sockets the process has open.", "", v);
    }
    gauge("play_uptime_seconds", "gauge", "Seconds since ipx-play started.", "", started.elapsed().as_secs() as f64);
    gauge(
        "play_build_info",
        "gauge",
        "The version running, as a label.",
        &format!("{{version=\"{}\"}}", env!("CARGO_PKG_VERSION")),
        1.0,
    );
    gauge("play_shares_total", "counter", "Episodes ipx servers sent to be shared.", "", SHARES.load(Relaxed) as f64);
    REQUESTS.render(
        &mut out,
        "play_http_request_duration_seconds",
        "Requests answered and the seconds each took, by method, route and status.",
    );
    out
}

/// Serves /metrics on a listener of its own: the public port is behind the tunnel with no Access
/// in front, and these are for the LAN.
// ponytail: no gzip, unlike the directory's (#253): a few dozen lines a scrape. Add tower-http's
// CompressionLayer if the routes grow.
pub async fn serve(listener: TcpListener) {
    let started = Instant::now();
    let app = Router::new().route(
        "/metrics",
        axum::routing::get(move || async move {
            ([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(started))
        }),
    );
    if let Err(e) = axum::serve(listener, app).await {
        tracing::error!(error = %e, "metrics stopped");
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn every_play_metric_grafana_queries_is_served() {
        // A row for each name, so a histogram's lines are there to find.
        REQUESTS.observe("method=\"GET\",route=\"/e/{id}\",status=\"200\"", 0.01);
        let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
        let at = l.local_addr().unwrap();
        tokio::spawn(serve(l));
        let text = reqwest::get(format!("http://{at}/metrics")).await.unwrap().text().await.unwrap();
        let src = [include_str!("../../../grafana/dashboard.py"), include_str!("../../../grafana/alerts.py")].concat();
        let names: std::collections::BTreeSet<&str> = src
            .match_indices("play_")
            .filter(|(i, _)| !src[..*i].ends_with(|c: char| c.is_ascii_alphanumeric() || c == '_'))
            .filter_map(|(i, _)| {
                let rest = &src[i..];
                let end = rest.find(|c: char| !(c.is_ascii_lowercase() || c == '_')).unwrap_or(rest.len());
                // A query names one with its labels, or the dashboard keeps it in a string; ipx's
                // events, such as play_start in a panel's description, are neither.
                rest[end..].starts_with(['{', '"']).then(|| &rest[..end])
            })
            // A histogram is queried by its series' names and served under its own.
            .map(|n| n.trim_end_matches("_bucket").trim_end_matches("_count").trim_end_matches("_sum"))
            .collect();
        assert!(names.len() >= 3, "{names:?}");
        for n in names {
            // Memory comes from /proc, which a Mac has not.
            if n.starts_with("play_process_") && process().0.is_none() {
                continue;
            }
            assert!(text.contains(&format!("# TYPE {n} ")), "{n} is queried by grafana/ but not served: {text}");
        }
    }
}