//! ipx-play's /metrics (#458), in Prometheus's text format: memory, open files, the requests it //! answered and the episodes it was given to share. The histogram and /proc reading are the //! directory's, so the two read alike on the dashboard. use axum::Router; use ipx_directory::metrics::{Histograms, SECONDS, process}; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering::Relaxed; use std::time::Instant; use tokio::net::TcpListener; /// Requests answered, by method, route and status: a page opened is `route="/e/{id}"`. pub static REQUESTS: Histograms = Histograms::new(SECONDS); /// Episodes an ipx server sent to be shared, the same one again included. pub static SHARES: AtomicU64 = AtomicU64::new(0); /// Nothing from the database, so a scrape costs nothing and answers while it is down. fn render(started: Instant) -> String { let mut out = String::new(); let mut gauge = |name: &str, kind: &str, help: &str, labels: &str, v: f64| { out.push_str(&format!("# HELP {name} {help}\n# TYPE {name} {kind}\n{name}{labels} {v}\n")); }; let (rss, fds) = process(); if let Some(v) = rss { gauge("play_process_resident_bytes", "gauge", "Memory the process holds.", "", v); } if let Some(v) = fds { gauge("play_process_open_fds", "gauge", "Files and sockets the process has open.", "", v); } gauge("play_uptime_seconds", "gauge", "Seconds since ipx-play started.", "", started.elapsed().as_secs() as f64); gauge( "play_build_info", "gauge", "The version running, as a label.", &format!("{{version=\"{}\"}}", env!("CARGO_PKG_VERSION")), 1.0, ); gauge("play_shares_total", "counter", "Episodes ipx servers sent to be shared.", "", SHARES.load(Relaxed) as f64); REQUESTS.render( &mut out, "play_http_request_duration_seconds", "Requests answered and the seconds each took, by method, route and status.", ); out } /// Serves /metrics on a listener of its own: the public port is behind the tunnel with no Access /// in front, and these are for the LAN. // ponytail: no gzip, unlike the directory's (#253): a few dozen lines a scrape. Add tower-http's // CompressionLayer if the routes grow. pub async fn serve(listener: TcpListener) { let started = Instant::now(); let app = Router::new().route( "/metrics", axum::routing::get(move || async move { ([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(started)) }), ); if let Err(e) = axum::serve(listener, app).await { tracing::error!(error = %e, "metrics stopped"); } } #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn every_play_metric_grafana_queries_is_served() { // A row for each name, so a histogram's lines are there to find. REQUESTS.observe("method=\"GET\",route=\"/e/{id}\",status=\"200\"", 0.01); let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let at = l.local_addr().unwrap(); tokio::spawn(serve(l)); let text = reqwest::get(format!("http://{at}/metrics")).await.unwrap().text().await.unwrap(); let src = [include_str!("../../../grafana/dashboard.py"), include_str!("../../../grafana/alerts.py")].concat(); let names: std::collections::BTreeSet<&str> = src .match_indices("play_") .filter(|(i, _)| !src[..*i].ends_with(|c: char| c.is_ascii_alphanumeric() || c == '_')) .filter_map(|(i, _)| { let rest = &src[i..]; let end = rest.find(|c: char| !(c.is_ascii_lowercase() || c == '_')).unwrap_or(rest.len()); // A query names one with its labels, or the dashboard keeps it in a string; ipx's // events, such as play_start in a panel's description, are neither. rest[end..].starts_with(['{', '"']).then(|| &rest[..end]) }) // A histogram is queried by its series' names and served under its own. .map(|n| n.trim_end_matches("_bucket").trim_end_matches("_count").trim_end_matches("_sum")) .collect(); assert!(names.len() >= 3, "{names:?}"); for n in names { // Memory comes from /proc, which a Mac has not. if n.starts_with("play_process_") && process().0.is_none() { continue; } assert!(text.contains(&format!("# TYPE {n} ")), "{n} is queried by grafana/ but not served: {text}"); } } }