mimalloc for ipx and the directory, and metrics and traces for memory (#224)

ipx's resident memory climbed from about 40 MB to 500-850 MB each night while scans ran five
times as often, and stayed there until a restart; the directory sat at 1.25 GiB idle. glibc's
allocator keeps what each thread's arena freed. mimalloc gives freed pages back, so resident
memory now follows what is in use. jemalloc was the first choice for its in-use figures, but its
upstream was archived in 2025.

Declared once in the directory crate, which ipx builds in, so both binaries get it.

What was missing to see it: the directory served no /metrics (now on metrics_bind, a port apart
from the tunnel's), nothing counted the feed bytes fetched, and the fetch span had no body size.
The feed crate's spans never reached Tempo either: both trace exporters filtered to their own
crate, so ipx_feed's fetch and parse spans were dropped.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/directory/src/config.rs

@@ -14,6 +14,10 @@ pub struct Config {
     /// The environment variable holding the database URL, so the file holds no secret.
     #[serde(default = "default_database_url_env")]
     pub database_url_env: String,
+    /// Where to serve /metrics for Prometheus (#224), apart from `bind`, which the tunnel
+    /// publishes: `0.0.0.0:9188`. Empty serves none.
+    #[serde(default)]
+    pub metrics_bind: String,
     #[serde(default)]
     pub tokens: Vec<TokenCfg>,
 }

crates/directory/src/lib.rs

@@ -8,6 +8,7 @@
 //! people's behalf.
 
 pub mod config;
+pub mod metrics;
 pub mod otel;
 mod site;
 pub mod store;

crates/directory/src/main.rs

@@ -82,7 +82,7 @@ async fn serve(path: &std::path::Path) -> Result<()> {
         .with(json.then(|| tracing_subscriber::fmt::layer().json().flatten_event(true).with_writer(std::io::stderr).with_filter(filter())))
         .with(otel.as_ref().map(|p| {
             use opentelemetry::trace::TracerProvider;
-            tracing_opentelemetry::layer().with_tracer(p.tracer("ipx-directory")).with_filter(tracing_subscriber::EnvFilter::new("ipx_directory=info"))
+            tracing_opentelemetry::layer().with_tracer(p.tracer("ipx-directory")).with_filter(tracing_subscriber::EnvFilter::new("ipx_directory=info,ipx_feed=info"))
         }))
         .init();
 
@@ -98,6 +98,11 @@ async fn serve(path: &std::path::Path) -> Result<()> {
 
     let listener = tokio::net::TcpListener::bind(&cfg.bind).await.with_context(|| format!("listening on {}", cfg.bind))?;
     tracing::info!(version = env!("CARGO_PKG_VERSION"), bind = %cfg.bind, tokens = cfg.tokens.len(), dropped, "directory started");
+    if !cfg.metrics_bind.is_empty() {
+        let metrics = tokio::net::TcpListener::bind(&cfg.metrics_bind).await.with_context(|| format!("listening for metrics on {}", cfg.metrics_bind))?;
+        tracing::info!(bind = %cfg.metrics_bind, "serving metrics for Prometheus at /metrics");
+        tokio::spawn(ipx_directory::metrics::serve(metrics));
+    }
     tokio::spawn(ipx_directory::refresher(dir.clone()));
     tokio::spawn(ipx_directory::reloader(dir.clone(), path.to_owned()));
     axum::serve(listener, ipx_directory::router(dir))

crates/directory/src/metrics.rs

@@ -0,0 +1,67 @@
+//! Memory, for ipx and the directory both (#224), and the directory's own /metrics.
+//!
+//! mimalloc is the allocator of every binary that builds this crate in, ipx included: with
+//! glibc's, ipx kept the 500-850 MB a night of scans had needed until it restarted, and the
+//! directory sat at 1.25 GiB doing nothing, each thread's arena holding what it had freed.
+//! mimalloc gives freed pages back, so resident memory follows what is in use: a spike that
+//! does not come down again is a leak. Not jemalloc, whose upstream was archived in 2025.
+
+use std::sync::atomic::Ordering::Relaxed;
+
+#[global_allocator]
+static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
+/// The process's resident memory in bytes and its open files, from /proc; None elsewhere.
+pub fn process() -> (Option<f64>, Option<f64>) {
+    let rss = std::fs::read_to_string("/proc/self/status").ok().and_then(|s| {
+        let kb = s.lines().find_map(|l| l.strip_prefix("VmRSS:"))?.trim().strip_suffix("kB")?.trim().parse::<f64>().ok()?;
+        Some(kb * 1024.0)
+    });
+    let fds = std::fs::read_dir("/proc/self/fd").ok().map(|d| d.count() as f64);
+    (rss, fds)
+}
+
+/// The directory's /metrics, in Prometheus's text format: memory, open files, and the feeds it
+/// has fetched. Nothing from the database, so a scrape costs nothing and answers while it is down.
+fn render(started: std::time::Instant) -> String {
+    let mut out = String::new();
+    let mut gauge = |name: &str, kind: &str, help: &str, labels: &str, v: f64| {
+        out.push_str(&format!("# HELP {name} {help}\n# TYPE {name} {kind}\n{name}{labels} {v}\n"));
+    };
+    let (rss, fds) = process();
+    if let Some(v) = rss {
+        gauge("directory_process_resident_bytes", "gauge", "Memory the process holds.", "", v);
+    }
+    if let Some(v) = fds {
+        gauge("directory_process_open_fds", "gauge", "Files and sockets the process has open.", "", v);
+    }
+    gauge("directory_feed_fetches_total", "counter", "Feeds fetched, answered 304 included.", "", ipx_feed::FETCHES.load(Relaxed) as f64);
+    gauge("directory_feed_bytes_total", "counter", "Bytes of feed bodies fetched.", "", ipx_feed::FETCHED_BYTES.load(Relaxed) as f64);
+    gauge("directory_uptime_seconds", "gauge", "Seconds since the directory started.", "", started.elapsed().as_secs() as f64);
+    gauge("directory_build_info", "gauge", "The version running, as a label.", &format!("{{version=\"{}\"}}", env!("CARGO_PKG_VERSION")), 1.0);
+    out
+}
+
+/// Serves /metrics on a listener of its own: the public port is behind the tunnel with no
+/// Access in front, and these are for the LAN.
+pub async fn serve(listener: tokio::net::TcpListener) {
+    let started = std::time::Instant::now();
+    let app = axum::Router::new().route(
+        "/metrics",
+        axum::routing::get(move || async move {
+            ([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(started))
+        }),
+    );
+    if let Err(e) = axum::serve(listener, app).await {
+        tracing::error!(error = %e, "metrics stopped");
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn metrics_say_memory_and_feeds() {
+        let text = super::render(std::time::Instant::now());
+        assert!(text.contains("directory_process_resident_bytes ") && text.contains("directory_feed_bytes_total "), "{text}");
+    }
+}