rays/ipx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 18 Wiki Activity

ipx-directory traces, and ipx's trace follows its calls there (#213)

Browse Source 
The exporter setup and the traceparent parser move from ipx into ipx_directory::otel, which ipx
already depends on, so both binaries share them. The directory makes a span a request, a child of
the caller's traceparent, and a trace a refresher round with a span a feed; ipx sends the current
span's traceparent on every call to the directory and wraps each share in a span. The dashboard's
Traces row and the prod-check skill cover all three services.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-10-12 16:19:32 +00:00
parent a3708769a4
commit 44e75df99e
12 changed files with 188 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/directory/Cargo.toml
View File

@@ -12,6 +12,9 @@ anyhow.workspace = true
axum.workspace = true
clap.workspace = true
ipx-feed.workspace = true
opentelemetry.workspace = true
opentelemetry-otlp.workspace = true
opentelemetry_sdk.workspace = true
reqwest.workspace = true
sea-orm.workspace = true
serde.workspace = true
@@ -20,6 +23,7 @@ sha2.workspace = true
tokio.workspace = true
toml.workspace = true
tracing.workspace = true
tracing-opentelemetry.workspace = true
tracing-subscriber.workspace = true
url.workspace = true

45
crates/directory/src/lib.rs
View File

@@ -8,6 +8,7 @@
//! people's behalf.
pub mod config;
pub mod otel;
pub mod store;
use std::sync::{Arc, RwLock};
@@ -177,14 +178,23 @@ pub async fn refresher(dir: Shared) {
loop {
match dir.store.due(REFRESH_EVERY, 25).await {
Ok(due) => {
for d in &due {
if let Err(e) = dir.refresh(d).await {
tracing::warn!(feed = %d.id, error = %format!("{e:#}"), "could not store a feed check");
}
}
if !due.is_empty() {
// One line a round, for the dashboard's refresher panel (#210); failures have their own.
tracing::info!(checked = due.len(), "feeds checked");
// A round is a trace, each feed's check a span in it (#213).
let round = tracing::info_span!("refresh", feeds = due.len());
tracing::Instrument::instrument(
async {
for d in &due {
let check = tracing::info_span!("feed", feed = %d.id);
if let Err(e) = tracing::Instrument::instrument(dir.refresh(d), check).await {
tracing::warn!(feed = %d.id, error = %format!("{e:#}"), "could not store a feed check");
}
}
// One line a round, for the dashboard's refresher panel (#210); failures have their own.
tracing::info!(checked = due.len(), "feeds checked");
},
round,
)
.await;
continue;
}
}
@@ -264,7 +274,26 @@ struct Who(String);
async fn access_log(req: Request, next: Next) -> Response {
let (method, path) = (req.method().clone(), req.uri().path().to_owned());
let start = Instant::now();
let resp = next.run(req).await;
// A span for each request but the healthcheck's, a child of the caller's when it sent a
// traceparent, as ipx does on its calls here (#213).
let span = if path == "/healthz" {
tracing::Span::none()
} else {
tracing::info_span!(
"http",
otel.name = %format!("{method} {path}"),
http.request.method = %method,
url.path = %path,
http.response.status_code = tracing::field::Empty,
)
};
if let Some(parent) = otel::remote_parent(req.headers(), None) {
use tracing_opentelemetry::OpenTelemetrySpanExt;
let _ = span.set_parent(parent);
}
let resp = tracing::Instrument::instrument(next.run(req), span.clone()).await;
span.record("http.response.status_code", resp.status().as_u16());
let _in = span.enter();
let who = resp.extensions().get::<Who>().map_or("", |w| w.0.as_str()).to_owned();
let status = resp.status().as_u16();
if path != "/healthz" {

17
crates/directory/src/main.rs
View File

@@ -75,11 +75,16 @@ async fn main() -> Result<()> {
async fn serve(path: &std::path::Path) -> Result<()> {
// One JSON object a line for Loki when DIRECTORY_LOG_FORMAT=json, as ipx does; text otherwise.
let json = std::env::var("DIRECTORY_LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json"));
let filter = tracing_subscriber::EnvFilter::try_from_env("DIRECTORY_LOG").unwrap_or_else(|_| "ipx_directory=info".into());
let filter = || tracing_subscriber::EnvFilter::try_from_env("DIRECTORY_LOG").unwrap_or_else(|_| "ipx_directory=info".into());
// Traces to Tempo when OTEL_EXPORTER_OTLP_ENDPOINT is set, joining ipx's (#213).
let otel = ipx_directory::otel::provider("ipx-directory")?;
tracing_subscriber::registry()
.with(filter)
.with((!json).then(|| tracing_subscriber::fmt::layer().with_writer(std::io::stderr)))
.with(json.then(|| tracing_subscriber::fmt::layer().json().flatten_event(true).with_writer(std::io::stderr)))
.with((!json).then(|| tracing_subscriber::fmt::layer().with_writer(std::io::stderr).with_filter(filter())))
.with(json.then(|| tracing_subscriber::fmt::layer().json().flatten_event(true).with_writer(std::io::stderr).with_filter(filter())))
.with(otel.as_ref().map(|p| {
use opentelemetry::trace::TracerProvider;
tracing_opentelemetry::layer().with_tracer(p.tracer("ipx-directory")).with_filter(tracing_subscriber::EnvFilter::new("ipx_directory=info"))
}))
.init();
let cfg = config::Config::load(path)?;
@@ -105,5 +110,9 @@ async fn serve(path: &std::path::Path) -> Result<()> {
}
})
.await?;
// The batch exporter holds the last few seconds of spans; without this they are lost.
if let Some(p) = otel {
let _ = p.shutdown();
}
Ok(())
}

83
crates/directory/src/otel.rs Normal file
View File

@@ -0,0 +1,83 @@
//! Traces, for ipx and the directory both (#213): the exporter, and W3C `traceparent` in from a
//! caller and out to a callee, so one trace follows a request from the browser through ipx-web
//! and ipx into the directory.
/// Traces over OTLP, to Tempo for one, when OTEL_EXPORTER_OTLP_ENDPOINT names a collector
/// (`http://host:4318`: the exporter speaks OTLP over HTTP and adds `/v1/traces`). The exporter
/// reads that and the other `OTEL_` variables itself.
pub fn provider(service: &'static str) -> anyhow::Result<Option<opentelemetry_sdk::trace::SdkTracerProvider>> {
if std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").unwrap_or_default().is_empty() {
return Ok(None);
}
let exporter = opentelemetry_otlp::SpanExporter::builder().with_http().build()?;
let resource = opentelemetry_sdk::Resource::builder().with_service_name(service).build();
Ok(Some(
opentelemetry_sdk::trace::SdkTracerProvider::builder()
.with_batch_exporter(exporter)
.with_resource(resource)
.build(),
))
}
/// The trace a request from the page belongs to, from its `traceparent` (#190), so the requests
/// one action sets off are one trace. None for a missing or malformed one. In the query where no
/// header can be set: a position saved by sendBeacon, and the playlist Safari's player fetches
/// itself (#192).
pub fn remote_parent(headers: &axum::http::HeaderMap, query: Option<&str>) -> Option<opentelemetry::Context> {
use opentelemetry::trace::{SpanContext, SpanId, TraceContextExt, TraceFlags, TraceId, TraceState};
let v = match headers.get("traceparent") {
Some(h) => h.to_str().ok()?,
None => query?.split('&').find_map(|kv| kv.strip_prefix("traceparent="))?,
};
let mut parts = v.split('-');
let (version, trace, span) = (parts.next()?, parts.next()?, parts.next()?);
if version != "00" || trace.len() != 32 || span.len() != 16 {
return None;
}
let sc = SpanContext::new(
TraceId::from_hex(trace).ok()?,
SpanId::from_hex(span).ok()?,
TraceFlags::SAMPLED,
true,
TraceState::default(),
);
sc.is_valid().then(|| opentelemetry::Context::new().with_remote_span_context(sc))
}
/// The current span as a `traceparent`, for a request to another service, so its spans join
/// this trace. None outside a traced span.
pub fn traceparent() -> Option<String> {
use opentelemetry::trace::TraceContextExt;
use tracing_opentelemetry::OpenTelemetrySpanExt;
let cx = tracing::Span::current().context();
let sc = cx.span().span_context().clone();
sc.is_valid().then(|| format!("00-{}-{}-{:02x}", sc.trace_id(), sc.span_id(), sc.trace_flags().to_u8()))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_requests_trace_is_the_pages_from_its_traceparent() {
use opentelemetry::trace::TraceContextExt;
let with = |v: &str| {
let mut h = axum::http::HeaderMap::new();
h.insert("traceparent", v.parse().unwrap());
remote_parent(&h, None)
};
let cx = with("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01").expect("parsed");
assert_eq!(cx.span().span_context().trace_id().to_string(), "4bf92f3577b34da6a3ce929d0e0e4736");
assert!(cx.span().span_context().is_remote());
// Anything else from the browser starts a trace of its own.
for bad in ["", "00-xyz-00f067aa0ba902b7-01", "01-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
"00-00000000000000000000000000000000-00f067aa0ba902b7-01", "00-4bf92f3577b34da6a3ce929d0e0e4736"] {
assert!(with(bad).is_none(), "{bad}");
}
assert!(remote_parent(&axum::http::HeaderMap::new(), None).is_none());
// From the query, where a beacon or Safari's player cannot send a header (#192).
let q = remote_parent(&axum::http::HeaderMap::new(), Some("x=1&traceparent=00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"));
assert_eq!(q.unwrap().span().span_context().trace_id().to_string(), "4bf92f3577b34da6a3ce929d0e0e4736");
assert!(remote_parent(&axum::http::HeaderMap::new(), Some("traceparent=nonsense")).is_none());
}
}