Trace ids, failure kinds and one line per event in the JSON log (#91)

From Dash0's structured logging guide, what applies here:

- Each JSON line inside a traced span ends with its trace_id and span_id, so a line in Loki leads
  to its trace in Tempo; the access log is written inside its request's span so it has one too.
  The JSON formatter takes no extra fields, so WithTrace appends them to the object it writes.
- A feed or download failure carries error.type (the HTTP status, or dns, redirect_loop,
  timeout, ...) and http.response.status_code, from failure_kind beside explain_failure, so
  failures group by kind without a regex over msg.
- Each event was logged twice: words under ipx::scan and fields under ipx::io. It is now one
  line under ipx::scan with both; the wire copy is at debug, for the admin page's Daemon I/O tab,
  and out of production's log. The healthcheck's status reply stays under ipx::io.
- The access log's ms is duration_ms. The dashboard and the prod-check skill follow.
- error fields are Display with the anyhow chain everywhere, not a mix of Debug and Display.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/main.rs

@@ -196,10 +196,10 @@ async fn main() -> Result<()> {
             }))
             .with(json.then(|| {
                 tracing_subscriber::fmt::layer()
-                    .json()
-                    .flatten_event(true)
-                    .with_current_span(true)
-                    .with_span_list(false)
+                    .fmt_fields(tracing_subscriber::fmt::format::JsonFields::new())
+                    .event_format(WithTrace(
+                        tracing_subscriber::fmt::format().json().flatten_event(true).with_current_span(true).with_span_list(false),
+                    ))
                     .with_writer(std::io::stderr)
                     .with_filter(stderr_filter())
             }))
@@ -275,6 +275,39 @@ async fn main() -> Result<()> {
     result
 }
 
+/// The JSON log line with the trace and span it belongs to (#91), so a line in Loki leads to its
+/// trace in Tempo. The JSON formatter cannot take a field of its own, so the ids go on the end of
+/// the object it writes. A line outside any traced span, or with no trace exporter, is unchanged.
+struct WithTrace<F>(F);
+
+impl<S, N, F> tracing_subscriber::fmt::FormatEvent<S, N> for WithTrace<F>
+where
+    F: tracing_subscriber::fmt::FormatEvent<S, N>,
+    S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
+    N: for<'w> tracing_subscriber::fmt::FormatFields<'w> + 'static,
+{
+    fn format_event(
+        &self,
+        ctx: &tracing_subscriber::fmt::FmtContext<'_, S, N>,
+        mut w: tracing_subscriber::fmt::format::Writer<'_>,
+        ev: &tracing::Event<'_>,
+    ) -> std::fmt::Result {
+        use opentelemetry::trace::TraceContextExt;
+        use tracing_opentelemetry::OpenTelemetrySpanExt;
+        let current = tracing::Span::current();
+        let sc = if current.is_none() { None } else { Some(current.context().span().span_context().clone()) };
+        let Some(sc) = sc.filter(|c| c.is_valid()) else {
+            return self.0.format_event(ctx, w, ev);
+        };
+        let mut line = String::new();
+        self.0.format_event(ctx, tracing_subscriber::fmt::format::Writer::new(&mut line), ev)?;
+        match line.trim_end().strip_suffix('}') {
+            Some(body) => writeln!(w, r#"{body},"trace_id":"{}","span_id":"{}"}}"#, sc.trace_id(), sc.span_id()),
+            None => w.write_str(&line),
+        }
+    }
+}
+
 /// Traces over OTLP, to Tempo for one, when OTEL_EXPORTER_OTLP_ENDPOINT names a collector
 /// (`http://host:4318`: the exporter speaks OTLP over HTTP and adds `/v1/traces`). The exporter
 /// reads that and the other `OTEL_` variables itself.
@@ -449,13 +482,13 @@ async fn daemon(
     match ctx.db.requeue_interrupted().await {
         Ok(n) if n > 0 => tracing::info!(count = n, "requeued downloads interrupted by a restart"),
         Ok(_) => {}
-        Err(e) => tracing::warn!(error = ?e, "could not requeue interrupted downloads"),
+        Err(e) => tracing::warn!(error = %format!("{e:#}"), "could not requeue interrupted downloads"),
     }
 
     match retire_stranded(&ctx).await {
         Ok(0) => {}
         Ok(n) => tracing::info!(feeds = n, "retired feeds whose OPML is no longer in config"),
-        Err(e) => tracing::warn!(error = ?e, "could not retire feeds whose OPML is no longer in config"),
+        Err(e) => tracing::warn!(error = %format!("{e:#}"), "could not retire feeds whose OPML is no longer in config"),
     }
 
     let (tx_cmd, mut rx_cmd) = mpsc::channel::<Cmd>(64);
@@ -603,7 +636,7 @@ async fn start_web(
     };
     Ok(Some(tokio::spawn(async move {
         if let Err(e) = web::serve(state, &bind).await {
-            tracing::error!(error = ?e, "web ui stopped");
+            tracing::error!(error = %format!("{e:#}"), "web ui stopped");
         }
     })))
 }
@@ -1649,7 +1682,7 @@ fn spawn_torrent(ctx: &Arc<Ctx>, feed_id: String, enclosure: i64, url: String, d
         match outcome {
             Ok((path, bytes)) => {
                 if let Err(e) = db.mark_downloaded(&url, &path, bytes).await {
-                    tracing::warn!(error = ?e, "could not record the finished torrent");
+                    tracing::warn!(error = %format!("{e:#}"), "could not record the finished torrent");
                 }
                 ctx.out.emit(Event::DownloadDone {
                     feed: feed_id,