Trace ids, failure kinds and one line per event in the JSON log (#91)

From Dash0's structured logging guide, what applies here:

- Each JSON line inside a traced span ends with its trace_id and span_id, so a line in Loki leads
  to its trace in Tempo; the access log is written inside its request's span so it has one too.
  The JSON formatter takes no extra fields, so WithTrace appends them to the object it writes.
- A feed or download failure carries error.type (the HTTP status, or dns, redirect_loop,
  timeout, ...) and http.response.status_code, from failure_kind beside explain_failure, so
  failures group by kind without a regex over msg.
- Each event was logged twice: words under ipx::scan and fields under ipx::io. It is now one
  line under ipx::scan with both; the wire copy is at debug, for the admin page's Daemon I/O tab,
  and out of production's log. The healthcheck's status reply stays under ipx::io.
- The access log's ms is duration_ms. The dashboard and the prod-check skill follow.
- error fields are Display with the anyhow chain everywhere, not a mix of Debug and Display.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

grafana/dashboard.py

@@ -4,7 +4,7 @@
 
 Grafana reads that file on its own within a minute; edits made in Grafana are refused. The panels
 read the fields of ipx's JSON log (IPX_LOG_FORMAT=json): renaming a field in web.rs access_log or
-ipc.rs log_wire has to be matched here. `dashboard.py queries` prints each
+ipc.rs log_event has to be matched here. `dashboard.py queries` prints each
 query, to try against Loki.
 """
 import json, sys
@@ -13,8 +13,10 @@ LOKI = {"type": "loki", "uid": "${loki}"}
 TEMPO = {"type": "tempo", "uid": "${tempo}"}
 SEL = '{container="iPX"}'
 # ipx logs one JSON object a line (IPX_LOG_FORMAT=json). Each scan and download event carries its
-# fields (ev, feed, new, bytes, msg, ...); each request its method, path, route, status and ms.
-EV = SEL + ' |= "\\"target\\":\\"ipx::io\\"" | json | __error__="" | ev != ""'
+# fields (ev, feed, new, bytes, msg, error.type, ...); each request its method, path, route, status
+# and duration_ms. Events are logged under ipx::scan, the healthcheck's status under ipx::io, so
+# the filter is on the ev field, not the target.
+EV = SEL + ' |= "\\"ev\\":\\"" | json | __error__="" | ev != ""'
 HTTP = SEL + ' |= "\\"target\\":\\"ipx::http\\"" | json | __error__="" | path != "/api/events"'
 BAD = SEL + ' | json | __error__="" | level =~ "WARN|ERROR"'
 TEXT = ' | line_format "{{.level}} {{.target}}: {{.message}}"'
@@ -122,12 +124,12 @@ y += 8
 row("Web")
 ts("Requests by status", [loki(f'sum by (status) (count_over_time({HTTP} [$__interval]))', legend="{{status}}")],
    0, bars=True, stack=True, desc="The event stream the page keeps open is left out.")
-ts("Response time", [loki(f'quantile_over_time(0.5, {HTTP} | unwrap ms [$__interval]) by ()', legend="median"),
-                     loki(f'quantile_over_time(0.95, {HTTP} | unwrap ms [$__interval]) by ()', ref="B", legend="95th percentile"),
-                     loki(f'max_over_time({HTTP} | unwrap ms [$__interval]) by ()', ref="C", legend="slowest")],
+ts("Response time", [loki(f'quantile_over_time(0.5, {HTTP} | unwrap duration_ms [$__interval]) by ()', legend="median"),
+                     loki(f'quantile_over_time(0.95, {HTTP} | unwrap duration_ms [$__interval]) by ()', ref="B", legend="95th percentile"),
+                     loki(f'max_over_time({HTTP} | unwrap duration_ms [$__interval]) by ()', ref="C", legend="slowest")],
    12, unit="ms")
 y += 8
-table("Slowest routes", [loki(f'topk(15, avg_over_time({HTTP} | route != "" | unwrap ms [$__range]) by (method, route))', instant=True)],
+table("Slowest routes", [loki(f'topk(15, avg_over_time({HTTP} | route != "" | unwrap duration_ms [$__range]) by (method, route))', instant=True)],
       0, rename={"method": "Method", "route": "Route", "Value": "Average ms"}, sort=[{"displayName": "Average ms", "desc": True}])
 table("Busiest routes", [loki(f'topk(15, sum by (method, route) (count_over_time({HTTP} | route != "" [$__range])))', instant=True)],
       12, rename={"method": "Method", "route": "Route", "Value": "Requests"}, sort=[{"displayName": "Requests", "desc": True}])