rays/ipx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 20 Wiki Activity

Requests are counted in /metrics, and only a failed one is logged (#364)

Browse Source 
ipx logged a line for every request: 2,306 a day, over half its log by
size, repeating Caddy's access line and the request's span. It stayed
because ipx had no request metric and the dashboard's API row was built
on the line. /metrics now has ipx_http_request_duration_seconds, a
histogram by method, route and status (5 ms to 10 s), the event stream
left out as the panels left it out. The route is the pattern, "" for an
unrouted path, and an unusual method is "other", so a scanner cannot make
a row a request. A request that went well is logged at debug, which the
admin page's log view keeps; a failed one keeps its warning in Loki.

The dashboard's API row reads the histogram (the slowest-request series
becomes the 99th percentile, a bucket's bound; the slowest themselves are
in Tempo). prom() takes instant queries and table() a unit for it.

The prod-check skill, reviewed whole: check 3 reads the histogram; the
directory has metrics, which it said it had not; "all gauges" was half
counters; the healthcheck removed in #218 was still named three times;
Caddy's /api/events aborts (600 a week, a page closing its stream) are
filtered as by design; the monitoring section checks Tempo's block poll,
which failed on 2026-10-25 on one empty block and left every search with
only the last 15 minutes; and the lines to expect from each container
are counted over a day, now that iPX can be quiet for an hour.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-10-25 15:11:20 +00:00
parent 6352324bcb
commit c788112190
6 changed files with 154 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
grafana/dashboard.py
View File

@@ -8,7 +8,7 @@ top-down as a request does: an overview, Tower, then each container (ipx-web, ip
worker and its process, the directory), then traces and logs. Prometheus scrapes ipx's /metrics
(job ipx, #194) and Tower's node exporter (job node); nothing scrapes the containers themselves,
so ipx-web and the directory are read from their logs. The log panels read fields: ipx's
(web.rs access_log, ipc.rs log_event), the directory's and Caddy's; renaming one has to be
(ipc.rs log_event), ipx's metrics (metrics.rs), the directory's and Caddy's; renaming one has to be
matched here. `dashboard.py queries` prints each Loki query, to try against Loki.
"""
import json, sys
@@ -18,11 +18,13 @@ TEMPO = {"type": "tempo", "uid": "${tempo}"}
PROM = {"type": "prometheus", "uid": "${prometheus}"}
SEL = '{container="iPX"}'
# ipx logs one JSON object a line (IPX_LOG_FORMAT=json). Each scan and download event carries its
# fields (ev, feed, new, bytes, msg, error.type, ...); each request its method, path, route, status
# and duration_ms. Events are logged under ipx::scan, the healthcheck's status under ipx::io, so
# the filter is on the ev field, not the target.
# fields (ev, feed, new, bytes, msg, error.type, ...); a failed request its method, path, route,
# status and duration_ms. Events are logged under ipx::scan, `ipx status`'s answer under ipx::io,
# so the filter is on the ev field, not the target.
EV = SEL + ' |= "\\"ev\\":\\"" | json | __error__="" | ev != ""'
HTTP = SEL + ' |= "\\"target\\":\\"ipx::http\\"" | json | __error__="" | path != "/api/events"'
# Each request ipx answers, the event stream aside, as a histogram (#364): its line in the log is
# debug, so only a failed request's is in Loki.
REQ = "ipx_http_request_duration_seconds"
BAD = SEL + ' | json | __error__="" | level =~ "WARN|ERROR"'
TEXT = ' | line_format "{{.level}} {{.target}}: {{.message}}"'
# Every production span: ipx-web's, ipx's and the directory's, one trace from the page to the
@@ -70,9 +72,11 @@ def loki(expr, ref="A", legend=None, instant=False, kind=None):
return t
def prom(expr, ref="A", legend=None):
def prom(expr, ref="A", legend=None, instant=False):
# ipx's /metrics (#194), scraped by Prometheus as job ipx.
t = {"refId": ref, "datasource": PROM, "expr": expr}
if instant:
t["instant"], t["range"], t["format"] = True, False, "table"
if legend:
t["legendFormat"] = legend
return t
@@ -105,7 +109,7 @@ def ts(title, targets, x, w=12, h=8, unit="short", bars=False, stack=False, desc
"tooltip": {"mode": "multi", "sort": "desc"}})
def table(title, targets, x, w=12, h=8, rename=None, sort=None, desc=""):
def table(title, targets, x, w=12, h=8, rename=None, sort=None, desc="", unit=None):
return panel("table", title, w, h, x, targets, description=desc,
transformations=[{"id": "labelsToFields", "options": {"mode": "columns"}},
{"id": "organize", "options": {
@@ -113,7 +117,7 @@ def table(title, targets, x, w=12, h=8, rename=None, sort=None, desc=""):
"service_name": True},
"renameByName": rename or {}}}],
options={"showHeader": True, "sortBy": sort or []},
fieldConfig={"defaults": {}, "overrides": []})
fieldConfig={"defaults": {"unit": unit} if unit else {}, "overrides": []})
GOOD_BAD = [{"color": "green", "value": None}, {"color": "orange", "value": 1}]
@@ -197,16 +201,16 @@ y += 8
# ---- ipx: the API it answers behind ipx-web
row("ipx: API")
ts("Requests by status", [loki(f'sum by (status) (count_over_time({HTTP} [$__interval]))', legend="{{status}}")],
ts("Requests by status", [prom(f'sum by (status) (increase({REQ}_count{{job="ipx"}}[$__interval]))', legend="{{status}}")],
0, bars=True, stack=True, desc="The event stream the page keeps open is left out.")
ts("Response time", [loki(f'quantile_over_time(0.5, {HTTP} | unwrap duration_ms [$__interval]) by ()', legend="median"),
loki(f'quantile_over_time(0.95, {HTTP} | unwrap duration_ms [$__interval]) by ()', ref="B", legend="95th percentile"),
loki(f'max_over_time({HTTP} | unwrap duration_ms [$__interval]) by ()', ref="C", legend="slowest")],
12, unit="ms")
ts("Response time", [prom(f'histogram_quantile(0.5, sum by (le) (rate({REQ}_bucket{{job="ipx"}}[$__rate_interval])))', legend="median"),
prom(f'histogram_quantile(0.95, sum by (le) (rate({REQ}_bucket{{job="ipx"}}[$__rate_interval])))', ref="B", legend="95th percentile"),
prom(f'histogram_quantile(0.99, sum by (le) (rate({REQ}_bucket{{job="ipx"}}[$__rate_interval])))', ref="C", legend="99th percentile")],
12, unit="s", desc="From ipx_http_request_duration_seconds, whose buckets run from 5 ms to 10 s; the slowest requests themselves are in Tempo.")
y += 8
table("Slowest routes", [loki(f'topk(15, avg_over_time({HTTP} | route != "" | unwrap duration_ms [$__range]) by (method, route))', instant=True)],
0, w=8, rename={"method": "Method", "route": "Route", "Value": "Average ms"}, sort=[{"displayName": "Average ms", "desc": True}])
table("Busiest routes", [loki(f'topk(15, sum by (method, route) (count_over_time({HTTP} | route != "" [$__range])))', instant=True)],
table("Slowest routes", [prom(f'topk(15, sum by (method, route) (increase({REQ}_sum{{job="ipx",route!=""}}[$__range])) / sum by (method, route) (increase({REQ}_count{{job="ipx",route!=""}}[$__range])))', instant=True)],
0, w=8, unit="s", rename={"method": "Method", "route": "Route", "Value": "Average"}, sort=[{"displayName": "Average", "desc": True}])
table("Busiest routes", [prom(f'topk(15, sum by (method, route) (increase({REQ}_count{{job="ipx",route!=""}}[$__range])))', instant=True)],
8, w=8, rename={"method": "Method", "route": "Route", "Value": "Requests"}, sort=[{"displayName": "Requests", "desc": True}])
ts("Open connections", [prom('max(ipx_http_requests_in_flight{job="ipx"})', legend="requests in flight"),
prom('max(ipx_event_listeners{job="ipx"})', ref="B", legend="live-update listeners"),