A Grafana dashboard for iPX
Built on what the monitoring project already collects: the container's log in Loki (through Alloy) and the traces in Tempo. It parses the access log and the event log's wire JSON, so there are no metrics to add to ipx: what is waiting and downloaded (from the healthcheck's status), new items, downloads and bytes, failing feeds and downloads, requests by status and response time, the slowest and busiest paths, recent and slow traces, and the log. Provisioned from a file, so it is regenerated here, not edited in Grafana. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
169
grafana/dashboard.py
Normal file
169
grafana/dashboard.py
Normal file
@@ -0,0 +1,169 @@
|
||||
"""The iPX dashboard in Grafana, from Loki (the container's log, shipped by Alloy) and Tempo.
|
||||
|
||||
python3 grafana/dashboard.py > /mnt/fast/arcane/projects/monitoring/grafana-provisioning/dashboards/ipx.json
|
||||
|
||||
Grafana reads that file on its own within a minute; edits made in Grafana are refused. The panels
|
||||
parse ipx's log lines, so a change to what the access log or the event log prints (web.rs
|
||||
access_log, ipc.rs Emitter::emit) has to be matched here. `dashboard.py queries` prints each
|
||||
query, to try against Loki.
|
||||
"""
|
||||
import json, sys
|
||||
|
||||
LOKI = {"type": "loki", "uid": "${loki}"}
|
||||
TEMPO = {"type": "tempo", "uid": "${tempo}"}
|
||||
SEL = '{container="iPX"}'
|
||||
# Every scan and download event is logged as its wire JSON: "ipx::io: <- {...}".
|
||||
EV = SEL + ' |= "ipx::io: <- {" | regexp "<- (?P<j>\\\\{.*\\\\})$" | line_format "{{.j}}" | json | __error__=""'
|
||||
HTTP = (SEL + ' |= "ipx::http: " | regexp "ipx::http: (?P<method>[A-Z]+) (?P<path>\\\\S+) -> (?P<status>\\\\d+) in (?P<ms>\\\\d+)ms"'
|
||||
' | path != "/api/events"')
|
||||
TRACES = '{resource.service.name="ipx" && resource.deployment.environment.name="production"'
|
||||
|
||||
|
||||
def status(field):
|
||||
return f'max(max_over_time({SEL} |= "\\"ev\\":\\"status\\"" | regexp "\\"{field}\\":(?P<v>\\\\d+)" | unwrap v [10m]))'
|
||||
|
||||
|
||||
QUERIES = {}
|
||||
panels, y = [], 0
|
||||
pid = 0
|
||||
|
||||
|
||||
def panel(kind, title, w, h, x, targets, **extra):
|
||||
global pid
|
||||
pid += 1
|
||||
p = {"id": pid, "type": kind, "title": title, "gridPos": {"x": x, "y": y, "w": w, "h": h},
|
||||
"datasource": targets[0].get("datasource", LOKI), "targets": targets}
|
||||
p.update(extra)
|
||||
panels.append(p)
|
||||
return p
|
||||
|
||||
|
||||
def loki(expr, ref="A", legend=None, instant=False, kind=None):
|
||||
QUERIES[expr] = instant
|
||||
t = {"refId": ref, "datasource": LOKI, "expr": expr, "queryType": "instant" if instant else "range"}
|
||||
if legend:
|
||||
t["legendFormat"] = legend
|
||||
return t
|
||||
|
||||
|
||||
def row(title):
|
||||
global y, pid
|
||||
pid += 1
|
||||
panels.append({"id": pid, "type": "row", "title": title, "collapsed": False,
|
||||
"gridPos": {"x": 0, "y": y, "w": 24, "h": 1}, "panels": []})
|
||||
y += 1
|
||||
|
||||
|
||||
def stat(title, expr, x, unit="short", color="blue", thresholds=None, desc=None):
|
||||
steps = thresholds or [{"color": color, "value": None}]
|
||||
return panel("stat", title, 4, 4, x, [loki(expr, instant=True)], description=desc or "",
|
||||
fieldConfig={"defaults": {"unit": unit, "color": {"mode": "thresholds"},
|
||||
"thresholds": {"mode": "absolute", "steps": steps}}, "overrides": []},
|
||||
options={"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
|
||||
"colorMode": "value", "graphMode": "none", "textMode": "value"})
|
||||
|
||||
|
||||
def ts(title, targets, x, w=12, h=8, unit="short", bars=False, stack=False, desc=""):
|
||||
custom = {"drawStyle": "bars" if bars else "line", "fillOpacity": 60 if bars else 10,
|
||||
"lineWidth": 1, "showPoints": "never", "stacking": {"mode": "normal" if stack else "none"}}
|
||||
return panel("timeseries", title, w, h, x, targets, description=desc,
|
||||
fieldConfig={"defaults": {"unit": unit, "custom": custom}, "overrides": []},
|
||||
options={"legend": {"displayMode": "list", "placement": "bottom"},
|
||||
"tooltip": {"mode": "multi", "sort": "desc"}})
|
||||
|
||||
|
||||
def table(title, targets, x, w=12, h=8, rename=None, sort=None, desc=""):
|
||||
return panel("table", title, w, h, x, targets, description=desc,
|
||||
transformations=[{"id": "labelsToFields", "options": {"mode": "columns"}},
|
||||
{"id": "organize", "options": {
|
||||
"excludeByName": {"Time": True, "container": True, "compose_project": True,
|
||||
"service_name": True},
|
||||
"renameByName": rename or {}}}],
|
||||
options={"showHeader": True, "sortBy": sort or []},
|
||||
fieldConfig={"defaults": {}, "overrides": []})
|
||||
|
||||
|
||||
# ---- Now
|
||||
row("Now")
|
||||
stat("Feeds", status("feeds"), 0, desc="From the healthcheck's status answer, every 30 seconds.")
|
||||
stat("Waiting to download", status("pending"), 4)
|
||||
stat("Downloaded", status("downloaded"), 8, color="green")
|
||||
stat("Feed failures", f'sum(count_over_time({EV} | ev="feed_error" [$__range])) or vector(0)', 12,
|
||||
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}],
|
||||
desc="Failed feed checks in the time range.")
|
||||
stat("Download failures", f'sum(count_over_time({EV} | ev="download_error" [$__range])) or vector(0)', 16,
|
||||
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}])
|
||||
stat("Warnings and errors", f'sum(count_over_time({SEL} |~ "^\\\\S+\\\\s+(WARN|ERROR) " [$__range])) or vector(0)', 20,
|
||||
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}, {"color": "red", "value": 50}])
|
||||
y += 4
|
||||
|
||||
# ---- Scans
|
||||
row("Scans and downloads")
|
||||
ts("New items found", [loki(f'sum(sum_over_time({EV} | ev="feed_done" | unwrap new [$__interval]))', legend="new items")],
|
||||
0, bars=True)
|
||||
ts("Downloads", [loki(f'sum(count_over_time({EV} | ev="download_done" [$__interval]))', legend="saved"),
|
||||
loki(f'sum(count_over_time({EV} | ev="download_error" [$__interval]))', ref="B", legend="failed")],
|
||||
12, bars=True)
|
||||
y += 8
|
||||
ts("Bytes downloaded", [loki(f'sum(sum_over_time({EV} | ev="download_done" | unwrap bytes [$__interval]))', legend="bytes")],
|
||||
0, unit="bytes", bars=True)
|
||||
ts("Feeds checked per scan", [loki(f'sum(sum_over_time({EV} | ev="scan_done" | unwrap feeds [$__interval]))', legend="feeds checked")],
|
||||
12, bars=True, desc="Feeds that were due and fetched; the rest were skipped as not due.")
|
||||
y += 8
|
||||
table("Failing feeds", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="feed_error" [$__range]))', instant=True)],
|
||||
0, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}])
|
||||
table("Failed downloads", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="download_error" [$__range]))', instant=True)],
|
||||
12, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}])
|
||||
y += 8
|
||||
|
||||
# ---- Web
|
||||
row("Web")
|
||||
ts("Requests by status", [loki(f'sum by (status) (count_over_time({HTTP} [$__interval]))', legend="{{status}}")],
|
||||
0, bars=True, stack=True, desc="The event stream the page keeps open is left out.")
|
||||
ts("Response time", [loki(f'quantile_over_time(0.5, {HTTP} | unwrap ms [$__interval]) by ()', legend="median"),
|
||||
loki(f'quantile_over_time(0.95, {HTTP} | unwrap ms [$__interval]) by ()', ref="B", legend="95th percentile"),
|
||||
loki(f'max_over_time({HTTP} | unwrap ms [$__interval]) by ()', ref="C", legend="slowest")],
|
||||
12, unit="ms")
|
||||
y += 8
|
||||
table("Slowest requests", [loki(f'topk(15, avg_over_time({HTTP} | unwrap ms [$__range]) by (method, path))', instant=True)],
|
||||
0, rename={"method": "Method", "path": "Path", "Value": "Average ms"}, sort=[{"displayName": "Average ms", "desc": True}])
|
||||
table("Busiest paths", [loki(f'topk(15, sum by (method, path) (count_over_time({HTTP} [$__range])))', instant=True)],
|
||||
12, rename={"method": "Method", "path": "Path", "Value": "Requests"}, sort=[{"displayName": "Requests", "desc": True}])
|
||||
y += 8
|
||||
|
||||
# ---- Traces
|
||||
row("Traces")
|
||||
for x, title, q in [(0, "Recent traces", TRACES + "}"),
|
||||
(12, "Slow traces (over 2s)", TRACES + " && duration > 2s}")]:
|
||||
panel("table", title, 12, 10, x,
|
||||
[{"refId": "A", "datasource": TEMPO, "queryType": "traceql", "query": q, "limit": 50,
|
||||
"tableType": "traces"}],
|
||||
fieldConfig={"defaults": {}, "overrides": []})
|
||||
y += 10
|
||||
|
||||
# ---- Log
|
||||
row("Log")
|
||||
panel("logs", "Warnings and errors", 24, 10, 0, [loki(f'{SEL} |~ "^\\\\S+\\\\s+(WARN|ERROR) "')],
|
||||
options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True})
|
||||
y += 10
|
||||
panel("logs", "Log", 24, 12, 0,
|
||||
[loki(f'{SEL} != "GET /api/events" != "\\"ev\\":\\"feed_skip\\"" != "{{\\"cmd\\":\\"status\\"}}"')],
|
||||
description="Without the event stream's requests, not-due skips and healthcheck status calls.",
|
||||
options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True})
|
||||
|
||||
dash = {
|
||||
"uid": "ipx", "title": "iPX", "tags": ["ipx"], "timezone": "browser", "schemaVersion": 39,
|
||||
"time": {"from": "now-24h", "to": "now"}, "refresh": "1m", "editable": True,
|
||||
"templating": {"list": [
|
||||
{"name": "loki", "label": "Logs", "type": "datasource", "query": "loki", "current": {}, "hide": 0},
|
||||
{"name": "tempo", "label": "Traces", "type": "datasource", "query": "tempo", "current": {}, "hide": 0},
|
||||
]},
|
||||
"links": [{"title": "iPX", "type": "link", "url": "https://ipodderx.sdf1.net", "targetBlank": True}],
|
||||
"panels": panels,
|
||||
}
|
||||
|
||||
if sys.argv[1:] == ["queries"]:
|
||||
for q, instant in QUERIES.items():
|
||||
print(json.dumps([q, instant]))
|
||||
else:
|
||||
print(json.dumps(dash, indent=2))
|
||||
Reference in New Issue
Block a user