A Grafana dashboard for iPX
Built on what the monitoring project already collects: the container's log in Loki (through Alloy) and the traces in Tempo. It parses the access log and the event log's wire JSON, so there are no metrics to add to ipx: what is waiting and downloaded (from the healthcheck's status), new items, downloads and bytes, failing feeds and downloads, requests by status and response time, the slowest and busiest paths, recent and slow traces, and the log. Provisioned from a file, so it is regenerated here, not edited in Grafana. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Added
|
||||
|
||||
- A Grafana dashboard for iPX, from its log in Loki and its traces in Tempo (`grafana/dashboard.py`).
|
||||
- With `OTEL_EXPORTER_OTLP_ENDPOINT` set, the daemon sends traces of its scans, downloads and web
|
||||
requests to a collector such as Tempo.
|
||||
- A feed that has no artwork of its own shows its website's icon instead.
|
||||
|
||||
@@ -170,6 +170,9 @@ Non-trivial logic leaves one runnable check behind. Pure functions (`merge_polic
|
||||
watch the shutdown channel itself; the daemon ignored SIGTERM for exactly this reason.
|
||||
* Only one daemon per socket. Removing the socket file defeats the guard and you get two daemons
|
||||
fighting over the database, with the stale one still holding the port.
|
||||
* **The Grafana dashboard parses the log** (`grafana/dashboard.py`): the access log's
|
||||
`GET /path -> 200 in 3ms` and the events' `ipx::io: <- {json}`. Change either and the panels go
|
||||
blank without an error; regenerate the dashboard with the new pattern.
|
||||
* `/api/settings` answering `200` does **not** mean the daemon is well — the web server is a
|
||||
different task. `ipx status` checks the control socket and the database; to see the worker
|
||||
getting through its jobs, watch for `scan complete` in the log.
|
||||
|
||||
169
grafana/dashboard.py
Normal file
169
grafana/dashboard.py
Normal file
@@ -0,0 +1,169 @@
|
||||
"""The iPX dashboard in Grafana, from Loki (the container's log, shipped by Alloy) and Tempo.
|
||||
|
||||
python3 grafana/dashboard.py > /mnt/fast/arcane/projects/monitoring/grafana-provisioning/dashboards/ipx.json
|
||||
|
||||
Grafana reads that file on its own within a minute; edits made in Grafana are refused. The panels
|
||||
parse ipx's log lines, so a change to what the access log or the event log prints (web.rs
|
||||
access_log, ipc.rs Emitter::emit) has to be matched here. `dashboard.py queries` prints each
|
||||
query, to try against Loki.
|
||||
"""
|
||||
import json, sys
|
||||
|
||||
LOKI = {"type": "loki", "uid": "${loki}"}
|
||||
TEMPO = {"type": "tempo", "uid": "${tempo}"}
|
||||
SEL = '{container="iPX"}'
|
||||
# Every scan and download event is logged as its wire JSON: "ipx::io: <- {...}".
|
||||
EV = SEL + ' |= "ipx::io: <- {" | regexp "<- (?P<j>\\\\{.*\\\\})$" | line_format "{{.j}}" | json | __error__=""'
|
||||
HTTP = (SEL + ' |= "ipx::http: " | regexp "ipx::http: (?P<method>[A-Z]+) (?P<path>\\\\S+) -> (?P<status>\\\\d+) in (?P<ms>\\\\d+)ms"'
|
||||
' | path != "/api/events"')
|
||||
TRACES = '{resource.service.name="ipx" && resource.deployment.environment.name="production"'
|
||||
|
||||
|
||||
def status(field):
|
||||
return f'max(max_over_time({SEL} |= "\\"ev\\":\\"status\\"" | regexp "\\"{field}\\":(?P<v>\\\\d+)" | unwrap v [10m]))'
|
||||
|
||||
|
||||
QUERIES = {}
|
||||
panels, y = [], 0
|
||||
pid = 0
|
||||
|
||||
|
||||
def panel(kind, title, w, h, x, targets, **extra):
|
||||
global pid
|
||||
pid += 1
|
||||
p = {"id": pid, "type": kind, "title": title, "gridPos": {"x": x, "y": y, "w": w, "h": h},
|
||||
"datasource": targets[0].get("datasource", LOKI), "targets": targets}
|
||||
p.update(extra)
|
||||
panels.append(p)
|
||||
return p
|
||||
|
||||
|
||||
def loki(expr, ref="A", legend=None, instant=False, kind=None):
|
||||
QUERIES[expr] = instant
|
||||
t = {"refId": ref, "datasource": LOKI, "expr": expr, "queryType": "instant" if instant else "range"}
|
||||
if legend:
|
||||
t["legendFormat"] = legend
|
||||
return t
|
||||
|
||||
|
||||
def row(title):
|
||||
global y, pid
|
||||
pid += 1
|
||||
panels.append({"id": pid, "type": "row", "title": title, "collapsed": False,
|
||||
"gridPos": {"x": 0, "y": y, "w": 24, "h": 1}, "panels": []})
|
||||
y += 1
|
||||
|
||||
|
||||
def stat(title, expr, x, unit="short", color="blue", thresholds=None, desc=None):
|
||||
steps = thresholds or [{"color": color, "value": None}]
|
||||
return panel("stat", title, 4, 4, x, [loki(expr, instant=True)], description=desc or "",
|
||||
fieldConfig={"defaults": {"unit": unit, "color": {"mode": "thresholds"},
|
||||
"thresholds": {"mode": "absolute", "steps": steps}}, "overrides": []},
|
||||
options={"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
|
||||
"colorMode": "value", "graphMode": "none", "textMode": "value"})
|
||||
|
||||
|
||||
def ts(title, targets, x, w=12, h=8, unit="short", bars=False, stack=False, desc=""):
|
||||
custom = {"drawStyle": "bars" if bars else "line", "fillOpacity": 60 if bars else 10,
|
||||
"lineWidth": 1, "showPoints": "never", "stacking": {"mode": "normal" if stack else "none"}}
|
||||
return panel("timeseries", title, w, h, x, targets, description=desc,
|
||||
fieldConfig={"defaults": {"unit": unit, "custom": custom}, "overrides": []},
|
||||
options={"legend": {"displayMode": "list", "placement": "bottom"},
|
||||
"tooltip": {"mode": "multi", "sort": "desc"}})
|
||||
|
||||
|
||||
def table(title, targets, x, w=12, h=8, rename=None, sort=None, desc=""):
|
||||
return panel("table", title, w, h, x, targets, description=desc,
|
||||
transformations=[{"id": "labelsToFields", "options": {"mode": "columns"}},
|
||||
{"id": "organize", "options": {
|
||||
"excludeByName": {"Time": True, "container": True, "compose_project": True,
|
||||
"service_name": True},
|
||||
"renameByName": rename or {}}}],
|
||||
options={"showHeader": True, "sortBy": sort or []},
|
||||
fieldConfig={"defaults": {}, "overrides": []})
|
||||
|
||||
|
||||
# ---- Now
|
||||
row("Now")
|
||||
stat("Feeds", status("feeds"), 0, desc="From the healthcheck's status answer, every 30 seconds.")
|
||||
stat("Waiting to download", status("pending"), 4)
|
||||
stat("Downloaded", status("downloaded"), 8, color="green")
|
||||
stat("Feed failures", f'sum(count_over_time({EV} | ev="feed_error" [$__range])) or vector(0)', 12,
|
||||
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}],
|
||||
desc="Failed feed checks in the time range.")
|
||||
stat("Download failures", f'sum(count_over_time({EV} | ev="download_error" [$__range])) or vector(0)', 16,
|
||||
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}])
|
||||
stat("Warnings and errors", f'sum(count_over_time({SEL} |~ "^\\\\S+\\\\s+(WARN|ERROR) " [$__range])) or vector(0)', 20,
|
||||
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}, {"color": "red", "value": 50}])
|
||||
y += 4
|
||||
|
||||
# ---- Scans
|
||||
row("Scans and downloads")
|
||||
ts("New items found", [loki(f'sum(sum_over_time({EV} | ev="feed_done" | unwrap new [$__interval]))', legend="new items")],
|
||||
0, bars=True)
|
||||
ts("Downloads", [loki(f'sum(count_over_time({EV} | ev="download_done" [$__interval]))', legend="saved"),
|
||||
loki(f'sum(count_over_time({EV} | ev="download_error" [$__interval]))', ref="B", legend="failed")],
|
||||
12, bars=True)
|
||||
y += 8
|
||||
ts("Bytes downloaded", [loki(f'sum(sum_over_time({EV} | ev="download_done" | unwrap bytes [$__interval]))', legend="bytes")],
|
||||
0, unit="bytes", bars=True)
|
||||
ts("Feeds checked per scan", [loki(f'sum(sum_over_time({EV} | ev="scan_done" | unwrap feeds [$__interval]))', legend="feeds checked")],
|
||||
12, bars=True, desc="Feeds that were due and fetched; the rest were skipped as not due.")
|
||||
y += 8
|
||||
table("Failing feeds", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="feed_error" [$__range]))', instant=True)],
|
||||
0, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}])
|
||||
table("Failed downloads", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="download_error" [$__range]))', instant=True)],
|
||||
12, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}])
|
||||
y += 8
|
||||
|
||||
# ---- Web
|
||||
row("Web")
|
||||
ts("Requests by status", [loki(f'sum by (status) (count_over_time({HTTP} [$__interval]))', legend="{{status}}")],
|
||||
0, bars=True, stack=True, desc="The event stream the page keeps open is left out.")
|
||||
ts("Response time", [loki(f'quantile_over_time(0.5, {HTTP} | unwrap ms [$__interval]) by ()', legend="median"),
|
||||
loki(f'quantile_over_time(0.95, {HTTP} | unwrap ms [$__interval]) by ()', ref="B", legend="95th percentile"),
|
||||
loki(f'max_over_time({HTTP} | unwrap ms [$__interval]) by ()', ref="C", legend="slowest")],
|
||||
12, unit="ms")
|
||||
y += 8
|
||||
table("Slowest requests", [loki(f'topk(15, avg_over_time({HTTP} | unwrap ms [$__range]) by (method, path))', instant=True)],
|
||||
0, rename={"method": "Method", "path": "Path", "Value": "Average ms"}, sort=[{"displayName": "Average ms", "desc": True}])
|
||||
table("Busiest paths", [loki(f'topk(15, sum by (method, path) (count_over_time({HTTP} [$__range])))', instant=True)],
|
||||
12, rename={"method": "Method", "path": "Path", "Value": "Requests"}, sort=[{"displayName": "Requests", "desc": True}])
|
||||
y += 8
|
||||
|
||||
# ---- Traces
|
||||
row("Traces")
|
||||
for x, title, q in [(0, "Recent traces", TRACES + "}"),
|
||||
(12, "Slow traces (over 2s)", TRACES + " && duration > 2s}")]:
|
||||
panel("table", title, 12, 10, x,
|
||||
[{"refId": "A", "datasource": TEMPO, "queryType": "traceql", "query": q, "limit": 50,
|
||||
"tableType": "traces"}],
|
||||
fieldConfig={"defaults": {}, "overrides": []})
|
||||
y += 10
|
||||
|
||||
# ---- Log
|
||||
row("Log")
|
||||
panel("logs", "Warnings and errors", 24, 10, 0, [loki(f'{SEL} |~ "^\\\\S+\\\\s+(WARN|ERROR) "')],
|
||||
options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True})
|
||||
y += 10
|
||||
panel("logs", "Log", 24, 12, 0,
|
||||
[loki(f'{SEL} != "GET /api/events" != "\\"ev\\":\\"feed_skip\\"" != "{{\\"cmd\\":\\"status\\"}}"')],
|
||||
description="Without the event stream's requests, not-due skips and healthcheck status calls.",
|
||||
options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True})
|
||||
|
||||
dash = {
|
||||
"uid": "ipx", "title": "iPX", "tags": ["ipx"], "timezone": "browser", "schemaVersion": 39,
|
||||
"time": {"from": "now-24h", "to": "now"}, "refresh": "1m", "editable": True,
|
||||
"templating": {"list": [
|
||||
{"name": "loki", "label": "Logs", "type": "datasource", "query": "loki", "current": {}, "hide": 0},
|
||||
{"name": "tempo", "label": "Traces", "type": "datasource", "query": "tempo", "current": {}, "hide": 0},
|
||||
]},
|
||||
"links": [{"title": "iPX", "type": "link", "url": "https://ipodderx.sdf1.net", "targetBlank": True}],
|
||||
"panels": panels,
|
||||
}
|
||||
|
||||
if sys.argv[1:] == ["queries"]:
|
||||
for q, instant in QUERIES.items():
|
||||
print(json.dumps([q, instant]))
|
||||
else:
|
||||
print(json.dumps(dash, indent=2))
|
||||
Reference in New Issue
Block a user