diff --git a/CHANGELOG.md b/CHANGELOG.md index e8960b9..117d02d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- A Grafana dashboard for iPX, from its log in Loki and its traces in Tempo (`grafana/dashboard.py`). - With `OTEL_EXPORTER_OTLP_ENDPOINT` set, the daemon sends traces of its scans, downloads and web requests to a collector such as Tempo. - A feed that has no artwork of its own shows its website's icon instead. diff --git a/CLAUDE.md b/CLAUDE.md index 8c1dd5a..131ad3b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -170,6 +170,9 @@ Non-trivial logic leaves one runnable check behind. Pure functions (`merge_polic watch the shutdown channel itself; the daemon ignored SIGTERM for exactly this reason. * Only one daemon per socket. Removing the socket file defeats the guard and you get two daemons fighting over the database, with the stale one still holding the port. +* **The Grafana dashboard parses the log** (`grafana/dashboard.py`): the access log's + `GET /path -> 200 in 3ms` and the events' `ipx::io: <- {json}`. Change either and the panels go + blank without an error; regenerate the dashboard with the new pattern. * `/api/settings` answering `200` does **not** mean the daemon is well — the web server is a different task. `ipx status` checks the control socket and the database; to see the worker getting through its jobs, watch for `scan complete` in the log. diff --git a/grafana/dashboard.py b/grafana/dashboard.py new file mode 100644 index 0000000..db4ecd4 --- /dev/null +++ b/grafana/dashboard.py @@ -0,0 +1,169 @@ +"""The iPX dashboard in Grafana, from Loki (the container's log, shipped by Alloy) and Tempo. + + python3 grafana/dashboard.py > /mnt/fast/arcane/projects/monitoring/grafana-provisioning/dashboards/ipx.json + +Grafana reads that file on its own within a minute; edits made in Grafana are refused. The panels +parse ipx's log lines, so a change to what the access log or the event log prints (web.rs +access_log, ipc.rs Emitter::emit) has to be matched here. `dashboard.py queries` prints each +query, to try against Loki. +""" +import json, sys + +LOKI = {"type": "loki", "uid": "${loki}"} +TEMPO = {"type": "tempo", "uid": "${tempo}"} +SEL = '{container="iPX"}' +# Every scan and download event is logged as its wire JSON: "ipx::io: <- {...}". +EV = SEL + ' |= "ipx::io: <- {" | regexp "<- (?P\\\\{.*\\\\})$" | line_format "{{.j}}" | json | __error__=""' +HTTP = (SEL + ' |= "ipx::http: " | regexp "ipx::http: (?P[A-Z]+) (?P\\\\S+) -> (?P\\\\d+) in (?P\\\\d+)ms"' + ' | path != "/api/events"') +TRACES = '{resource.service.name="ipx" && resource.deployment.environment.name="production"' + + +def status(field): + return f'max(max_over_time({SEL} |= "\\"ev\\":\\"status\\"" | regexp "\\"{field}\\":(?P\\\\d+)" | unwrap v [10m]))' + + +QUERIES = {} +panels, y = [], 0 +pid = 0 + + +def panel(kind, title, w, h, x, targets, **extra): + global pid + pid += 1 + p = {"id": pid, "type": kind, "title": title, "gridPos": {"x": x, "y": y, "w": w, "h": h}, + "datasource": targets[0].get("datasource", LOKI), "targets": targets} + p.update(extra) + panels.append(p) + return p + + +def loki(expr, ref="A", legend=None, instant=False, kind=None): + QUERIES[expr] = instant + t = {"refId": ref, "datasource": LOKI, "expr": expr, "queryType": "instant" if instant else "range"} + if legend: + t["legendFormat"] = legend + return t + + +def row(title): + global y, pid + pid += 1 + panels.append({"id": pid, "type": "row", "title": title, "collapsed": False, + "gridPos": {"x": 0, "y": y, "w": 24, "h": 1}, "panels": []}) + y += 1 + + +def stat(title, expr, x, unit="short", color="blue", thresholds=None, desc=None): + steps = thresholds or [{"color": color, "value": None}] + return panel("stat", title, 4, 4, x, [loki(expr, instant=True)], description=desc or "", + fieldConfig={"defaults": {"unit": unit, "color": {"mode": "thresholds"}, + "thresholds": {"mode": "absolute", "steps": steps}}, "overrides": []}, + options={"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + "colorMode": "value", "graphMode": "none", "textMode": "value"}) + + +def ts(title, targets, x, w=12, h=8, unit="short", bars=False, stack=False, desc=""): + custom = {"drawStyle": "bars" if bars else "line", "fillOpacity": 60 if bars else 10, + "lineWidth": 1, "showPoints": "never", "stacking": {"mode": "normal" if stack else "none"}} + return panel("timeseries", title, w, h, x, targets, description=desc, + fieldConfig={"defaults": {"unit": unit, "custom": custom}, "overrides": []}, + options={"legend": {"displayMode": "list", "placement": "bottom"}, + "tooltip": {"mode": "multi", "sort": "desc"}}) + + +def table(title, targets, x, w=12, h=8, rename=None, sort=None, desc=""): + return panel("table", title, w, h, x, targets, description=desc, + transformations=[{"id": "labelsToFields", "options": {"mode": "columns"}}, + {"id": "organize", "options": { + "excludeByName": {"Time": True, "container": True, "compose_project": True, + "service_name": True}, + "renameByName": rename or {}}}], + options={"showHeader": True, "sortBy": sort or []}, + fieldConfig={"defaults": {}, "overrides": []}) + + +# ---- Now +row("Now") +stat("Feeds", status("feeds"), 0, desc="From the healthcheck's status answer, every 30 seconds.") +stat("Waiting to download", status("pending"), 4) +stat("Downloaded", status("downloaded"), 8, color="green") +stat("Feed failures", f'sum(count_over_time({EV} | ev="feed_error" [$__range])) or vector(0)', 12, + thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}], + desc="Failed feed checks in the time range.") +stat("Download failures", f'sum(count_over_time({EV} | ev="download_error" [$__range])) or vector(0)', 16, + thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}]) +stat("Warnings and errors", f'sum(count_over_time({SEL} |~ "^\\\\S+\\\\s+(WARN|ERROR) " [$__range])) or vector(0)', 20, + thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}, {"color": "red", "value": 50}]) +y += 4 + +# ---- Scans +row("Scans and downloads") +ts("New items found", [loki(f'sum(sum_over_time({EV} | ev="feed_done" | unwrap new [$__interval]))', legend="new items")], + 0, bars=True) +ts("Downloads", [loki(f'sum(count_over_time({EV} | ev="download_done" [$__interval]))', legend="saved"), + loki(f'sum(count_over_time({EV} | ev="download_error" [$__interval]))', ref="B", legend="failed")], + 12, bars=True) +y += 8 +ts("Bytes downloaded", [loki(f'sum(sum_over_time({EV} | ev="download_done" | unwrap bytes [$__interval]))', legend="bytes")], + 0, unit="bytes", bars=True) +ts("Feeds checked per scan", [loki(f'sum(sum_over_time({EV} | ev="scan_done" | unwrap feeds [$__interval]))', legend="feeds checked")], + 12, bars=True, desc="Feeds that were due and fetched; the rest were skipped as not due.") +y += 8 +table("Failing feeds", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="feed_error" [$__range]))', instant=True)], + 0, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}]) +table("Failed downloads", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="download_error" [$__range]))', instant=True)], + 12, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}]) +y += 8 + +# ---- Web +row("Web") +ts("Requests by status", [loki(f'sum by (status) (count_over_time({HTTP} [$__interval]))', legend="{{status}}")], + 0, bars=True, stack=True, desc="The event stream the page keeps open is left out.") +ts("Response time", [loki(f'quantile_over_time(0.5, {HTTP} | unwrap ms [$__interval]) by ()', legend="median"), + loki(f'quantile_over_time(0.95, {HTTP} | unwrap ms [$__interval]) by ()', ref="B", legend="95th percentile"), + loki(f'max_over_time({HTTP} | unwrap ms [$__interval]) by ()', ref="C", legend="slowest")], + 12, unit="ms") +y += 8 +table("Slowest requests", [loki(f'topk(15, avg_over_time({HTTP} | unwrap ms [$__range]) by (method, path))', instant=True)], + 0, rename={"method": "Method", "path": "Path", "Value": "Average ms"}, sort=[{"displayName": "Average ms", "desc": True}]) +table("Busiest paths", [loki(f'topk(15, sum by (method, path) (count_over_time({HTTP} [$__range])))', instant=True)], + 12, rename={"method": "Method", "path": "Path", "Value": "Requests"}, sort=[{"displayName": "Requests", "desc": True}]) +y += 8 + +# ---- Traces +row("Traces") +for x, title, q in [(0, "Recent traces", TRACES + "}"), + (12, "Slow traces (over 2s)", TRACES + " && duration > 2s}")]: + panel("table", title, 12, 10, x, + [{"refId": "A", "datasource": TEMPO, "queryType": "traceql", "query": q, "limit": 50, + "tableType": "traces"}], + fieldConfig={"defaults": {}, "overrides": []}) +y += 10 + +# ---- Log +row("Log") +panel("logs", "Warnings and errors", 24, 10, 0, [loki(f'{SEL} |~ "^\\\\S+\\\\s+(WARN|ERROR) "')], + options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True}) +y += 10 +panel("logs", "Log", 24, 12, 0, + [loki(f'{SEL} != "GET /api/events" != "\\"ev\\":\\"feed_skip\\"" != "{{\\"cmd\\":\\"status\\"}}"')], + description="Without the event stream's requests, not-due skips and healthcheck status calls.", + options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True}) + +dash = { + "uid": "ipx", "title": "iPX", "tags": ["ipx"], "timezone": "browser", "schemaVersion": 39, + "time": {"from": "now-24h", "to": "now"}, "refresh": "1m", "editable": True, + "templating": {"list": [ + {"name": "loki", "label": "Logs", "type": "datasource", "query": "loki", "current": {}, "hide": 0}, + {"name": "tempo", "label": "Traces", "type": "datasource", "query": "tempo", "current": {}, "hide": 0}, + ]}, + "links": [{"title": "iPX", "type": "link", "url": "https://ipodderx.sdf1.net", "targetBlank": True}], + "panels": panels, +} + +if sys.argv[1:] == ["queries"]: + for q, instant in QUERIES.items(): + print(json.dumps([q, instant])) +else: + print(json.dumps(dash, indent=2))