rays
9fc1e7e248
Logs and traces show requests, scans and downloads; nothing showed what the daemon holds meanwhile. /metrics, behind sign-in, gives database connections in use and idle against the pool's maximum, commands waiting for the worker, live-update listeners, free torrent slots, episodes playing through DeadAir, the library's counts, the tokio runtime's workers, tasks and queue, and the process's memory and open files, in Prometheus's text format, written by hand rather than with a client library. Not logged or traced: Prometheus asks every 15 s. The iPX dashboard gains a Daemon row of them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
205 lines
11 KiB
Python
205 lines
11 KiB
Python
"""The iPX dashboard in Grafana, from Loki (the container's log, shipped by Alloy) and Tempo.
|
|
|
|
python3 grafana/dashboard.py > /mnt/fast/arcane/projects/monitoring/grafana-provisioning/dashboards/ipx.json
|
|
|
|
Grafana reads that file on its own within a minute; edits made in Grafana are refused. The
|
|
Daemon row reads ipx's /metrics from Prometheus (job ipx, #194); the rest read the fields of
|
|
ipx's JSON log (IPX_LOG_FORMAT=json): renaming a field in web.rs access_log or ipc.rs log_event
|
|
has to be matched here. `dashboard.py queries` prints each
|
|
query, to try against Loki.
|
|
"""
|
|
import json, sys
|
|
|
|
LOKI = {"type": "loki", "uid": "${loki}"}
|
|
TEMPO = {"type": "tempo", "uid": "${tempo}"}
|
|
PROM = {"type": "prometheus", "uid": "${prometheus}"}
|
|
SEL = '{container="iPX"}'
|
|
# ipx logs one JSON object a line (IPX_LOG_FORMAT=json). Each scan and download event carries its
|
|
# fields (ev, feed, new, bytes, msg, error.type, ...); each request its method, path, route, status
|
|
# and duration_ms. Events are logged under ipx::scan, the healthcheck's status under ipx::io, so
|
|
# the filter is on the ev field, not the target.
|
|
EV = SEL + ' |= "\\"ev\\":\\"" | json | __error__="" | ev != ""'
|
|
HTTP = SEL + ' |= "\\"target\\":\\"ipx::http\\"" | json | __error__="" | path != "/api/events"'
|
|
BAD = SEL + ' | json | __error__="" | level =~ "WARN|ERROR"'
|
|
TEXT = ' | line_format "{{.level}} {{.target}}: {{.message}}"'
|
|
TRACES = '{resource.service.name="ipx" && resource.deployment.environment.name="production"'
|
|
|
|
|
|
def status(field):
|
|
return f'max(max_over_time({EV} | ev = "status" | unwrap {field} [10m]))'
|
|
|
|
|
|
QUERIES = {}
|
|
panels, y = [], 0
|
|
pid = 0
|
|
|
|
|
|
def panel(kind, title, w, h, x, targets, **extra):
|
|
global pid
|
|
pid += 1
|
|
p = {"id": pid, "type": kind, "title": title, "gridPos": {"x": x, "y": y, "w": w, "h": h},
|
|
"datasource": targets[0].get("datasource", LOKI), "targets": targets}
|
|
p.update(extra)
|
|
panels.append(p)
|
|
return p
|
|
|
|
|
|
def loki(expr, ref="A", legend=None, instant=False, kind=None):
|
|
QUERIES[expr] = instant
|
|
t = {"refId": ref, "datasource": LOKI, "expr": expr, "queryType": "instant" if instant else "range"}
|
|
if legend:
|
|
t["legendFormat"] = legend
|
|
return t
|
|
|
|
|
|
def prom(expr, ref="A", legend=None):
|
|
# ipx's /metrics (#194), scraped by Prometheus as job ipx.
|
|
t = {"refId": ref, "datasource": PROM, "expr": expr}
|
|
if legend:
|
|
t["legendFormat"] = legend
|
|
return t
|
|
|
|
|
|
def row(title):
|
|
global y, pid
|
|
pid += 1
|
|
panels.append({"id": pid, "type": "row", "title": title, "collapsed": False,
|
|
"gridPos": {"x": 0, "y": y, "w": 24, "h": 1}, "panels": []})
|
|
y += 1
|
|
|
|
|
|
def stat(title, expr, x, unit="short", color="blue", thresholds=None, desc=None):
|
|
steps = thresholds or [{"color": color, "value": None}]
|
|
return panel("stat", title, 4, 4, x, [loki(expr, instant=True)], description=desc or "",
|
|
fieldConfig={"defaults": {"unit": unit, "color": {"mode": "thresholds"},
|
|
"thresholds": {"mode": "absolute", "steps": steps}}, "overrides": []},
|
|
options={"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
|
|
"colorMode": "value", "graphMode": "none", "textMode": "value"})
|
|
|
|
|
|
def ts(title, targets, x, w=12, h=8, unit="short", bars=False, stack=False, desc=""):
|
|
custom = {"drawStyle": "bars" if bars else "line", "fillOpacity": 60 if bars else 10,
|
|
"lineWidth": 1, "showPoints": "never", "stacking": {"mode": "normal" if stack else "none"}}
|
|
return panel("timeseries", title, w, h, x, targets, description=desc,
|
|
fieldConfig={"defaults": {"unit": unit, "custom": custom}, "overrides": []},
|
|
options={"legend": {"displayMode": "list", "placement": "bottom"},
|
|
"tooltip": {"mode": "multi", "sort": "desc"}})
|
|
|
|
|
|
def table(title, targets, x, w=12, h=8, rename=None, sort=None, desc=""):
|
|
return panel("table", title, w, h, x, targets, description=desc,
|
|
transformations=[{"id": "labelsToFields", "options": {"mode": "columns"}},
|
|
{"id": "organize", "options": {
|
|
"excludeByName": {"Time": True, "container": True, "compose_project": True,
|
|
"service_name": True},
|
|
"renameByName": rename or {}}}],
|
|
options={"showHeader": True, "sortBy": sort or []},
|
|
fieldConfig={"defaults": {}, "overrides": []})
|
|
|
|
|
|
# ---- Now
|
|
row("Now")
|
|
stat("Feeds", status("feeds"), 0, desc="From the healthcheck's status answer, every 30 seconds.")
|
|
stat("Waiting to download", status("pending"), 4)
|
|
stat("Downloaded", status("downloaded"), 8, color="green")
|
|
stat("Feed failures", f'sum(count_over_time({EV} | ev="feed_error" [$__range])) or vector(0)', 12,
|
|
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}],
|
|
desc="Failed feed checks in the time range.")
|
|
stat("Download failures", f'sum(count_over_time({EV} | ev="download_error" [$__range])) or vector(0)', 16,
|
|
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}])
|
|
stat("Warnings and errors", f'sum(count_over_time({BAD} [$__range])) or vector(0)', 20,
|
|
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}, {"color": "red", "value": 50}])
|
|
y += 4
|
|
|
|
# ---- Scans
|
|
row("Scans and downloads")
|
|
ts("New items found", [loki(f'sum(sum_over_time({EV} | ev="feed_done" | unwrap new [$__interval]))', legend="new items")],
|
|
0, bars=True)
|
|
ts("Downloads", [loki(f'sum(count_over_time({EV} | ev="download_done" [$__interval]))', legend="saved"),
|
|
loki(f'sum(count_over_time({EV} | ev="download_error" [$__interval]))', ref="B", legend="failed")],
|
|
12, bars=True)
|
|
y += 8
|
|
ts("Bytes downloaded", [loki(f'sum(sum_over_time({EV} | ev="download_done" | unwrap bytes [$__interval]))', legend="bytes")],
|
|
0, unit="bytes", bars=True)
|
|
ts("Feeds checked per scan", [loki(f'sum(sum_over_time({EV} | ev="scan_done" | unwrap feeds [$__interval]))', legend="feeds checked")],
|
|
12, bars=True, desc="Feeds that were due and fetched; the rest were skipped as not due.")
|
|
y += 8
|
|
table("Failing feeds", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="feed_error" [$__range]))', instant=True)],
|
|
0, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}])
|
|
table("Failed downloads", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="download_error" [$__range]))', instant=True)],
|
|
12, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}])
|
|
y += 8
|
|
|
|
# ---- Web
|
|
row("Web")
|
|
ts("Requests by status", [loki(f'sum by (status) (count_over_time({HTTP} [$__interval]))', legend="{{status}}")],
|
|
0, bars=True, stack=True, desc="The event stream the page keeps open is left out.")
|
|
ts("Response time", [loki(f'quantile_over_time(0.5, {HTTP} | unwrap duration_ms [$__interval]) by ()', legend="median"),
|
|
loki(f'quantile_over_time(0.95, {HTTP} | unwrap duration_ms [$__interval]) by ()', ref="B", legend="95th percentile"),
|
|
loki(f'max_over_time({HTTP} | unwrap duration_ms [$__interval]) by ()', ref="C", legend="slowest")],
|
|
12, unit="ms")
|
|
y += 8
|
|
table("Slowest routes", [loki(f'topk(15, avg_over_time({HTTP} | route != "" | unwrap duration_ms [$__range]) by (method, route))', instant=True)],
|
|
0, rename={"method": "Method", "route": "Route", "Value": "Average ms"}, sort=[{"displayName": "Average ms", "desc": True}])
|
|
table("Busiest routes", [loki(f'topk(15, sum by (method, route) (count_over_time({HTTP} | route != "" [$__range])))', instant=True)],
|
|
12, rename={"method": "Method", "route": "Route", "Value": "Requests"}, sort=[{"displayName": "Requests", "desc": True}])
|
|
y += 8
|
|
|
|
# ---- Daemon, from Prometheus: what the daemon holds between requests (#194)
|
|
row("Daemon")
|
|
ts("Database connections", [prom('ipx_db_connections{job="ipx", state="in_use"}', legend="in use"),
|
|
prom('ipx_db_connections{job="ipx", state="idle"}', ref="B", legend="idle"),
|
|
prom('ipx_db_connections_max{job="ipx"}', ref="C", legend="most the pool opens")],
|
|
0, w=8, desc="In use reaching the most the pool opens means requests and scans are waiting for a connection.")
|
|
ts("Waiting", [prom('ipx_worker_queue{job="ipx"}', legend="commands for the worker"),
|
|
prom('ipx_runtime_queue{job="ipx"}', ref="B", legend="tasks for a thread"),
|
|
prom('ipx_downloads_pending{job="ipx"}', ref="C", legend="downloads")],
|
|
8, w=8, desc="Scans and downloads run one at a time; a queue that only grows means the worker is stuck.")
|
|
ts("Listeners and tasks", [prom('ipx_event_listeners{job="ipx"}', legend="live-update listeners"),
|
|
prom('ipx_runtime_tasks{job="ipx"}', ref="B", legend="runtime tasks"),
|
|
prom('ipx_deadair_playing{job="ipx"}', ref="C", legend="playing through DeadAir")],
|
|
16, w=8)
|
|
y += 8
|
|
ts("Memory", [prom('ipx_process_resident_bytes{job="ipx"}', legend="resident")], 0, w=12, unit="bytes")
|
|
ts("Open files and sockets", [prom('ipx_process_open_fds{job="ipx"}', legend="open")], 12, w=12)
|
|
y += 8
|
|
|
|
# ---- Traces
|
|
row("Traces")
|
|
for x, title, q in [(0, "Recent traces", TRACES + "}"),
|
|
(12, "Slow traces (over 2s)", TRACES + " && duration > 2s}")]:
|
|
panel("table", title, 12, 10, x,
|
|
[{"refId": "A", "datasource": TEMPO, "queryType": "traceql", "query": q, "limit": 50,
|
|
"tableType": "traces"}],
|
|
fieldConfig={"defaults": {}, "overrides": []})
|
|
y += 10
|
|
|
|
# ---- Log
|
|
row("Log")
|
|
panel("logs", "Warnings and errors", 24, 10, 0, [loki(BAD + TEXT)],
|
|
options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True})
|
|
y += 10
|
|
panel("logs", "Log", 24, 12, 0,
|
|
[loki(SEL + ' | json | __error__="" | path != "/api/events" | ev != "feed_skip" | ev != "status"'
|
|
' | message != "-> {\\"cmd\\":\\"status\\"}"' + TEXT)],
|
|
description="Without the event stream's requests, not-due skips and healthcheck status calls.",
|
|
options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True})
|
|
|
|
dash = {
|
|
"uid": "ipx", "title": "iPX", "tags": ["ipx"], "timezone": "browser", "schemaVersion": 39,
|
|
"time": {"from": "now-24h", "to": "now"}, "refresh": "1m", "editable": True,
|
|
"templating": {"list": [
|
|
{"name": "loki", "label": "Logs", "type": "datasource", "query": "loki", "current": {}, "hide": 0},
|
|
{"name": "tempo", "label": "Traces", "type": "datasource", "query": "tempo", "current": {}, "hide": 0},
|
|
{"name": "prometheus", "label": "Metrics", "type": "datasource", "query": "prometheus", "current": {}, "hide": 0},
|
|
]},
|
|
"links": [{"title": "iPX", "type": "link", "url": "https://ipodderx.sdf1.net", "targetBlank": True}],
|
|
"panels": panels,
|
|
}
|
|
|
|
if sys.argv[1:] == ["queries"]:
|
|
for q, instant in QUERIES.items():
|
|
print(json.dumps([q, instant]))
|
|
else:
|
|
print(json.dumps(dash, indent=2))
|