rays
c28accb8e4
The Database connections panel drew idle connections as their own line, which sits on the max for ten minutes after any burst and read as the pool maxing out, while at most two were ever in use. It now draws in use against the max, with open dashed behind them. The pool itself goes from sqlx's ten to twenty: the server allows 100 and had about 13 open across every app. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
245 lines
14 KiB
Python
245 lines
14 KiB
Python
"""The iPX dashboard in Grafana, from Loki (the container's log, shipped by Alloy) and Tempo.
|
|
|
|
python3 grafana/dashboard.py > /mnt/fast/arcane/projects/monitoring/grafana-provisioning/dashboards/ipx.json
|
|
|
|
Grafana reads that file on its own within a minute; edits made in Grafana are refused. The
|
|
Daemon row reads ipx's /metrics from Prometheus (job ipx, #194); the rest read the fields of
|
|
ipx's JSON log (IPX_LOG_FORMAT=json): renaming a field in web.rs access_log or ipc.rs log_event
|
|
has to be matched here. `dashboard.py queries` prints each
|
|
query, to try against Loki.
|
|
"""
|
|
import json, sys
|
|
|
|
LOKI = {"type": "loki", "uid": "${loki}"}
|
|
TEMPO = {"type": "tempo", "uid": "${tempo}"}
|
|
PROM = {"type": "prometheus", "uid": "${prometheus}"}
|
|
SEL = '{container="iPX"}'
|
|
# ipx logs one JSON object a line (IPX_LOG_FORMAT=json). Each scan and download event carries its
|
|
# fields (ev, feed, new, bytes, msg, error.type, ...); each request its method, path, route, status
|
|
# and duration_ms. Events are logged under ipx::scan, the healthcheck's status under ipx::io, so
|
|
# the filter is on the ev field, not the target.
|
|
EV = SEL + ' |= "\\"ev\\":\\"" | json | __error__="" | ev != ""'
|
|
HTTP = SEL + ' |= "\\"target\\":\\"ipx::http\\"" | json | __error__="" | path != "/api/events"'
|
|
BAD = SEL + ' | json | __error__="" | level =~ "WARN|ERROR"'
|
|
TEXT = ' | line_format "{{.level}} {{.target}}: {{.message}}"'
|
|
TRACES = '{resource.service.name="ipx" && resource.deployment.environment.name="production"'
|
|
|
|
|
|
def status(field):
|
|
return f'max(max_over_time({EV} | ev = "status" | unwrap {field} [10m]))'
|
|
|
|
|
|
QUERIES = {}
|
|
panels, y = [], 0
|
|
pid = 0
|
|
|
|
|
|
def panel(kind, title, w, h, x, targets, **extra):
|
|
global pid
|
|
pid += 1
|
|
p = {"id": pid, "type": kind, "title": title, "gridPos": {"x": x, "y": y, "w": w, "h": h},
|
|
"datasource": targets[0].get("datasource", LOKI), "targets": targets}
|
|
p.update(extra)
|
|
panels.append(p)
|
|
return p
|
|
|
|
|
|
def loki(expr, ref="A", legend=None, instant=False, kind=None):
|
|
QUERIES[expr] = instant
|
|
t = {"refId": ref, "datasource": LOKI, "expr": expr, "queryType": "instant" if instant else "range"}
|
|
if legend:
|
|
t["legendFormat"] = legend
|
|
return t
|
|
|
|
|
|
def prom(expr, ref="A", legend=None):
|
|
# ipx's /metrics (#194), scraped by Prometheus as job ipx.
|
|
t = {"refId": ref, "datasource": PROM, "expr": expr}
|
|
if legend:
|
|
t["legendFormat"] = legend
|
|
return t
|
|
|
|
|
|
def row(title):
|
|
global y, pid
|
|
pid += 1
|
|
panels.append({"id": pid, "type": "row", "title": title, "collapsed": False,
|
|
"gridPos": {"x": 0, "y": y, "w": 24, "h": 1}, "panels": []})
|
|
y += 1
|
|
|
|
|
|
def stat(title, expr, x, unit="short", color="blue", thresholds=None, desc=None):
|
|
steps = thresholds or [{"color": color, "value": None}]
|
|
return panel("stat", title, 4, 4, x, [loki(expr, instant=True)], description=desc or "",
|
|
fieldConfig={"defaults": {"unit": unit, "color": {"mode": "thresholds"},
|
|
"thresholds": {"mode": "absolute", "steps": steps}}, "overrides": []},
|
|
options={"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
|
|
"colorMode": "value", "graphMode": "none", "textMode": "value"})
|
|
|
|
|
|
def ts(title, targets, x, w=12, h=8, unit="short", bars=False, stack=False, desc=""):
|
|
custom = {"drawStyle": "bars" if bars else "line", "fillOpacity": 60 if bars else 10,
|
|
"lineWidth": 1, "showPoints": "never", "stacking": {"mode": "normal" if stack else "none"}}
|
|
return panel("timeseries", title, w, h, x, targets, description=desc,
|
|
fieldConfig={"defaults": {"unit": unit, "custom": custom}, "overrides": []},
|
|
options={"legend": {"displayMode": "list", "placement": "bottom"},
|
|
"tooltip": {"mode": "multi", "sort": "desc"}})
|
|
|
|
|
|
def table(title, targets, x, w=12, h=8, rename=None, sort=None, desc=""):
|
|
return panel("table", title, w, h, x, targets, description=desc,
|
|
transformations=[{"id": "labelsToFields", "options": {"mode": "columns"}},
|
|
{"id": "organize", "options": {
|
|
"excludeByName": {"Time": True, "container": True, "compose_project": True,
|
|
"service_name": True},
|
|
"renameByName": rename or {}}}],
|
|
options={"showHeader": True, "sortBy": sort or []},
|
|
fieldConfig={"defaults": {}, "overrides": []})
|
|
|
|
|
|
# ---- Now
|
|
row("Now")
|
|
stat("Feeds", status("feeds"), 0, desc="From the healthcheck's status answer, every 30 seconds.")
|
|
stat("Waiting to download", status("pending"), 4)
|
|
stat("Downloaded", status("downloaded"), 8, color="green")
|
|
stat("Feed failures", f'sum(count_over_time({EV} | ev="feed_error" [$__range])) or vector(0)', 12,
|
|
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}],
|
|
desc="Failed feed checks in the time range.")
|
|
stat("Download failures", f'sum(count_over_time({EV} | ev="download_error" [$__range])) or vector(0)', 16,
|
|
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}])
|
|
stat("Warnings and errors", f'sum(count_over_time({BAD} [$__range])) or vector(0)', 20,
|
|
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}, {"color": "red", "value": 50}])
|
|
y += 4
|
|
|
|
# ---- Scans
|
|
row("Scans and downloads")
|
|
ts("New items found", [loki(f'sum(sum_over_time({EV} | ev="feed_done" | unwrap new [$__interval]))', legend="new items")],
|
|
0, bars=True)
|
|
ts("Downloads", [loki(f'sum(count_over_time({EV} | ev="download_done" [$__interval]))', legend="saved"),
|
|
loki(f'sum(count_over_time({EV} | ev="download_error" [$__interval]))', ref="B", legend="failed")],
|
|
12, bars=True)
|
|
y += 8
|
|
ts("Bytes downloaded", [loki(f'sum(sum_over_time({EV} | ev="download_done" | unwrap bytes [$__interval]))', legend="bytes")],
|
|
0, unit="bytes", bars=True)
|
|
ts("Feeds checked per scan", [loki(f'sum(sum_over_time({EV} | ev="scan_done" | unwrap feeds [$__interval]))', legend="feeds checked")],
|
|
12, bars=True, desc="Feeds that were due and fetched; the rest were skipped as not due.")
|
|
y += 8
|
|
table("Failing feeds", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="feed_error" [$__range]))', instant=True)],
|
|
0, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}])
|
|
table("Failed downloads", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="download_error" [$__range]))', instant=True)],
|
|
12, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}])
|
|
y += 8
|
|
|
|
# ---- Web
|
|
row("Web")
|
|
ts("Requests by status", [loki(f'sum by (status) (count_over_time({HTTP} [$__interval]))', legend="{{status}}")],
|
|
0, bars=True, stack=True, desc="The event stream the page keeps open is left out.")
|
|
ts("Response time", [loki(f'quantile_over_time(0.5, {HTTP} | unwrap duration_ms [$__interval]) by ()', legend="median"),
|
|
loki(f'quantile_over_time(0.95, {HTTP} | unwrap duration_ms [$__interval]) by ()', ref="B", legend="95th percentile"),
|
|
loki(f'max_over_time({HTTP} | unwrap duration_ms [$__interval]) by ()', ref="C", legend="slowest")],
|
|
12, unit="ms")
|
|
y += 8
|
|
table("Slowest routes", [loki(f'topk(15, avg_over_time({HTTP} | route != "" | unwrap duration_ms [$__range]) by (method, route))', instant=True)],
|
|
0, rename={"method": "Method", "route": "Route", "Value": "Average ms"}, sort=[{"displayName": "Average ms", "desc": True}])
|
|
table("Busiest routes", [loki(f'topk(15, sum by (method, route) (count_over_time({HTTP} | route != "" [$__range])))', instant=True)],
|
|
12, rename={"method": "Method", "route": "Route", "Value": "Requests"}, sort=[{"displayName": "Requests", "desc": True}])
|
|
y += 8
|
|
|
|
# ---- Daemon, from Prometheus: what the daemon holds between requests (#194)
|
|
row("Daemon")
|
|
# Open connections stay open, idle, for ten minutes after a burst, so open at the max is not
|
|
# pressure; drawn as a line of its own it read as the pool maxing out (#199).
|
|
ts("Database connections", [prom('max(max_over_time(ipx_db_connections{job="ipx", state="in_use"}[$__interval]))', legend="in use"),
|
|
prom('sum(ipx_db_connections{job="ipx"})', ref="B", legend="open"),
|
|
prom('max(ipx_db_connections_max{job="ipx"})', ref="C", legend="most the pool opens")],
|
|
0, w=8, desc="In use reaching the most the pool opens means requests and scans are waiting for a connection. "
|
|
"Open connections stay open, idle, for ten minutes after a burst.")["fieldConfig"]["overrides"] = [
|
|
{"matcher": {"id": "byName", "options": "open"},
|
|
"properties": [{"id": "custom.lineStyle", "value": {"fill": "dash", "dash": [4, 4]}},
|
|
{"id": "custom.fillOpacity", "value": 0}, {"id": "color", "value": {"mode": "fixed", "fixedColor": "#8e8e8e"}}]}]
|
|
ts("Waiting", [prom('ipx_worker_queue{job="ipx"}', legend="commands for the worker"),
|
|
prom('ipx_runtime_queue{job="ipx"}', ref="B", legend="tasks for a thread"),
|
|
prom('ipx_downloads_pending{job="ipx"}', ref="C", legend="downloads")],
|
|
8, w=8, desc="Scans and downloads run one at a time; a queue that only grows means the worker is stuck.")
|
|
ts("Listeners and tasks", [prom('ipx_event_listeners{job="ipx"}', legend="live-update listeners"),
|
|
prom('ipx_runtime_tasks{job="ipx"}', ref="B", legend="runtime tasks"),
|
|
prom('ipx_deadair_playing{job="ipx"}', ref="C", legend="playing through DeadAir")],
|
|
16, w=8)
|
|
y += 8
|
|
ts("Memory", [prom('ipx_process_resident_bytes{job="ipx"}', legend="resident")], 0, w=12, unit="bytes")
|
|
ts("Open files and sockets", [prom('ipx_process_open_fds{job="ipx"}', legend="open")], 12, w=12)
|
|
y += 8
|
|
# Jobs and queues (#195)
|
|
ts("Worker", [prom('max by (kind) (ipx_worker_job_seconds{job="ipx"})', legend="on a {{kind}} for"),
|
|
prom('max(ipx_last_scan_age_seconds{job="ipx"})', ref="B", legend="since the last scan finished")],
|
|
0, w=8, unit="s", desc="A job's time that only climbs, or no scan for over 10 minutes, means the worker is stuck; the healthcheck still passes.")
|
|
ts("Jobs done", [prom('sum by (kind) (increase(ipx_jobs_total{job="ipx"}[$__rate_interval]))', legend="{{kind}}")],
|
|
8, w=8, bars=True, stack=True)
|
|
ts("Time on jobs", [prom('sum by (kind) (increase(ipx_job_seconds_total{job="ipx"}[$__rate_interval]))', legend="{{kind}}")],
|
|
16, w=8, unit="s", bars=True, stack=True, desc="Seconds the worker spent on each kind of job in each interval.")
|
|
y += 8
|
|
ts("In flight", [prom('max(ipx_http_requests_in_flight{job="ipx"})', legend="web requests"),
|
|
prom('max(ipx_scan_feeds_fetching{job="ipx"})', ref="B", legend="feeds fetching"),
|
|
prom('max(ipx_downloads_active{job="ipx"})', ref="C", legend="downloads"),
|
|
prom('2 - max(ipx_torrent_slots_free{job="ipx"})', ref="D", legend="torrents")],
|
|
0, w=8)
|
|
ts("Download speed", [prom('sum(rate(ipx_download_bytes_total{job="ipx"}[$__rate_interval]))', legend="downloading")],
|
|
8, w=8, unit="Bps")
|
|
ts("Backlogs", [prom('max(ipx_deadair_backlog{job="ipx"})', legend="DeadAir to look at"),
|
|
prom('max(ipx_feed_retries{job="ipx"})', ref="B", legend="feeds to retry a lookup"),
|
|
prom('max(ipx_event_queue{job="ipx"})', ref="C", legend="live updates unread"),
|
|
prom('sum(increase(ipx_events_missed_total{job="ipx"}[$__rate_interval]))', ref="D", legend="live updates missed")],
|
|
16, w=8)
|
|
y += 8
|
|
ts("DeadAir on a file", [prom('max(ipx_deadair_busy_seconds{job="ipx"})', legend="for")], 0, w=8, unit="s",
|
|
desc="How long DeadAir has been on the file it is analysing; a few seconds each.")
|
|
ts("Artwork", [prom('max(ipx_art_cache_bytes{job="ipx"}) / max(ipx_art_cache_limit_bytes{job="ipx"})', legend="full"),
|
|
prom('sum(increase(ipx_art_requests_total{job="ipx", result="hit"}[$__rate_interval])) / '
|
|
'sum(increase(ipx_art_requests_total{job="ipx"}[$__rate_interval]))', ref="B", legend="served from disk")],
|
|
8, w=8, unit="percentunit",
|
|
desc="Full sits near 100% once the cache has filled, since each sweep trims it to fit. Served from disk is the hit rate: "
|
|
"the rest was fetched from the publisher.")
|
|
ts("Artwork evicted", [prom('sum(increase(ipx_art_evicted_total{job="ipx"}[$__rate_interval]))', legend="images")],
|
|
16, w=8, bars=True,
|
|
desc="Images each sweep threw out to stay under the limit. Many at every sweep, with a falling hit rate, means the limit is too small.")
|
|
y += 8
|
|
|
|
# ---- Traces
|
|
row("Traces")
|
|
for x, title, q in [(0, "Recent traces", TRACES + "}"),
|
|
(12, "Slow traces (over 2s)", TRACES + " && duration > 2s}")]:
|
|
panel("table", title, 12, 10, x,
|
|
[{"refId": "A", "datasource": TEMPO, "queryType": "traceql", "query": q, "limit": 50,
|
|
"tableType": "traces"}],
|
|
fieldConfig={"defaults": {}, "overrides": []})
|
|
y += 10
|
|
|
|
# ---- Log
|
|
row("Log")
|
|
panel("logs", "Warnings and errors", 24, 10, 0, [loki(BAD + TEXT)],
|
|
options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True})
|
|
y += 10
|
|
panel("logs", "Log", 24, 12, 0,
|
|
[loki(SEL + ' | json | __error__="" | path != "/api/events" | ev != "feed_skip" | ev != "status"'
|
|
' | message != "-> {\\"cmd\\":\\"status\\"}"' + TEXT)],
|
|
description="Without the event stream's requests, not-due skips and healthcheck status calls.",
|
|
options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True})
|
|
|
|
dash = {
|
|
"uid": "ipx", "title": "iPX", "tags": ["ipx"], "timezone": "browser", "schemaVersion": 39,
|
|
"time": {"from": "now-24h", "to": "now"}, "refresh": "1m", "editable": True,
|
|
"templating": {"list": [
|
|
{"name": "loki", "label": "Logs", "type": "datasource", "query": "loki", "current": {}, "hide": 0},
|
|
{"name": "tempo", "label": "Traces", "type": "datasource", "query": "tempo", "current": {}, "hide": 0},
|
|
{"name": "prometheus", "label": "Metrics", "type": "datasource", "query": "prometheus", "current": {}, "hide": 0},
|
|
]},
|
|
"links": [{"title": "iPX", "type": "link", "url": "https://ipodderx.sdf1.net", "targetBlank": True}],
|
|
"panels": panels,
|
|
}
|
|
|
|
if sys.argv[1:] == ["queries"]:
|
|
for q, instant in QUERIES.items():
|
|
print(json.dumps([q, instant]))
|
|
else:
|
|
print(json.dumps(dash, indent=2))
|