rays
41fc2e3933
They had no person, and drew as " ( key)". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
418 lines
31 KiB
Python
418 lines
31 KiB
Python
"""The iPX dashboard in Grafana: the three containers of the Arcane project ipx (ipx, ipx-web,
|
|
ipx-directory) and Tower under them, from Prometheus, Loki (their logs, shipped by Alloy) and Tempo.
|
|
|
|
python3 grafana/dashboard.py > /mnt/fast/arcane/projects/monitoring/grafana-provisioning/dashboards/ipx.json
|
|
|
|
Grafana reads that file on its own within a minute; edits made in Grafana are refused. Rows go
|
|
top-down as a request does: an overview, Tower, then each container (ipx-web, ipx's API, its
|
|
worker and its process, the directory), then traces and logs. Prometheus scrapes ipx's /metrics
|
|
(job ipx, #194) and Tower's node exporter (job node); nothing scrapes the containers themselves,
|
|
so ipx-web and the directory are read from their logs. The log panels read fields: ipx's
|
|
(web.rs access_log, ipc.rs log_event), the directory's and Caddy's; renaming one has to be
|
|
matched here. `dashboard.py queries` prints each Loki query, to try against Loki.
|
|
"""
|
|
import json, sys
|
|
|
|
LOKI = {"type": "loki", "uid": "${loki}"}
|
|
TEMPO = {"type": "tempo", "uid": "${tempo}"}
|
|
PROM = {"type": "prometheus", "uid": "${prometheus}"}
|
|
SEL = '{container="iPX"}'
|
|
# ipx logs one JSON object a line (IPX_LOG_FORMAT=json). Each scan and download event carries its
|
|
# fields (ev, feed, new, bytes, msg, error.type, ...); each request its method, path, route, status
|
|
# and duration_ms. Events are logged under ipx::scan, the healthcheck's status under ipx::io, so
|
|
# the filter is on the ev field, not the target.
|
|
EV = SEL + ' |= "\\"ev\\":\\"" | json | __error__="" | ev != ""'
|
|
HTTP = SEL + ' |= "\\"target\\":\\"ipx::http\\"" | json | __error__="" | path != "/api/events"'
|
|
BAD = SEL + ' | json | __error__="" | level =~ "WARN|ERROR"'
|
|
TEXT = ' | line_format "{{.level}} {{.target}}: {{.message}}"'
|
|
# Every production span: ipx-web's, ipx's and the directory's, one trace from the page to the
|
|
# directory where a request goes that far (#213). A daemon run by hand has no such tag.
|
|
TRACES = '{resource.deployment.environment.name="production"'
|
|
# The directory's JSON log (#204): a line a request (method, path, status, duration_ms, token), a
|
|
# line a snapshot (instance, accepted, refused_*), a line a refresher round (checked) and one a
|
|
# failed feed check (feed, error).
|
|
DIR = '{container="ipx-directory"} | json | __error__=""'
|
|
DIR_HTTP = '{container="ipx-directory"} |= "ipx_directory::http" | json | __error__=""'
|
|
# ipx-web's Caddy access log (#203, #210): request.uri, status, duration in seconds. The page and
|
|
# its files only: /api and /media are ipx's, and in its own log already.
|
|
WEB = ('{container="ipx-web"} |= "http.log.access" | json | __error__=""'
|
|
' | request_uri !~ "/(api|media)/.*" | request_remote_ip != "127.0.0.1"') # not its healthcheck
|
|
# Less what Caddy says at every start, by design: no admin endpoint, and no HTTPS or HTTP/3
|
|
# behind the tunnel; and its goodbye on a restart.
|
|
OTHERS_BAD = ('{container=~"ipx-web|ipx-directory"} | json | __error__="" | level =~ "(?i)warn|error"'
|
|
' | msg !~ "admin endpoint disabled|automatic HTTPS is completely disabled|HTTP/[23] skipped.*|exiting; byeee.*"')
|
|
|
|
|
|
QUERIES = {}
|
|
panels, y = [], 0
|
|
pid = 0
|
|
|
|
|
|
def panel(kind, title, w, h, x, targets, **extra):
|
|
global pid
|
|
pid += 1
|
|
sources = {t.get("datasource", LOKI)["uid"] for t in targets}
|
|
# A panel with queries on more than one data source must be Mixed, or Grafana runs them all on
|
|
# the panel's and Downloads, Loki's and Prometheus's, failed (#226).
|
|
ds = {"type": "datasource", "uid": "-- Mixed --"} if len(sources) > 1 else targets[0].get("datasource", LOKI)
|
|
p = {"id": pid, "type": kind, "title": title, "gridPos": {"x": x, "y": y, "w": w, "h": h},
|
|
"datasource": ds, "targets": targets}
|
|
p.update(extra)
|
|
panels.append(p)
|
|
return p
|
|
|
|
|
|
def loki(expr, ref="A", legend=None, instant=False, kind=None):
|
|
QUERIES[expr] = instant
|
|
t = {"refId": ref, "datasource": LOKI, "expr": expr, "queryType": "instant" if instant else "range"}
|
|
if legend:
|
|
t["legendFormat"] = legend
|
|
return t
|
|
|
|
|
|
def prom(expr, ref="A", legend=None):
|
|
# ipx's /metrics (#194), scraped by Prometheus as job ipx.
|
|
t = {"refId": ref, "datasource": PROM, "expr": expr}
|
|
if legend:
|
|
t["legendFormat"] = legend
|
|
return t
|
|
|
|
|
|
def row(title):
|
|
global y, pid
|
|
pid += 1
|
|
panels.append({"id": pid, "type": "row", "title": title, "collapsed": False,
|
|
"gridPos": {"x": 0, "y": y, "w": 24, "h": 1}, "panels": []})
|
|
y += 1
|
|
|
|
|
|
def stat(title, target, x, unit="short", color="blue", thresholds=None, desc=None):
|
|
"""target: loki(..., instant=True) or prom(...)."""
|
|
steps = thresholds or [{"color": color, "value": None}]
|
|
return panel("stat", title, 4, 4, x, [target], description=desc or "",
|
|
fieldConfig={"defaults": {"unit": unit, "color": {"mode": "thresholds"},
|
|
"thresholds": {"mode": "absolute", "steps": steps}}, "overrides": []},
|
|
options={"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
|
|
"colorMode": "value", "graphMode": "none", "textMode": "value"})
|
|
|
|
|
|
def ts(title, targets, x, w=12, h=8, unit="short", bars=False, stack=False, desc=""):
|
|
custom = {"drawStyle": "bars" if bars else "line", "fillOpacity": 60 if bars else 10,
|
|
"lineWidth": 1, "showPoints": "never", "stacking": {"mode": "normal" if stack else "none"}}
|
|
return panel("timeseries", title, w, h, x, targets, description=desc,
|
|
fieldConfig={"defaults": {"unit": unit, "custom": custom}, "overrides": []},
|
|
options={"legend": {"displayMode": "list", "placement": "bottom"},
|
|
"tooltip": {"mode": "multi", "sort": "desc"}})
|
|
|
|
|
|
def table(title, targets, x, w=12, h=8, rename=None, sort=None, desc=""):
|
|
return panel("table", title, w, h, x, targets, description=desc,
|
|
transformations=[{"id": "labelsToFields", "options": {"mode": "columns"}},
|
|
{"id": "organize", "options": {
|
|
"excludeByName": {"Time": True, "container": True, "compose_project": True,
|
|
"service_name": True},
|
|
"renameByName": rename or {}}}],
|
|
options={"showHeader": True, "sortBy": sort or []},
|
|
fieldConfig={"defaults": {}, "overrides": []})
|
|
|
|
|
|
GOOD_BAD = [{"color": "green", "value": None}, {"color": "orange", "value": 1}]
|
|
# Caddy's line for each request, what kind it is by its address.
|
|
WEB_ALL = ('{container="ipx-web"} |= "http.log.access" | json | __error__="" | request_remote_ip != "127.0.0.1"'
|
|
' | label_format kind=`{{ if hasPrefix "/api/" .request_uri }}API{{ else if hasPrefix "/media/" .request_uri }}'
|
|
'media{{ else }}page and files{{ end }}`')
|
|
LOG_OPTS = {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True}
|
|
|
|
|
|
def bar_overrides(*names):
|
|
return [{"matcher": {"id": "byName", "options": n},
|
|
"properties": [{"id": "unit", "value": "short"}, {"id": "custom.axisPlacement", "value": "right"},
|
|
{"id": "custom.drawStyle", "value": "bars"}, {"id": "custom.fillOpacity", "value": 60}]}
|
|
for n in names]
|
|
|
|
|
|
# ---- Overview: is each part up, and is anything wrong
|
|
row("Overview")
|
|
stat("ipx", prom('max(up{job="ipx"})'), 0, thresholds=[{"color": "red", "value": None}, {"color": "green", "value": 1}],
|
|
desc="Prometheus reaching ipx's /metrics: 1 is up. Its healthcheck asks the daemon and the database too.")["fieldConfig"]["defaults"]["mappings"] = [
|
|
{"type": "value", "options": {"0": {"text": "down"}, "1": {"text": "up"}}}]
|
|
stat("Feeds", prom('max(ipx_feeds{job="ipx"})'), 4)
|
|
stat("Waiting to download", prom('max(ipx_downloads_pending{job="ipx"})'), 8)
|
|
stat("Since the last scan", prom('max(ipx_last_scan_age_seconds{job="ipx"})'), 12, unit="s",
|
|
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 600}, {"color": "red", "value": 1800}],
|
|
desc="Over ten minutes means the worker may be stuck; the healthcheck still passes.")
|
|
stat("Page and API errors", loki(f'sum(count_over_time({WEB_ALL} | status >= 500 [$__range])) or vector(0)', instant=True), 16,
|
|
thresholds=GOOD_BAD, desc="Answers of 500 and up from ipx-web, for the page or passed on from ipx: 502 is ipx not answering.")
|
|
stat("Warnings and errors", loki(f'sum(count_over_time({BAD} [$__range])) + sum(count_over_time({OTHERS_BAD} [$__range])) or vector(0)', instant=True), 20,
|
|
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}, {"color": "red", "value": 50}],
|
|
desc="All three containers, in the time range.")
|
|
y += 4
|
|
stat("Feed failures", loki(f'sum(count_over_time({EV} | ev="feed_error" [$__range])) or vector(0)', instant=True), 0,
|
|
thresholds=GOOD_BAD, desc="Failed feed checks in the time range.")
|
|
stat("Download failures", loki(f'sum(count_over_time({EV} | ev="download_error" [$__range])) or vector(0)', instant=True), 4,
|
|
thresholds=GOOD_BAD)
|
|
stat("Downloaded", prom('max(ipx_files_downloaded{job="ipx"})'), 8, color="green")
|
|
stat("ipx running for", prom('max(ipx_uptime_seconds{job="ipx"})'), 12, unit="s")
|
|
stat("Directory: feeds shared", loki('sum(max by (instance) (max_over_time(' + DIR + ' | message="snapshot" | unwrap accepted [6h])))', instant=True), 16,
|
|
desc="The feeds each server's latest snapshot gave, added up.")
|
|
stat("Directory: refused as private", loki(f'sum(sum_over_time({DIR} | message="snapshot" | unwrap refused_private [$__range])) or vector(0)', instant=True), 20,
|
|
thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}],
|
|
desc="Feeds a server sent that the directory caught as private. Not 0 means a server's own check missed one.")
|
|
y += 4
|
|
|
|
# ---- Tower: the host all three run on, from the node exporter
|
|
row("Tower")
|
|
ts("Load", [prom('node_load1{job="node"}', legend="1 minute"), prom('node_load15{job="node"}', ref="B", legend="15 minutes")],
|
|
0, w=8, desc="Twelve threads. Load far above that with little CPU used is processes waiting on a disk or the user shares (shfs).")
|
|
ts("CPU", [prom('sum by (mode) (rate(node_cpu_seconds_total{job="node", mode=~"user|system|iowait"}[$__rate_interval]))', legend="{{mode}}")],
|
|
8, w=8, stack=True, desc="Cores busy, by kind. Much system time with little user time is the kernel, often FUSE (shfs).")
|
|
ts("Memory available", [prom('node_memory_MemAvailable_bytes{job="node"}', legend="available")], 16, w=8, unit="bytes")
|
|
y += 8
|
|
ts("Disk busy", [prom('rate(node_disk_io_time_seconds_total{job="node", device=~"nvme.*|md.*"}[$__rate_interval])', legend="{{device}}")],
|
|
0, w=12, unit="percentunit", desc="nvme1n1 is /mnt/fast (appdata, Postgres's data); md* are the array's disks behind /mnt/user (downloads).")
|
|
ts("Network", [prom('sum(rate(node_network_receive_bytes_total{job="node", device!~"lo|veth.*|docker.*|br-.*"}[$__rate_interval]))', legend="in"),
|
|
prom('sum(rate(node_network_transmit_bytes_total{job="node", device!~"lo|veth.*|docker.*|br-.*"}[$__rate_interval]))', ref="B", legend="out")],
|
|
12, w=12, unit="Bps")
|
|
y += 8
|
|
|
|
# ---- ipx-web: Caddy, the way in. Serves the page itself, passes /api and /media to ipx
|
|
row("ipx-web")
|
|
ts("Requests by kind", [loki(f'sum by (kind) (count_over_time({WEB_ALL} [$__interval]))', legend="{{kind}}")],
|
|
0, w=8, bars=True, stack=True, desc="Everything through Caddy, from the LAN and the tunnel. The API and media go on to ipx.")
|
|
ts("Page and files by status", [loki(f'sum by (status) (count_over_time({WEB} [$__interval]))', legend="{{status}}")],
|
|
8, w=8, bars=True, stack=True, desc="What Caddy served itself. A page answered 401 or 403 is ipx refusing it in /api/page-check.")
|
|
ts("Page response time", [loki(f'quantile_over_time(0.95, {WEB} | unwrap duration [$__interval]) by ()', legend="95th percentile"),
|
|
loki(f'max_over_time({WEB} | unwrap duration [$__interval]) by ()', ref="B", legend="slowest")],
|
|
16, w=8, unit="s", desc="A page waits on ipx's /api/page-check before it is sent.")
|
|
y += 8
|
|
# What the person waits for, from the page itself (#246): a click to the list drawn.
|
|
LIST = SEL + ' |= "\\"ev\\":\\"list_shown\\"" | json | __error__=""'
|
|
ts("A click to the list shown", [loki(f'quantile_over_time(0.95, {LIST} | unwrap ms [$__interval]) by ()', legend="95th percentile"),
|
|
loki(f'quantile_over_time(0.5, {LIST} | unwrap ms [$__interval]) by ()', ref="B", legend="median"),
|
|
loki(f'max_over_time({LIST} | unwrap ms [$__interval]) by ()', ref="C", legend="slowest")],
|
|
0, w=12, unit="ms", desc="Measured in the browser, network and drawing included. The API's part is in the click's trace.")
|
|
table("Slowest lists by view", [loki(f'topk(10, max_over_time({LIST} | unwrap ms [$__range]) by (feed, filter))', instant=True)],
|
|
12, rename={"feed": "View", "filter": "Filter", "Value": "Slowest (ms)"}, sort=[{"displayName": "Slowest (ms)", "desc": True}])
|
|
y += 8
|
|
|
|
# ---- ipx: the API it answers behind ipx-web
|
|
row("ipx: API")
|
|
ts("Requests by status", [loki(f'sum by (status) (count_over_time({HTTP} [$__interval]))', legend="{{status}}")],
|
|
0, bars=True, stack=True, desc="The event stream the page keeps open is left out.")
|
|
ts("Response time", [loki(f'quantile_over_time(0.5, {HTTP} | unwrap duration_ms [$__interval]) by ()', legend="median"),
|
|
loki(f'quantile_over_time(0.95, {HTTP} | unwrap duration_ms [$__interval]) by ()', ref="B", legend="95th percentile"),
|
|
loki(f'max_over_time({HTTP} | unwrap duration_ms [$__interval]) by ()', ref="C", legend="slowest")],
|
|
12, unit="ms")
|
|
y += 8
|
|
table("Slowest routes", [loki(f'topk(15, avg_over_time({HTTP} | route != "" | unwrap duration_ms [$__range]) by (method, route))', instant=True)],
|
|
0, w=8, rename={"method": "Method", "route": "Route", "Value": "Average ms"}, sort=[{"displayName": "Average ms", "desc": True}])
|
|
table("Busiest routes", [loki(f'topk(15, sum by (method, route) (count_over_time({HTTP} | route != "" [$__range])))', instant=True)],
|
|
8, w=8, rename={"method": "Method", "route": "Route", "Value": "Requests"}, sort=[{"displayName": "Requests", "desc": True}])
|
|
ts("Open connections", [prom('max(ipx_http_requests_in_flight{job="ipx"})', legend="requests in flight"),
|
|
prom('max(ipx_event_listeners{job="ipx"})', ref="B", legend="live-update listeners"),
|
|
prom('max(ipx_event_queue{job="ipx"})', ref="C", legend="live updates unread"),
|
|
prom('sum(increase(ipx_events_missed_total{job="ipx"}[$__rate_interval]))', ref="D", legend="live updates missed")],
|
|
16, w=8, desc="Each open page holds one live-update listener. Missed updates mean a page fell behind and reloads its lists.")
|
|
y += 8
|
|
|
|
# ---- ipx: the worker, one job at a time: scans, downloads, DeadAir
|
|
row("ipx: worker")
|
|
ts("Worker", [prom('max by (kind) (ipx_worker_job_seconds{job="ipx"})', legend="on a {{kind}} for"),
|
|
prom('max(ipx_last_scan_age_seconds{job="ipx"})', ref="B", legend="since the last scan finished")],
|
|
0, w=8, unit="s", desc="A job's time that only climbs, or no scan for over 10 minutes, means the worker is stuck; the healthcheck still passes.")
|
|
ts("Jobs done", [prom('sum by (kind) (increase(ipx_jobs_total{job="ipx"}[$__rate_interval]))', legend="{{kind}}")],
|
|
8, w=8, bars=True, stack=True)
|
|
ts("Time on jobs", [prom('sum by (kind) (increase(ipx_job_seconds_total{job="ipx"}[$__rate_interval]))', legend="{{kind}}")],
|
|
16, w=8, unit="s", bars=True, stack=True, desc="Seconds the worker spent on each kind of job in each interval.")
|
|
y += 8
|
|
ts("Waiting", [prom('ipx_worker_queue{job="ipx"}', legend="commands for the worker"),
|
|
prom('ipx_downloads_pending{job="ipx"}', ref="B", legend="downloads"),
|
|
prom('max(ipx_feed_retries{job="ipx"})', ref="C", legend="feeds to retry a lookup"),
|
|
prom('max(ipx_deadair_backlog{job="ipx"})', ref="D", legend="files for DeadAir")],
|
|
0, w=8, desc="A queue that only grows means the worker is stuck or behind.")
|
|
ts("Feeds checked and new items", [loki(f'sum(sum_over_time({EV} | ev="scan_done" | unwrap feeds [$__interval]))', legend="feeds checked"),
|
|
loki(f'sum(sum_over_time({EV} | ev="feed_done" | unwrap new [$__interval]))', ref="B", legend="new items")],
|
|
8, w=8, bars=True, desc="Feeds that were due and fetched; the rest were skipped as not due.")
|
|
ts("Downloads", [loki(f'sum(count_over_time({EV} | ev="download_done" [$__interval]))', legend="saved"),
|
|
loki(f'sum(count_over_time({EV} | ev="download_error" [$__interval]))', ref="B", legend="failed"),
|
|
prom('max(ipx_downloads_active{job="ipx"})', ref="C", legend="at once"),
|
|
prom('2 - max(ipx_torrent_slots_free{job="ipx"})', ref="D", legend="torrents")],
|
|
16, w=8, bars=True)
|
|
y += 8
|
|
ts("Download speed", [prom('sum(rate(ipx_download_bytes_total{job="ipx"}[$__rate_interval]))', legend="downloading")],
|
|
0, w=8, unit="Bps")
|
|
ts("Feeds fetching at once", [prom('max(ipx_scan_feeds_fetching{job="ipx"})', legend="fetching")], 8, w=8)
|
|
ts("DeadAir on a file", [prom('max(ipx_deadair_busy_seconds{job="ipx"})', legend="for"),
|
|
prom('max(ipx_deadair_playing{job="ipx"})', ref="B", legend="playing through DeadAir")],
|
|
16, w=8, unit="s", desc="How long DeadAir has been on the file it is analysing; a few seconds each.")["fieldConfig"]["overrides"] = [
|
|
{"matcher": {"id": "byName", "options": "playing through DeadAir"},
|
|
"properties": [{"id": "unit", "value": "short"}, {"id": "custom.axisPlacement", "value": "right"}]}]
|
|
y += 8
|
|
table("Failing feeds", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="feed_error" [$__range]))', instant=True)],
|
|
0, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}])
|
|
table("Failed downloads", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="download_error" [$__range]))', instant=True)],
|
|
12, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}])
|
|
y += 8
|
|
|
|
# ---- ipx: Signals, items put to TypeSafe's Jev on each person's own key (#254)
|
|
row("ipx: Signals")
|
|
ts("Items judged", [prom('sum(increase(ipx_signals_judged_total{job="ipx"}[$__rate_interval]))', legend="judged"),
|
|
prom('sum(increase(ipx_signals_hits_total{job="ipx"}[$__rate_interval]))', ref="B", legend="fit a Signal")],
|
|
0, w=8, bars=True, desc="Each item is judged once against every Signal of its subscribers that has not had it; a burst follows a scan or a new Signal.")
|
|
ts("Calls to Jev", [prom('sum by (result) (increase(ipx_jev_requests_total{job="ipx"}[$__rate_interval]))', legend="{{result}}"),
|
|
prom('(sum(increase(ipx_jev_seconds_total{job="ipx"}[$__rate_interval])) - sum(increase(ipx_jev_paced_seconds_total{job="ipx"}[$__rate_interval]))) / '
|
|
'sum(increase(ipx_jev_requests_total{job="ipx"}[$__rate_interval]))', ref="B", legend="seconds a call")],
|
|
8, w=8, bars=True, stack=True,
|
|
desc="key_refused stops that person's Signals until they save another key; malformed stops the pass and is ipx's to fix; "
|
|
"rate_limited is TypeSafe saying slow down (#277), and every call waits; "
|
|
"error is TypeSafe failing or slow, and the item is asked again next pass. Seconds a call is TypeSafe's, without the wait for a turn.")["fieldConfig"]["overrides"] = [
|
|
{"matcher": {"id": "byName", "options": "seconds a call"},
|
|
"properties": [{"id": "unit", "value": "s"}, {"id": "custom.axisPlacement", "value": "right"},
|
|
{"id": "custom.drawStyle", "value": "line"}, {"id": "custom.stacking", "value": {"mode": "none"}}]}]
|
|
ts("Signals judge", [prom('max(ipx_signals_last_pass_age_seconds{job="ipx"})', legend="since the last pass"),
|
|
prom('max(ipx_signals_keys_refused{job="ipx"})', ref="B", legend="keys refused")],
|
|
16, w=8, unit="s", desc="It passes at least every five minutes; much longer means it is stuck on Jev. Keys refused, on the right, "
|
|
"are people whose Signals wait for a new key.")["fieldConfig"]["overrides"] = [
|
|
{"matcher": {"id": "byName", "options": "keys refused"},
|
|
"properties": [{"id": "unit", "value": "short"}, {"id": "custom.axisPlacement", "value": "right"}]}]
|
|
y += 8
|
|
ts("Tokens", [prom('sum by (kind) (increase(ipx_jev_tokens_total{job="ipx"}[$__rate_interval]))', legend="{{kind}}")],
|
|
0, w=8, bars=True, stack=True, desc="What TypeSafe counted for each call, as its answers say: what Signals, icons and key checks cost.")
|
|
ts("Tokens a call", [prom('sum by (kind) (increase(ipx_jev_tokens_total{job="ipx"}[$__rate_interval])) / ignoring(kind) group_left '
|
|
'sum(increase(ipx_jev_requests_total{job="ipx", result="ok"}[$__rate_interval]))', legend="{{kind}}")],
|
|
8, w=8, desc="Most of the input is the questions, one for each Signal, then the item's text, cut at 1,000 characters.")
|
|
# TypeSafe's default limits (#277): ipx paces itself to 60 calls and 80,000 input tokens a second.
|
|
LIMIT = lambda n: [{"color": "transparent", "value": None}, {"color": "red", "value": n}]
|
|
p = ts("Against TypeSafe's limits", [prom('sum(rate(ipx_jev_requests_total{job="ipx"}[$__rate_interval]))', legend="calls a second"),
|
|
prom('sum(rate(ipx_jev_tokens_total{job="ipx", kind="input"}[$__rate_interval])) / 1000', ref="B", legend="thousand input tokens a second"),
|
|
prom('sum(rate(ipx_jev_paced_seconds_total{job="ipx"}[$__rate_interval]))', ref="C", legend="seconds waited for a turn, a second")],
|
|
16, w=8, desc="TypeSafe allows 80 calls and 100,000 input tokens a second; the red line is 80 on both scales (calls, and thousands of tokens). "
|
|
"Waiting for a turn means ipx is holding itself under them, and is the time a new Signal's backfill takes longer for it.")
|
|
p["fieldConfig"]["defaults"]["custom"]["thresholdsStyle"] = {"mode": "line"}
|
|
p["fieldConfig"]["defaults"]["thresholds"] = {"mode": "absolute", "steps": LIMIT(80)}
|
|
y += 8
|
|
# By person and whose key paid (#278): who the server's TypeSafe bill is for. An item shared on the
|
|
# server's key is split by each person's share of its questions.
|
|
# Counts from before #278 have no person; user!="" leaves them out rather than drawing " ( key)".
|
|
ts("Input tokens by person", [prom('sum by (user, key) (increase(ipx_jev_tokens_total{job="ipx", kind="input", user!=""}[$__rate_interval]))',
|
|
legend="{{user}} ({{key}} key)")],
|
|
0, w=8, bars=True, stack=True, desc="server is the server's TypeSafe key, what the server pays for; own is the person's own key.")
|
|
ts("Items judged by person", [prom('sum by (user, key) (increase(ipx_signals_judged_total{job="ipx", user!=""}[$__rate_interval]))', legend="{{user}} ({{key}} key)"),
|
|
prom('sum by (user) (increase(ipx_signals_hits_total{job="ipx", user!=""}[$__rate_interval]))', ref="B", legend="{{user}}: fit a Signal")],
|
|
8, w=8, bars=True)
|
|
def ptab(expr, ref):
|
|
return {"refId": ref, "datasource": PROM, "expr": expr, "instant": True, "format": "table"}
|
|
panel("table", "Who used what", 8, 8, 16,
|
|
[ptab('sum by (user, key) (increase(ipx_jev_tokens_total{job="ipx", kind="input", user!=""}[$__range]))', "A"),
|
|
ptab('sum by (user, key) (increase(ipx_jev_tokens_total{job="ipx", kind="output", user!=""}[$__range]))', "B"),
|
|
ptab('sum by (user, key) (increase(ipx_signals_judged_total{job="ipx", user!=""}[$__range]))', "C")],
|
|
description="Over the dashboard's time range: each person's input and output tokens and items judged, and whose key paid. "
|
|
"Icon picks are on the server's key, key checks on the person's own.",
|
|
transformations=[{"id": "merge", "options": {}},
|
|
{"id": "organize", "options": {"excludeByName": {"Time": True},
|
|
"renameByName": {"user": "Person", "key": "Key", "Value #A": "Input tokens",
|
|
"Value #B": "Output tokens", "Value #C": "Items judged"}}}],
|
|
options={"showHeader": True, "sortBy": [{"displayName": "Input tokens", "desc": True}]},
|
|
fieldConfig={"defaults": {"decimals": 0}, "overrides": []})
|
|
y += 8
|
|
|
|
# ---- ipx: the process, what it holds between requests and jobs (#194)
|
|
row("ipx: process")
|
|
# Open connections stay open, idle, for ten minutes after a burst, so open at the max is not
|
|
# pressure; drawn as a line of its own it read as the pool maxing out (#199).
|
|
ts("Database connections", [prom('max(max_over_time(ipx_db_connections{job="ipx", state="in_use"}[$__interval]))', legend="in use"),
|
|
prom('min(min_over_time(ipx_db_connections{job="ipx", state="idle"}[$__interval]))', ref="B", legend="ready"),
|
|
prom('max(ipx_db_connections_max{job="ipx"})', ref="C", legend="the pool")],
|
|
0, w=8, desc="The daemon keeps the whole pool open (#241), so in use and ready add up to the pool. In use reaching "
|
|
"the pool, ready at 0, means requests and scans are waiting for a connection (#251).")
|
|
ts("Memory", [prom('ipx_process_resident_bytes{job="ipx"}', legend="resident")], 8, w=8, unit="bytes")
|
|
ts("Open files and sockets", [prom('ipx_process_open_fds{job="ipx"}', legend="open")], 16, w=8)
|
|
y += 8
|
|
ts("Runtime", [prom('ipx_runtime_tasks{job="ipx"}', legend="tasks"),
|
|
prom('ipx_runtime_queue{job="ipx"}', ref="B", legend="tasks waiting for a thread"),
|
|
prom('ipx_runtime_workers{job="ipx"}', ref="C", legend="threads")],
|
|
0, w=12, desc="Tasks waiting for a thread for long means something is blocking the runtime.")
|
|
ts("Artwork", [prom('max(ipx_art_cache_bytes{job="ipx"}) / max(ipx_art_cache_limit_bytes{job="ipx"})', legend="full"),
|
|
prom('sum(increase(ipx_art_requests_total{job="ipx", result="hit"}[$__rate_interval])) / '
|
|
'sum(increase(ipx_art_requests_total{job="ipx"}[$__rate_interval]))', ref="B", legend="hit rate"),
|
|
prom('sum(increase(ipx_art_evicted_total{job="ipx"}[$__rate_interval]))', ref="C", legend="images evicted")],
|
|
12, w=12, unit="percentunit",
|
|
desc="Full sits near 100% once the cache has filled, since each sweep trims it to fit. Hit rate is the share served from disk: "
|
|
"the rest was fetched from the publisher. Evicted, on the right, is images each sweep threw out to stay under the "
|
|
"limit: many at every sweep, with a falling hit rate, means the limit is too small.")["fieldConfig"]["overrides"] = bar_overrides("images evicted")
|
|
y += 8
|
|
|
|
# ---- ipx-directory: public at directory.ipxreader.app, shared by ipx servers (#204)
|
|
row("ipx-directory")
|
|
stat("Snapshots", loki(f'sum(count_over_time({DIR} | message="snapshot" [$__range])) or vector(0)', instant=True), 0,
|
|
desc="ipx sends one when what it shares changes, and every half hour (#211).")
|
|
stat("Feed checks", loki(f'sum(sum_over_time({DIR} | message="feeds checked" | unwrap checked [$__range])) or vector(0)', instant=True), 4,
|
|
desc="Each feed every twelve hours, five at a time.")
|
|
stat("Failed feed checks", loki(f'sum(count_over_time({DIR} | message="feed check failed" [$__range])) or vector(0)', instant=True), 8,
|
|
thresholds=GOOD_BAD, desc="Mostly the publishers': a feed gone or moved.")
|
|
stat("Requests", loki(f'sum(count_over_time({DIR_HTTP} [$__range])) or vector(0)', instant=True), 12)
|
|
stat("Requests refused", loki(f'sum(count_over_time({DIR_HTTP} | status =~ "401|403" [$__range])) or vector(0)', instant=True), 16,
|
|
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 100}],
|
|
desc="Without a token, or with one that has not the access asked for. Now and then is someone trying a public address.")
|
|
stat("Slowest request", loki(f'max(max_over_time({DIR_HTTP} | unwrap duration_ms [$__range])) or vector(0)', instant=True), 20, unit="ms")
|
|
y += 4
|
|
ts("Requests by token", [loki(f'sum by (status, token) (count_over_time({DIR_HTTP} [$__interval]))', legend="{{status}} {{token}}")],
|
|
0, bars=True, stack=True)
|
|
ts("Feed checks over time", [loki(f'sum(sum_over_time({DIR} | message="feeds checked" | unwrap checked [$__interval]))', legend="checked"),
|
|
loki(f'sum(count_over_time({DIR} | message="feed check failed" [$__interval]))', ref="B", legend="failed")],
|
|
12, bars=True, desc="A burst follows a snapshot with new feeds.")
|
|
y += 8
|
|
table("Failing feeds in the directory", [loki(f'sum by (feed, error) (count_over_time({DIR} | message="feed check failed" [$__range]))', instant=True)],
|
|
0, rename={"feed": "Feed id", "error": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}])
|
|
table("Snapshots by server", [loki(f'sum by (instance) (count_over_time({DIR} | message="snapshot" [$__range]))', instant=True)],
|
|
12, rename={"instance": "Token", "Value": "Snapshots"}, sort=[{"displayName": "Snapshots", "desc": True}])
|
|
y += 8
|
|
# The directory's own /metrics, on a port apart from the tunnel's (#224).
|
|
ts("Directory memory", [prom('directory_process_resident_bytes{job="ipx-directory"}', legend="resident")], 0, unit="bytes",
|
|
desc="mimalloc gives back what is freed, so this follows what is in use: a rise that never comes down is a leak.")
|
|
ts("Feed bytes fetched", [prom('rate(ipx_feed_bytes_total{job="ipx"}[$__rate_interval])', legend="ipx"),
|
|
prom('rate(directory_feed_bytes_total{job="ipx-directory"}[$__rate_interval])', ref="B", legend="directory")],
|
|
12, unit="Bps", desc="Feeds are read whole, so this is what scans and checks allocate; set it beside memory.")
|
|
y += 8
|
|
|
|
# ---- Traces: all three send them to Tempo (#213)
|
|
row("Traces")
|
|
for x, title, q in [(0, "Recent", TRACES + "}"),
|
|
(8, "Slow (over 2s)", TRACES + " && duration > 2s}"),
|
|
(16, "Failed", TRACES + " && status = error}")]:
|
|
panel("table", title, 8, 10, x,
|
|
[{"refId": "A", "datasource": TEMPO, "queryType": "traceql", "query": q, "limit": 50,
|
|
"tableType": "traces"}],
|
|
fieldConfig={"defaults": {}, "overrides": []})
|
|
y += 10
|
|
|
|
# ---- Logs
|
|
row("Logs")
|
|
panel("logs", "ipx: warnings and errors", 12, 10, 0, [loki(BAD + TEXT)], options=LOG_OPTS)
|
|
panel("logs", "ipx-web and ipx-directory: warnings and errors", 12, 10, 12,
|
|
[loki(OTHERS_BAD + ' | line_format "{{.container}} {{.level}}: {{.message}}{{.msg}}"')], options=LOG_OPTS)
|
|
y += 10
|
|
panel("logs", "ipx log", 24, 12, 0,
|
|
[loki(SEL + ' | json | __error__="" | path != "/api/events" | ev != "feed_skip" | ev != "status"'
|
|
' | message != "-> {\\"cmd\\":\\"status\\"}"' + TEXT)],
|
|
description="Without the event stream's requests, not-due skips and healthcheck status calls.", options=LOG_OPTS)
|
|
y += 12
|
|
|
|
dash = {
|
|
"uid": "ipx", "title": "iPX", "tags": ["ipx"], "timezone": "browser", "schemaVersion": 39,
|
|
"time": {"from": "now-24h", "to": "now"}, "refresh": "1m", "editable": True,
|
|
"templating": {"list": [
|
|
{"name": "loki", "label": "Logs", "type": "datasource", "query": "loki", "current": {}, "hide": 0},
|
|
{"name": "tempo", "label": "Traces", "type": "datasource", "query": "tempo", "current": {}, "hide": 0},
|
|
{"name": "prometheus", "label": "Metrics", "type": "datasource", "query": "prometheus", "current": {}, "hide": 0},
|
|
]},
|
|
"links": [{"title": "iPX", "type": "link", "url": "https://ipodderx.sdf1.net", "targetBlank": True}],
|
|
"panels": panels,
|
|
}
|
|
|
|
if sys.argv[1:] == ["queries"]:
|
|
for q, instant in QUERIES.items():
|
|
print(json.dumps([q, instant]))
|
|
else:
|
|
print(json.dumps(dash, indent=2))
|