From 7ff73f1a8b3fec6e9fd299120f7cf925fdf3c8a8 Mon Sep 17 00:00:00 2001 From: rays Date: Mon, 12 Oct 2026 15:43:56 +0000 Subject: [PATCH] Dashboard laid out by container: overview, Tower, ipx-web, ipx, directory (#210) The Daemon row was named for when ipx was the only thing running. Rows now follow how a request goes: an overview of all three containers, Tower under them (the node exporter, so a night like 2026-10-12's shfs stall shows up), ipx-web, ipx's API, worker and process, then the directory, traces and logs. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- grafana/dashboard.py | 303 ++++++++++++++++++++++++------------------- 2 files changed, 169 insertions(+), 136 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5892f5..fbda4ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- The Grafana dashboard has a Page row, from ipx-web's new request log, and a Directory row: snapshots, feed checks and failures, and requests by token (#210). +- The Grafana dashboard is laid out by container: an overview of all three, Tower's load, CPU, memory and disks, then ipx-web (from its new request log), ipx's API, worker and process, and the directory, each with its own warnings (#210). - `ipx-directory`, a directory of public podcasts that ipx servers share: each sends the public feeds its people subscribe to, and any server with a token browses them. Tokens are in its config file, each read-only, read-write or admin, and revoked one at a time (#204). ### Changed diff --git a/grafana/dashboard.py b/grafana/dashboard.py index 97eedc7..03e0df0 100644 --- a/grafana/dashboard.py +++ b/grafana/dashboard.py @@ -1,13 +1,15 @@ -"""The iPX dashboard in Grafana, from Loki (the containers' logs, shipped by Alloy) and Tempo: -ipx, and since #210 ipx-web and ipx-directory. +"""The iPX dashboard in Grafana: the three containers of the Arcane project ipx (ipx, ipx-web, +ipx-directory) and Tower under them, from Prometheus, Loki (their logs, shipped by Alloy) and Tempo. python3 grafana/dashboard.py > /mnt/fast/arcane/projects/monitoring/grafana-provisioning/dashboards/ipx.json -Grafana reads that file on its own within a minute; edits made in Grafana are refused. The -Daemon row reads ipx's /metrics from Prometheus (job ipx, #194); the rest read the fields of -ipx's JSON log (IPX_LOG_FORMAT=json): renaming a field in web.rs access_log or ipc.rs log_event -has to be matched here. `dashboard.py queries` prints each -query, to try against Loki. +Grafana reads that file on its own within a minute; edits made in Grafana are refused. Rows go +top-down as a request does: an overview, Tower, then each container (ipx-web, ipx's API, its +worker and its process, the directory), then traces and logs. Prometheus scrapes ipx's /metrics +(job ipx, #194) and Tower's node exporter (job node); nothing scrapes the containers themselves, +so ipx-web and the directory are read from their logs. The log panels read fields: ipx's +(web.rs access_log, ipc.rs log_event), the directory's and Caddy's; renaming one has to be +matched here. `dashboard.py queries` prints each Loki query, to try against Loki. """ import json, sys @@ -39,10 +41,6 @@ OTHERS_BAD = ('{container=~"ipx-web|ipx-directory"} | json | __error__="" | leve ' | msg !~ "admin endpoint disabled|automatic HTTPS is completely disabled|HTTP/[23] skipped.*|exiting; byeee.*"') -def status(field): - return f'max(max_over_time({EV} | ev = "status" | unwrap {field} [10m]))' - - QUERIES = {} panels, y = [], 0 pid = 0 @@ -82,9 +80,10 @@ def row(title): y += 1 -def stat(title, expr, x, unit="short", color="blue", thresholds=None, desc=None): +def stat(title, target, x, unit="short", color="blue", thresholds=None, desc=None): + """target: loki(..., instant=True) or prom(...).""" steps = thresholds or [{"color": color, "value": None}] - return panel("stat", title, 4, 4, x, [loki(expr, instant=True)], description=desc or "", + return panel("stat", title, 4, 4, x, [target], description=desc or "", fieldConfig={"defaults": {"unit": unit, "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": steps}}, "overrides": []}, options={"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, @@ -111,41 +110,78 @@ def table(title, targets, x, w=12, h=8, rename=None, sort=None, desc=""): fieldConfig={"defaults": {}, "overrides": []}) -# ---- Now -row("Now") -stat("Feeds", status("feeds"), 0, desc="From the healthcheck's status answer, every 30 seconds.") -stat("Waiting to download", status("pending"), 4) -stat("Downloaded", status("downloaded"), 8, color="green") -stat("Feed failures", f'sum(count_over_time({EV} | ev="feed_error" [$__range])) or vector(0)', 12, - thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}], - desc="Failed feed checks in the time range.") -stat("Download failures", f'sum(count_over_time({EV} | ev="download_error" [$__range])) or vector(0)', 16, - thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}]) -stat("Warnings and errors", f'sum(count_over_time({BAD} [$__range])) or vector(0)', 20, - thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}, {"color": "red", "value": 50}]) +GOOD_BAD = [{"color": "green", "value": None}, {"color": "orange", "value": 1}] +# Caddy's line for each request, what kind it is by its address. +WEB_ALL = ('{container="ipx-web"} |= "http.log.access" | json | __error__="" | request_remote_ip != "127.0.0.1"' + ' | label_format kind=`{{ if hasPrefix "/api/" .request_uri }}API{{ else if hasPrefix "/media/" .request_uri }}' + 'media{{ else }}page and files{{ end }}`') +LOG_OPTS = {"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True} + + +def bar_overrides(*names): + return [{"matcher": {"id": "byName", "options": n}, + "properties": [{"id": "unit", "value": "short"}, {"id": "custom.axisPlacement", "value": "right"}, + {"id": "custom.drawStyle", "value": "bars"}, {"id": "custom.fillOpacity", "value": 60}]} + for n in names] + + +# ---- Overview: is each part up, and is anything wrong +row("Overview") +stat("ipx", prom('max(up{job="ipx"})'), 0, thresholds=[{"color": "red", "value": None}, {"color": "green", "value": 1}], + desc="Prometheus reaching ipx's /metrics: 1 is up. Its healthcheck asks the daemon and the database too.")["fieldConfig"]["defaults"]["mappings"] = [ + {"type": "value", "options": {"0": {"text": "down"}, "1": {"text": "up"}}}] +stat("Feeds", prom('max(ipx_feeds{job="ipx"})'), 4) +stat("Waiting to download", prom('max(ipx_downloads_pending{job="ipx"})'), 8) +stat("Since the last scan", prom('max(ipx_last_scan_age_seconds{job="ipx"})'), 12, unit="s", + thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 600}, {"color": "red", "value": 1800}], + desc="Over ten minutes means the worker may be stuck; the healthcheck still passes.") +stat("Page and API errors", loki(f'sum(count_over_time({WEB_ALL} | status >= 500 [$__range])) or vector(0)', instant=True), 16, + thresholds=GOOD_BAD, desc="Answers of 500 and up from ipx-web, for the page or passed on from ipx: 502 is ipx not answering.") +stat("Warnings and errors", loki(f'sum(count_over_time({BAD} [$__range])) + sum(count_over_time({OTHERS_BAD} [$__range])) or vector(0)', instant=True), 20, + thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}, {"color": "red", "value": 50}], + desc="All three containers, in the time range.") +y += 4 +stat("Feed failures", loki(f'sum(count_over_time({EV} | ev="feed_error" [$__range])) or vector(0)', instant=True), 0, + thresholds=GOOD_BAD, desc="Failed feed checks in the time range.") +stat("Download failures", loki(f'sum(count_over_time({EV} | ev="download_error" [$__range])) or vector(0)', instant=True), 4, + thresholds=GOOD_BAD) +stat("Downloaded", prom('max(ipx_files_downloaded{job="ipx"})'), 8, color="green") +stat("ipx running for", prom('max(ipx_uptime_seconds{job="ipx"})'), 12, unit="s") +stat("Directory: feeds shared", loki('sum(max by (instance) (max_over_time(' + DIR + ' | message="snapshot" | unwrap accepted [6h])))', instant=True), 16, + desc="The feeds each server's latest snapshot gave, added up.") +stat("Directory: refused as private", loki(f'sum(sum_over_time({DIR} | message="snapshot" | unwrap refused_private [$__range])) or vector(0)', instant=True), 20, + thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}], + desc="Feeds a server sent that the directory caught as private. Not 0 means a server's own check missed one.") y += 4 -# ---- Scans -row("Scans and downloads") -ts("New items found", [loki(f'sum(sum_over_time({EV} | ev="feed_done" | unwrap new [$__interval]))', legend="new items")], - 0, bars=True) -ts("Downloads", [loki(f'sum(count_over_time({EV} | ev="download_done" [$__interval]))', legend="saved"), - loki(f'sum(count_over_time({EV} | ev="download_error" [$__interval]))', ref="B", legend="failed")], - 12, bars=True) +# ---- Tower: the host all three run on, from the node exporter +row("Tower") +ts("Load", [prom('node_load1{job="node"}', legend="1 minute"), prom('node_load15{job="node"}', ref="B", legend="15 minutes")], + 0, w=8, desc="Twelve threads. Load far above that with little CPU used is processes waiting on a disk or the user shares (shfs).") +ts("CPU", [prom('sum by (mode) (rate(node_cpu_seconds_total{job="node", mode=~"user|system|iowait"}[$__rate_interval]))', legend="{{mode}}")], + 8, w=8, stack=True, desc="Cores busy, by kind. Much system time with little user time is the kernel, often FUSE (shfs).") +ts("Memory available", [prom('node_memory_MemAvailable_bytes{job="node"}', legend="available")], 16, w=8, unit="bytes") y += 8 -ts("Bytes downloaded", [loki(f'sum(sum_over_time({EV} | ev="download_done" | unwrap bytes [$__interval]))', legend="bytes")], - 0, unit="bytes", bars=True) -ts("Feeds checked per scan", [loki(f'sum(sum_over_time({EV} | ev="scan_done" | unwrap feeds [$__interval]))', legend="feeds checked")], - 12, bars=True, desc="Feeds that were due and fetched; the rest were skipped as not due.") -y += 8 -table("Failing feeds", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="feed_error" [$__range]))', instant=True)], - 0, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}]) -table("Failed downloads", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="download_error" [$__range]))', instant=True)], - 12, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}]) +ts("Disk busy", [prom('rate(node_disk_io_time_seconds_total{job="node", device=~"nvme.*|md.*"}[$__rate_interval])', legend="{{device}}")], + 0, w=12, unit="percentunit", desc="nvme1n1 is /mnt/fast (appdata, Postgres's data); md* are the array's disks behind /mnt/user (downloads).") +ts("Network", [prom('sum(rate(node_network_receive_bytes_total{job="node", device!~"lo|veth.*|docker.*|br-.*"}[$__rate_interval]))', legend="in"), + prom('sum(rate(node_network_transmit_bytes_total{job="node", device!~"lo|veth.*|docker.*|br-.*"}[$__rate_interval]))', ref="B", legend="out")], + 12, w=12, unit="Bps") y += 8 -# ---- Web -row("Web") +# ---- ipx-web: Caddy, the way in. Serves the page itself, passes /api and /media to ipx +row("ipx-web: the page, and the way in") +ts("Requests by kind", [loki(f'sum by (kind) (count_over_time({WEB_ALL} [$__interval]))', legend="{{kind}}")], + 0, w=8, bars=True, stack=True, desc="Everything through Caddy, from the LAN and the tunnel. The API and media go on to ipx.") +ts("Page and files by status", [loki(f'sum by (status) (count_over_time({WEB} [$__interval]))', legend="{{status}}")], + 8, w=8, bars=True, stack=True, desc="What Caddy served itself. A page answered 401 or 403 is ipx refusing it in /api/page-check.") +ts("Page response time", [loki(f'quantile_over_time(0.95, {WEB} | unwrap duration [$__interval]) by ()', legend="95th percentile"), + loki(f'max_over_time({WEB} | unwrap duration [$__interval]) by ()', ref="B", legend="slowest")], + 16, w=8, unit="s", desc="A page waits on ipx's /api/page-check before it is sent.") +y += 8 + +# ---- ipx: the API it answers behind ipx-web +row("ipx: API") ts("Requests by status", [loki(f'sum by (status) (count_over_time({HTTP} [$__interval]))', legend="{{status}}")], 0, bars=True, stack=True, desc="The event stream the page keeps open is left out.") ts("Response time", [loki(f'quantile_over_time(0.5, {HTTP} | unwrap duration_ms [$__interval]) by ()', legend="median"), @@ -154,52 +190,57 @@ ts("Response time", [loki(f'quantile_over_time(0.5, {HTTP} | unwrap duration_ms 12, unit="ms") y += 8 table("Slowest routes", [loki(f'topk(15, avg_over_time({HTTP} | route != "" | unwrap duration_ms [$__range]) by (method, route))', instant=True)], - 0, rename={"method": "Method", "route": "Route", "Value": "Average ms"}, sort=[{"displayName": "Average ms", "desc": True}]) + 0, w=8, rename={"method": "Method", "route": "Route", "Value": "Average ms"}, sort=[{"displayName": "Average ms", "desc": True}]) table("Busiest routes", [loki(f'topk(15, sum by (method, route) (count_over_time({HTTP} | route != "" [$__range])))', instant=True)], - 12, rename={"method": "Method", "route": "Route", "Value": "Requests"}, sort=[{"displayName": "Requests", "desc": True}]) + 8, w=8, rename={"method": "Method", "route": "Route", "Value": "Requests"}, sort=[{"displayName": "Requests", "desc": True}]) +ts("Open connections", [prom('max(ipx_http_requests_in_flight{job="ipx"})', legend="requests in flight"), + prom('max(ipx_event_listeners{job="ipx"})', ref="B", legend="live-update listeners"), + prom('max(ipx_event_queue{job="ipx"})', ref="C", legend="live updates unread"), + prom('sum(increase(ipx_events_missed_total{job="ipx"}[$__rate_interval]))', ref="D", legend="live updates missed")], + 16, w=8, desc="Each open page holds one live-update listener. Missed updates mean a page fell behind and reloads its lists.") y += 8 -# ---- Page, from ipx-web's log -row("Page") -ts("Page and file requests by status", [loki(f'sum by (status) (count_over_time({WEB} [$__interval]))', legend="{{status}}")], - 0, bars=True, stack=True, desc="What Caddy served itself: the app, the admin and sign-in pages, scripts, styles, icons. " - "A page answered 401 or 403 is ipx refusing it in /api/page-check.") -ts("Page response time", [loki(f'quantile_over_time(0.95, {WEB} | unwrap duration [$__interval]) by ()', legend="95th percentile"), - loki(f'max_over_time({WEB} | unwrap duration [$__interval]) by ()', ref="B", legend="slowest")], - 12, unit="s", desc="A page waits on ipx's /api/page-check before it is sent.") +# ---- ipx: the worker, one job at a time: scans, downloads, DeadAir +row("ipx: worker (scans and downloads)") +ts("Worker", [prom('max by (kind) (ipx_worker_job_seconds{job="ipx"})', legend="on a {{kind}} for"), + prom('max(ipx_last_scan_age_seconds{job="ipx"})', ref="B", legend="since the last scan finished")], + 0, w=8, unit="s", desc="A job's time that only climbs, or no scan for over 10 minutes, means the worker is stuck; the healthcheck still passes.") +ts("Jobs done", [prom('sum by (kind) (increase(ipx_jobs_total{job="ipx"}[$__rate_interval]))', legend="{{kind}}")], + 8, w=8, bars=True, stack=True) +ts("Time on jobs", [prom('sum by (kind) (increase(ipx_job_seconds_total{job="ipx"}[$__rate_interval]))', legend="{{kind}}")], + 16, w=8, unit="s", bars=True, stack=True, desc="Seconds the worker spent on each kind of job in each interval.") +y += 8 +ts("Waiting", [prom('ipx_worker_queue{job="ipx"}', legend="commands for the worker"), + prom('ipx_downloads_pending{job="ipx"}', ref="B", legend="downloads"), + prom('max(ipx_feed_retries{job="ipx"})', ref="C", legend="feeds to retry a lookup"), + prom('max(ipx_deadair_backlog{job="ipx"})', ref="D", legend="files for DeadAir")], + 0, w=8, desc="A queue that only grows means the worker is stuck or behind.") +ts("Feeds checked and new items", [loki(f'sum(sum_over_time({EV} | ev="scan_done" | unwrap feeds [$__interval]))', legend="feeds checked"), + loki(f'sum(sum_over_time({EV} | ev="feed_done" | unwrap new [$__interval]))', ref="B", legend="new items")], + 8, w=8, bars=True, desc="Feeds that were due and fetched; the rest were skipped as not due.") +ts("Downloads", [loki(f'sum(count_over_time({EV} | ev="download_done" [$__interval]))', legend="saved"), + loki(f'sum(count_over_time({EV} | ev="download_error" [$__interval]))', ref="B", legend="failed"), + prom('max(ipx_downloads_active{job="ipx"})', ref="C", legend="at once"), + prom('2 - max(ipx_torrent_slots_free{job="ipx"})', ref="D", legend="torrents")], + 16, w=8, bars=True) +y += 8 +ts("Download speed", [prom('sum(rate(ipx_download_bytes_total{job="ipx"}[$__rate_interval]))', legend="downloading")], + 0, w=8, unit="Bps") +ts("Feeds fetching at once", [prom('max(ipx_scan_feeds_fetching{job="ipx"})', legend="fetching")], 8, w=8) +ts("DeadAir on a file", [prom('max(ipx_deadair_busy_seconds{job="ipx"})', legend="for"), + prom('max(ipx_deadair_playing{job="ipx"})', ref="B", legend="playing through DeadAir")], + 16, w=8, unit="s", desc="How long DeadAir has been on the file it is analysing; a few seconds each.")["fieldConfig"]["overrides"] = [ + {"matcher": {"id": "byName", "options": "playing through DeadAir"}, + "properties": [{"id": "unit", "value": "short"}, {"id": "custom.axisPlacement", "value": "right"}]}] +y += 8 +table("Failing feeds", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="feed_error" [$__range]))', instant=True)], + 0, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}]) +table("Failed downloads", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="download_error" [$__range]))', instant=True)], + 12, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}]) y += 8 -# ---- Directory, from ipx-directory's log -row("Directory") -stat("Feeds shared", 'sum(max by (instance) (max_over_time(' + DIR + ' | message="snapshot" | unwrap accepted [6h])))', 0, - desc="The feeds each server's latest snapshot gave, added up: the same feed from two servers counts twice.") -stat("Snapshots", f'sum(count_over_time({DIR} | message="snapshot" [$__range])) or vector(0)', 4, - desc="ipx sends one when what it shares changes, and every half hour (#211).") -stat("Refused as private", f'sum(sum_over_time({DIR} | message="snapshot" | unwrap refused_private [$__range])) or vector(0)', 8, - thresholds=[{"color": "green", "value": None}, {"color": "red", "value": 1}], - desc="Feeds a server sent that the directory caught as private. Not 0 means a server's own check missed one.") -stat("Feed checks", f'sum(sum_over_time({DIR} | message="feeds checked" | unwrap checked [$__range])) or vector(0)', 12) -stat("Failed feed checks", f'sum(count_over_time({DIR} | message="feed check failed" [$__range])) or vector(0)', 16, - thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}], - desc="Mostly the publishers': a feed gone or moved.") -stat("Requests refused", f'sum(count_over_time({DIR_HTTP} | status =~ "401|403" [$__range])) or vector(0)', 20, - thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 100}], - desc="Without a token, or with one that has not the access asked for. Now and then is someone trying a public address.") -y += 4 -ts("Directory requests", [loki(f'sum by (status, token) (count_over_time({DIR_HTTP} [$__interval]))', legend="{{status}} {{token}}")], - 0, bars=True, stack=True) -ts("Feed checks", [loki(f'sum(sum_over_time({DIR} | message="feeds checked" | unwrap checked [$__interval]))', legend="checked"), - loki(f'sum(count_over_time({DIR} | message="feed check failed" [$__interval]))', ref="B", legend="failed")], - 12, bars=True, desc="Each feed every six hours, 25 at a time; a burst follows a snapshot with new feeds.") -y += 8 -table("Directory: failing feeds", [loki(f'sum by (feed, error) (count_over_time({DIR} | message="feed check failed" [$__range]))', instant=True)], - 0, rename={"feed": "Feed id", "error": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}]) -table("Snapshots by server", [loki(f'sum by (instance) (count_over_time({DIR} | message="snapshot" [$__range]))', instant=True)], - 12, rename={"instance": "Token", "Value": "Snapshots"}, sort=[{"displayName": "Snapshots", "desc": True}]) -y += 8 - -# ---- Daemon, from Prometheus: what the daemon holds between requests (#194) -row("Daemon") +# ---- ipx: the process, what it holds between requests and jobs (#194) +row("ipx: process") # Open connections stay open, idle, for ten minutes after a burst, so open at the max is not # pressure; drawn as a line of its own it read as the pool maxing out (#199). ts("Database connections", [prom('max(max_over_time(ipx_db_connections{job="ipx", state="in_use"}[$__interval]))', legend="in use"), @@ -210,42 +251,13 @@ ts("Database connections", [prom('max(max_over_time(ipx_db_connections{job="ipx" {"matcher": {"id": "byName", "options": "open"}, "properties": [{"id": "custom.lineStyle", "value": {"fill": "dash", "dash": [4, 4]}}, {"id": "custom.fillOpacity", "value": 0}, {"id": "color", "value": {"mode": "fixed", "fixedColor": "#8e8e8e"}}]}] -ts("Waiting", [prom('ipx_worker_queue{job="ipx"}', legend="commands for the worker"), - prom('ipx_runtime_queue{job="ipx"}', ref="B", legend="tasks for a thread"), - prom('ipx_downloads_pending{job="ipx"}', ref="C", legend="downloads")], - 8, w=8, desc="Scans and downloads run one at a time; a queue that only grows means the worker is stuck.") -ts("Listeners and tasks", [prom('ipx_event_listeners{job="ipx"}', legend="live-update listeners"), - prom('ipx_runtime_tasks{job="ipx"}', ref="B", legend="runtime tasks"), - prom('ipx_deadair_playing{job="ipx"}', ref="C", legend="playing through DeadAir")], - 16, w=8) +ts("Memory", [prom('ipx_process_resident_bytes{job="ipx"}', legend="resident")], 8, w=8, unit="bytes") +ts("Open files and sockets", [prom('ipx_process_open_fds{job="ipx"}', legend="open")], 16, w=8) y += 8 -ts("Memory", [prom('ipx_process_resident_bytes{job="ipx"}', legend="resident")], 0, w=12, unit="bytes") -ts("Open files and sockets", [prom('ipx_process_open_fds{job="ipx"}', legend="open")], 12, w=12) -y += 8 -# Jobs and queues (#195) -ts("Worker", [prom('max by (kind) (ipx_worker_job_seconds{job="ipx"})', legend="on a {{kind}} for"), - prom('max(ipx_last_scan_age_seconds{job="ipx"})', ref="B", legend="since the last scan finished")], - 0, w=8, unit="s", desc="A job's time that only climbs, or no scan for over 10 minutes, means the worker is stuck; the healthcheck still passes.") -ts("Jobs done", [prom('sum by (kind) (increase(ipx_jobs_total{job="ipx"}[$__rate_interval]))', legend="{{kind}}")], - 8, w=8, bars=True, stack=True) -ts("Time on jobs", [prom('sum by (kind) (increase(ipx_job_seconds_total{job="ipx"}[$__rate_interval]))', legend="{{kind}}")], - 16, w=8, unit="s", bars=True, stack=True, desc="Seconds the worker spent on each kind of job in each interval.") -y += 8 -ts("In flight", [prom('max(ipx_http_requests_in_flight{job="ipx"})', legend="web requests"), - prom('max(ipx_scan_feeds_fetching{job="ipx"})', ref="B", legend="feeds fetching"), - prom('max(ipx_downloads_active{job="ipx"})', ref="C", legend="downloads"), - prom('2 - max(ipx_torrent_slots_free{job="ipx"})', ref="D", legend="torrents")], - 0, w=8) -ts("Download speed", [prom('sum(rate(ipx_download_bytes_total{job="ipx"}[$__rate_interval]))', legend="downloading")], - 8, w=8, unit="Bps") -ts("Backlogs", [prom('max(ipx_deadair_backlog{job="ipx"})', legend="DeadAir to look at"), - prom('max(ipx_feed_retries{job="ipx"})', ref="B", legend="feeds to retry a lookup"), - prom('max(ipx_event_queue{job="ipx"})', ref="C", legend="live updates unread"), - prom('sum(increase(ipx_events_missed_total{job="ipx"}[$__rate_interval]))', ref="D", legend="live updates missed")], - 16, w=8) -y += 8 -ts("DeadAir on a file", [prom('max(ipx_deadair_busy_seconds{job="ipx"})', legend="for")], 0, w=12, unit="s", - desc="How long DeadAir has been on the file it is analysing; a few seconds each.") +ts("Runtime", [prom('ipx_runtime_tasks{job="ipx"}', legend="tasks"), + prom('ipx_runtime_queue{job="ipx"}', ref="B", legend="tasks waiting for a thread"), + prom('ipx_runtime_workers{job="ipx"}', ref="C", legend="threads")], + 0, w=12, desc="Tasks waiting for a thread for long means something is blocking the runtime.") ts("Artwork", [prom('max(ipx_art_cache_bytes{job="ipx"}) / max(ipx_art_cache_limit_bytes{job="ipx"})', legend="full"), prom('sum(increase(ipx_art_requests_total{job="ipx", result="hit"}[$__rate_interval])) / ' 'sum(increase(ipx_art_requests_total{job="ipx"}[$__rate_interval]))', ref="B", legend="hit rate"), @@ -253,14 +265,37 @@ ts("Artwork", [prom('max(ipx_art_cache_bytes{job="ipx"}) / max(ipx_art_cache_lim 12, w=12, unit="percentunit", desc="Full sits near 100% once the cache has filled, since each sweep trims it to fit. Hit rate is the share served from disk: " "the rest was fetched from the publisher. Evicted, on the right, is images each sweep threw out to stay under the " - "limit: many at every sweep, with a falling hit rate, means the limit is too small.")["fieldConfig"]["overrides"] = [ - {"matcher": {"id": "byName", "options": "images evicted"}, - "properties": [{"id": "unit", "value": "short"}, {"id": "custom.axisPlacement", "value": "right"}, - {"id": "custom.drawStyle", "value": "bars"}, {"id": "custom.fillOpacity", "value": 60}]}] + "limit: many at every sweep, with a falling hit rate, means the limit is too small.")["fieldConfig"]["overrides"] = bar_overrides("images evicted") y += 8 -# ---- Traces -row("Traces") +# ---- ipx-directory: public at ipx-dir.sdf1.net, shared by ipx servers (#204) +row("ipx-directory") +stat("Snapshots", loki(f'sum(count_over_time({DIR} | message="snapshot" [$__range])) or vector(0)', instant=True), 0, + desc="ipx sends one when what it shares changes, and every half hour (#211).") +stat("Feed checks", loki(f'sum(sum_over_time({DIR} | message="feeds checked" | unwrap checked [$__range])) or vector(0)', instant=True), 4, + desc="Each feed every six hours, 25 at a time.") +stat("Failed feed checks", loki(f'sum(count_over_time({DIR} | message="feed check failed" [$__range])) or vector(0)', instant=True), 8, + thresholds=GOOD_BAD, desc="Mostly the publishers': a feed gone or moved.") +stat("Requests", loki(f'sum(count_over_time({DIR_HTTP} [$__range])) or vector(0)', instant=True), 12) +stat("Requests refused", loki(f'sum(count_over_time({DIR_HTTP} | status =~ "401|403" [$__range])) or vector(0)', instant=True), 16, + thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 100}], + desc="Without a token, or with one that has not the access asked for. Now and then is someone trying a public address.") +stat("Slowest request", loki(f'max(max_over_time({DIR_HTTP} | unwrap duration_ms [$__range])) or vector(0)', instant=True), 20, unit="ms") +y += 4 +ts("Requests by token", [loki(f'sum by (status, token) (count_over_time({DIR_HTTP} [$__interval]))', legend="{{status}} {{token}}")], + 0, bars=True, stack=True) +ts("Feed checks over time", [loki(f'sum(sum_over_time({DIR} | message="feeds checked" | unwrap checked [$__interval]))', legend="checked"), + loki(f'sum(count_over_time({DIR} | message="feed check failed" [$__interval]))', ref="B", legend="failed")], + 12, bars=True, desc="A burst follows a snapshot with new feeds.") +y += 8 +table("Failing feeds in the directory", [loki(f'sum by (feed, error) (count_over_time({DIR} | message="feed check failed" [$__range]))', instant=True)], + 0, rename={"feed": "Feed id", "error": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}]) +table("Snapshots by server", [loki(f'sum by (instance) (count_over_time({DIR} | message="snapshot" [$__range]))', instant=True)], + 12, rename={"instance": "Token", "Value": "Snapshots"}, sort=[{"displayName": "Snapshots", "desc": True}]) +y += 8 + +# ---- Traces: ipx's, to Tempo; ipx-web and the directory send none +row("Traces (ipx)") for x, title, q in [(0, "Recent traces", TRACES + "}"), (12, "Slow traces (over 2s)", TRACES + " && duration > 2s}")]: panel("table", title, 12, 10, x, @@ -269,19 +304,17 @@ for x, title, q in [(0, "Recent traces", TRACES + "}"), fieldConfig={"defaults": {}, "overrides": []}) y += 10 -# ---- Log -row("Log") -panel("logs", "Warnings and errors", 24, 10, 0, [loki(BAD + TEXT)], - options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True}) +# ---- Logs +row("Logs") +panel("logs", "ipx: warnings and errors", 12, 10, 0, [loki(BAD + TEXT)], options=LOG_OPTS) +panel("logs", "ipx-web and ipx-directory: warnings and errors", 12, 10, 12, + [loki(OTHERS_BAD + ' | line_format "{{.container}} {{.level}}: {{.message}}{{.msg}}"')], options=LOG_OPTS) y += 10 -panel("logs", "Page and directory: warnings and errors", 24, 8, 0, [loki(OTHERS_BAD + ' | line_format "{{.container}} {{.level}}: {{.message}}{{.msg}}"')], - options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True}) -y += 8 -panel("logs", "Log", 24, 12, 0, +panel("logs", "ipx log", 24, 12, 0, [loki(SEL + ' | json | __error__="" | path != "/api/events" | ev != "feed_skip" | ev != "status"' ' | message != "-> {\\"cmd\\":\\"status\\"}"' + TEXT)], - description="Without the event stream's requests, not-due skips and healthcheck status calls.", - options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True}) + description="Without the event stream's requests, not-due skips and healthcheck status calls.", options=LOG_OPTS) +y += 12 dash = { "uid": "ipx", "title": "iPX", "tags": ["ipx"], "timezone": "browser", "schemaVersion": 39,