Prometheus metrics for the daemon's state between requests (#194)

Logs and traces show requests, scans and downloads; nothing showed what
the daemon holds meanwhile. /metrics, behind sign-in, gives database
connections in use and idle against the pool's maximum, commands waiting
for the worker, live-update listeners, free torrent slots, episodes
playing through DeadAir, the library's counts, the tokio runtime's
workers, tasks and queue, and the process's memory and open files, in
Prometheus's text format, written by hand rather than with a client
library. Not logged or traced: Prometheus asks every 15 s. The iPX
dashboard gains a Daemon row of them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

grafana/dashboard.py

@@ -2,15 +2,17 @@
 
     python3 grafana/dashboard.py > /mnt/fast/arcane/projects/monitoring/grafana-provisioning/dashboards/ipx.json
 
-Grafana reads that file on its own within a minute; edits made in Grafana are refused. The panels
-read the fields of ipx's JSON log (IPX_LOG_FORMAT=json): renaming a field in web.rs access_log or
-ipc.rs log_event has to be matched here. `dashboard.py queries` prints each
+Grafana reads that file on its own within a minute; edits made in Grafana are refused. The
+Daemon row reads ipx's /metrics from Prometheus (job ipx, #194); the rest read the fields of
+ipx's JSON log (IPX_LOG_FORMAT=json): renaming a field in web.rs access_log or ipc.rs log_event
+has to be matched here. `dashboard.py queries` prints each
 query, to try against Loki.
 """
 import json, sys
 
 LOKI = {"type": "loki", "uid": "${loki}"}
 TEMPO = {"type": "tempo", "uid": "${tempo}"}
+PROM = {"type": "prometheus", "uid": "${prometheus}"}
 SEL = '{container="iPX"}'
 # ipx logs one JSON object a line (IPX_LOG_FORMAT=json). Each scan and download event carries its
 # fields (ev, feed, new, bytes, msg, error.type, ...); each request its method, path, route, status
@@ -50,6 +52,14 @@ def loki(expr, ref="A", legend=None, instant=False, kind=None):
     return t
 
 
+def prom(expr, ref="A", legend=None):
+    # ipx's /metrics (#194), scraped by Prometheus as job ipx.
+    t = {"refId": ref, "datasource": PROM, "expr": expr}
+    if legend:
+        t["legendFormat"] = legend
+    return t
+
+
 def row(title):
     global y, pid
     pid += 1
@@ -135,6 +145,25 @@ table("Busiest routes", [loki(f'topk(15, sum by (method, route) (count_over_time
       12, rename={"method": "Method", "route": "Route", "Value": "Requests"}, sort=[{"displayName": "Requests", "desc": True}])
 y += 8
 
+# ---- Daemon, from Prometheus: what the daemon holds between requests (#194)
+row("Daemon")
+ts("Database connections", [prom('ipx_db_connections{job="ipx", state="in_use"}', legend="in use"),
+                            prom('ipx_db_connections{job="ipx", state="idle"}', ref="B", legend="idle"),
+                            prom('ipx_db_connections_max{job="ipx"}', ref="C", legend="most the pool opens")],
+   0, w=8, desc="In use reaching the most the pool opens means requests and scans are waiting for a connection.")
+ts("Waiting", [prom('ipx_worker_queue{job="ipx"}', legend="commands for the worker"),
+               prom('ipx_runtime_queue{job="ipx"}', ref="B", legend="tasks for a thread"),
+               prom('ipx_downloads_pending{job="ipx"}', ref="C", legend="downloads")],
+   8, w=8, desc="Scans and downloads run one at a time; a queue that only grows means the worker is stuck.")
+ts("Listeners and tasks", [prom('ipx_event_listeners{job="ipx"}', legend="live-update listeners"),
+                           prom('ipx_runtime_tasks{job="ipx"}', ref="B", legend="runtime tasks"),
+                           prom('ipx_deadair_playing{job="ipx"}', ref="C", legend="playing through DeadAir")],
+   16, w=8)
+y += 8
+ts("Memory", [prom('ipx_process_resident_bytes{job="ipx"}', legend="resident")], 0, w=12, unit="bytes")
+ts("Open files and sockets", [prom('ipx_process_open_fds{job="ipx"}', legend="open")], 12, w=12)
+y += 8
+
 # ---- Traces
 row("Traces")
 for x, title, q in [(0, "Recent traces", TRACES + "}"),
@@ -162,6 +191,7 @@ dash = {
     "templating": {"list": [
         {"name": "loki", "label": "Logs", "type": "datasource", "query": "loki", "current": {}, "hide": 0},
         {"name": "tempo", "label": "Traces", "type": "datasource", "query": "tempo", "current": {}, "hide": 0},
+        {"name": "prometheus", "label": "Metrics", "type": "datasource", "query": "prometheus", "current": {}, "hide": 0},
     ]},
     "links": [{"title": "iPX", "type": "link", "url": "https://ipodderx.sdf1.net", "targetBlank": True}],
     "panels": panels,