Prod check reads the daemon's metrics from Prometheus (#194)

query.py prom asks Prometheus as it asks Loki, and a sixth check looks for
a pool at its max, a worker or runtime queue that only grows, memory, open
files or tasks that only climb, and restarts by the uptime's resets.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.claude/skills/ipx-prod-check/query.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
-"""Ask production's Loki or Tempo a question, from anywhere that can run docker on Tower.
+"""Ask production's Loki, Tempo or Prometheus a question, from anywhere that can run docker on Tower.
 
     query.py logs    '<LogQL log query>'    [since]   lines, newest first
     query.py metric  '<LogQL metric query>' [since]   one value per series; $range is `since`
     query.py traces  '<TraceQL query>'      [since]   matching traces, slowest first
     query.py trace   <trace id>                       one trace's spans, as a tree
+    query.py prom    '<PromQL query>'       [since]   one value per series, now; $range is `since`
 
 `since` is 1h, 24h, 7d and the like (default 24h). Loki and Tempo publish no query port on the
 host, so each call runs a throwaway alpine container on the monitoring project's network.
@@ -55,6 +56,14 @@ def main():
             when = time.strftime("%m-%d %H:%M:%S", time.gmtime(int(t["startTimeUnixNano"]) / 1e9))
             print(f"{t.get('durationMs', 0):>8}ms  {when}Z  {t['traceID']}  {t.get('rootTraceName', '')}")
         print(f"-- {len(traces)} trace(s)", file=sys.stderr)
+    elif kind == "prom":
+        # ipx's /metrics (#194), scraped as job ipx; through the network like the others.
+        r = fetch(f"http://prometheus:9090/api/v1/query?query={enc(q.replace('$range', since))}&time={now}")
+        rows = sorted(r["data"]["result"], key=lambda s: -float(s["value"][1]))
+        for s in rows:
+            labels = {k: v for k, v in s["metric"].items() if k not in ("job", "instance")}
+            print(f"{float(s['value'][1]):>14g}  {json.dumps(labels) if labels else ''}")
+        print(f"-- {len(rows)} series", file=sys.stderr)
     elif kind == "trace":
         r = fetch(f"http://tempo:3200/api/traces/{q}")
         spans = [sp for b in r.get("batches", r.get("resourceSpans", []))