From ca73b8136fc37b875a61671e5317616338428e74 Mon Sep 17 00:00:00 2001 From: rays Date: Thu, 15 Oct 2026 21:25:50 +0000 Subject: [PATCH] Grafana: the connections panel shows in use and ready, not a pool that looks full (#251) Since #241 the daemon keeps all 20 connections open, so the panel's open line sat on the pool's maximum and read as exhausted; in use has been at most 4 in a week. It draws in use, ready (idle) and the pool now, and the prod check reads idle the same way. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/ipx-prod-check/SKILL.md | 4 ++-- grafana/dashboard.py | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.claude/skills/ipx-prod-check/SKILL.md b/.claude/skills/ipx-prod-check/SKILL.md index 578649e..decd425 100644 --- a/.claude/skills/ipx-prod-check/SKILL.md +++ b/.claude/skills/ipx-prod-check/SKILL.md @@ -196,8 +196,8 @@ a count says something happened, the lines and traces say why. $Q prom 'max(max_over_time(ipx_deadair_busy_seconds{job="ipx"}[$range]))' 24h ``` `up` 0 means Prometheus cannot reach the port: the container is down or `metrics_bind` is - unset. Connections in use at the pool's max (20 on Postgres) means something is waiting for - one: find the slow requests or scan of that time in check 3 and 5. A worker queue that only + unset. The daemon keeps all 20 connections open on Postgres (#241), so `idle` is the pool's 20 less + those in use; in use at 20, idle at 0, means something is waiting for one: find the slow requests or scan of that time in check 3 and 5. A worker queue that only grows means downloads are not keeping up, or the worker is stuck on one job. A runtime queue above 0 for more than a scrape or two means every thread is busy: something is blocking a thread or burning one. Memory, open files or tasks only climbing across a day with no restart diff --git a/grafana/dashboard.py b/grafana/dashboard.py index 513f3cd..e7ac53c 100644 --- a/grafana/dashboard.py +++ b/grafana/dashboard.py @@ -259,13 +259,10 @@ row("ipx: process") # Open connections stay open, idle, for ten minutes after a burst, so open at the max is not # pressure; drawn as a line of its own it read as the pool maxing out (#199). ts("Database connections", [prom('max(max_over_time(ipx_db_connections{job="ipx", state="in_use"}[$__interval]))', legend="in use"), - prom('sum(ipx_db_connections{job="ipx"})', ref="B", legend="open"), - prom('max(ipx_db_connections_max{job="ipx"})', ref="C", legend="most the pool opens")], - 0, w=8, desc="In use reaching the most the pool opens means requests and scans are waiting for a connection. " - "Open connections stay open, idle, for ten minutes after a burst.")["fieldConfig"]["overrides"] = [ - {"matcher": {"id": "byName", "options": "open"}, - "properties": [{"id": "custom.lineStyle", "value": {"fill": "dash", "dash": [4, 4]}}, - {"id": "custom.fillOpacity", "value": 0}, {"id": "color", "value": {"mode": "fixed", "fixedColor": "#8e8e8e"}}]}] + prom('min(min_over_time(ipx_db_connections{job="ipx", state="idle"}[$__interval]))', ref="B", legend="ready"), + prom('max(ipx_db_connections_max{job="ipx"})', ref="C", legend="the pool")], + 0, w=8, desc="The daemon keeps the whole pool open (#241), so in use and ready add up to the pool. In use reaching " + "the pool, ready at 0, means requests and scans are waiting for a connection (#251).") ts("Memory", [prom('ipx_process_resident_bytes{job="ipx"}', legend="resident")], 8, w=8, unit="bytes") ts("Open files and sockets", [prom('ipx_process_open_fds{job="ipx"}', legend="open")], 16, w=8) y += 8