"""The iPX alert rules in Grafana (#365): what should be looked at now rather than at the next prod check, written out as Grafana's alerting provisioning, beside the dashboard. python3 grafana/alerts.py > /mnt/fast/arcane/projects/monitoring/grafana-provisioning/alerting/ipx.json Grafana reads it at start, or at once from POST /api/admin/provisioning/alerting/reload. There is no contact point yet: a rule that fires shows in Grafana's Alerting page, and adding one (email, ntfy, a webhook) is a file in the same folder. Each rule is a query and a threshold on it; the query's value is in the alert, so it says how bad as well as what. """ import json # Prometheus was added in Grafana's settings, not provisioned, so its uid is Grafana's own. PROM = "eenybu3ao2t4wb" LOKI = "P8E80F9AEF21F6940" def rule(uid, title, expr, op, value, for_, summary, *, ds=PROM, window=600, no_data="OK"): """Fires when `expr` is `op` (gt, lt) `value` for `for_`. No data is fine by default: most of these metrics are absent until there is something to say (no job running, no scan yet).""" return { "uid": f"ipx-{uid}", "title": title, "condition": "C", "for": for_, "data": [ {"refId": "A", "relativeTimeRange": {"from": window, "to": 0}, "datasourceUid": ds, # Loki's instant query is a queryType; Prometheus's a flag. "model": {"refId": "A", "expr": expr, **({"queryType": "instant"} if ds == LOKI else {"instant": True})}}, {"refId": "C", "datasourceUid": "__expr__", "model": {"refId": "C", "type": "threshold", "expression": "A", "conditions": [{"evaluator": {"type": op, "params": [value]}}]}}, ], "noDataState": no_data, "execErrState": "Error", "annotations": {"summary": summary}, "labels": {"service": "ipx"}, } rules = [ # Down: Prometheus cannot reach it. A container gone is no data at all, so that fires too. rule("down", "ipx is down", 'up{job="ipx"}', "lt", 1, "2m", "Prometheus cannot reach ipx's /metrics: the iPX container is down or restarting.", no_data="Alerting"), rule("directory-down", "ipx-directory is down", 'up{job="ipx-directory"}', "lt", 1, "5m", "Prometheus cannot reach the directory's /metrics.", no_data="Alerting"), rule("web-down", "ipx-web is down", 'up{job="ipx-web"}', "lt", 1, "5m", "Prometheus cannot reach Caddy's /metrics: the page is likely down too.", no_data="Alerting"), # The worker: the daemon answers `ipx status` while it is stuck, so only these say so. rule("scan-stale", "ipx's scans have stopped", 'max(ipx_last_scan_age_seconds{job="ipx"})', "gt", 900, "5m", "No scan has finished for over 15 minutes; scans run at least every 10. The worker is stuck: its job and trace say on what."), rule("job-long", "ipx's worker is stuck on one job", 'max(ipx_worker_job_seconds{job="ipx"})', "gt", 1800, "1m", "One job has run for over 30 minutes (ipx_worker_job_seconds says which kind)."), rule("pool-full", "ipx's database connections are all in use", 'max_over_time(ipx_db_connections{job="ipx",state="idle"}[5m])', "lt", 1, "5m", "No idle connection for five minutes: requests and scans are waiting for one (ipx_db_wait_seconds)."), rule("5xx", "ipx is answering with server errors", 'sum(increase(ipx_http_request_duration_seconds_count{job="ipx",status=~"5.."}[10m]))', "gt", 5, "0s", "More than five 5xx answers in ten minutes; ipx's log has a warning for each, with its trace."), rule("502", "ipx-web cannot reach ipx", 'sum(increase(caddy_http_request_duration_seconds_count{job="ipx-web",code=~"502|504"}[10m]))', "gt", 5, "0s", "Caddy answered 502 or 504 for ipx more than five times in ten minutes: ipx is not answering it."), rule("jev", "ipx's calls to Jev are refused", 'sum(increase(ipx_jev_requests_total{job="ipx",result=~"rate_limited|malformed"}[15m]))', "gt", 0, "0s", "TypeSafe refused calls as over its limits (#277) or malformed (#257): ipx is asking wrongly."), rule("signals-stale", "Signals have stopped", 'max(ipx_signals_last_pass_age_seconds{job="ipx"})', "gt", 900, "5m", "The Signals judge has not finished a pass for 15 minutes; its signals trace shows which Jev call holds it."), # Only once it has been up 13 hours: its rounds come in bursts about twelve hours apart, and the # count starts again at each restart, so a deploy would read as stopped. rule("directory-refresh", "The directory has stopped checking feeds", 'sum(increase(directory_feed_checks_total{job="ipx-directory"}[13h])) < 1' ' and on() max(directory_uptime_seconds{job="ipx-directory"}) > 46800', "lt", 1, "30m", "No feed checked in 13 hours; each is due every 12.", window=46800), # Tower, which everything shares: a stall here is every request slow (2026-10-18, load 160). rule("load", "Tower is overloaded", 'max(node_load5{job="node"})', "gt", 48, "10m", "Load over 48 on twelve threads for ten minutes: builds, or processes blocked on a disk."), rule("memory", "Tower is out of memory", 'min(node_memory_MemAvailable_bytes{job="node"})', "lt", 2e9, "5m", "Under 2 GB available, with no swap: next comes stalling on page faults, then the OOM killer."), # The monitoring itself (#366): nothing else watches it. rule("monitoring-down", "Part of the monitoring is down", 'min by (job) (up{job=~"prometheus|tempo|loki|alloy"})', "lt", 1, "5m", "Prometheus cannot reach Tempo, Loki or Alloy."), rule("tempo-blocks", "Tempo cannot read its stored traces", '(max(tempodb_blocklist_tenant_index_age_seconds) > bool 900) + (max(tempodb_blocklist_length) < bool 1)', "gt", 0, "10m", "Tempo's look over its stored blocks is failing (2026-10-25: one empty block); searches see only the last 15 " "minutes. Its log says which block: move it out of /mnt/user/logs/tempo/blocks/single-tenant."), rule("logs-stopped", "No lines from iPX reach Loki", 'sum(count_over_time({container="iPX"}[3h]))', "lt", 1, "0s", "Loki has nothing from iPX in three hours: Alloy is not shipping it, or the container is down.", ds=LOKI, window=10800, no_data="Alerting"), ] print(json.dumps({"apiVersion": 1, "groups": [ {"orgId": 1, "name": "ipx", "folder": "iPX", "interval": "1m", "rules": rules}, ]}, indent=2))