rays
c4f0a362d8
Its rounds come in bursts about twelve hours apart and its count starts again at each restart, so for hours after a deploy no check had been counted and the rule would have fired. It judges only once the directory has been up 13 hours. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
92 lines
6.4 KiB
Python
92 lines
6.4 KiB
Python
"""The iPX alert rules in Grafana (#365): what should be looked at now rather than at the next prod
|
|
check, written out as Grafana's alerting provisioning, beside the dashboard.
|
|
|
|
python3 grafana/alerts.py > /mnt/fast/arcane/projects/monitoring/grafana-provisioning/alerting/ipx.json
|
|
|
|
Grafana reads it at start, or at once from POST /api/admin/provisioning/alerting/reload. There is
|
|
no contact point yet: a rule that fires shows in Grafana's Alerting page, and adding one (email,
|
|
ntfy, a webhook) is a file in the same folder. Each rule is a query and a threshold on it; the
|
|
query's value is in the alert, so it says how bad as well as what.
|
|
"""
|
|
import json
|
|
|
|
# Prometheus was added in Grafana's settings, not provisioned, so its uid is Grafana's own.
|
|
PROM = "eenybu3ao2t4wb"
|
|
LOKI = "P8E80F9AEF21F6940"
|
|
|
|
|
|
def rule(uid, title, expr, op, value, for_, summary, *, ds=PROM, window=600, no_data="OK"):
|
|
"""Fires when `expr` is `op` (gt, lt) `value` for `for_`. No data is fine by default: most of
|
|
these metrics are absent until there is something to say (no job running, no scan yet)."""
|
|
return {
|
|
"uid": f"ipx-{uid}", "title": title, "condition": "C", "for": for_,
|
|
"data": [
|
|
{"refId": "A", "relativeTimeRange": {"from": window, "to": 0}, "datasourceUid": ds,
|
|
# Loki's instant query is a queryType; Prometheus's a flag.
|
|
"model": {"refId": "A", "expr": expr,
|
|
**({"queryType": "instant"} if ds == LOKI else {"instant": True})}},
|
|
{"refId": "C", "datasourceUid": "__expr__",
|
|
"model": {"refId": "C", "type": "threshold", "expression": "A",
|
|
"conditions": [{"evaluator": {"type": op, "params": [value]}}]}},
|
|
],
|
|
"noDataState": no_data, "execErrState": "Error",
|
|
"annotations": {"summary": summary},
|
|
"labels": {"service": "ipx"},
|
|
}
|
|
|
|
|
|
rules = [
|
|
# Down: Prometheus cannot reach it. A container gone is no data at all, so that fires too.
|
|
rule("down", "ipx is down", 'up{job="ipx"}', "lt", 1, "2m",
|
|
"Prometheus cannot reach ipx's /metrics: the iPX container is down or restarting.", no_data="Alerting"),
|
|
rule("directory-down", "ipx-directory is down", 'up{job="ipx-directory"}', "lt", 1, "5m",
|
|
"Prometheus cannot reach the directory's /metrics.", no_data="Alerting"),
|
|
rule("web-down", "ipx-web is down", 'up{job="ipx-web"}', "lt", 1, "5m",
|
|
"Prometheus cannot reach Caddy's /metrics: the page is likely down too.", no_data="Alerting"),
|
|
# The worker: the daemon answers `ipx status` while it is stuck, so only these say so.
|
|
rule("scan-stale", "ipx's scans have stopped", 'max(ipx_last_scan_age_seconds{job="ipx"})', "gt", 900, "5m",
|
|
"No scan has finished for over 15 minutes; scans run at least every 10. The worker is stuck: its job and trace say on what."),
|
|
rule("job-long", "ipx's worker is stuck on one job", 'max(ipx_worker_job_seconds{job="ipx"})', "gt", 1800, "1m",
|
|
"One job has run for over 30 minutes (ipx_worker_job_seconds says which kind)."),
|
|
rule("pool-full", "ipx's database connections are all in use",
|
|
'max_over_time(ipx_db_connections{job="ipx",state="idle"}[5m])', "lt", 1, "5m",
|
|
"No idle connection for five minutes: requests and scans are waiting for one (ipx_db_wait_seconds)."),
|
|
rule("5xx", "ipx is answering with server errors",
|
|
'sum(increase(ipx_http_request_duration_seconds_count{job="ipx",status=~"5.."}[10m]))', "gt", 5, "0s",
|
|
"More than five 5xx answers in ten minutes; ipx's log has a warning for each, with its trace."),
|
|
rule("502", "ipx-web cannot reach ipx",
|
|
'sum(increase(caddy_http_request_duration_seconds_count{job="ipx-web",code=~"502|504"}[10m]))', "gt", 5, "0s",
|
|
"Caddy answered 502 or 504 for ipx more than five times in ten minutes: ipx is not answering it."),
|
|
rule("jev", "ipx's calls to Jev are refused",
|
|
'sum(increase(ipx_jev_requests_total{job="ipx",result=~"rate_limited|malformed"}[15m]))', "gt", 0, "0s",
|
|
"TypeSafe refused calls as over its limits (#277) or malformed (#257): ipx is asking wrongly."),
|
|
rule("signals-stale", "Signals have stopped", 'max(ipx_signals_last_pass_age_seconds{job="ipx"})', "gt", 900, "5m",
|
|
"The Signals judge has not finished a pass for 15 minutes; its signals trace shows which Jev call holds it."),
|
|
# Only once it has been up 13 hours: its rounds come in bursts about twelve hours apart, and the
|
|
# count starts again at each restart, so a deploy would read as stopped.
|
|
rule("directory-refresh", "The directory has stopped checking feeds",
|
|
'sum(increase(directory_feed_checks_total{job="ipx-directory"}[13h])) < 1'
|
|
' and on() max(directory_uptime_seconds{job="ipx-directory"}) > 46800', "lt", 1, "30m",
|
|
"No feed checked in 13 hours; each is due every 12.", window=46800),
|
|
# Tower, which everything shares: a stall here is every request slow (2026-10-18, load 160).
|
|
rule("load", "Tower is overloaded", 'max(node_load5{job="node"})', "gt", 48, "10m",
|
|
"Load over 48 on twelve threads for ten minutes: builds, or processes blocked on a disk."),
|
|
rule("memory", "Tower is out of memory", 'min(node_memory_MemAvailable_bytes{job="node"})', "lt", 2e9, "5m",
|
|
"Under 2 GB available, with no swap: next comes stalling on page faults, then the OOM killer."),
|
|
# The monitoring itself (#366): nothing else watches it.
|
|
rule("monitoring-down", "Part of the monitoring is down", 'min by (job) (up{job=~"prometheus|tempo|loki|alloy"})',
|
|
"lt", 1, "5m", "Prometheus cannot reach Tempo, Loki or Alloy."),
|
|
rule("tempo-blocks", "Tempo cannot read its stored traces",
|
|
'(max(tempodb_blocklist_tenant_index_age_seconds) > bool 900) + (max(tempodb_blocklist_length) < bool 1)',
|
|
"gt", 0, "10m",
|
|
"Tempo's look over its stored blocks is failing (2026-10-25: one empty block); searches see only the last 15 "
|
|
"minutes. Its log says which block: move it out of /mnt/user/logs/tempo/blocks/single-tenant."),
|
|
rule("logs-stopped", "No lines from iPX reach Loki", 'sum(count_over_time({container="iPX"}[3h]))', "lt", 1, "0s",
|
|
"Loki has nothing from iPX in three hours: Alloy is not shipping it, or the container is down.",
|
|
ds=LOKI, window=10800, no_data="Alerting"),
|
|
]
|
|
|
|
print(json.dumps({"apiVersion": 1, "groups": [
|
|
{"orgId": 1, "name": "ipx", "folder": "iPX", "interval": "1m", "rules": rules},
|
|
]}, indent=2))
|