diff --git a/grafana/alerts.py b/grafana/alerts.py index 5ae55e4..5f97a13 100644 --- a/grafana/alerts.py +++ b/grafana/alerts.py @@ -62,8 +62,11 @@ rules = [ "TypeSafe refused calls as over its limits (#277) or malformed (#257): ipx is asking wrongly."), rule("signals-stale", "Signals have stopped", 'max(ipx_signals_last_pass_age_seconds{job="ipx"})', "gt", 900, "5m", "The Signals judge has not finished a pass for 15 minutes; its signals trace shows which Jev call holds it."), + # Only once it has been up 13 hours: its rounds come in bursts about twelve hours apart, and the + # count starts again at each restart, so a deploy would read as stopped. rule("directory-refresh", "The directory has stopped checking feeds", - 'sum(increase(directory_feed_checks_total{job="ipx-directory"}[13h]))', "lt", 1, "30m", + 'sum(increase(directory_feed_checks_total{job="ipx-directory"}[13h])) < 1' + ' and on() max(directory_uptime_seconds{job="ipx-directory"}) > 46800', "lt", 1, "30m", "No feed checked in 13 hours; each is due every 12.", window=46800), # Tower, which everything shares: a stall here is every request slow (2026-10-18, load 160). rule("load", "Tower is overloaded", 'max(node_load5{job="node"})', "gt", 48, "10m",