rays/ipx
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases 20 Wiki Activity

ipx-play: metrics for Prometheus, a dashboard row and a down alert (#458)

Browse Source 
ipx-play sent traces and logged JSON but served no /metrics, so nothing showed whether
shared links were opening or failing. It now serves them on metrics_bind (9189 in
production, off the tunnel's port), with the directory's histogram and /proc reading:
requests by method, route and status (by route, so each shared episode is not a row of its
own, and an unknown address is route=""), episodes shared, memory, open files, uptime and
version.

The iPX dashboard gets an ipx-play row, its warnings join the other containers' log panel,
and alerts.py fires when Prometheus cannot reach it. The prod-check skill and docs/play.md
name it, with the scrape job and the OTEL_RESOURCE_ATTRIBUTES tag the trace panels need.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-11-13 22:08:11 -05:00
parent 47e093e313
commit 5d0ff1ac57
10 changed files with 209 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
grafana/alerts.py
View File

@@ -41,6 +41,8 @@ rules = [
"Prometheus cannot reach ipx's /metrics: the iPX container is down or restarting.", no_data="Alerting"),
rule("directory-down", "ipx-directory is down", 'up{job="ipx-directory"}', "lt", 1, "5m",
"Prometheus cannot reach the directory's /metrics.", no_data="Alerting"),
rule("play-down", "ipx-play is down", 'up{job="ipx-play"}', "lt", 1, "5m",
"Prometheus cannot reach ipx-play's /metrics: shared links are likely not opening.", no_data="Alerting"),
rule("web-down", "ipx-web is down", 'up{job="ipx-web"}', "lt", 1, "5m",
"Prometheus cannot reach Caddy's /metrics: the page is likely down too.", no_data="Alerting"),
# The worker: the daemon answers `ipxd status` while it is stuck, so only these say so.

42
grafana/dashboard.py
View File

@@ -1,11 +1,11 @@
"""The iPX dashboard in Grafana: the three containers of the Arcane project ipx (ipx, ipx-web,
ipx-directory) and Tower under them, from Prometheus, Loki (their logs, shipped by Alloy) and Tempo.
"""The iPX dashboard in Grafana: the four containers of the Arcane project ipx (ipx, ipx-web,
ipx-directory, ipx-play) and Tower under them, from Prometheus, Loki (their logs, shipped by Alloy) and Tempo.
python3 grafana/dashboard.py > /mnt/fast/arcane/projects/monitoring/grafana-provisioning/dashboards/ipx.json
Grafana reads that file on its own within a minute; edits made in Grafana are refused. Rows go
top-down as a request does: an overview, Tower, then each container (ipx-web, ipx's API, its
worker and its process, the directory), then traces and logs. Prometheus scrapes ipx's /metrics
worker and its process, the directory, ipx-play), then traces and logs. Prometheus scrapes ipx's /metrics
(job ipx, #194) and Tower's node exporter (job node); nothing scrapes the containers themselves,
so ipx-web and the directory are read from their logs. The log panels read fields: ipx's
(ipc.rs log_event), ipx's metrics (metrics.rs), the directory's and Caddy's; renaming one has to be
@@ -41,7 +41,9 @@ WEB = ('{container="ipx-web"} |= "http.log.access" | json | __error__=""'
' | request_uri !~ "/(api|media)/.*" | request_remote_ip != "127.0.0.1"') # not its healthcheck
# Less what Caddy says at every start, by design: no admin endpoint, and no HTTPS or HTTP/3
# behind the tunnel; and its goodbye on a restart.
OTHERS_BAD = ('{container=~"ipx-web|ipx-directory"} | json | __error__="" | level =~ "(?i)warn|error"'
# ipx-play's (#447): a line a request (method, path, status, duration_ms, token), and one a share.
PLAY_REQ = "play_http_request_duration_seconds"
OTHERS_BAD = ('{container=~"ipx-web|ipx-directory|ipx-play"} | json | __error__="" | level =~ "(?i)warn|error"'
' | msg !~ "admin endpoint disabled|automatic HTTPS is completely disabled|HTTP/[23] skipped.*|exiting; byeee.*"')
@@ -149,7 +151,7 @@ stat("Page and API errors", loki(f'sum(count_over_time({WEB_ALL} | status >= 500
thresholds=GOOD_BAD, desc="Answers of 500 and up from ipx-web, for the page or passed on from ipx: 502 is ipx not answering.")
stat("Warnings and errors", loki(f'sum(count_over_time({BAD} [$__range])) + sum(count_over_time({OTHERS_BAD} [$__range])) or vector(0)', instant=True), 20,
thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}, {"color": "red", "value": 50}],
desc="All three containers, in the time range.")
desc="All four containers, in the time range.")
y += 4
stat("Feed failures", loki(f'sum(count_over_time({EV} | ev="feed_error" [$__range])) or vector(0)', instant=True), 0,
thresholds=GOOD_BAD, desc="Failed feed checks in the time range.")
@@ -394,7 +396,33 @@ ts("Feed bytes fetched", [prom('rate(ipx_feed_bytes_total{job="ipx"}[$__rate_int
12, unit="Bps", desc="Feeds are read whole, so this is what scans and checks allocate; set it beside memory.")
y += 8
# ---- Traces: all three send them to Tempo (#213)
# ---- ipx-play: the page a shared episode opens, public at play.ipxreader.app (#447), from its
# /metrics on a port apart from the tunnel's (#458)
row("ipx-play")
stat("ipx-play", prom('max(up{job="ipx-play"})'), 0, thresholds=[{"color": "red", "value": None}, {"color": "green", "value": 1}],
desc="Prometheus reaching ipx-play's /metrics: 1 is up.")["fieldConfig"]["defaults"]["mappings"] = [
{"type": "value", "options": {"0": {"text": "down"}, "1": {"text": "up"}}}]
stat("Episodes shared", prom('sum(increase(play_shares_total{job="ipx-play"}[$__range])) or vector(0)', instant=True), 4,
desc="Links ipx servers asked for, the same episode again included.")
stat("Pages opened", prom(f'sum(increase({PLAY_REQ}_count{{job="ipx-play",route="/e/{{id}}",status="200"}}[$__range])) or vector(0)', instant=True), 8,
desc="Shared episodes someone opened, a link preview's fetch included.")
stat("Links not found", prom(f'sum(increase({PLAY_REQ}_count{{job="ipx-play",status="404"}}[$__range])) or vector(0)', instant=True), 12,
desc="A link mistyped, or someone trying addresses.")
stat("Server errors", prom(f'sum(increase({PLAY_REQ}_count{{job="ipx-play",status=~"5.."}}[$__range])) or vector(0)', instant=True), 16,
thresholds=GOOD_BAD, desc="Mostly its database not answering; its log has an error for each.")
stat("Requests refused", prom(f'sum(increase({PLAY_REQ}_count{{job="ipx-play",status=~"401|422"}}[$__range])) or vector(0)', instant=True), 20,
thresholds=GOOD_BAD, desc="A share without a valid token (401), or of an episode that looked private (422): ipx's own check should have kept it.")
y += 4
ts("Requests by route", [prom(f'sum by (route, status) (increase({PLAY_REQ}_count{{job="ipx-play"}}[$__interval]))', legend="{{status}} {{route}}")],
0, w=8, bars=True, stack=True, desc="An empty route is an address no route has.")
ts("Response time", [prom(f'histogram_quantile(0.5, sum by (le) (rate({PLAY_REQ}_bucket{{job="ipx-play"}}[$__rate_interval])))', legend="median"),
prom(f'histogram_quantile(0.95, sum by (le) (rate({PLAY_REQ}_bucket{{job="ipx-play"}}[$__rate_interval])))', ref="B", legend="95th percentile")],
8, w=8, unit="s")
ts("ipx-play memory", [prom('play_process_resident_bytes{job="ipx-play"}', legend="resident")], 16, w=8, unit="bytes",
desc="mimalloc, as the directory's: a rise that never comes down is a leak.")
y += 8
# ---- Traces: all four send them to Tempo (#213)
row("Traces")
for x, title, q in [(0, "Recent", TRACES + "}"),
(8, "Slow (over 2s)", TRACES + " && duration > 2s}"),
@@ -408,7 +436,7 @@ y += 10
# ---- Logs
row("Logs")
panel("logs", "ipx: warnings and errors", 12, 10, 0, [loki(BAD + TEXT)], options=LOG_OPTS)
panel("logs", "ipx-web and ipx-directory: warnings and errors", 12, 10, 12,
panel("logs", "ipx-web, ipx-directory and ipx-play: warnings and errors", 12, 10, 12,
[loki(OTHERS_BAD + ' | line_format "{{.container}} {{.level}}: {{.message}}{{.msg}}"')], options=LOG_OPTS)
y += 10
panel("logs", "ipx log", 24, 12, 0,