"""The iPX dashboard in Grafana, from Loki (the container's log, shipped by Alloy) and Tempo. python3 grafana/dashboard.py > /mnt/fast/arcane/projects/monitoring/grafana-provisioning/dashboards/ipx.json Grafana reads that file on its own within a minute; edits made in Grafana are refused. The panels read the fields of ipx's JSON log (IPX_LOG_FORMAT=json): renaming a field in web.rs access_log or ipc.rs log_wire has to be matched here. `dashboard.py queries` prints each query, to try against Loki. """ import json, sys LOKI = {"type": "loki", "uid": "${loki}"} TEMPO = {"type": "tempo", "uid": "${tempo}"} SEL = '{container="iPX"}' # ipx logs one JSON object a line (IPX_LOG_FORMAT=json). Each scan and download event carries its # fields (ev, feed, new, bytes, msg, ...); each request its method, path, route, status and ms. EV = SEL + ' |= "\\"target\\":\\"ipx::io\\"" | json | __error__="" | ev != ""' HTTP = SEL + ' |= "\\"target\\":\\"ipx::http\\"" | json | __error__="" | path != "/api/events"' BAD = SEL + ' | json | __error__="" | level =~ "WARN|ERROR"' TEXT = ' | line_format "{{.level}} {{.target}}: {{.message}}"' TRACES = '{resource.service.name="ipx" && resource.deployment.environment.name="production"' def status(field): return f'max(max_over_time({EV} | ev = "status" | unwrap {field} [10m]))' QUERIES = {} panels, y = [], 0 pid = 0 def panel(kind, title, w, h, x, targets, **extra): global pid pid += 1 p = {"id": pid, "type": kind, "title": title, "gridPos": {"x": x, "y": y, "w": w, "h": h}, "datasource": targets[0].get("datasource", LOKI), "targets": targets} p.update(extra) panels.append(p) return p def loki(expr, ref="A", legend=None, instant=False, kind=None): QUERIES[expr] = instant t = {"refId": ref, "datasource": LOKI, "expr": expr, "queryType": "instant" if instant else "range"} if legend: t["legendFormat"] = legend return t def row(title): global y, pid pid += 1 panels.append({"id": pid, "type": "row", "title": title, "collapsed": False, "gridPos": {"x": 0, "y": y, "w": 24, "h": 1}, "panels": []}) y += 1 def stat(title, expr, x, unit="short", color="blue", thresholds=None, desc=None): steps = thresholds or [{"color": color, "value": None}] return panel("stat", title, 4, 4, x, [loki(expr, instant=True)], description=desc or "", fieldConfig={"defaults": {"unit": unit, "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": steps}}, "overrides": []}, options={"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, "colorMode": "value", "graphMode": "none", "textMode": "value"}) def ts(title, targets, x, w=12, h=8, unit="short", bars=False, stack=False, desc=""): custom = {"drawStyle": "bars" if bars else "line", "fillOpacity": 60 if bars else 10, "lineWidth": 1, "showPoints": "never", "stacking": {"mode": "normal" if stack else "none"}} return panel("timeseries", title, w, h, x, targets, description=desc, fieldConfig={"defaults": {"unit": unit, "custom": custom}, "overrides": []}, options={"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi", "sort": "desc"}}) def table(title, targets, x, w=12, h=8, rename=None, sort=None, desc=""): return panel("table", title, w, h, x, targets, description=desc, transformations=[{"id": "labelsToFields", "options": {"mode": "columns"}}, {"id": "organize", "options": { "excludeByName": {"Time": True, "container": True, "compose_project": True, "service_name": True}, "renameByName": rename or {}}}], options={"showHeader": True, "sortBy": sort or []}, fieldConfig={"defaults": {}, "overrides": []}) # ---- Now row("Now") stat("Feeds", status("feeds"), 0, desc="From the healthcheck's status answer, every 30 seconds.") stat("Waiting to download", status("pending"), 4) stat("Downloaded", status("downloaded"), 8, color="green") stat("Feed failures", f'sum(count_over_time({EV} | ev="feed_error" [$__range])) or vector(0)', 12, thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}], desc="Failed feed checks in the time range.") stat("Download failures", f'sum(count_over_time({EV} | ev="download_error" [$__range])) or vector(0)', 16, thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}]) stat("Warnings and errors", f'sum(count_over_time({BAD} [$__range])) or vector(0)', 20, thresholds=[{"color": "green", "value": None}, {"color": "orange", "value": 1}, {"color": "red", "value": 50}]) y += 4 # ---- Scans row("Scans and downloads") ts("New items found", [loki(f'sum(sum_over_time({EV} | ev="feed_done" | unwrap new [$__interval]))', legend="new items")], 0, bars=True) ts("Downloads", [loki(f'sum(count_over_time({EV} | ev="download_done" [$__interval]))', legend="saved"), loki(f'sum(count_over_time({EV} | ev="download_error" [$__interval]))', ref="B", legend="failed")], 12, bars=True) y += 8 ts("Bytes downloaded", [loki(f'sum(sum_over_time({EV} | ev="download_done" | unwrap bytes [$__interval]))', legend="bytes")], 0, unit="bytes", bars=True) ts("Feeds checked per scan", [loki(f'sum(sum_over_time({EV} | ev="scan_done" | unwrap feeds [$__interval]))', legend="feeds checked")], 12, bars=True, desc="Feeds that were due and fetched; the rest were skipped as not due.") y += 8 table("Failing feeds", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="feed_error" [$__range]))', instant=True)], 0, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}]) table("Failed downloads", [loki(f'sum by (feed, msg) (count_over_time({EV} | ev="download_error" [$__range]))', instant=True)], 12, rename={"feed": "Feed", "msg": "Error", "Value": "Failures"}, sort=[{"displayName": "Failures", "desc": True}]) y += 8 # ---- Web row("Web") ts("Requests by status", [loki(f'sum by (status) (count_over_time({HTTP} [$__interval]))', legend="{{status}}")], 0, bars=True, stack=True, desc="The event stream the page keeps open is left out.") ts("Response time", [loki(f'quantile_over_time(0.5, {HTTP} | unwrap ms [$__interval]) by ()', legend="median"), loki(f'quantile_over_time(0.95, {HTTP} | unwrap ms [$__interval]) by ()', ref="B", legend="95th percentile"), loki(f'max_over_time({HTTP} | unwrap ms [$__interval]) by ()', ref="C", legend="slowest")], 12, unit="ms") y += 8 table("Slowest routes", [loki(f'topk(15, avg_over_time({HTTP} | route != "" | unwrap ms [$__range]) by (method, route))', instant=True)], 0, rename={"method": "Method", "route": "Route", "Value": "Average ms"}, sort=[{"displayName": "Average ms", "desc": True}]) table("Busiest routes", [loki(f'topk(15, sum by (method, route) (count_over_time({HTTP} | route != "" [$__range])))', instant=True)], 12, rename={"method": "Method", "route": "Route", "Value": "Requests"}, sort=[{"displayName": "Requests", "desc": True}]) y += 8 # ---- Traces row("Traces") for x, title, q in [(0, "Recent traces", TRACES + "}"), (12, "Slow traces (over 2s)", TRACES + " && duration > 2s}")]: panel("table", title, 12, 10, x, [{"refId": "A", "datasource": TEMPO, "queryType": "traceql", "query": q, "limit": 50, "tableType": "traces"}], fieldConfig={"defaults": {}, "overrides": []}) y += 10 # ---- Log row("Log") panel("logs", "Warnings and errors", 24, 10, 0, [loki(BAD + TEXT)], options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True}) y += 10 panel("logs", "Log", 24, 12, 0, [loki(SEL + ' | json | __error__="" | path != "/api/events" | ev != "feed_skip" | ev != "status"' ' | message != "-> {\\"cmd\\":\\"status\\"}"' + TEXT)], description="Without the event stream's requests, not-due skips and healthcheck status calls.", options={"showTime": True, "wrapLogMessage": True, "sortOrder": "Descending", "enableLogDetails": True}) dash = { "uid": "ipx", "title": "iPX", "tags": ["ipx"], "timezone": "browser", "schemaVersion": 39, "time": {"from": "now-24h", "to": "now"}, "refresh": "1m", "editable": True, "templating": {"list": [ {"name": "loki", "label": "Logs", "type": "datasource", "query": "loki", "current": {}, "hide": 0}, {"name": "tempo", "label": "Traces", "type": "datasource", "query": "tempo", "current": {}, "hide": 0}, ]}, "links": [{"title": "iPX", "type": "link", "url": "https://ipodderx.sdf1.net", "targetBlank": True}], "panels": panels, } if sys.argv[1:] == ["queries"]: for q, instant in QUERIES.items(): print(json.dumps([q, instant])) else: print(json.dumps(dash, indent=2))