rays
3df7c974e8
Check 5 listed traces over 5 s, which were all page traces holding minutes of live updates or audio, so a slow request never showed. query.py's new `spans` lists the matching spans with their own time; check 5 asks for ipx requests over a second, steps over half a second, and says to set the hours beside Tower's load: on 2026-10-18, 120 of 130 slow requests were the host stalling. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
117 lines
6.4 KiB
Python
Executable File
117 lines
6.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Ask production's Loki, Tempo or Prometheus a question, from anywhere that can run docker on Tower.
|
|
|
|
query.py logs '<LogQL log query>' [since] lines, newest first
|
|
query.py metric '<LogQL metric query>' [since] one value per series; $range is `since`
|
|
query.py traces '<TraceQL query>' [since] matching traces, slowest first
|
|
query.py spans '<TraceQL query>' [since] the matching spans, each with its own time
|
|
query.py trace <trace id> one trace's spans, as a tree
|
|
query.py prom '<PromQL query>' [since] one value per series, now; $range is `since`
|
|
|
|
`since` is 1h, 24h, 7d and the like (default 24h). Loki and Tempo publish no query port on the
|
|
host, so each call runs a throwaway alpine container on the monitoring project's network.
|
|
"""
|
|
import json, subprocess, sys, time, urllib.parse
|
|
|
|
NET = "monitoring_default"
|
|
|
|
|
|
def fetch(url):
|
|
out = subprocess.run(["docker", "run", "--rm", "--network", NET, "alpine", "wget", "-qO-", url],
|
|
capture_output=True, text=True, timeout=120)
|
|
if out.returncode:
|
|
sys.exit(f"query failed: {out.stderr.strip() or out.stdout.strip()}\n{url}")
|
|
return json.loads(out.stdout)
|
|
|
|
|
|
def seconds(since):
|
|
return int(since[:-1]) * {"m": 60, "h": 3600, "d": 86400}[since[-1]]
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 3:
|
|
sys.exit(__doc__)
|
|
kind, q = sys.argv[1], sys.argv[2]
|
|
since = sys.argv[3] if len(sys.argv) > 3 else "24h"
|
|
now = time.time()
|
|
start = now - seconds(since)
|
|
enc = urllib.parse.quote
|
|
if kind == "logs":
|
|
r = fetch(f"http://loki:3100/loki/api/v1/query_range?query={enc(q)}&limit=500"
|
|
f"&start={int(start * 1e9)}&end={int(now * 1e9)}&direction=backward")
|
|
lines = [(ts, line) for s in r["data"]["result"] for ts, line in s["values"]]
|
|
for ts, line in sorted(lines, reverse=True):
|
|
print(line)
|
|
print(f"-- {len(lines)} line(s){' (limit reached)' if len(lines) >= 500 else ''}", file=sys.stderr)
|
|
elif kind == "metric":
|
|
r = fetch(f"http://loki:3100/loki/api/v1/query?query={enc(q.replace('$range', since))}&time={int(now * 1e9)}")
|
|
rows = sorted(r["data"]["result"], key=lambda s: -float(s["value"][1]))
|
|
for s in rows:
|
|
labels = {k: v for k, v in s["metric"].items() if k not in ("container", "compose_project", "service_name")}
|
|
print(f"{s['value'][1]:>12} {json.dumps(labels) if labels else ''}")
|
|
print(f"-- {len(rows)} series", file=sys.stderr)
|
|
elif kind == "traces":
|
|
r = fetch(f"http://tempo:3200/api/search?q={enc(q)}&start={int(start)}&end={int(now)}&limit=100")
|
|
traces = sorted(r.get("traces", []), key=lambda t: -t.get("durationMs", 0))
|
|
for t in traces:
|
|
when = time.strftime("%m-%d %H:%M:%S", time.gmtime(int(t["startTimeUnixNano"]) / 1e9))
|
|
# No root name: its root span is a tap in the page, which the browser never sends.
|
|
name = t.get("rootTraceName") or "(from the page)"
|
|
print(f"{t.get('durationMs', 0):>8}ms {when}Z {t['traceID']} {name}")
|
|
print(f"-- {len(traces)} trace(s)", file=sys.stderr)
|
|
elif kind == "spans":
|
|
# The spans that matched, each with its own time: a trace's length says nothing about a
|
|
# slow request in it when the trace also holds a page's live updates or an audio stream.
|
|
r = fetch(f"http://tempo:3200/api/search?q={enc(q)}&start={int(start)}&end={int(now)}&limit=200&spss=20")
|
|
rows = []
|
|
for t in r.get("traces", []):
|
|
for ss in t.get("spanSets") or [t.get("spanSet") or {}]:
|
|
for sp in ss.get("spans", []):
|
|
attrs = {a["key"]: next(iter(a["value"].values()), None) for a in sp.get("attributes", [])}
|
|
what = attrs.get("http.route") or attrs.get("of") or sp.get("name") or ""
|
|
rows.append((int(sp.get("durationNanos", 0)) / 1e6, int(sp["startTimeUnixNano"]), t["traceID"], what))
|
|
for ms, ts, tid, what in sorted(rows, reverse=True):
|
|
when = time.strftime("%m-%d %H:%M:%S", time.gmtime(ts / 1e9))
|
|
print(f"{ms:>8.0f}ms {when}Z {tid} {what}")
|
|
print(f"-- {len(rows)} span(s)", file=sys.stderr)
|
|
elif kind == "prom":
|
|
# ipx's /metrics (#194), scraped as job ipx; through the network like the others.
|
|
r = fetch(f"http://prometheus:9090/api/v1/query?query={enc(q.replace('$range', since))}&time={now}")
|
|
rows = sorted(r["data"]["result"], key=lambda s: -float(s["value"][1]))
|
|
for s in rows:
|
|
labels = {k: v for k, v in s["metric"].items() if k not in ("job", "instance")}
|
|
print(f"{float(s['value'][1]):>14g} {json.dumps(labels) if labels else ''}")
|
|
print(f"-- {len(rows)} series", file=sys.stderr)
|
|
elif kind == "trace":
|
|
r = fetch(f"http://tempo:3200/api/traces/{q}")
|
|
spans = [sp for b in r.get("batches", r.get("resourceSpans", []))
|
|
for ss in b.get("scopeSpans", b.get("instrumentationLibrarySpans", [])) for sp in ss["spans"]]
|
|
kids = {}
|
|
for sp in spans:
|
|
kids.setdefault(sp.get("parentSpanId", ""), []).append(sp)
|
|
|
|
def show(sp, depth):
|
|
ms = (int(sp["endTimeUnixNano"]) - int(sp["startTimeUnixNano"])) / 1e6
|
|
attrs = {a["key"]: next(iter(a["value"].values()), None) for a in sp.get("attributes", [])
|
|
if not a["key"].startswith(("code.", "thread.")) and a["key"] not in ("busy_ns", "idle_ns", "target")}
|
|
err = " ERROR" if sp.get("status", {}).get("code") in (2, "STATUS_CODE_ERROR") else ""
|
|
print(f"{' ' * depth}{sp['name']} {ms:.0f}ms{err} {json.dumps(attrs) if attrs else ''}")
|
|
for e in sp.get("events", []):
|
|
msg = next((a["value"].get("stringValue") for a in e.get("attributes", []) if a["key"] == "message"), e.get("name"))
|
|
# A scan logs a skip for every feed not due; they bury what happened.
|
|
if '"ev":"feed_skip"' in (msg or ""):
|
|
continue
|
|
print(f"{' ' * depth} - {msg}")
|
|
for k in sorted(kids.get(sp["spanId"], []), key=lambda s: int(s["startTimeUnixNano"])):
|
|
show(k, depth + 1)
|
|
|
|
ids = {sp["spanId"] for sp in spans}
|
|
for root in [sp for sp in spans if sp.get("parentSpanId", "") not in ids]:
|
|
show(root, 0)
|
|
else:
|
|
sys.exit(__doc__)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|