Add rolling 24h activity rollup to daily health heartbeat #1

Merged
paul merged 1 commits from daily-heartbeat-24h-summary into main 2026-07-01 09:39:39 +00:00
Showing only changes of commit 8a6a468bec - Show all commits
+42 -1
View File
@@ -127,6 +127,10 @@ def load_state(cfg):
"pending": None, # outstanding question awaiting admin reply "pending": None, # outstanding question awaiting admin reply
"last_event_check": None, # max kuma heartbeat id seen (outage history) "last_event_check": None, # max kuma heartbeat id seen (outage history)
"open_outages": {}, # monitor_id -> open DOWN window, cross-cycle "open_outages": {}, # monitor_id -> open DOWN window, cross-cycle
"since_heartbeat": { # rolling tally flushed into the daily heartbeat
"recovered": {}, # monitor name -> self-recovered blip count
"incidents": {}, # monitor name -> sustained-outage count
},
} }
@@ -603,10 +607,37 @@ def report_recovered_outages(cfg, state, recovered):
relay_result(cfg, state, "Outage post-mortem", result) relay_result(cfg, state, "Outage post-mortem", result)
def _bump(counts, names):
for n in names:
counts[n] = counts.get(n, 0) + 1
def _fmt_name_counts(counts):
return ", ".join(f"{n} ×{c}" for n, c in sorted(counts.items()))
def heartbeat_since_summary(tally):
"""One-line 'past 24h' rollup for the daily heartbeat, so an all-green
snapshot still says what happened between heartbeats."""
rec, inc = tally.get("recovered", {}), tally.get("incidents", {})
n_rec, n_inc = sum(rec.values()), sum(inc.values())
if not n_rec and not n_inc:
return "Past 24h: no downtime — nothing needed operational support."
parts = []
if n_inc:
parts.append(f"{n_inc} sustained outage(s) ({_fmt_name_counts(inc)})")
if n_rec:
parts.append(f"{n_rec} self-recovered blip(s) ({_fmt_name_counts(rec)})")
return "Past 24h: " + "; ".join(parts) + "."
def health_cycle(cfg, state): def health_cycle(cfg, state):
log("Health cycle: reading Uptime Kuma…") log("Health cycle: reading Uptime Kuma…")
monitors = read_kuma_monitors(cfg) monitors = read_kuma_monitors(cfg)
streak = state.setdefault("down_streak", {}) streak = state.setdefault("down_streak", {})
tally = state.setdefault("since_heartbeat", {})
tally.setdefault("recovered", {})
tally.setdefault("incidents", {})
down_now = [] down_now = []
for m in monitors: for m in monitors:
@@ -620,6 +651,13 @@ def health_cycle(cfg, state):
confirmed = [m for m in down_now confirmed = [m for m in down_now
if streak.get(m["name"], 0) >= cfg["down_confirmations"]] if streak.get(m["name"], 0) >= cfg["down_confirmations"]]
# Count each sustained outage once — the cycle its streak first crosses the
# confirmation threshold — so an outage spanning many cycles isn't tallied
# repeatedly in the daily rollup.
_bump(tally["incidents"],
[m["name"] for m in confirmed
if streak.get(m["name"]) == cfg["down_confirmations"]])
log(f" {len(monitors)} tracked, {len(down_now)} down, " log(f" {len(monitors)} tracked, {len(down_now)} down, "
f"{len(confirmed)} confirmed down") f"{len(confirmed)} confirmed down")
@@ -632,6 +670,7 @@ def health_cycle(cfg, state):
state["last_event_check"] = watermark state["last_event_check"] = watermark
if recovered: if recovered:
log(f" {len(recovered)} self-recovered outage(s) since last check") log(f" {len(recovered)} self-recovered outage(s) since last check")
_bump(tally["recovered"], [o["name"] for o in recovered])
report_recovered_outages(cfg, state, recovered) report_recovered_outages(cfg, state, recovered)
except Exception as e: except Exception as e:
log(f"WARN: outage-history check failed: {e}") log(f"WARN: outage-history check failed: {e}")
@@ -643,9 +682,11 @@ def health_cycle(cfg, state):
up = sum(1 for m in monitors if m.get("status") == KUMA_UP) up = sum(1 for m in monitors if m.get("status") == KUMA_UP)
matrix_send( matrix_send(
cfg, cfg,
f"✅ **Daily health** — {up}/{len(monitors)} tracked services up.", f"✅ **Daily health** — {up}/{len(monitors)} tracked services up.\n"
f"{heartbeat_since_summary(tally)}",
) )
state["last_heartbeat"] = time.time() state["last_heartbeat"] = time.time()
tally["recovered"], tally["incidents"] = {}, {}
return return
if cfg["autonomy"] == "notify": if cfg["autonomy"] == "notify":