Compare commits

..
2 Commits
Author SHA1 Message Date
paul 46a1ec1e01 Merge pull request 'Add rolling 24h activity rollup to daily health heartbeat' (#1) from daily-heartbeat-24h-summary into main
Reviewed-on: #1
2026-07-01 09:39:39 +00:00
paulandClaude Opus 4.8 8a6a468bec Add rolling 24h activity rollup to daily health heartbeat
The daily " Daily health — N/N up" message was a bare point-in-time
snapshot: it said nothing about downtime or operational support during
the preceding window. Self-recovered blips and sustained-outage
remediations were reported only as separate real-time messages, easy to
miss.

Accumulate a per-service tally in state["since_heartbeat"] and fold it
into the heartbeat line, then reset on send:
- self-recovered blips counted once per DOWN→UP window
- sustained outages counted once, at the cycle the down-streak first
  crosses down_confirmations (so a long outage isn't tallied each cycle)

Wording is autonomy-mode-neutral ("sustained outage(s)"). Existing state
files without the key are initialized via setdefault, so no reset needed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-01 21:37:45 +12:00
+42 -1
View File
@@ -127,6 +127,10 @@ def load_state(cfg):
"pending": None, # outstanding question awaiting admin reply
"last_event_check": None, # max kuma heartbeat id seen (outage history)
"open_outages": {}, # monitor_id -> open DOWN window, cross-cycle
"since_heartbeat": { # rolling tally flushed into the daily heartbeat
"recovered": {}, # monitor name -> self-recovered blip count
"incidents": {}, # monitor name -> sustained-outage count
},
}
@@ -603,10 +607,37 @@ def report_recovered_outages(cfg, state, recovered):
relay_result(cfg, state, "Outage post-mortem", result)
def _bump(counts, names):
for n in names:
counts[n] = counts.get(n, 0) + 1
def _fmt_name_counts(counts):
return ", ".join(f"{n} ×{c}" for n, c in sorted(counts.items()))
def heartbeat_since_summary(tally):
"""One-line 'past 24h' rollup for the daily heartbeat, so an all-green
snapshot still says what happened between heartbeats."""
rec, inc = tally.get("recovered", {}), tally.get("incidents", {})
n_rec, n_inc = sum(rec.values()), sum(inc.values())
if not n_rec and not n_inc:
return "Past 24h: no downtime — nothing needed operational support."
parts = []
if n_inc:
parts.append(f"{n_inc} sustained outage(s) ({_fmt_name_counts(inc)})")
if n_rec:
parts.append(f"{n_rec} self-recovered blip(s) ({_fmt_name_counts(rec)})")
return "Past 24h: " + "; ".join(parts) + "."
def health_cycle(cfg, state):
log("Health cycle: reading Uptime Kuma…")
monitors = read_kuma_monitors(cfg)
streak = state.setdefault("down_streak", {})
tally = state.setdefault("since_heartbeat", {})
tally.setdefault("recovered", {})
tally.setdefault("incidents", {})
down_now = []
for m in monitors:
@@ -620,6 +651,13 @@ def health_cycle(cfg, state):
confirmed = [m for m in down_now
if streak.get(m["name"], 0) >= cfg["down_confirmations"]]
# Count each sustained outage once — the cycle its streak first crosses the
# confirmation threshold — so an outage spanning many cycles isn't tallied
# repeatedly in the daily rollup.
_bump(tally["incidents"],
[m["name"] for m in confirmed
if streak.get(m["name"]) == cfg["down_confirmations"]])
log(f" {len(monitors)} tracked, {len(down_now)} down, "
f"{len(confirmed)} confirmed down")
@@ -632,6 +670,7 @@ def health_cycle(cfg, state):
state["last_event_check"] = watermark
if recovered:
log(f" {len(recovered)} self-recovered outage(s) since last check")
_bump(tally["recovered"], [o["name"] for o in recovered])
report_recovered_outages(cfg, state, recovered)
except Exception as e:
log(f"WARN: outage-history check failed: {e}")
@@ -643,9 +682,11 @@ def health_cycle(cfg, state):
up = sum(1 for m in monitors if m.get("status") == KUMA_UP)
matrix_send(
cfg,
f"✅ **Daily health** — {up}/{len(monitors)} tracked services up.",
f"✅ **Daily health** — {up}/{len(monitors)} tracked services up.\n"
f"{heartbeat_since_summary(tally)}",
)
state["last_heartbeat"] = time.time()
tally["recovered"], tally["incidents"] = {}, {}
return
if cfg["autonomy"] == "notify":