From 8a6a468becbf42995c0b1be255b9a6dda2e9e450 Mon Sep 17 00:00:00 2001 From: Semprini Date: Wed, 1 Jul 2026 21:37:45 +1200 Subject: [PATCH] Add rolling 24h activity rollup to daily health heartbeat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The daily "✅ Daily health — N/N up" message was a bare point-in-time snapshot: it said nothing about downtime or operational support during the preceding window. Self-recovered blips and sustained-outage remediations were reported only as separate real-time messages, easy to miss. Accumulate a per-service tally in state["since_heartbeat"] and fold it into the heartbeat line, then reset on send: - self-recovered blips counted once per DOWN→UP window - sustained outages counted once, at the cycle the down-streak first crosses down_confirmations (so a long outage isn't tallied each cycle) Wording is autonomy-mode-neutral ("sustained outage(s)"). Existing state files without the key are initialized via setdefault, so no reset needed. Co-Authored-By: Claude Opus 4.8 --- maintainer.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/maintainer.py b/maintainer.py index 41e6345..a274b77 100755 --- a/maintainer.py +++ b/maintainer.py @@ -127,6 +127,10 @@ def load_state(cfg): "pending": None, # outstanding question awaiting admin reply "last_event_check": None, # max kuma heartbeat id seen (outage history) "open_outages": {}, # monitor_id -> open DOWN window, cross-cycle + "since_heartbeat": { # rolling tally flushed into the daily heartbeat + "recovered": {}, # monitor name -> self-recovered blip count + "incidents": {}, # monitor name -> sustained-outage count + }, } @@ -603,10 +607,37 @@ def report_recovered_outages(cfg, state, recovered): relay_result(cfg, state, "Outage post-mortem", result) +def _bump(counts, names): + for n in names: + counts[n] = counts.get(n, 0) + 1 + + +def _fmt_name_counts(counts): + return ", ".join(f"{n} ×{c}" for n, c in sorted(counts.items())) + + +def heartbeat_since_summary(tally): + """One-line 'past 24h' rollup for the daily heartbeat, so an all-green + snapshot still says what happened between heartbeats.""" + rec, inc = tally.get("recovered", {}), tally.get("incidents", {}) + n_rec, n_inc = sum(rec.values()), sum(inc.values()) + if not n_rec and not n_inc: + return "Past 24h: no downtime — nothing needed operational support." + parts = [] + if n_inc: + parts.append(f"{n_inc} sustained outage(s) ({_fmt_name_counts(inc)})") + if n_rec: + parts.append(f"{n_rec} self-recovered blip(s) ({_fmt_name_counts(rec)})") + return "Past 24h: " + "; ".join(parts) + "." + + def health_cycle(cfg, state): log("Health cycle: reading Uptime Kuma…") monitors = read_kuma_monitors(cfg) streak = state.setdefault("down_streak", {}) + tally = state.setdefault("since_heartbeat", {}) + tally.setdefault("recovered", {}) + tally.setdefault("incidents", {}) down_now = [] for m in monitors: @@ -620,6 +651,13 @@ def health_cycle(cfg, state): confirmed = [m for m in down_now if streak.get(m["name"], 0) >= cfg["down_confirmations"]] + # Count each sustained outage once — the cycle its streak first crosses the + # confirmation threshold — so an outage spanning many cycles isn't tallied + # repeatedly in the daily rollup. + _bump(tally["incidents"], + [m["name"] for m in confirmed + if streak.get(m["name"]) == cfg["down_confirmations"]]) + log(f" {len(monitors)} tracked, {len(down_now)} down, " f"{len(confirmed)} confirmed down") @@ -632,6 +670,7 @@ def health_cycle(cfg, state): state["last_event_check"] = watermark if recovered: log(f" {len(recovered)} self-recovered outage(s) since last check") + _bump(tally["recovered"], [o["name"] for o in recovered]) report_recovered_outages(cfg, state, recovered) except Exception as e: log(f"WARN: outage-history check failed: {e}") @@ -643,9 +682,11 @@ def health_cycle(cfg, state): up = sum(1 for m in monitors if m.get("status") == KUMA_UP) matrix_send( cfg, - f"✅ **Daily health** — {up}/{len(monitors)} tracked services up.", + f"✅ **Daily health** — {up}/{len(monitors)} tracked services up.\n" + f"{heartbeat_since_summary(tally)}", ) state["last_heartbeat"] = time.time() + tally["recovered"], tally["incidents"] = {}, {} return if cfg["autonomy"] == "notify":