Merge pull request 'Add rolling 24h activity rollup to daily health heartbeat' (#1) from daily-heartbeat-24h-summary into main
Reviewed-on: #1
This commit was merged in pull request #1.
This commit is contained in:
+42
-1
@@ -127,6 +127,10 @@ def load_state(cfg):
|
|||||||
"pending": None, # outstanding question awaiting admin reply
|
"pending": None, # outstanding question awaiting admin reply
|
||||||
"last_event_check": None, # max kuma heartbeat id seen (outage history)
|
"last_event_check": None, # max kuma heartbeat id seen (outage history)
|
||||||
"open_outages": {}, # monitor_id -> open DOWN window, cross-cycle
|
"open_outages": {}, # monitor_id -> open DOWN window, cross-cycle
|
||||||
|
"since_heartbeat": { # rolling tally flushed into the daily heartbeat
|
||||||
|
"recovered": {}, # monitor name -> self-recovered blip count
|
||||||
|
"incidents": {}, # monitor name -> sustained-outage count
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -603,10 +607,37 @@ def report_recovered_outages(cfg, state, recovered):
|
|||||||
relay_result(cfg, state, "Outage post-mortem", result)
|
relay_result(cfg, state, "Outage post-mortem", result)
|
||||||
|
|
||||||
|
|
||||||
|
def _bump(counts, names):
|
||||||
|
for n in names:
|
||||||
|
counts[n] = counts.get(n, 0) + 1
|
||||||
|
|
||||||
|
|
||||||
|
def _fmt_name_counts(counts):
|
||||||
|
return ", ".join(f"{n} ×{c}" for n, c in sorted(counts.items()))
|
||||||
|
|
||||||
|
|
||||||
|
def heartbeat_since_summary(tally):
|
||||||
|
"""One-line 'past 24h' rollup for the daily heartbeat, so an all-green
|
||||||
|
snapshot still says what happened between heartbeats."""
|
||||||
|
rec, inc = tally.get("recovered", {}), tally.get("incidents", {})
|
||||||
|
n_rec, n_inc = sum(rec.values()), sum(inc.values())
|
||||||
|
if not n_rec and not n_inc:
|
||||||
|
return "Past 24h: no downtime — nothing needed operational support."
|
||||||
|
parts = []
|
||||||
|
if n_inc:
|
||||||
|
parts.append(f"{n_inc} sustained outage(s) ({_fmt_name_counts(inc)})")
|
||||||
|
if n_rec:
|
||||||
|
parts.append(f"{n_rec} self-recovered blip(s) ({_fmt_name_counts(rec)})")
|
||||||
|
return "Past 24h: " + "; ".join(parts) + "."
|
||||||
|
|
||||||
|
|
||||||
def health_cycle(cfg, state):
|
def health_cycle(cfg, state):
|
||||||
log("Health cycle: reading Uptime Kuma…")
|
log("Health cycle: reading Uptime Kuma…")
|
||||||
monitors = read_kuma_monitors(cfg)
|
monitors = read_kuma_monitors(cfg)
|
||||||
streak = state.setdefault("down_streak", {})
|
streak = state.setdefault("down_streak", {})
|
||||||
|
tally = state.setdefault("since_heartbeat", {})
|
||||||
|
tally.setdefault("recovered", {})
|
||||||
|
tally.setdefault("incidents", {})
|
||||||
|
|
||||||
down_now = []
|
down_now = []
|
||||||
for m in monitors:
|
for m in monitors:
|
||||||
@@ -620,6 +651,13 @@ def health_cycle(cfg, state):
|
|||||||
confirmed = [m for m in down_now
|
confirmed = [m for m in down_now
|
||||||
if streak.get(m["name"], 0) >= cfg["down_confirmations"]]
|
if streak.get(m["name"], 0) >= cfg["down_confirmations"]]
|
||||||
|
|
||||||
|
# Count each sustained outage once — the cycle its streak first crosses the
|
||||||
|
# confirmation threshold — so an outage spanning many cycles isn't tallied
|
||||||
|
# repeatedly in the daily rollup.
|
||||||
|
_bump(tally["incidents"],
|
||||||
|
[m["name"] for m in confirmed
|
||||||
|
if streak.get(m["name"]) == cfg["down_confirmations"]])
|
||||||
|
|
||||||
log(f" {len(monitors)} tracked, {len(down_now)} down, "
|
log(f" {len(monitors)} tracked, {len(down_now)} down, "
|
||||||
f"{len(confirmed)} confirmed down")
|
f"{len(confirmed)} confirmed down")
|
||||||
|
|
||||||
@@ -632,6 +670,7 @@ def health_cycle(cfg, state):
|
|||||||
state["last_event_check"] = watermark
|
state["last_event_check"] = watermark
|
||||||
if recovered:
|
if recovered:
|
||||||
log(f" {len(recovered)} self-recovered outage(s) since last check")
|
log(f" {len(recovered)} self-recovered outage(s) since last check")
|
||||||
|
_bump(tally["recovered"], [o["name"] for o in recovered])
|
||||||
report_recovered_outages(cfg, state, recovered)
|
report_recovered_outages(cfg, state, recovered)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log(f"WARN: outage-history check failed: {e}")
|
log(f"WARN: outage-history check failed: {e}")
|
||||||
@@ -643,9 +682,11 @@ def health_cycle(cfg, state):
|
|||||||
up = sum(1 for m in monitors if m.get("status") == KUMA_UP)
|
up = sum(1 for m in monitors if m.get("status") == KUMA_UP)
|
||||||
matrix_send(
|
matrix_send(
|
||||||
cfg,
|
cfg,
|
||||||
f"✅ **Daily health** — {up}/{len(monitors)} tracked services up.",
|
f"✅ **Daily health** — {up}/{len(monitors)} tracked services up.\n"
|
||||||
|
f"{heartbeat_since_summary(tally)}",
|
||||||
)
|
)
|
||||||
state["last_heartbeat"] = time.time()
|
state["last_heartbeat"] = time.time()
|
||||||
|
tally["recovered"], tally["incidents"] = {}, {}
|
||||||
return
|
return
|
||||||
|
|
||||||
if cfg["autonomy"] == "notify":
|
if cfg["autonomy"] == "notify":
|
||||||
|
|||||||
Reference in New Issue
Block a user