Compare commits
2
Commits
00195ee6d5
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
46a1ec1e01 | ||
|
|
8a6a468bec |
+42
-1
@@ -127,6 +127,10 @@ def load_state(cfg):
|
||||
"pending": None, # outstanding question awaiting admin reply
|
||||
"last_event_check": None, # max kuma heartbeat id seen (outage history)
|
||||
"open_outages": {}, # monitor_id -> open DOWN window, cross-cycle
|
||||
"since_heartbeat": { # rolling tally flushed into the daily heartbeat
|
||||
"recovered": {}, # monitor name -> self-recovered blip count
|
||||
"incidents": {}, # monitor name -> sustained-outage count
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -603,10 +607,37 @@ def report_recovered_outages(cfg, state, recovered):
|
||||
relay_result(cfg, state, "Outage post-mortem", result)
|
||||
|
||||
|
||||
def _bump(counts, names):
|
||||
for n in names:
|
||||
counts[n] = counts.get(n, 0) + 1
|
||||
|
||||
|
||||
def _fmt_name_counts(counts):
|
||||
return ", ".join(f"{n} ×{c}" for n, c in sorted(counts.items()))
|
||||
|
||||
|
||||
def heartbeat_since_summary(tally):
|
||||
"""One-line 'past 24h' rollup for the daily heartbeat, so an all-green
|
||||
snapshot still says what happened between heartbeats."""
|
||||
rec, inc = tally.get("recovered", {}), tally.get("incidents", {})
|
||||
n_rec, n_inc = sum(rec.values()), sum(inc.values())
|
||||
if not n_rec and not n_inc:
|
||||
return "Past 24h: no downtime — nothing needed operational support."
|
||||
parts = []
|
||||
if n_inc:
|
||||
parts.append(f"{n_inc} sustained outage(s) ({_fmt_name_counts(inc)})")
|
||||
if n_rec:
|
||||
parts.append(f"{n_rec} self-recovered blip(s) ({_fmt_name_counts(rec)})")
|
||||
return "Past 24h: " + "; ".join(parts) + "."
|
||||
|
||||
|
||||
def health_cycle(cfg, state):
|
||||
log("Health cycle: reading Uptime Kuma…")
|
||||
monitors = read_kuma_monitors(cfg)
|
||||
streak = state.setdefault("down_streak", {})
|
||||
tally = state.setdefault("since_heartbeat", {})
|
||||
tally.setdefault("recovered", {})
|
||||
tally.setdefault("incidents", {})
|
||||
|
||||
down_now = []
|
||||
for m in monitors:
|
||||
@@ -620,6 +651,13 @@ def health_cycle(cfg, state):
|
||||
confirmed = [m for m in down_now
|
||||
if streak.get(m["name"], 0) >= cfg["down_confirmations"]]
|
||||
|
||||
# Count each sustained outage once — the cycle its streak first crosses the
|
||||
# confirmation threshold — so an outage spanning many cycles isn't tallied
|
||||
# repeatedly in the daily rollup.
|
||||
_bump(tally["incidents"],
|
||||
[m["name"] for m in confirmed
|
||||
if streak.get(m["name"]) == cfg["down_confirmations"]])
|
||||
|
||||
log(f" {len(monitors)} tracked, {len(down_now)} down, "
|
||||
f"{len(confirmed)} confirmed down")
|
||||
|
||||
@@ -632,6 +670,7 @@ def health_cycle(cfg, state):
|
||||
state["last_event_check"] = watermark
|
||||
if recovered:
|
||||
log(f" {len(recovered)} self-recovered outage(s) since last check")
|
||||
_bump(tally["recovered"], [o["name"] for o in recovered])
|
||||
report_recovered_outages(cfg, state, recovered)
|
||||
except Exception as e:
|
||||
log(f"WARN: outage-history check failed: {e}")
|
||||
@@ -643,9 +682,11 @@ def health_cycle(cfg, state):
|
||||
up = sum(1 for m in monitors if m.get("status") == KUMA_UP)
|
||||
matrix_send(
|
||||
cfg,
|
||||
f"✅ **Daily health** — {up}/{len(monitors)} tracked services up.",
|
||||
f"✅ **Daily health** — {up}/{len(monitors)} tracked services up.\n"
|
||||
f"{heartbeat_since_summary(tally)}",
|
||||
)
|
||||
state["last_heartbeat"] = time.time()
|
||||
tally["recovered"], tally["incidents"] = {}, {}
|
||||
return
|
||||
|
||||
if cfg["autonomy"] == "notify":
|
||||
|
||||
Reference in New Issue
Block a user