#!/usr/bin/env python3
"""
semprini-maintainer — autonomous stack maintenance daemon.

Runs as a systemd service. Periodically:
  * (frequent)   reads Uptime Kuma to learn which services are tracked and their
                 up/down status, and — when something is down — invokes the
                 canonical stack-support agent (Claude Code, headless) to
                 diagnose and remediate fully autonomously.
  * (infrequent) resolves the containers behind the tracked services, finds the
                 compose file that owns each, and invokes the stack-support
                 agent to research + apply upgrades (test, then rollback on
                 failure). Ambiguous / breaking upgrades are escalated to the
                 admin over Matrix and resumed once they reply.

All operator communication happens through the core stack's Matrix
(chat.semprini.me) via a dedicated bot account.

The scope ("which services to maintain") is whatever Uptime Kuma is tracking at
the moment the health check runs — it is re-read every cycle, never hard-coded.

Design notes
------------
* Pure Python stdlib only (urllib, sqlite via a throwaway container, subprocess)
  so the service deploys without a venv.
* The daemon is intentionally "dumb": it gathers state and hands the fuzzy work
  (mapping a monitor to a container, choosing a fix, planning an upgrade) to the
  LLM agent defined in agents/stack-support.agent.md. The agent returns a small
  JSON envelope which the daemon relays / acts on.
"""

import argparse
import json
import os
import re
import subprocess
import sys
import time
import traceback
import urllib.parse
import urllib.request
from datetime import datetime, timezone

# --------------------------------------------------------------------------- #
# Config / state
# --------------------------------------------------------------------------- #

DEFAULTS = {
    "repo_dir": "/home/paul/Dev/semprini-core",
    # Extra directories the agent may read/operate on (other projects that use
    # the core stack, e.g. semprini-blog). Passed to claude via --add-dir.
    "extra_dirs": ["/home/paul/Dev"],
    "agent_file": "agents/stack-support.agent.md",
    "runbook_file": "docs/stack-support-runbook.md",
    "claude_bin": "/home/paul/.local/bin/claude",
    # Pin a standard 200K-window model. Leaving this None uses the account
    # default, which may be a 1M-context ([1m]) model; once a cycle's context
    # crosses 200K that escalates to the 1M tier and fails on subscription
    # plans with "Usage credits are required for long context requests" (429).
    "claude_model": "claude-opus-4-8",
    "claude_extra_args": [],
    "agent_timeout_seconds": 1800,
    "kuma_volume": "uptime-kuma-data",
    "sqlite_image": "keinos/sqlite3",
    # autonomy: "full" (fix + upgrade), "fix-only" (remediate, only report
    # upgrades), or "notify" (never act, only report).
    "autonomy": "full",
    "health_interval_seconds": 300,        # 5 min
    "upgrade_interval_seconds": 604800,    # 7 days
    "matrix_poll_seconds": 30,
    # A monitor must be seen DOWN this many consecutive cycles before we act,
    # to ride out restarts / flapping.
    "down_confirmations": 2,
    # Send a quiet "all healthy" heartbeat to Matrix at most this often.
    "heartbeat_interval_seconds": 86400,   # daily
    "state_file": "/home/paul/.local/state/semprini-maintainer/state.json",
    "matrix": {
        "homeserver": "https://matrix.semprini.me",
        "user_id": "",
        "access_token": "",
        "room_id": "",
        "admin_user_id": "@paul:semprini.me",
    },
}


def load_config(path):
    cfg = json.loads(json.dumps(DEFAULTS))  # deep copy
    if path and os.path.exists(path):
        with open(path) as fh:
            user = json.load(fh)
        for k, v in user.items():
            if k == "matrix" and isinstance(v, dict):
                cfg["matrix"].update(v)
            else:
                cfg[k] = v
    return cfg


def load_state(cfg):
    path = cfg["state_file"]
    if os.path.exists(path):
        try:
            with open(path) as fh:
                return json.load(fh)
        except Exception:
            log("WARN: could not parse state file, starting fresh")
    return {
        "last_health": 0,
        "last_upgrade": 0,
        "last_heartbeat": 0,
        "down_streak": {},      # monitor name -> consecutive-down count
        "matrix_since": None,   # /sync pagination token
        "pending": None,        # outstanding question awaiting admin reply
    }


def save_state(cfg, state):
    path = cfg["state_file"]
    os.makedirs(os.path.dirname(path), exist_ok=True)
    tmp = path + ".tmp"
    with open(tmp, "w") as fh:
        json.dump(state, fh, indent=2)
    os.replace(tmp, path)


def log(msg):
    print(f"[{datetime.now(timezone.utc).isoformat()}] {msg}", flush=True)


# --------------------------------------------------------------------------- #
# Uptime Kuma — read tracked monitors + latest status
# --------------------------------------------------------------------------- #

KUMA_QUERY = (
    "select json_group_array(json_object("
    "'id', m.id, 'name', m.name, 'type', m.type, 'url', m.url, "
    "'hostname', m.hostname, 'port', m.port, "
    "'status', (select h.status from heartbeat h where h.monitor_id=m.id "
    "order by h.time desc limit 1), "
    "'msg', (select h.msg from heartbeat h where h.monitor_id=m.id "
    "order by h.time desc limit 1))) "
    "from monitor m where m.active=1 order by m.id;"
)

# Uptime Kuma heartbeat status codes.
KUMA_DOWN, KUMA_UP, KUMA_PENDING, KUMA_MAINT = 0, 1, 2, 3


def read_kuma_monitors(cfg):
    """Return list of active monitors with their latest status.

    Reads kuma.db out of the named Docker volume via a throwaway sqlite
    container (the file is root-owned; the daemon user only has docker access).
    """
    script = f"cp /src/kuma.db /tmp/k.db && sqlite3 /tmp/k.db \"{KUMA_QUERY}\""
    out = subprocess.run(
        [
            "docker", "run", "--rm",
            "-v", f"{cfg['kuma_volume']}:/src:ro",
            cfg["sqlite_image"], "sh", "-c", script,
        ],
        capture_output=True, text=True, timeout=120,
    )
    if out.returncode != 0:
        raise RuntimeError(f"kuma read failed: {out.stderr.strip()}")
    data = out.stdout.strip() or "[]"
    return json.loads(data)


def docker_ps():
    out = subprocess.run(
        ["docker", "ps", "-a", "--format",
         "table {{.Names}}\t{{.Status}}\t{{.Image}}"],
        capture_output=True, text=True, timeout=60,
    )
    return out.stdout.strip()


# --------------------------------------------------------------------------- #
# Matrix client (client-server API over stdlib)
# --------------------------------------------------------------------------- #

def _matrix_req(cfg, method, path, body=None, params=None):
    base = cfg["matrix"]["homeserver"].rstrip("/")
    url = f"{base}{path}"
    if params:
        url += "?" + urllib.parse.urlencode(params)
    data = json.dumps(body).encode() if body is not None else None
    req = urllib.request.Request(url, data=data, method=method)
    req.add_header("Authorization", f"Bearer {cfg['matrix']['access_token']}")
    req.add_header("Content-Type", "application/json")
    with urllib.request.urlopen(req, timeout=60) as resp:
        raw = resp.read().decode()
    return json.loads(raw) if raw else {}


def _md_to_html(text):
    """Very small markdown -> HTML for Matrix formatted bodies."""
    import html
    esc = html.escape(text)
    esc = re.sub(r"\*\*(.+?)\*\*", r"<b>\1</b>", esc)
    esc = re.sub(r"`(.+?)`", r"<code>\1</code>", esc)
    return esc.replace("\n", "<br>")


def matrix_send(cfg, text):
    if not cfg["matrix"].get("room_id") or not cfg["matrix"].get("access_token"):
        log(f"MATRIX (not configured) >> {text}")
        return
    txn = str(int(time.time() * 1000))
    room = urllib.parse.quote(cfg["matrix"]["room_id"])
    try:
        _matrix_req(
            cfg, "PUT",
            f"/_matrix/client/v3/rooms/{room}/send/m.room.message/{txn}",
            body={
                "msgtype": "m.text",
                "body": text,
                "format": "org.matrix.custom.html",
                "formatted_body": _md_to_html(text),
            },
        )
    except Exception as e:
        log(f"ERROR sending to Matrix: {e}")


def matrix_poll_admin(cfg, state):
    """Return new text messages from the admin in our room since last sync."""
    if not cfg["matrix"].get("access_token"):
        return []
    params = {"timeout": "0"}
    if state.get("matrix_since"):
        params["since"] = state["matrix_since"]
    try:
        resp = _matrix_req(cfg, "GET", "/_matrix/client/v3/sync", params=params)
    except Exception as e:
        log(f"ERROR polling Matrix: {e}")
        return []
    state["matrix_since"] = resp.get("next_batch", state.get("matrix_since"))
    messages = []
    room_id = cfg["matrix"]["room_id"]
    admin = cfg["matrix"]["admin_user_id"]
    rooms = resp.get("rooms", {}).get("join", {})
    room = rooms.get(room_id)
    if not room:
        return messages
    for ev in room.get("timeline", {}).get("events", []):
        if ev.get("sender") != admin:
            continue
        # This bot is stdlib-only and cannot do E2E encryption. If the room is
        # encrypted, the admin's replies arrive as undecryptable m.room.encrypted
        # events — warn loudly rather than silently ignoring the operator. The
        # room should be created unencrypted (see register-matrix-bot.sh).
        if ev.get("type") == "m.room.encrypted":
            log("WARN: received an ENCRYPTED message from the admin — this bot "
                "cannot decrypt it. The maintenance room must be unencrypted.")
            continue
        if ev.get("type") != "m.room.message":
            continue
        content = ev.get("content", {})
        if content.get("msgtype") == "m.text":
            messages.append(content.get("body", "").strip())
    return messages


# --------------------------------------------------------------------------- #
# Stack-support agent invocation (Claude Code, headless)
# --------------------------------------------------------------------------- #

RESULT_PROTOCOL = """
When you have finished, end your reply with EXACTLY ONE fenced json block:

```json
{"status": "ok|action_taken|escalate|blocked",
 "summary": "one or two sentence operator summary (markdown ok)",
 "details": "optional longer detail",
 "question": "optional — only if status is escalate: the decision you need",
 "options": ["optional", "list", "of", "choices"]}
```

status meanings:
  ok           — everything healthy, nothing to do
  action_taken — you fixed/changed something; describe it in summary
  escalate     — you need a human decision (set question + options)
  blocked      — hard blocker, cannot proceed safely (explain in summary)
""".strip()


def run_agent(cfg, task):
    """Invoke the canonical stack-support agent headless and return its output."""
    agent_path = os.path.join(cfg["repo_dir"], cfg["agent_file"])
    with open(agent_path) as fh:
        agent_md = fh.read()

    system = (
        "You are the semprini-core Stack Maintenance Agent, running unattended "
        "as a systemd service. The following is your canonical agent definition; "
        "follow it as the source of truth, together with "
        f"{cfg['runbook_file']} and CLAUDE.md in this repository.\n\n"
        "You operate FULLY AUTONOMOUSLY per the CLAUDE.md sysadmin role: own the "
        "fix end to end. Make the smallest safe change that restores service. "
        "Pin versions after successful upgrades; auto-rollback on failure. "
        "Only set status=escalate when a genuine human decision is required "
        "(e.g. a major/breaking upgrade with trade-offs).\n\n"
        "Services in scope are exactly those Uptime Kuma tracks. Other projects "
        "(e.g. semprini-blog) live alongside semprini-core and use it for "
        "IAM/reverse-proxy; maintain their containers too when they are tracked.\n\n"
        "--- BEGIN agents/stack-support.agent.md ---\n"
        f"{agent_md}\n"
        "--- END agents/stack-support.agent.md ---\n\n"
        f"{RESULT_PROTOCOL}"
    )

    cmd = [
        cfg["claude_bin"], "-p", task,
        "--append-system-prompt", system,
        "--permission-mode", "bypassPermissions",
        "--output-format", "text",
    ]
    if cfg.get("claude_model"):
        cmd += ["--model", cfg["claude_model"]]
    for d in cfg.get("extra_dirs", []):
        cmd += ["--add-dir", d]
    cmd += list(cfg.get("claude_extra_args", []))

    log(f"Invoking stack-support agent ({len(task)} char task)…")
    proc = subprocess.run(
        cmd, cwd=cfg["repo_dir"], capture_output=True, text=True,
        timeout=cfg["agent_timeout_seconds"],
    )
    if proc.returncode != 0:
        log(f"WARN: agent exited {proc.returncode}: {proc.stderr.strip()[:500]}")
    return proc.stdout.strip()


def parse_agent_result(output):
    """Pull the trailing JSON envelope out of the agent's reply."""
    blocks = re.findall(r"```json\s*(\{.*?\})\s*```", output, re.DOTALL)
    if blocks:
        try:
            return json.loads(blocks[-1])
        except Exception:
            pass
    # Fallback: try a bare trailing {...}
    m = re.search(r"(\{[^{}]*\"status\"[^{}]*\})\s*$", output, re.DOTALL)
    if m:
        try:
            return json.loads(m.group(1))
        except Exception:
            pass
    return {"status": "unknown",
            "summary": (output[-800:] or "no output from agent")}


def relay_result(cfg, state, header, result):
    """Post an agent result to Matrix and register any escalation."""
    status = result.get("status", "unknown")
    summary = result.get("summary", "")
    icon = {"ok": "✅", "action_taken": "🔧", "escalate": "❓",
            "blocked": "⛔", "unknown": "⚠️"}.get(status, "•")
    msg = f"{icon} **{header}** — {status}\n{summary}"
    if result.get("details"):
        msg += f"\n\n{result['details']}"

    if status == "escalate" and result.get("question"):
        opts = result.get("options") or []
        msg += f"\n\n**Decision needed:** {result['question']}"
        if opts:
            msg += "\n" + "\n".join(f"  {i+1}. {o}" for i, o in enumerate(opts))
            msg += "\n\n_Reply with a number or a short answer._"
        state["pending"] = {
            "header": header,
            "question": result["question"],
            "options": opts,
            "asked_at": time.time(),
        }
    matrix_send(cfg, msg)


# --------------------------------------------------------------------------- #
# Cycles
# --------------------------------------------------------------------------- #

def status_label(code):
    return {KUMA_DOWN: "DOWN", KUMA_UP: "UP", KUMA_PENDING: "PENDING",
            KUMA_MAINT: "MAINTENANCE"}.get(code, f"?{code}")


def health_cycle(cfg, state):
    log("Health cycle: reading Uptime Kuma…")
    monitors = read_kuma_monitors(cfg)
    streak = state.setdefault("down_streak", {})

    down_now = []
    for m in monitors:
        name = m["name"]
        if m.get("status") == KUMA_DOWN:
            streak[name] = streak.get(name, 0) + 1
            down_now.append(m)
        else:
            streak.pop(name, None)

    confirmed = [m for m in down_now
                 if streak.get(m["name"], 0) >= cfg["down_confirmations"]]

    log(f"  {len(monitors)} tracked, {len(down_now)} down, "
        f"{len(confirmed)} confirmed down")

    if not confirmed:
        # quiet heartbeat
        if (time.time() - state.get("last_heartbeat", 0)
                >= cfg["heartbeat_interval_seconds"]):
            up = sum(1 for m in monitors if m.get("status") == KUMA_UP)
            matrix_send(
                cfg,
                f"✅ **Daily health** — {up}/{len(monitors)} tracked services up.",
            )
            state["last_heartbeat"] = time.time()
        return

    if cfg["autonomy"] == "notify":
        lines = "\n".join(
            f"  • **{m['name']}** ({m.get('url') or m.get('hostname')}): "
            f"{m.get('msg') or 'down'}" for m in confirmed
        )
        matrix_send(cfg, f"⚠️ **Services down** (notify-only mode):\n{lines}")
        return

    incident = "\n".join(
        f"  • {m['name']} | type={m['type']} | "
        f"target={m.get('url') or (m.get('hostname') or '') + ':' + str(m.get('port') or '')} | "
        f"last_msg={m.get('msg') or 'n/a'}"
        for m in confirmed
    )
    task = (
        "ROUTINE HEALTH CHECK — Uptime Kuma reports the following tracked "
        "services DOWN (confirmed over multiple cycles):\n\n"
        f"{incident}\n\n"
        "Current containers:\n"
        f"{docker_ps()}\n\n"
        "For each down service: map the monitor target (a compose service "
        "name / network alias) to its container, diagnose using the runbook, "
        "and remediate autonomously. Re-verify recovery. If a service is down "
        "by design or needs a human decision, set status=escalate."
    )
    result = parse_agent_result(run_agent(cfg, task))
    relay_result(cfg, state, "Health remediation", result)


def resolve_tracked_images(cfg, monitors):
    """Best-effort map of tracked services -> running image (for context)."""
    ps = subprocess.run(
        ["docker", "ps", "--format", "{{.Names}}\t{{.Image}}"],
        capture_output=True, text=True, timeout=60,
    ).stdout.strip().splitlines()
    name_to_image = {}
    for line in ps:
        if "\t" in line:
            n, img = line.split("\t", 1)
            name_to_image[n] = img
    rows = []
    for m in monitors:
        target = (m.get("hostname") or m.get("url") or "").lower()
        token = re.sub(r"^https?://", "", target).split("/")[0].split(":")[0]
        matches = [f"{n} -> {img}" for n, img in name_to_image.items()
                   if token and token in n]
        rows.append(f"  • {m['name']} (target={token or 'n/a'}): "
                    f"{', '.join(matches) or 'no obvious container match'}")
    return "\n".join(rows)


def upgrade_cycle(cfg, state):
    if cfg["autonomy"] == "notify":
        autonomy_note = (
            "Mode is NOTIFY-ONLY: do NOT change anything. Report available "
            "upgrades and recommendations only; set status=escalate with the list."
        )
    elif cfg["autonomy"] == "fix-only":
        autonomy_note = (
            "Mode is FIX-ONLY for upgrades: research and report available "
            "upgrades; do NOT apply them. Set status=escalate to ask which to apply."
        )
    else:
        autonomy_note = (
            "Mode is FULL: apply safe upgrades autonomously (pull, test for "
            "stability, pin the new tag in the owning compose file, redeploy, "
            "rollback on failure). For MAJOR/breaking upgrades with trade-offs, "
            "set status=escalate with options instead of applying."
        )

    log("Upgrade cycle…")
    monitors = read_kuma_monitors(cfg)
    context = resolve_tracked_images(cfg, monitors)
    task = (
        "PERIODIC UPGRADE REVIEW. Scope = services Uptime Kuma currently tracks. "
        "Tracked services and their running images:\n\n"
        f"{context}\n\n"
        "Use the upgrade-services skill (agents/skills/upgrade-services/SKILL.md). "
        "For each tracked service: find the compose file that owns it (search "
        "core_stack/compose*.yml and other project repos under the added dirs, "
        "e.g. semprini-blog/docker-compose.yml), determine the current pinned "
        "tag, check Docker Hub / GitHub for newer stable releases, and assess "
        "breaking changes.\n\n"
        f"{autonomy_note}\n\n"
        "Summarise what is up to date, what you upgraded, and anything needing "
        "a decision."
    )
    result = parse_agent_result(run_agent(cfg, task))
    relay_result(cfg, state, "Upgrade review", result)


def handle_replies(cfg, state):
    msgs = matrix_poll_admin(cfg, state)
    if not msgs:
        return
    pending = state.get("pending")
    for body in msgs:
        log(f"Admin reply: {body!r}")
        if not pending:
            # Unprompted instruction from the admin — treat as an ad-hoc task.
            task = (
                "The operator sent this instruction over Matrix; act on it "
                f"autonomously and report back:\n\n{body}"
            )
            result = parse_agent_result(run_agent(cfg, task))
            relay_result(cfg, state, "Operator request", result)
            pending = state.get("pending")
            continue

        # Resolve the outstanding escalation with the admin's answer.
        answer = body
        opts = pending.get("options") or []
        if body.isdigit() and 1 <= int(body) <= len(opts):
            answer = opts[int(body) - 1]
        task = (
            f"Earlier you escalated this decision:\n"
            f"  Question: {pending['question']}\n"
            f"  Options: {opts}\n\n"
            f"The operator decided: {answer}\n\n"
            "Carry out that decision now, autonomously, with full testing and "
            "rollback on failure. Report the outcome."
        )
        state["pending"] = None
        save_state(cfg, state)
        result = parse_agent_result(run_agent(cfg, task))
        relay_result(cfg, state, "Decision carried out", result)
        pending = state.get("pending")


# --------------------------------------------------------------------------- #
# Main loop
# --------------------------------------------------------------------------- #

def main():
    ap = argparse.ArgumentParser(description="semprini stack maintenance daemon")
    ap.add_argument("--config",
                    default="/home/paul/Dev/semprini-maintainer/config.json")
    ap.add_argument("--once", choices=["health", "upgrade", "replies"],
                    help="run a single cycle and exit (for testing)")
    args = ap.parse_args()

    cfg = load_config(args.config)
    state = load_state(cfg)

    if args.once:
        log(f"Running single '{args.once}' cycle…")
        {"health": health_cycle, "upgrade": upgrade_cycle,
         "replies": handle_replies}[args.once](cfg, state)
        save_state(cfg, state)
        return

    log("semprini-maintainer starting up.")
    matrix_send(cfg, "🤖 **stack-maintainer online** — monitoring tracked "
                     "services via Uptime Kuma.")

    while True:
        now = time.time()
        try:
            handle_replies(cfg, state)

            if now - state.get("last_health", 0) >= cfg["health_interval_seconds"]:
                health_cycle(cfg, state)
                state["last_health"] = now

            # Hold upgrades while an escalation is unanswered.
            if (not state.get("pending")
                    and now - state.get("last_upgrade", 0)
                    >= cfg["upgrade_interval_seconds"]):
                upgrade_cycle(cfg, state)
                state["last_upgrade"] = now

            save_state(cfg, state)
        except Exception as e:
            log("ERROR in main loop:\n" + traceback.format_exc())
            try:
                matrix_send(cfg, f"⛔ **maintainer error**: {e}")
            except Exception:
                pass

        time.sleep(cfg["matrix_poll_seconds"])


if __name__ == "__main__":
    sys.exit(main())