Nexus/server/application/services/watch_service.py

"""Watch slot business logic — pin CRUD and live metrics assembly."""

from __future__ import annotations

import json
from datetime import datetime, timedelta, timezone
from typing import Any

from sqlalchemy.ext.asyncio import AsyncSession

from server.domain.models import Admin, AuditLog, Server
from server.infrastructure.database.audit_log_repo import AuditLogRepositoryImpl
from server.infrastructure.database.server_repo import ServerRepositoryImpl
from server.infrastructure.database.watch_repo import WatchRepositoryImpl
from server.infrastructure.redis.client import get_redis
from server.background.watch_probe_runner import trigger_immediate_watch_probe
from server.utils.watch_state import clear_watch_redis_snapshot
from server.utils.watch_metrics import (
    WATCH_MAX_SLOTS,
    WATCH_PIN_TTL_DEFAULT_MINUTES,
    normalize_watch_ttl_minutes,
    resolve_pin_ttl_minutes,
    server_can_be_watched,
)

REDIS_WATCH_LIVE_PREFIX = "watch:live:"
REDIS_WATCH_PROC_PREFIX = "watch:proc:"
WATCH_SPARKLINE_MAX = 180  # 15 min @ 5s


def _utc_naive_now() -> datetime:
    return datetime.now(timezone.utc).replace(tzinfo=None)


def _remaining_sec(expires_at: datetime) -> int:
    now = _utc_naive_now()
    return max(0, int((expires_at - now).total_seconds()))


async def _load_live_metrics(server_id: int) -> dict[str, Any] | None:
    redis = get_redis()
    raw = await redis.lindex(f"{REDIS_WATCH_LIVE_PREFIX}{server_id}", 0)
    if not raw:
        return None
    try:
        data = json.loads(raw)
        return data if isinstance(data, dict) else None
    except json.JSONDecodeError:
        return None


async def _load_sparkline(server_id: int) -> list[dict[str, Any]]:
    redis = get_redis()
    rows = await redis.lrange(f"{REDIS_WATCH_LIVE_PREFIX}{server_id}", 0, WATCH_SPARKLINE_MAX - 1)
    out: list[dict[str, Any]] = []
    for raw in reversed(rows):
        try:
            point = json.loads(raw)
            if isinstance(point, dict):
                out.append(point)
        except json.JSONDecodeError:
            continue
    return out


async def _load_processes(server_id: int) -> dict[str, Any] | list[dict[str, Any]] | None:
    redis = get_redis()
    raw = await redis.get(f"{REDIS_WATCH_PROC_PREFIX}{server_id}")
    if not raw:
        return None
    try:
        data = json.loads(raw)
    except json.JSONDecodeError:
        return None
    if isinstance(data, dict):
        from server.utils.watch_metrics import sanitize_process_bundle

        return sanitize_process_bundle(data)
    if isinstance(data, list):
        from server.utils.watch_metrics import sanitize_processes

        legacy = sanitize_processes(data)
        return {"top_cpu": legacy or [], "top_mem": []} if legacy else None
    return None


async def _restore_idle_agent_watch(watch_repo: WatchRepositoryImpl, server_id: int) -> None:
    """Stop 5s watch snapshots when no slot actively monitors this server (Agent stays 60s)."""
    if await watch_repo.count_monitoring_pins_for_server(server_id) == 0:
        await clear_watch_redis_snapshot(server_id)


async def _eager_watch_probe(server_id: int) -> None:
    """Fill Redis + WS immediately when monitoring starts (avoid empty-metrics gap after resume)."""
    await trigger_immediate_watch_probe(server_id)


async def _finalize_due_pins(watch_repo: WatchRepositoryImpl, db: AsyncSession) -> None:
    """Pause TTL-expired pins on read/write paths (probe loop also runs this every 5s)."""
    expired_servers = await watch_repo.expire_due_pins()
    if not expired_servers:
        return
    for server_id in expired_servers:
        if await watch_repo.count_monitoring_pins_for_server(server_id) == 0:
            await clear_watch_redis_snapshot(server_id)
    await db.commit()


async def _slot_payload(
    *,
    slot_index: int,
    pin_row: dict[str, Any] | None,
) -> dict[str, Any]:
    if not pin_row:
        return {"slot_index": slot_index, "empty": True}
    monitoring_enabled = pin_row.get("monitoring_enabled", True)
    metrics = None
    sparkline: list[dict[str, Any]] = []
    processes = None
    if monitoring_enabled:
        metrics = await _load_live_metrics(pin_row["server_id"])
        sparkline = await _load_sparkline(pin_row["server_id"])
        processes = await _load_processes(pin_row["server_id"])
        if metrics and processes:
            metrics = {**metrics, "processes": processes}
    remaining = _remaining_sec(pin_row["expires_at"]) if monitoring_enabled else None
    return {
        "slot_index": slot_index,
        "empty": False,
        "pin_id": pin_row["pin_id"],
        "server_id": pin_row["server_id"],
        "server_name": pin_row.get("server_name") or "",
        "session_id": pin_row["session_id"],
        "pinned_at": pin_row["pinned_at"].isoformat() if pin_row.get("pinned_at") else None,
        "expires_at": pin_row["expires_at"].isoformat() if pin_row.get("expires_at") else None,
        "remaining_sec": remaining,
        "monitoring_enabled": monitoring_enabled,
        "ttl_minutes": int(pin_row.get("ttl_minutes") or WATCH_PIN_TTL_DEFAULT_MINUTES),
        "ttl_hours": int(pin_row.get("ttl_hours") or max(1, (int(pin_row.get("ttl_minutes") or 30) + 59) // 60)),
        "metrics": metrics,
        "sparkline": sparkline,
        "processes": processes,
    }


class WatchService:
    def __init__(self, db: AsyncSession):
        self.db = db
        self.watch_repo = WatchRepositoryImpl(db)
        self.server_repo = ServerRepositoryImpl(db)
        self.audit_repo = AuditLogRepositoryImpl(db)

    async def list_slots(self, admin_id: int) -> dict[str, Any]:
        await _finalize_due_pins(self.watch_repo, self.db)
        pins = await self.watch_repo.list_active_pins_for_admin(admin_id)
        by_slot = {p["slot_index"]: p for p in pins}
        slots = [
            await _slot_payload(slot_index=i, pin_row=by_slot.get(i))
            for i in range(WATCH_MAX_SLOTS)
        ]
        return {"slots": slots}

    async def pin_server(
        self,
        *,
        admin: Admin,
        server_id: int,
        slot_index: int | None = None,
        replace_slot: int | None = None,
        ttl_minutes: int = WATCH_PIN_TTL_DEFAULT_MINUTES,
        ip_address: str = "",
    ) -> dict[str, Any]:
        await _finalize_due_pins(self.watch_repo, self.db)
        minutes = normalize_watch_ttl_minutes(ttl_minutes)
        server = await self.server_repo.get_by_id(server_id)
        if not server:
            raise ValueError("服务器不存在")

        ok, reason = server_can_be_watched(server)
        if not ok:
            raise ValueError(reason)

        existing = await self.watch_repo.get_pin_by_server(admin.id, server_id)
        if existing:
            await self.watch_repo.reset_pin_ttl(admin.id, server_id, minutes)
            await self._audit(
                admin=admin,
                action="watch_pin_add",
                server=server,
                detail=f"重置监测 {minutes}min · 槽 {existing.slot_index + 1}",
                ip_address=ip_address,
            )
            await self.db.commit()
            return await self.list_slots(admin.id)

        if slot_index is not None:
            await self.watch_repo.replace_pin_server(
                admin_id=admin.id,
                slot_index=slot_index,
                server_id=server_id,
                ttl_minutes=minutes,
            )
            await self._audit(
                admin=admin,
                action="watch_pin_replace" if replace_slot is not None else "watch_pin_add",
                server=server,
                detail=f"加入监测 · 槽 {slot_index + 1} · {minutes}min",
                ip_address=ip_address,
            )
            await self.db.commit()
            await _eager_watch_probe(server_id)
            return await self.list_slots(admin.id)

        empty = await self.watch_repo.first_empty_slot(admin.id)
        if empty is not None:
            await self.watch_repo.create_pin(
                admin_id=admin.id,
                server_id=server_id,
                slot_index=empty,
                ttl_minutes=minutes,
            )
            await self._audit(
                admin=admin,
                action="watch_pin_add",
                server=server,
                detail=f"加入监测 · 槽 {empty + 1} · {minutes}min",
                ip_address=ip_address,
            )
            await self.db.commit()
            await _eager_watch_probe(server_id)
            return await self.list_slots(admin.id)

        if replace_slot is not None:
            await self.watch_repo.replace_pin_server(
                admin_id=admin.id,
                slot_index=replace_slot,
                server_id=server_id,
                ttl_minutes=minutes,
            )
            await self._audit(
                admin=admin,
                action="watch_pin_replace",
                server=server,
                detail=f"替换监测 · 槽 {replace_slot + 1} · {minutes}min",
                ip_address=ip_address,
            )
            await self.db.commit()
            await _eager_watch_probe(server_id)
            return await self.list_slots(admin.id)

        raise SlotsFullError()

    async def replace_slot(
        self,
        *,
        admin: Admin,
        slot_index: int,
        server_id: int,
        ip_address: str = "",
    ) -> dict[str, Any]:
        await _finalize_due_pins(self.watch_repo, self.db)
        if slot_index < 0 or slot_index >= WATCH_MAX_SLOTS:
            raise ValueError("槽位无效")
        server = await self.server_repo.get_by_id(server_id)
        if not server:
            raise ValueError("服务器不存在")
        ok, reason = server_can_be_watched(server)
        if not ok:
            raise ValueError(reason)
        other = await self.watch_repo.get_pin_by_server(admin.id, server_id)
        if other and other.slot_index != slot_index:
            await self.watch_repo.end_session(other.session_id, "replaced")
            await self.watch_repo.delete_pin(other.id)
        existing = await self.watch_repo.get_pin_by_slot(admin.id, slot_index)
        slot_ttl = (
            resolve_pin_ttl_minutes(
                ttl_minutes=getattr(existing, "ttl_minutes", None),
                ttl_hours=existing.ttl_hours,
            )
            if existing
            else WATCH_PIN_TTL_DEFAULT_MINUTES
        )
        await self.watch_repo.replace_pin_server(
            admin_id=admin.id,
            slot_index=slot_index,
            server_id=server_id,
            ttl_minutes=slot_ttl,
        )
        await self._audit(
            admin=admin,
            action="watch_pin_replace",
            server=server,
            detail=f"替换监测 · 槽 {slot_index + 1} · {slot_ttl}min",
            ip_address=ip_address,
        )
        await self.db.commit()
        await _eager_watch_probe(server_id)
        return await self.list_slots(admin.id)

    async def remove_slot(
        self,
        *,
        admin: Admin,
        slot_index: int,
        ip_address: str = "",
    ) -> dict[str, Any]:
        await _finalize_due_pins(self.watch_repo, self.db)
        if slot_index < 0 or slot_index >= WATCH_MAX_SLOTS:
            raise ValueError("槽位无效")
        pin = await self.watch_repo.remove_pin_by_slot(admin.id, slot_index)
        if not pin:
            raise ValueError("槽位为空")
        server_id = pin.server_id
        server = await self.server_repo.get_by_id(server_id)
        await self._audit(
            admin=admin,
            action="watch_pin_remove",
            server=server,
            detail=f"移除监测 · 槽 {slot_index + 1}",
            ip_address=ip_address,
        )
        await self.db.commit()
        await _restore_idle_agent_watch(self.watch_repo, server_id)
        return await self.list_slots(admin.id)

    async def set_slot_monitoring(
        self,
        *,
        admin: Admin,
        slot_index: int,
        monitoring_enabled: bool,
        ip_address: str = "",
    ) -> dict[str, Any]:
        await _finalize_due_pins(self.watch_repo, self.db)
        if slot_index < 0 or slot_index >= WATCH_MAX_SLOTS:
            raise ValueError("槽位无效")
        pin = await self.watch_repo.set_monitoring_enabled(
            admin.id,
            slot_index,
            monitoring_enabled,
        )
        if not pin:
            raise ValueError("槽位为空")
        server = await self.server_repo.get_by_id(pin.server_id)
        minutes = int(pin.ttl_minutes or WATCH_PIN_TTL_DEFAULT_MINUTES)
        action = "watch_pin_resume" if monitoring_enabled else "watch_pin_pause"
        detail = (
            f"开启实时监测 · 槽 {slot_index + 1} · 重置 {minutes}min"
            if monitoring_enabled
            else f"暂停实时监测 · 槽 {slot_index + 1} · 恢复 Agent 60s"
        )
        await self._audit(
            admin=admin,
            action=action,
            server=server,
            detail=detail,
            ip_address=ip_address,
        )
        await self.db.commit()
        if not monitoring_enabled:
            await _restore_idle_agent_watch(self.watch_repo, pin.server_id)
        else:
            await _eager_watch_probe(pin.server_id)
        return await self.list_slots(admin.id)

    async def set_slot_ttl(
        self,
        *,
        admin: Admin,
        slot_index: int,
        ttl_minutes: int,
        ip_address: str = "",
    ) -> dict[str, Any]:
        await _finalize_due_pins(self.watch_repo, self.db)
        if slot_index < 0 or slot_index >= WATCH_MAX_SLOTS:
            raise ValueError("槽位无效")
        minutes = normalize_watch_ttl_minutes(ttl_minutes)
        pin = await self.watch_repo.set_pin_ttl_minutes(admin.id, slot_index, minutes)
        if not pin:
            raise ValueError("槽位为空")
        server = await self.server_repo.get_by_id(pin.server_id)
        await self._audit(
            admin=admin,
            action="watch_pin_ttl",
            server=server,
            detail=f"监测时长 · 槽 {slot_index + 1} · {minutes}min",
            ip_address=ip_address,
        )
        await self.db.commit()
        return await self.list_slots(admin.id)

    async def get_live_for_servers(self, server_ids: list[int]) -> dict[str, Any]:
        items: list[dict[str, Any]] = []
        for sid in server_ids:
            metrics = await _load_live_metrics(sid)
            sparkline = await _load_sparkline(sid)
            items.append({"server_id": sid, "metrics": metrics, "sparkline": sparkline})
        return {"items": items}

    async def pinned_server_map(self, admin_id: int) -> dict[int, int]:
        """server_id → slot_index (0-based) for list UI."""
        pins = await self.watch_repo.list_active_pins_for_admin(admin_id)
        return {p["server_id"]: p["slot_index"] for p in pins}

    async def get_metrics(
        self,
        admin_id: int,
        *,
        server_ids: list[int],
        hours: int = 24,
    ) -> dict[str, Any]:
        since = _utc_naive_now() - timedelta(hours=max(1, min(hours, 168)))
        points = await self.watch_repo.list_metrics_series(
            admin_id,
            server_ids=server_ids,
            since=since,
            ok_only=True,
        )
        return {"points": points, "hours": hours}

    async def get_probe_records(
        self,
        admin_id: int,
        *,
        server_id: int | None = None,
        session_id: int | None = None,
        probe_status: str | None = None,
        hours: int = 24,
        page: int = 1,
        per_page: int = 50,
    ) -> dict[str, Any]:
        since = _utc_naive_now() - timedelta(hours=max(1, min(hours, 168)))
        items, total = await self.watch_repo.list_probe_records(
            admin_id,
            server_id=server_id,
            session_id=session_id,
            probe_status=probe_status,
            since=since,
            page=page,
            per_page=per_page,
        )
        return {"items": items, "total": total, "page": page, "per_page": per_page}

    async def get_sessions(self, admin_id: int, *, hours: int = 168) -> dict[str, Any]:
        since = _utc_naive_now() - timedelta(hours=max(1, min(hours, 168)))
        sessions = await self.watch_repo.list_sessions(admin_id, since=since)
        return {"sessions": sessions, "hours": hours}

    async def get_processes(self, server_id: int) -> dict[str, Any]:
        processes = await _load_processes(server_id)
        return {
            "server_id": server_id,
            "processes": processes if processes is not None else {"top_cpu": [], "top_mem": []},
        }

    async def install_psutil_ssh(
        self,
        *,
        admin: Admin,
        server_id: int,
        ip_address: str = "",
    ) -> dict[str, Any]:
        pinned = await self.pinned_server_map(admin.id)
        if server_id not in pinned:
            raise ValueError("该服务器不在当前监测槽中")
        server = await self.server_repo.get_by_id(server_id)
        if not server:
            raise ValueError("服务器不存在")

        from server.utils.psutil_install import install_psutil_via_ssh

        try:
            result = await install_psutil_via_ssh(server)
        except Exception as e:
            raise ValueError(f"SSH 连接失败: {e}") from e

        if not result.get("success"):
            msg = str(result.get("message") or "psutil 安装失败")
            raise ValueError(msg)

        await self._audit(
            admin=admin,
            action="watch_install_psutil",
            server=server,
            detail=f"SSH 安装 psutil · {server.name or server_id}",
            ip_address=ip_address,
        )
        await self.db.commit()
        return {
            "success": True,
            "server_id": server_id,
            "message": result.get("message") or "psutil 已就绪",
        }

    async def _audit(
        self,
        *,
        admin: Admin,
        action: str,
        server: Server | None,
        detail: str,
        ip_address: str,
    ) -> None:
        await self.audit_repo.create(
            AuditLog(
                admin_username=admin.username,
                action=action,
                target_type="server",
                target_id=server.id if server else 0,
                detail=detail,
                ip_address=ip_address,
            )
        )


class SlotsFullError(Exception):
    """All 4 watch slots occupied."""