Nexus/server/utils/watch_metrics.py

"""Watch slot probe — rate calculation and metric normalization."""

from __future__ import annotations

import json
from dataclasses import dataclass
from typing import Any

from server.utils.fleet_metrics import _metric_from_system_info, _safe_pct

WATCH_MAX_SLOTS = 4
WATCH_PIN_TTL_HOURS = 8
WATCH_PIN_TTL_EXTENDED_HOURS = 24
WATCH_PIN_TTL_ALLOWED = frozenset({WATCH_PIN_TTL_HOURS, WATCH_PIN_TTL_EXTENDED_HOURS})


def normalize_watch_ttl_hours(hours: int) -> int:
    """Only 8h (default) or 24h (explicit opt-in) are allowed."""
    if hours == WATCH_PIN_TTL_EXTENDED_HOURS:
        return WATCH_PIN_TTL_EXTENDED_HOURS
    return WATCH_PIN_TTL_HOURS
WATCH_PROBE_INTERVAL_SEC = 5
WATCH_REDIS_FRESH_SEC = 90
WATCH_PROBE_TIMEOUT_SEC = 8

PROBE_OK = "ok"
PROBE_SSH_TIMEOUT = "ssh_timeout"
PROBE_SSH_AUTH = "ssh_auth"
PROBE_NO_CRED = "no_cred"
PROBE_REDIS_STALE = "redis_stale"
PROBE_PARSE_ERROR = "parse_error"
PROBE_OFFLINE = "offline"


@dataclass
class WatchProbeSample:
    probe_status: str
    source: str
    is_online: bool = False
    duration_ms: int = 0
    error: str | None = None
    cpu_pct: int | None = None
    mem_pct: int | None = None
    disk_pct: int | None = None
    disk_mount: str = "/"
    load_1: float | None = None
    load_5: float | None = None
    load_15: float | None = None
    net_bytes_sent: int | None = None
    net_bytes_recv: int | None = None
    disk_read_bytes: int | None = None
    disk_write_bytes: int | None = None
    net_up_bps: int | None = None
    net_down_bps: int | None = None
    disk_read_bps: int | None = None
    disk_write_bps: int | None = None
    processes_json: list[dict[str, Any]] | None = None

    def to_live_dict(self, *, server_id: int) -> dict[str, Any]:
        out = {
            "server_id": server_id,
            "probe_status": self.probe_status,
            "source": self.source,
            "is_online": self.is_online,
            "cpu_pct": self.cpu_pct,
            "mem_pct": self.mem_pct,
            "disk_pct": self.disk_pct,
            "load_1": self.load_1,
            "load_5": self.load_5,
            "load_15": self.load_15,
            "net_up_bps": self.net_up_bps,
            "net_down_bps": self.net_down_bps,
            "disk_read_bps": self.disk_read_bps,
            "disk_write_bps": self.disk_write_bps,
            "error": self.error,
        }
        if self.processes_json:
            out["processes"] = self.processes_json
        return out


def redis_sample_has_io(sample: WatchProbeSample | None) -> bool:
    """True when Agent/Redis provides cumulative net + disk IO (M3: skip SSH)."""
    if sample is None or sample.probe_status != PROBE_OK:
        return False
    return (
        sample.net_bytes_sent is not None
        and sample.net_bytes_recv is not None
        and sample.disk_read_bytes is not None
        and sample.disk_write_bytes is not None
        and sample.cpu_pct is not None
        and sample.mem_pct is not None
    )


def redis_sample_has_basic_metrics(sample: WatchProbeSample | None) -> bool:
    """True when Agent heartbeat has CPU or memory — IO optional."""
    if sample is None or sample.probe_status != PROBE_OK:
        return False
    return sample.cpu_pct is not None or sample.mem_pct is not None


def sanitize_processes(raw: Any) -> list[dict[str, Any]] | None:
    if not isinstance(raw, list):
        return None
    out: list[dict[str, Any]] = []
    for item in raw[:5]:
        if not isinstance(item, dict):
            continue
        name = str(item.get("name") or "")[:80]
        out.append(
            {
                "pid": item.get("pid"),
                "name": name,
                "cpu_pct": item.get("cpu_pct"),
                "mem_pct": item.get("mem_pct"),
            }
        )
    return out or None


@dataclass
class _RateState:
    net_bytes_sent: int | None = None
    net_bytes_recv: int | None = None
    disk_read_bytes: int | None = None
    disk_write_bytes: int | None = None
    recorded_at_ts: float | None = None


def server_can_be_watched(server: Any) -> tuple[bool, str]:
    """Return whether server has Agent or SSH credentials for watch probes."""
    if getattr(server, "agent_version", None):
        return True, ""
    if getattr(server, "password", None) or getattr(server, "ssh_key_private", None):
        return True, ""
    if getattr(server, "ssh_key_configured", None) or getattr(server, "preset_id", None):
        return True, ""
    if getattr(server, "ssh_key_preset_id", None):
        return True, ""
    return False, "需要已安装 Agent 或配置 SSH 凭据"


def _int_pct(value: Any) -> int | None:
    pct = _safe_pct(value)
    return round(pct) if pct is not None else None


def _positive_int(value: Any) -> int | None:
    if value is None:
        return None
    try:
        num = int(value)
    except (TypeError, ValueError):
        return None
    return num if num >= 0 else None


def compute_rates(
    prev: _RateState | None,
    *,
    net_bytes_sent: int | None,
    net_bytes_recv: int | None,
    disk_read_bytes: int | None,
    disk_write_bytes: int | None,
    now_ts: float,
) -> tuple[int | None, int | None, int | None, int | None]:
    if prev is None or prev.recorded_at_ts is None:
        return None, None, None, None
    dt = now_ts - prev.recorded_at_ts
    if dt <= 0:
        return None, None, None, None

    def _rate(curr: int | None, old: int | None) -> int | None:
        if curr is None or old is None:
            return None
        delta = curr - old
        if delta < 0:
            return None
        return int(delta / dt)

    return (
        _rate(net_bytes_sent, prev.net_bytes_sent),
        _rate(net_bytes_recv, prev.net_bytes_recv),
        _rate(disk_read_bytes, prev.disk_read_bytes),
        _rate(disk_write_bytes, prev.disk_write_bytes),
    )


def apply_rates_to_sample(sample: WatchProbeSample, prev: _RateState | None, now_ts: float) -> _RateState:
    up, down, dr, dw = compute_rates(
        prev,
        net_bytes_sent=sample.net_bytes_sent,
        net_bytes_recv=sample.net_bytes_recv,
        disk_read_bytes=sample.disk_read_bytes,
        disk_write_bytes=sample.disk_write_bytes,
        now_ts=now_ts,
    )
    sample.net_up_bps = up
    sample.net_down_bps = down
    sample.disk_read_bps = dr
    sample.disk_write_bps = dw
    return _RateState(
        net_bytes_sent=sample.net_bytes_sent,
        net_bytes_recv=sample.net_bytes_recv,
        disk_read_bytes=sample.disk_read_bytes,
        disk_write_bytes=sample.disk_write_bytes,
        recorded_at_ts=now_ts,
    )


def parse_ssh_watch_payload(data: dict[str, Any]) -> WatchProbeSample:
    load = data.get("load_avg") or data.get("load") or []
    if isinstance(load, (list, tuple)) and len(load) >= 3:
        load_1, load_5, load_15 = float(load[0]), float(load[1]), float(load[2])
    else:
        load_1 = float(data["load_1"]) if data.get("load_1") is not None else None
        load_5 = float(data["load_5"]) if data.get("load_5") is not None else None
        load_15 = float(data["load_15"]) if data.get("load_15") is not None else None

    net = data.get("net_io") or {}
    disk_io = data.get("disk_io") or {}
    processes = sanitize_processes(data.get("top_processes"))

    return WatchProbeSample(
        probe_status=PROBE_OK,
        source="ssh",
        is_online=True,
        cpu_pct=_int_pct(data.get("cpu_usage")),
        mem_pct=_int_pct(data.get("mem_usage")),
        disk_pct=_int_pct(data.get("disk_usage")),
        disk_mount=str(data.get("disk_mount") or "/"),
        load_1=load_1,
        load_5=load_5,
        load_15=load_15,
        net_bytes_sent=_positive_int(net.get("bytes_sent")),
        net_bytes_recv=_positive_int(net.get("bytes_recv")),
        disk_read_bytes=_positive_int(disk_io.get("read_bytes")),
        disk_write_bytes=_positive_int(disk_io.get("write_bytes")),
        processes_json=processes,
    )


def parse_redis_watch_payload(heartbeat: dict[str, str]) -> WatchProbeSample | None:
    if heartbeat.get("is_online") != "True":
        return None
    raw = heartbeat.get("system_info") or "{}"
    try:
        info = json.loads(raw) if isinstance(raw, str) else (raw or {})
    except json.JSONDecodeError:
        return None
    if not isinstance(info, dict):
        return None

    cpu = _metric_from_system_info(info, "cpu_usage", "cpu")
    mem = _metric_from_system_info(info, "mem_usage", "mem")
    disk = _metric_from_system_info(info, "disk_usage", "disk")
    net = info.get("net_io") if isinstance(info.get("net_io"), dict) else {}
    disk_io = info.get("disk_io") if isinstance(info.get("disk_io"), dict) else {}
    load = info.get("load_avg") or []

    load_1 = load_5 = load_15 = None
    if isinstance(load, (list, tuple)) and len(load) >= 3:
        load_1, load_5, load_15 = float(load[0]), float(load[1]), float(load[2])

    sample = WatchProbeSample(
        probe_status=PROBE_OK,
        source="redis",
        is_online=True,
        cpu_pct=round(cpu) if cpu is not None else None,
        mem_pct=round(mem) if mem is not None else None,
        disk_pct=round(disk) if disk is not None else None,
        load_1=load_1,
        load_5=load_5,
        load_15=load_15,
        net_bytes_sent=_positive_int(net.get("bytes_sent")),
        net_bytes_recv=_positive_int(net.get("bytes_recv")),
        disk_read_bytes=_positive_int(disk_io.get("read_bytes")),
        disk_write_bytes=_positive_int(disk_io.get("write_bytes")),
    )
    return sample if (cpu is not None or mem is not None) else None


def merge_redis_and_ssh(redis_sample: WatchProbeSample | None, ssh_sample: WatchProbeSample) -> WatchProbeSample:
    if redis_sample is None:
        return ssh_sample
    if ssh_sample.probe_status != PROBE_OK:
        return redis_sample
    out = WatchProbeSample(
        probe_status=PROBE_OK,
        source="mixed",
        is_online=True,
        duration_ms=ssh_sample.duration_ms,
        cpu_pct=redis_sample.cpu_pct if redis_sample.cpu_pct is not None else ssh_sample.cpu_pct,
        mem_pct=redis_sample.mem_pct if redis_sample.mem_pct is not None else ssh_sample.mem_pct,
        disk_pct=redis_sample.disk_pct if redis_sample.disk_pct is not None else ssh_sample.disk_pct,
        disk_mount=ssh_sample.disk_mount or redis_sample.disk_mount,
        load_1=ssh_sample.load_1 if ssh_sample.load_1 is not None else redis_sample.load_1,
        load_5=ssh_sample.load_5 if ssh_sample.load_5 is not None else redis_sample.load_5,
        load_15=ssh_sample.load_15 if ssh_sample.load_15 is not None else redis_sample.load_15,
        net_bytes_sent=ssh_sample.net_bytes_sent,
        net_bytes_recv=ssh_sample.net_bytes_recv,
        disk_read_bytes=ssh_sample.disk_read_bytes,
        disk_write_bytes=ssh_sample.disk_write_bytes,
    )
    return out


def failure_sample(
    *,
    status: str,
    source: str,
    error: str,
    duration_ms: int = 0,
    is_online: bool = False,
) -> WatchProbeSample:
    return WatchProbeSample(
        probe_status=status,
        source=source,
        is_online=is_online,
        duration_ms=duration_ms,
        error=error[:255],
    )