Nexus/server/api/servers.py

"""Nexus — Servers API Routes (CRUD + heartbeat + health check + push)
Presentation layer — receives HTTP requests, delegates to ServerService/SyncService.

Live server status comes from Redis (real-time), base info from MySQL.
All write operations require JWT authentication and produce audit logs.
"""

import json
import logging
from typing import Optional

from fastapi import APIRouter, Depends, HTTPException, Query, Request

from server.api.dependencies import get_server_service, get_sync_service, get_db
from server.api.auth_jwt import get_current_admin
from server.api.schemas import ServerCreate, ServerUpdate, ServerCheck
from server.application.services.server_service import ServerService
from server.application.services.sync_service import SyncService
from server.domain.models import Server, SyncLog, Admin, AuditLog
from server.config import settings
from server.infrastructure.redis.client import get_redis
from server.infrastructure.database.audit_log_repo import AuditLogRepositoryImpl

from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

logger = logging.getLogger("nexus.servers")

router = APIRouter(prefix="/api/servers", tags=["servers"])

REDIS_KEY_PREFIX = "heartbeat:"


# ── CRUD ──

@router.get("/", response_model=dict)
async def list_servers(
    category: Optional[str] = Query(None, description="Filter by category"),
    page: int = Query(1, ge=1, description="Page number"),
    per_page: int = Query(50, ge=1, le=200, description="Items per page"),
    admin: Admin = Depends(get_current_admin),
    service: ServerService = Depends(get_server_service),
    db: AsyncSession = Depends(get_db),
):
    """List servers with live status from Redis (DB-level pagination)

    Base info (name/IP/category) from MySQL with DB-level pagination,
    real-time status from Redis. If Redis heartbeat key exists,
    override is_online/system_info/last_heartbeat.
    """
    from server.infrastructure.database.server_repo import ServerRepositoryImpl

    offset = (page - 1) * per_page
    repo = ServerRepositoryImpl(db)
    page_servers, total = await repo.get_paginated(category, offset, per_page)
    redis = get_redis()

    result = []
    for server in page_servers:
        server_data = _server_to_dict(server)

        # Overlay Redis heartbeat data (real-time)
        try:
            heartbeat = await redis.hgetall(f"{REDIS_KEY_PREFIX}{server.id}")
            if heartbeat:
                server_data["is_online"] = heartbeat.get("is_online") == "True"
                if heartbeat.get("system_info"):
                    try:
                        server_data["system_info"] = json.loads(heartbeat.get("system_info", "{}"))
                    except (json.JSONDecodeError, TypeError):
                        pass  # Keep MySQL value
                server_data["last_heartbeat"] = heartbeat.get("last_heartbeat", server_data.get("last_heartbeat"))
                server_data["agent_version"] = heartbeat.get("agent_version", server_data.get("agent_version", ""))
                # Time drift detection fields
                try:
                    server_data["time_drift_seconds"] = float(heartbeat.get("time_drift_seconds", 0))
                except (ValueError, TypeError):
                    server_data["time_drift_seconds"] = 0.0
                server_data["drift_level"] = heartbeat.get("drift_level", "ok")
                server_data["_source"] = "redis"
        except Exception as e:
            logger.warning(f"Redis read failed for server {server.id}: {e}")

        result.append(server_data)

    return {
        "items": result,
        "total": total,
        "page": page,
        "per_page": per_page,
        "pages": (total + per_page - 1) // per_page,
    }


# ── Dashboard Stats (must be before /{id} to avoid path collision) ──

@router.get("/stats", response_model=dict)
async def server_stats(
    admin: Admin = Depends(get_current_admin),
    db: AsyncSession = Depends(get_db),
):
    """Aggregated server stats for dashboard — SQL aggregation + Redis alert count"""
    from sqlalchemy import func, text

    # Total + category breakdown in a single query
    result = await db.execute(
        select(Server.category, func.count(Server.id))
        .group_by(Server.category)
    )
    categories = {}
    total = 0
    for cat, cnt in result.all():
        key = cat or "uncategorized"
        categories[key] = cnt
        total += cnt

    # Online count from Redis heartbeats (real-time); MySQL is stale between flushes
    redis = get_redis()
    online = 0
    try:
        cursor = 0
        while True:
            cursor, keys = await redis.scan(cursor, match="heartbeat:*", count=200)
            for key in keys:
                data = await redis.hgetall(key)
                if data.get("is_online") == "True":
                    online += 1
            if cursor == 0:
                break
    except Exception as e:
        logger.warning(f"Failed to count Redis heartbeats in server_stats: {e}")
        result = await db.execute(
            select(func.count(Server.id)).where(Server.is_online == True)
        )
        online = int(result.scalar() or 0)

    alerts = 0
    try:
        cursor = 0
        while True:
            cursor, keys = await redis.scan(cursor, match="alerts:*", count=200)
            alerts += len(keys)
            if cursor == 0:
                break
    except Exception as e:
        logger.warning(f"Failed to count Redis alerts in server_stats: {e}")

    return {
        "total": total,
        "online": online,
        "offline": max(0, total - online),
        "alerts": alerts,
        "categories": categories,
    }


# ── Sync Logs ──

@router.get("/logs", response_model=dict)
async def get_all_sync_logs(
    server_id: Optional[int] = Query(None, description="Filter by server ID"),
    status: Optional[str] = Query(None, description="success|failed|running"),
    sync_mode: Optional[str] = Query(None, description="incremental|full|overwrite|checksum|command"),
    trigger_type: Optional[str] = Query(None, description="manual|schedule|retry|batch"),
    offset: int = Query(0, ge=0),
    limit: int = Query(50, ge=1, le=500),
    admin: Admin = Depends(get_current_admin),
    db: AsyncSession = Depends(get_db),
):
    """Get paginated sync logs across all servers with optional filters."""
    from server.infrastructure.database.sync_log_repo import SyncLogRepositoryImpl
    repo = SyncLogRepositoryImpl(db)
    rows, total = await repo.get_all_paginated(
        server_id=server_id, status=status, sync_mode=sync_mode,
        trigger_type=trigger_type, offset=offset, limit=limit,
    )
    items = [_sync_log_to_dict(log, server_name=name) for log, name in rows]
    return {"items": items, "total": total, "offset": offset, "limit": limit}


@router.get("/{id}", response_model=dict)
async def get_server(
    id: int,
    admin: Admin = Depends(get_current_admin),
    service: ServerService = Depends(get_server_service),
):
    """Get a single server by ID with live status from Redis"""
    server = await service.get_server(id)
    if not server:
        raise HTTPException(status_code=404, detail="Server not found")

    server_data = _server_to_dict(server)

    # Overlay Redis heartbeat data
    redis = get_redis()
    try:
        heartbeat = await redis.hgetall(f"{REDIS_KEY_PREFIX}{id}")
        if heartbeat:
            server_data["is_online"] = heartbeat.get("is_online") == "True"
            if heartbeat.get("system_info"):
                try:
                    server_data["system_info"] = json.loads(heartbeat.get("system_info", "{}"))
                except (json.JSONDecodeError, TypeError):
                    pass
            server_data["last_heartbeat"] = heartbeat.get("last_heartbeat", server_data.get("last_heartbeat"))
            server_data["agent_version"] = heartbeat.get("agent_version", server_data.get("agent_version", ""))
            try:
                server_data["time_drift_seconds"] = float(heartbeat.get("time_drift_seconds", 0))
            except (ValueError, TypeError):
                server_data["time_drift_seconds"] = 0.0
            server_data["drift_level"] = heartbeat.get("drift_level", "ok")
            server_data["_source"] = "redis"
    except Exception as e:
        logger.warning(f"Redis read failed for server {id}: {e}")

    return server_data


@router.post("/", response_model=dict, status_code=201)
async def create_server(
    request: Request,
    payload: ServerCreate,
    admin: Admin = Depends(get_current_admin),
    service: ServerService = Depends(get_server_service),
    db: AsyncSession = Depends(get_db),
):
    """Create a new server (sensitive fields encrypted at rest)"""
    from server.infrastructure.database.crypto import encrypt_value

    data = payload.model_dump(exclude_none=True)
    # Encrypt sensitive fields before storing
    if data.get("password"):
        data["password"] = encrypt_value(data["password"])
    if data.get("ssh_key_private"):
        data["ssh_key_private"] = encrypt_value(data["ssh_key_private"])

    server = Server(**data)
    created = await service.create_server(server)

    ip_address = request.client.host if request.client else ""
    audit_repo = AuditLogRepositoryImpl(db)
    await audit_repo.create(AuditLog(
        admin_username=admin.username,
        action="create_server",
        target_type="server",
        target_id=created.id,
        detail=f"name={created.name} domain={created.domain}",
        ip_address=ip_address,
    ))

    return _server_to_dict(created)


@router.put("/{id}", response_model=dict)
async def update_server(
    request: Request,
    id: int,
    payload: ServerUpdate,
    admin: Admin = Depends(get_current_admin),
    service: ServerService = Depends(get_server_service),
    db: AsyncSession = Depends(get_db),
):
    """Update an existing server (sensitive fields encrypted at rest)"""
    from server.infrastructure.database.crypto import encrypt_value

    server = await service.get_server(id)
    if not server:
        raise HTTPException(status_code=404, detail="Server not found")
    for key, value in payload.model_dump(exclude_unset=True).items():
        if hasattr(server, key) and key != "id":
            # Encrypt sensitive fields on update
            if key == "password" and value:
                value = encrypt_value(value)
            elif key == "ssh_key_private" and value:
                value = encrypt_value(value)
            setattr(server, key, value)
    updated = await service.update_server(server)

    ip_address = request.client.host if request.client else ""
    audit_repo = AuditLogRepositoryImpl(db)
    await audit_repo.create(AuditLog(
        admin_username=admin.username,
        action="update_server",
        target_type="server",
        target_id=id,
        detail=f"name={updated.name}",
        ip_address=ip_address,
    ))

    return _server_to_dict(updated)


@router.delete("/{id}", status_code=204)
async def delete_server(
    request: Request,
    id: int,
    admin: Admin = Depends(get_current_admin),
    service: ServerService = Depends(get_server_service),
    db: AsyncSession = Depends(get_db),
):
    """Delete a server"""
    server = await service.get_server(id)
    if not server:
        raise HTTPException(status_code=404, detail="Server not found")
    result = await service.delete_server(id)
    if not result:
        raise HTTPException(status_code=404, detail="Server not found")

    ip_address = request.client.host if request.client else ""
    audit_repo = AuditLogRepositoryImpl(db)
    await audit_repo.create(AuditLog(
        admin_username=admin.username,
        action="delete_server",
        target_type="server",
        target_id=id,
        detail=f"name={server.name}",
        ip_address=ip_address,
    ))


# ── Agent API Key ──

@router.post("/{id}/agent-key", response_model=dict)
async def generate_agent_api_key(
    request: Request,
    id: int,
    admin: Admin = Depends(get_current_admin),
    service: ServerService = Depends(get_server_service),
    db: AsyncSession = Depends(get_db),
):
    """Generate a new per-server Agent API key"""
    server = await service.get_server(id)
    if not server:
        raise HTTPException(status_code=404, detail="Server not found")

    import secrets
    new_key = f"nxs-{secrets.token_urlsafe(32)}"
    server.agent_api_key = new_key
    await service.update_server(server)

    ip_address = request.client.host if request.client else ""
    audit_repo = AuditLogRepositoryImpl(db)
    await audit_repo.create(AuditLog(
        admin_username=admin.username,
        action="generate_agent_key",
        target_type="server",
        target_id=id,
        detail=f"Generated new Agent API key for {server.name}",
        ip_address=ip_address,
    ))

    return {
        "server_id": id,
        "agent_api_key": new_key,
        "agent_api_key_preview": f"{new_key[:12]}...",
        "display_once": True,
        "message": "完整密钥仅在本响应中返回一次，请立即复制保存",
    }


# ── Agent Install ──

@router.post("/{id}/install-agent", response_model=dict)
async def install_agent_remote(
    request: Request,
    id: int,
    admin: Admin = Depends(get_current_admin),
    service: ServerService = Depends(get_server_service),
    db: AsyncSession = Depends(get_db),
):
    """Install Nexus Agent on the managed server via SSH.

    Uses the server's stored SSH credentials to connect and run install.sh.
    Requires NEXUS_API_BASE_URL to be configured (used in the install command).
    """
    from server.infrastructure.ssh.asyncssh_pool import exec_ssh_command
    from server.infrastructure.database.crypto import decrypt_value

    server = await service.get_server(id)
    if not server:
        raise HTTPException(status_code=404, detail="Server not found")

    # Validate prerequisites
    base_url = (settings.API_BASE_URL or "").strip().rstrip("/")
    if not base_url:
        raise HTTPException(
            status_code=400,
            detail="NEXUS_API_BASE_URL 未配置，无法生成安装命令。请在系统设置中配置主站对外 URL。",
        )

    api_key = server.agent_api_key or settings.API_KEY
    if not api_key:
        raise HTTPException(status_code=400, detail="请先为该服务器生成 Agent API Key")

    agent_port = server.agent_port or 8601

    # Build install command
    install_cmd = (
        f"curl -fsSL {base_url}/agent/install.sh | bash -s -- "
        f"--url {base_url} --key {api_key} --id {id} --port {agent_port}"
    )

    # Execute via SSH
    try:
        result = await exec_ssh_command(server, install_cmd, timeout=120)
    except Exception as e:
        raise HTTPException(status_code=502, detail=f"SSH 连接失败: {e}")

    if result["exit_code"] != 0:
        raise HTTPException(
            status_code=400,
            detail=f"安装脚本失败 (exit {result['exit_code']}): {result['stderr'][:500]}",
        )

    # Audit
    ip_address = request.client.host if request.client else ""
    audit_repo = AuditLogRepositoryImpl(db)
    await audit_repo.create(AuditLog(
        admin_username=admin.username,
        action="install_agent",
        target_type="server",
        target_id=id,
        detail=f"Agent 远程安装: {server.name} ({server.domain})",
        ip_address=ip_address,
    ))

    return {
        "success": True,
        "server_id": id,
        "server_name": server.name,
        "stdout": result["stdout"][:3000],
        "exit_code": result["exit_code"],
    }


@router.get("/{id}/agent-install-cmd", response_model=dict)
async def get_agent_install_cmd(
    id: int,
    admin: Admin = Depends(get_current_admin),
    service: ServerService = Depends(get_server_service),
):
    """Return the curl install command for this server (for manual copy-paste)."""
    server = await service.get_server(id)
    if not server:
        raise HTTPException(status_code=404, detail="Server not found")

    base_url = (settings.API_BASE_URL or "").strip().rstrip("/")
    api_key = server.agent_api_key or settings.API_KEY
    agent_port = server.agent_port or 8601

    has_key = bool(server.agent_api_key)
    cmd = None
    if base_url and api_key:
        cmd = (
            f"curl -fsSL {base_url}/agent/install.sh | bash -s -- "
            f"--url {base_url} --key {api_key} --id {id} --port {agent_port}"
        )

    return {
        "server_id": id,
        "server_name": server.name,
        "base_url": base_url,
        "has_agent_key": has_key,
        "agent_port": agent_port,
        "install_cmd": cmd,
        "is_online": server.is_online,
    }


# ── Agent Upgrade ──

@router.post("/{id}/upgrade-agent", response_model=dict)
async def upgrade_agent(
    request: Request,
    id: int,
    admin: Admin = Depends(get_current_admin),
    service: ServerService = Depends(get_server_service),
    db: AsyncSession = Depends(get_db),
):
    """Upgrade Nexus Agent on the managed server via SSH.

    Downloads the latest agent.py from the central server and restarts
    the nexus-agent systemd service. Does NOT touch config.json.
    """
    from server.infrastructure.ssh.asyncssh_pool import exec_ssh_command
    import shlex

    server = await service.get_server(id)
    if not server:
        raise HTTPException(status_code=404, detail="Server not found")

    base_url = (settings.API_BASE_URL or "").strip().rstrip("/")
    if not base_url:
        raise HTTPException(
            status_code=400,
            detail="NEXUS_API_BASE_URL 未配置，无法构建下载地址",
        )

    agent_url = f"{base_url}/agent/agent.py"
    install_dir = "/opt/nexus-agent"
    venv_python = f"{install_dir}/.venv/bin/python"

    # Step 1: Check Python version in venv
    pyver_cmd = (
        f"{venv_python} -c '"
        "import sys; v=sys.version_info; "
        "print(f\"py_ok_{v.major}.{v.minor}.{v.micro}\") "
        "if v.major==3 and v.minor>=10 else sys.exit(1)'"
    )
    try:
        pyver = await exec_ssh_command(server, pyver_cmd, timeout=15)
    except Exception as e:
        raise HTTPException(status_code=502, detail=f"SSH 连接失败: {e}")

    if pyver["exit_code"] != 0 or "py_ok_" not in pyver["stdout"]:
        ver_info = pyver["stderr"].replace("\n", "; ").strip()[:200] if pyver["stderr"] else "venv 不存在或 Python 版本 < 3.10"
        raise HTTPException(
            status_code=424,
            detail=(
                f"子机 Python 版本不满足要求（需要 3.10+），请手动修复后重试。\n"
                f"  venv 路径: {venv_python}\n"
                f"  详情: {ver_info}"
            ),
        )

    # Step 2: Auto-update pip dependencies (locked versions)
    pip_cmd = (
        f"{venv_python} -m pip install -q "
        f"fastapi==0.115.6 uvicorn==0.34.0 httpx==0.28.1 psutil python-multipart==0.0.19"
    )

    # Step 3: Backup current agent.py
    backup_cmd = f"cp {install_dir}/agent.py {install_dir}/agent.py.bak"

    # Step 4: pip install + backup + download new agent.py → restart service
    upgrade_cmd = (
        f"{pip_cmd} "
        f"&& {backup_cmd} "
        f"&& curl -fsSL {shlex.quote(agent_url)} -o {install_dir}/agent.py "
        f"&& systemctl restart nexus-agent "
        f"&& sleep 2 "
        f"&& systemctl is-active nexus-agent "
        f"&& echo 'upgrade_ok'"
    )

    try:
        result = await exec_ssh_command(server, upgrade_cmd, timeout=120)
    except Exception as e:
        raise HTTPException(status_code=502, detail=f"SSH 连接失败: {e}")

    success = result["exit_code"] == 0 and "upgrade_ok" in result["stdout"]

    if not success:
        # Rollback: restore backup and restart
        rollback_cmd = (
            f"cp {install_dir}/agent.py.bak {install_dir}/agent.py "
            f"&& systemctl restart nexus-agent"
        )
        try:
            await exec_ssh_command(server, rollback_cmd, timeout=30)
        except Exception:
            pass
        raise HTTPException(
            status_code=400,
            detail=f"升级失败 (exit {result['exit_code']}): {result['stderr'][:300]}，已回滚至旧版本",
        )

    # Audit
    ip_address = request.client.host if request.client else ""
    audit_repo = AuditLogRepositoryImpl(db)
    await audit_repo.create(AuditLog(
        admin_username=admin.username,
        action="upgrade_agent",
        target_type="server",
        target_id=id,
        detail=f"Agent 升级: {server.name} ({server.domain})",
        ip_address=ip_address,
    ))

    return {"success": True, "server_id": id, "server_name": server.name, "stdout": result["stdout"][:500]}


# ── Health Check ──

@router.post("/check", response_model=dict)
async def check_servers(
    payload: ServerCheck,
    admin: Admin = Depends(get_current_admin),
    service: ServerService = Depends(get_server_service),
):
    """Check health status of specified servers via SSH probe (no Agent port required)"""
    return await service.check_all_servers(payload.server_ids)



@router.get("/{id}/logs", response_model=list)
async def get_server_logs(
    id: int,
    limit: int = Query(50, ge=1, le=200),
    admin: Admin = Depends(get_current_admin),
    sync_service: SyncService = Depends(get_sync_service),
):
    """Get sync logs for a specific server"""
    logs = await sync_service.get_sync_logs(id, limit)
    return [_sync_log_to_dict(log) for log in logs]


# ── Helper functions ──

def _server_to_dict(server: Server) -> dict:
    """Convert Server model to API response dict (hide sensitive fields)"""
    return {
        "id": server.id,
        "name": server.name,
        "domain": server.domain,
        "port": server.port,
        "username": server.username,
        "auth_method": server.auth_method,
        "password_set": bool(server.password),
        "ssh_key_path": server.ssh_key_path,
        "ssh_key_public": server.ssh_key_public or "",
        "ssh_key_private_set": bool(server.ssh_key_private),
        "agent_port": server.agent_port,
        "agent_api_key": (server.agent_api_key[:8] + "...") if server.agent_api_key and len(server.agent_api_key) > 8 else (server.agent_api_key or ""),
        "agent_api_key_set": bool(server.agent_api_key),
        "description": server.description,
        "target_path": server.target_path,
        "category": server.category,
        "platform_id": server.platform_id,
        "node_id": server.node_id,
        "protocols": server.protocols,
        "extra_attrs": server.extra_attrs,
        "connectivity": server.connectivity,
        "ssh_key_configured": server.ssh_key_configured,
        "is_online": server.is_online,
        "last_heartbeat": str(server.last_heartbeat) if server.last_heartbeat else None,
        "last_checked_at": str(server.last_checked_at) if server.last_checked_at else None,
        "system_info": server.system_info,
        "agent_version": server.agent_version,
        "created_at": str(server.created_at) if server.created_at else None,
        "updated_at": str(server.updated_at) if server.updated_at else None,
    }


def _sync_log_to_dict(log, server_name: str = None) -> dict:
    """Convert SyncLog model to API response dict"""
    return {
        "id": log.id,
        "server_id": log.server_id,
        "server_name": server_name,
        "source_path": log.source_path,
        "target_path": log.target_path,
        "trigger_type": log.trigger_type,
        "operator": log.operator,
        "status": log.status,
        "sync_mode": log.sync_mode,
        "files_total": log.files_total,
        "files_transferred": log.files_transferred,
        "duration_seconds": log.duration_seconds,
        "error_message": log.error_message,
        "started_at": str(log.started_at) if log.started_at else None,
        "finished_at": str(log.finished_at) if log.finished_at else None,
    }