Nexus Agent
7209f53af9
离线边沿检测推送一次 Telegram（设置开关 notify_alert_offline）；对齐仪表盘与服务器页统计卡、舰队趋势交互与告警叠加。 Co-authored-by: Cursor <cursoragent@cursor.com>
88 lines
2.9 KiB
Python
88 lines
2.9 KiB
Python
"""Nexus — Server offline alert background task.
|
|
|
|
Every 30 seconds: detect online→offline transitions among Agent-monitored servers
|
|
and push one Telegram alert per transition (edge-triggered).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
|
|
from sqlalchemy import select
|
|
|
|
from server.application.server_connectivity import (
|
|
REDIS_KEY_PREFIX,
|
|
heartbeat_indicates_online,
|
|
monitored_server_filter,
|
|
)
|
|
from server.domain.models import Server
|
|
from server.infrastructure.database.session import AsyncSessionLocal
|
|
from server.infrastructure.redis.client import get_redis
|
|
|
|
logger = logging.getLogger("nexus.server_offline_monitor")
|
|
|
|
MONITOR_INTERVAL = 30 # seconds — align with self_monitor
|
|
_BATCH_SIZE = 200
|
|
|
|
# server_id → last known online (None = cold start, no alert)
|
|
_prev_online: dict[int, bool] = {}
|
|
|
|
|
|
async def _load_monitored_servers() -> list[tuple[int, str]]:
|
|
async with AsyncSessionLocal() as session:
|
|
result = await session.execute(
|
|
select(Server.id, Server.name).where(monitored_server_filter())
|
|
)
|
|
return [(int(row[0]), row[1] or f"server-{row[0]}") for row in result.all()]
|
|
|
|
|
|
async def _check_offline_transitions() -> int:
|
|
"""Scan monitored servers; return count of new offline alerts fired."""
|
|
servers = await _load_monitored_servers()
|
|
if not servers:
|
|
return 0
|
|
|
|
redis = get_redis()
|
|
alerts_sent = 0
|
|
|
|
for i in range(0, len(servers), _BATCH_SIZE):
|
|
chunk = servers[i : i + _BATCH_SIZE]
|
|
pipe = redis.pipeline()
|
|
for server_id, _ in chunk:
|
|
pipe.hgetall(f"{REDIS_KEY_PREFIX}{server_id}")
|
|
heartbeats = await pipe.execute()
|
|
|
|
for (server_id, server_name), hb in zip(chunk, heartbeats):
|
|
currently_online = heartbeat_indicates_online(hb)
|
|
prev_online = _prev_online.get(server_id)
|
|
|
|
if prev_online is None:
|
|
_prev_online[server_id] = currently_online
|
|
continue
|
|
|
|
if prev_online and not currently_online:
|
|
from server.api.websocket import broadcast_offline_alert
|
|
|
|
last_hb = (hb or {}).get("last_heartbeat", "")
|
|
await broadcast_offline_alert(server_id, server_name, last_hb)
|
|
alerts_sent += 1
|
|
logger.warning("Server offline alert: id=%s name=%s", server_id, server_name)
|
|
|
|
_prev_online[server_id] = currently_online
|
|
|
|
return alerts_sent
|
|
|
|
|
|
async def server_offline_monitor_loop():
|
|
"""Background task: edge-detect server offline and notify Telegram once per event."""
|
|
logger.info("Server offline monitor started (interval: %ss)", MONITOR_INTERVAL)
|
|
while True:
|
|
await asyncio.sleep(MONITOR_INTERVAL)
|
|
try:
|
|
sent = await _check_offline_transitions()
|
|
if sent:
|
|
logger.info("Server offline monitor: %s alert(s) sent", sent)
|
|
except Exception:
|
|
logger.error("Server offline monitor tick failed", exc_info=True)
|