99 lines
3.3 KiB
Python
99 lines
3.3 KiB
Python
|
"""Nexus — Server offline alert background task.
|
||
|
|
|
||
|
|
Every 30 seconds: detect online→offline transitions among Agent-monitored servers
|
||
|
|
and push one Telegram alert per transition (edge-triggered).
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import asyncio
|
||
|
|
import logging
|
||
|
|
|
||
|
|
from sqlalchemy import select
|
||
|
|
|
||
|
|
from server.application.server_batch_common import batch_server_display_name
|
||
|
|
from server.application.server_connectivity import (
|
||
|
|
REDIS_KEY_PREFIX,
|
||
|
|
heartbeat_indicates_online,
|
||
|
|
monitored_server_filter,
|
||
|
|
)
|
||
|
|
from server.domain.models import Server
|
||
|
|
from server.infrastructure.database.session import AsyncSessionLocal
|
||
|
|
from server.infrastructure.redis.client import get_redis
|
||
|
|
|
||
|
|
logger = logging.getLogger("nexus.server_offline_monitor")
|
||
|
|
|
||
|
|
MONITOR_INTERVAL = 30 # seconds — align with self_monitor
|
||
|
|
_BATCH_SIZE = 200
|
||
|
|
|
||
|
|
# server_id → last known online (None = cold start, no alert)
|
||
|
|
_prev_online: dict[int, bool] = {}
|
||
|
|
|
||
|
|
|
||
|
|
async def _load_monitored_servers() -> list[tuple[int, str]]:
|
||
|
|
async with AsyncSessionLocal() as session:
|
||
|
|
result = await session.execute(
|
||
|
|
select(Server.id, Server.name, Server.domain, Server.description).where(
|
||
|
|
monitored_server_filter()
|
||
|
|
)
|
||
|
|
)
|
||
|
|
return [
|
||
|
|
(
|
||
|
|
int(row[0]),
|
||
|
|
batch_server_display_name(
|
||
|
|
Server(id=int(row[0]), name=row[1], domain=row[2], description=row[3])
|
||
|
|
),
|
||
|
|
)
|
||
|
|
for row in result.all()
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
async def _check_offline_transitions() -> int:
|
||
|
|
"""Scan monitored servers; return count of new offline alerts fired."""
|
||
|
|
servers = await _load_monitored_servers()
|
||
|
|
if not servers:
|
||
|
|
return 0
|
||
|
|
|
||
|
|
redis = get_redis()
|
||
|
|
alerts_sent = 0
|
||
|
|
|
||
|
|
for i in range(0, len(servers), _BATCH_SIZE):
|
||
|
|
chunk = servers[i : i + _BATCH_SIZE]
|
||
|
|
pipe = redis.pipeline()
|
||
|
|
for server_id, _ in chunk:
|
||
|
|
pipe.hgetall(f"{REDIS_KEY_PREFIX}{server_id}")
|
||
|
|
heartbeats = await pipe.execute()
|
||
|
|
|
||
|
|
for (server_id, server_name), hb in zip(chunk, heartbeats):
|
||
|
|
currently_online = heartbeat_indicates_online(hb)
|
||
|
|
prev_online = _prev_online.get(server_id)
|
||
|
|
|
||
|
|
if prev_online is None:
|
||
|
|
_prev_online[server_id] = currently_online
|
||
|
|
continue
|
||
|
|
|
||
|
|
if prev_online and not currently_online:
|
||
|
|
from server.api.websocket import broadcast_offline_alert
|
||
|
|
|
||
|
|
last_hb = (hb or {}).get("last_heartbeat", "")
|
||
|
|
await broadcast_offline_alert(server_id, server_name, last_hb)
|
||
|
|
alerts_sent += 1
|
||
|
|
logger.warning("Server offline alert: id=%s name=%s", server_id, server_name)
|
||
|
|
|
||
|
|
_prev_online[server_id] = currently_online
|
||
|
|
|
||
|
|
return alerts_sent
|
||
|
|
|
||
|
|
|
||
|
|
async def server_offline_monitor_loop():
|
||
|
|
"""Background task: edge-detect server offline and notify Telegram once per event."""
|
||
|
|
logger.info("Server offline monitor started (interval: %ss)", MONITOR_INTERVAL)
|
||
|
|
while True:
|
||
|
|
await asyncio.sleep(MONITOR_INTERVAL)
|
||
|
|
try:
|
||
|
|
sent = await _check_offline_transitions()
|
||
|
|
if sent:
|
||
|
|
logger.info("Server offline monitor: %s alert(s) sent", sent)
|
||
|
|
except Exception:
|
||
|
|
logger.error("Server offline monitor tick failed", exc_info=True)
|