"""Nexus — Server offline alert background task. Every 30 seconds: detect online→offline transitions among Agent-monitored servers and push one Telegram alert per transition (edge-triggered). """ from __future__ import annotations import asyncio import logging from sqlalchemy import select from server.application.server_batch_common import batch_server_display_name from server.application.server_connectivity import ( REDIS_KEY_PREFIX, heartbeat_indicates_online, monitored_server_filter, ) from server.domain.models import Server from server.infrastructure.database.session import AsyncSessionLocal from server.infrastructure.redis.client import get_redis logger = logging.getLogger("nexus.server_offline_monitor") MONITOR_INTERVAL = 30 # seconds — align with self_monitor _BATCH_SIZE = 200 # server_id → last known online (None = cold start, no alert) _prev_online: dict[int, bool] = {} async def _load_monitored_servers() -> list[tuple[int, str]]: async with AsyncSessionLocal() as session: result = await session.execute( select(Server.id, Server.name, Server.domain, Server.description).where( monitored_server_filter() ) ) return [ ( int(row[0]), batch_server_display_name( Server(id=int(row[0]), name=row[1], domain=row[2], description=row[3]) ), ) for row in result.all() ] async def _check_offline_transitions() -> int: """Scan monitored servers; return count of new offline alerts fired.""" servers = await _load_monitored_servers() if not servers: return 0 redis = get_redis() alerts_sent = 0 for i in range(0, len(servers), _BATCH_SIZE): chunk = servers[i : i + _BATCH_SIZE] pipe = redis.pipeline() for server_id, _ in chunk: pipe.hgetall(f"{REDIS_KEY_PREFIX}{server_id}") heartbeats = await pipe.execute() for (server_id, server_name), hb in zip(chunk, heartbeats): currently_online = heartbeat_indicates_online(hb) prev_online = _prev_online.get(server_id) if prev_online is None: _prev_online[server_id] = currently_online continue if prev_online and not currently_online: from server.api.websocket import broadcast_offline_alert last_hb = (hb or {}).get("last_heartbeat", "") await broadcast_offline_alert(server_id, server_name, last_hb) alerts_sent += 1 logger.warning("Server offline alert: id=%s name=%s", server_id, server_name) _prev_online[server_id] = currently_online return alerts_sent async def server_offline_monitor_loop(): """Background task: edge-detect server offline and notify Telegram once per event.""" logger.info("Server offline monitor started (interval: %ss)", MONITOR_INTERVAL) while True: await asyncio.sleep(MONITOR_INTERVAL) try: sent = await _check_offline_transitions() if sent: logger.info("Server offline monitor: %s alert(s) sent", sent) except Exception: logger.error("Server offline monitor tick failed", exc_info=True)