"""Nexus — Python Self-Monitor Background Task Every 30 seconds: check Redis, MySQL, WebSocket connections. If critical service fails → send Telegram system alert. If service recovers → send Telegram recovery notification. """ import asyncio import logging import time from sqlalchemy import text from server.infrastructure.database.session import AsyncSessionLocal from server.infrastructure.redis.client import get_redis from server.infrastructure.telegram import send_telegram_system_alert from server.api.websocket import manager as ws_manager from server.config import settings logger = logging.getLogger("nexus.self_monitor") MONITOR_INTERVAL = 30 # seconds _TELEGRAM_COOLDOWN = 300 # 5 minutes — same alert won't re-push Telegram # Track previous state for recovery detection _prev_redis_ok = True _prev_mysql_ok = True # Track last Telegram push time per alert key (prevent spam) _last_telegram_time: dict = {} async def self_monitor_loop(): """Background task: every 30s, self-check critical services. Checks: 1. Redis connection — if lost, send Telegram warning; if recovered, send recovery 2. MySQL connection — if lost, send Telegram warning; if recovered, send recovery 3. WebSocket client count (via ConnectionManager) """ global _prev_redis_ok, _prev_mysql_ok def _should_send(key: str) -> bool: """Rate-limit Telegram pushes to 1 per 5 minutes per alert key.""" now = time.monotonic() last = _last_telegram_time.get(key, 0) if now - last < _TELEGRAM_COOLDOWN: return False _last_telegram_time[key] = now return True logger.info("Self-monitor loop started (interval: 30s)") while True: await asyncio.sleep(MONITOR_INTERVAL) # ── Redis check ── redis_ok = True try: redis = get_redis() await redis.ping() except Exception: redis_ok = False logger.warning("Redis health check failed", exc_info=True) try: if settings.REDIS_URL and _should_send("NOTIFY_SYSTEM_REDIS"): await send_telegram_system_alert("⚠️ Redis连接丢失,心跳数据可能中断", notify_key="NOTIFY_SYSTEM_REDIS") except Exception: logger.error("Failed to send Redis alert via Telegram") # Redis recovery notification if redis_ok and not _prev_redis_ok: try: await send_telegram_system_alert("🟢 Redis连接已恢复正常", notify_key="NOTIFY_SYSTEM_REDIS") _last_telegram_time.pop("NOTIFY_SYSTEM_REDIS", None) # clear cooldown except Exception: logger.error("Failed to send Redis recovery via Telegram") logger.info("Redis connection recovered") _prev_redis_ok = redis_ok # ── MySQL check ── mysql_ok = True try: async with AsyncSessionLocal() as session: await session.execute(text("SELECT 1")) except Exception: mysql_ok = False logger.warning("MySQL health check failed", exc_info=True) try: if _should_send("NOTIFY_SYSTEM_MYSQL"): await send_telegram_system_alert("🔴 MySQL 连接异常,请检查数据库服务", notify_key="NOTIFY_SYSTEM_MYSQL") except Exception: logger.error("Failed to send MySQL alert via Telegram", exc_info=True) # MySQL recovery notification if mysql_ok and not _prev_mysql_ok: try: await send_telegram_system_alert("🟢 MySQL连接已恢复正常", notify_key="NOTIFY_SYSTEM_MYSQL") _last_telegram_time.pop("NOTIFY_SYSTEM_MYSQL", None) # clear cooldown except Exception: logger.error("Failed to send MySQL recovery via Telegram") logger.info("MySQL connection recovered") _prev_mysql_ok = mysql_ok # ── Log status ── logger.debug( f"Self-monitor: redis={'ok' if redis_ok else 'error'}, " f"mysql={'ok' if mysql_ok else 'error'}, " f"ws_clients={ws_manager.client_count}" )