Your Name
32feb1b6db
Fixes: - F821: install.py site_url 未定义(NameError)、sync_v2.py os 未导入、script_jobs.py LOG f-string - F401: 移除 22 个未使用导入（models/__init__.py、install.py、auth.py 等） - B904: 20 个 except 块 raise 添加 from e/from None - S110: 20 个有意的 try-except-pass 添加 noqa 注释（SSH清理/DDL幂等/WS断连） - B007: 3 个未使用循环变量重命名（dirs→_dirs、server_id→_server_id） - F541: 3 个无占位符 f-string 修正 - F841: auth.py 未使用 ip_address 变量移除 - S105: auth_service.py Redis key prefix 误报 noqa - B023: script_execution_flush.py 循环变量绑定修复 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
108 lines
4.2 KiB
Python
108 lines
4.2 KiB
Python
"""Nexus — Python Self-Monitor Background Task
|
|
Every 30 seconds: check Redis, MySQL, WebSocket connections.
|
|
If critical service fails → send Telegram system alert.
|
|
If service recovers → send Telegram recovery notification.
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
import time
|
|
|
|
from sqlalchemy import text
|
|
|
|
from server.infrastructure.database.session import AsyncSessionLocal
|
|
from server.infrastructure.redis.client import get_redis
|
|
from server.infrastructure.telegram import send_telegram_system_alert
|
|
from server.api.websocket import manager as ws_manager
|
|
from server.config import settings
|
|
|
|
logger = logging.getLogger("nexus.self_monitor")
|
|
|
|
MONITOR_INTERVAL = 30 # seconds
|
|
_TELEGRAM_COOLDOWN = 300 # 5 minutes — same alert won't re-push Telegram
|
|
|
|
# Track previous state for recovery detection
|
|
_prev_redis_ok = True
|
|
_prev_mysql_ok = True
|
|
# Track last Telegram push time per alert key (prevent spam)
|
|
_last_telegram_time: dict = {}
|
|
|
|
|
|
async def self_monitor_loop():
|
|
"""Background task: every 30s, self-check critical services.
|
|
|
|
Checks:
|
|
1. Redis connection — if lost, send Telegram warning; if recovered, send recovery
|
|
2. MySQL connection — if lost, send Telegram warning; if recovered, send recovery
|
|
3. WebSocket client count (via ConnectionManager)
|
|
"""
|
|
global _prev_redis_ok, _prev_mysql_ok
|
|
|
|
def _should_send(key: str) -> bool:
|
|
"""Rate-limit Telegram pushes to 1 per 5 minutes per alert key."""
|
|
now = time.monotonic()
|
|
last = _last_telegram_time.get(key, 0)
|
|
if now - last < _TELEGRAM_COOLDOWN:
|
|
return False
|
|
_last_telegram_time[key] = now
|
|
return True
|
|
|
|
logger.info("Self-monitor loop started (interval: 30s)")
|
|
while True:
|
|
await asyncio.sleep(MONITOR_INTERVAL)
|
|
|
|
# ── Redis check ──
|
|
redis_ok = True
|
|
try:
|
|
redis = get_redis()
|
|
await redis.ping()
|
|
except Exception:
|
|
redis_ok = False
|
|
logger.warning("Redis health check failed", exc_info=True)
|
|
try:
|
|
if settings.REDIS_URL and _should_send("NOTIFY_SYSTEM_REDIS"):
|
|
await send_telegram_system_alert("⚠️ Redis连接丢失，心跳数据可能中断", notify_key="NOTIFY_SYSTEM_REDIS")
|
|
except Exception:
|
|
logger.error("Failed to send Redis alert via Telegram")
|
|
|
|
# Redis recovery notification
|
|
if redis_ok and not _prev_redis_ok:
|
|
try:
|
|
await send_telegram_system_alert("🟢 Redis连接已恢复正常", notify_key="NOTIFY_SYSTEM_REDIS")
|
|
_last_telegram_time.pop("NOTIFY_SYSTEM_REDIS", None) # clear cooldown
|
|
except Exception:
|
|
logger.error("Failed to send Redis recovery via Telegram")
|
|
logger.info("Redis connection recovered")
|
|
_prev_redis_ok = redis_ok
|
|
|
|
# ── MySQL check ──
|
|
mysql_ok = True
|
|
try:
|
|
async with AsyncSessionLocal() as session:
|
|
await session.execute(text("SELECT 1"))
|
|
except Exception:
|
|
mysql_ok = False
|
|
logger.warning("MySQL health check failed", exc_info=True)
|
|
try:
|
|
if _should_send("NOTIFY_SYSTEM_MYSQL"):
|
|
await send_telegram_system_alert("🔴 MySQL 连接异常，请检查数据库服务", notify_key="NOTIFY_SYSTEM_MYSQL")
|
|
except Exception:
|
|
logger.error("Failed to send MySQL alert via Telegram", exc_info=True)
|
|
|
|
# MySQL recovery notification
|
|
if mysql_ok and not _prev_mysql_ok:
|
|
try:
|
|
await send_telegram_system_alert("🟢 MySQL连接已恢复正常", notify_key="NOTIFY_SYSTEM_MYSQL")
|
|
_last_telegram_time.pop("NOTIFY_SYSTEM_MYSQL", None) # clear cooldown
|
|
except Exception:
|
|
logger.error("Failed to send MySQL recovery via Telegram")
|
|
logger.info("MySQL connection recovered")
|
|
_prev_mysql_ok = mysql_ok
|
|
|
|
# ── Log status ──
|
|
logger.debug(
|
|
f"Self-monitor: redis={'ok' if redis_ok else 'error'}, "
|
|
f"mysql={'ok' if mysql_ok else 'error'}, "
|
|
f"ws_clients={ws_manager.client_count}"
|
|
)
|