Your Name
c9a99f4fb3
代码修复: - web/agent/agent.py: datetime.utcnow() → datetime.now(timezone.utc) - requirements.txt: 移除 paramiko==3.5.0 (已无活跃引用) - 删除 server/infrastructure/ssh/pool.py (DEPRECATED, 无引用) 连接池三层对齐 (config.py/.env.example/session.py): - DB_POOL_SIZE 100→160, DB_MAX_OVERFLOW 100→120 - 基于 MySQL max_connections=400, install.php 公式 文档修复 (21项): - P0: 硬编码域名/IP替换为配置变量占位符 - P1: 10个过时设计文档加归档标注 (引用旧文件结构) - P2: step-3-webssh报告 paramiko引用修正 - 6份审查报告连接池参数勘误 100/100→160/120 - ECC安全报告 EC5 datetime.utcnow 标记已修复 - docs/README.md 文档索引重写 - docs/memory/mem_nexus_overview.md 移除硬编码凭证 - docs/project/tech-stack-inventory.md paramiko标记已移除 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
91 lines
3.2 KiB
Python
91 lines
3.2 KiB
Python
"""Nexus — Python Self-Monitor Background Task
|
|
Every 30 seconds: check Redis, MySQL, WebSocket connections.
|
|
If critical service fails → send Telegram system alert.
|
|
If service recovers → send Telegram recovery notification.
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
|
|
from sqlalchemy import text
|
|
|
|
from server.infrastructure.database.session import AsyncSessionLocal
|
|
from server.infrastructure.redis.client import get_redis
|
|
from server.infrastructure.telegram import send_telegram_system_alert
|
|
from server.api.websocket import manager as ws_manager
|
|
from server.config import settings
|
|
|
|
logger = logging.getLogger("nexus.self_monitor")
|
|
|
|
MONITOR_INTERVAL = 30 # seconds
|
|
|
|
# Track previous state for recovery detection
|
|
_prev_redis_ok = True
|
|
_prev_mysql_ok = True
|
|
|
|
|
|
async def self_monitor_loop():
|
|
"""Background task: every 30s, self-check critical services.
|
|
|
|
Checks:
|
|
1. Redis connection — if lost, send Telegram warning; if recovered, send recovery
|
|
2. MySQL connection — if lost, send Telegram warning; if recovered, send recovery
|
|
3. WebSocket client count (via ConnectionManager)
|
|
"""
|
|
global _prev_redis_ok, _prev_mysql_ok
|
|
|
|
logger.info("Self-monitor loop started (interval: 30s)")
|
|
while True:
|
|
await asyncio.sleep(MONITOR_INTERVAL)
|
|
|
|
# ── Redis check ──
|
|
redis_ok = True
|
|
try:
|
|
redis = get_redis()
|
|
await redis.ping()
|
|
except Exception as e:
|
|
redis_ok = False
|
|
try:
|
|
if settings.REDIS_URL:
|
|
await send_telegram_system_alert(f"⚠️ Redis连接丢失，心跳数据可能中断")
|
|
except Exception:
|
|
logger.error("Failed to send Redis alert via Telegram")
|
|
|
|
# Redis recovery notification
|
|
if redis_ok and not _prev_redis_ok:
|
|
try:
|
|
if settings.REDIS_URL:
|
|
await send_telegram_system_alert(f"🟢 Redis连接已恢复正常")
|
|
except Exception:
|
|
logger.error("Failed to send Redis recovery via Telegram")
|
|
logger.info("Redis connection recovered")
|
|
_prev_redis_ok = redis_ok
|
|
|
|
# ── MySQL check ──
|
|
mysql_ok = True
|
|
try:
|
|
async with AsyncSessionLocal() as session:
|
|
await session.execute(text("SELECT 1"))
|
|
except Exception as e:
|
|
mysql_ok = False
|
|
try:
|
|
await send_telegram_system_alert(f"🔴 MySQL连接异常: {str(e)[:100]}")
|
|
except Exception:
|
|
logger.error("Failed to send MySQL alert via Telegram")
|
|
|
|
# MySQL recovery notification
|
|
if mysql_ok and not _prev_mysql_ok:
|
|
try:
|
|
await send_telegram_system_alert(f"🟢 MySQL连接已恢复正常")
|
|
except Exception:
|
|
logger.error("Failed to send MySQL recovery via Telegram")
|
|
logger.info("MySQL connection recovered")
|
|
_prev_mysql_ok = mysql_ok
|
|
|
|
# ── Log status ──
|
|
logger.debug(
|
|
f"Self-monitor: redis={'ok' if redis_ok else 'error'}, "
|
|
f"mysql={'ok' if mysql_ok else 'error'}, "
|
|
f"ws_clients={ws_manager.client_count}"
|
|
)
|