feat: Nexus 6.0 6-step implementation (Step 0-5)
Step 0: Infrastructure hardening — Redis BlockingConnectionPool, WebSocket two-layer (memory+Redis Pub/Sub), JWT auth, DB session leak fix Step 1: Data layer — 4 new tables (platforms/nodes/command_logs/ssh_sessions), data migration (category→Node), repos + API routes Step 2: Auth layer — JWT middleware on all routes, TOTP JWT integration Step 3: Web SSH — asyncssh connection pool, /ws/terminal endpoint, xterm.js frontend, Koko protocol Step 4: Sync engine v2 — file/command/config/SFTP modes, parallel execution Step 5: Frontend migration — 12 Tailwind CSS v4 + Alpine.js pages, PHP-FPM removal from nginx config 21 Python backend + 12 HTML frontend + 2 deploy configs + 3 test files Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+24
-25
@@ -10,7 +10,7 @@ Heartbeat flow:
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException
|
||||
@@ -19,7 +19,7 @@ from server.api.dependencies import get_server_service
|
||||
from server.api.websocket import broadcast_alert, broadcast_recovery
|
||||
from server.application.services.server_service import ServerService
|
||||
from server.config import settings
|
||||
from server.infrastructure.redis.client import get_redis
|
||||
from server.infrastructure.redis.client import get_redis, get_redis_sync
|
||||
from server.infrastructure.telegram import send_telegram_system_alert
|
||||
|
||||
logger = logging.getLogger("nexus.agent")
|
||||
@@ -113,22 +113,21 @@ async def receive_heartbeat(
|
||||
server_name = payload.get("server_name", "")
|
||||
|
||||
# ── 1. Write to Redis (real-time data) ──
|
||||
redis = await get_redis()
|
||||
if redis:
|
||||
redis = get_redis()
|
||||
try:
|
||||
redis_key = f"{REDIS_KEY_PREFIX}{server_id}"
|
||||
try:
|
||||
await redis.hset(redis_key, mapping={
|
||||
"is_online": str(is_online),
|
||||
"system_info": json.dumps(system_info),
|
||||
"last_heartbeat": datetime.utcnow().isoformat(),
|
||||
"agent_version": agent_version,
|
||||
"server_name": server_name,
|
||||
})
|
||||
# Set TTL = flush interval × 1.5 (prevent stale online status after key expires)
|
||||
await redis.expire(redis_key, REDIS_KEY_EXPIRE)
|
||||
except Exception as e:
|
||||
logger.error(f"Redis heartbeat write failed for server {server_id}: {e}")
|
||||
# Redis write failed — still continue with MySQL update
|
||||
await redis.hset(redis_key, mapping={
|
||||
"is_online": str(is_online),
|
||||
"system_info": json.dumps(system_info),
|
||||
"last_heartbeat": datetime.now(timezone.utc).isoformat(),
|
||||
"agent_version": agent_version,
|
||||
"server_name": server_name,
|
||||
})
|
||||
# Set TTL = flush interval × 1.5 (prevent stale online status after key expires)
|
||||
await redis.expire(redis_key, REDIS_KEY_EXPIRE)
|
||||
except Exception as e:
|
||||
logger.error(f"Redis heartbeat write failed for server {server_id}: {e}")
|
||||
# Redis write failed — still continue with MySQL update
|
||||
|
||||
# ── 2. Alert detection ──
|
||||
alert_thresholds = {
|
||||
@@ -144,16 +143,17 @@ async def receive_heartbeat(
|
||||
await broadcast_alert(server_id, alert_type, alert_value, server_name)
|
||||
|
||||
# Track active alert in Redis
|
||||
if redis:
|
||||
try:
|
||||
await redis.sadd(f"{REDIS_ALERT_KEY_PREFIX}{server_id}", f"{alert_type}:{alert_value}")
|
||||
await redis.expire(f"{REDIS_ALERT_KEY_PREFIX}{server_id}", 3600) # 1h TTL
|
||||
except Exception:
|
||||
pass # Non-critical
|
||||
try:
|
||||
redis = get_redis()
|
||||
await redis.sadd(f"{REDIS_ALERT_KEY_PREFIX}{server_id}", f"{alert_type}:{alert_value}")
|
||||
await redis.expire(f"{REDIS_ALERT_KEY_PREFIX}{server_id}", 3600) # 1h TTL
|
||||
except Exception:
|
||||
pass # Non-critical
|
||||
|
||||
# ── 3. Recovery detection ──
|
||||
if redis and system_info:
|
||||
if system_info:
|
||||
try:
|
||||
redis = get_redis()
|
||||
prev_alerts = await redis.smembers(f"{REDIS_ALERT_KEY_PREFIX}{server_id}")
|
||||
if prev_alerts:
|
||||
recoveries = _detect_recovery(system_info, prev_alerts, alert_thresholds)
|
||||
@@ -162,7 +162,6 @@ async def receive_heartbeat(
|
||||
await broadcast_recovery(server_id, metric, value, server_name)
|
||||
|
||||
# Remove recovered alert from Redis tracking
|
||||
# Find and remove the matching metric entry
|
||||
for prev in prev_alerts:
|
||||
if prev.startswith(f"{metric}:"):
|
||||
await redis.srem(f"{REDIS_ALERT_KEY_PREFIX}{server_id}", prev)
|
||||
|
||||
Reference in New Issue
Block a user