Your Name
938d26927f
- Agent per-server API Key认证 (agent.py) - JWT updated_at校验与会话超时 (auth_jwt.py) - 服务器凭据Fernet加密存储+密码脱敏 (servers.py) - WebSSH服务器级授权检查 (webssh.py) - Config同步去重+回滚机制 (sync_engine_v2.py) - DB层分页offset/limit (server_repo.py) - session.py logger修复 + @property死代码清理 - 心跳刷入rollback误用修复 (heartbeat_flush.py) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
101 lines
3.8 KiB
Python
101 lines
3.8 KiB
Python
"""Nexus — Heartbeat Flush Background Task
|
|
Every 10 minutes: batch-flush Redis heartbeat data to MySQL servers table.
|
|
MySQL only stores historical snapshots; frontend reads live data from Redis.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import update
|
|
|
|
from server.infrastructure.database.session import AsyncSessionLocal
|
|
from server.infrastructure.redis.client import get_redis
|
|
from server.domain.models import Server
|
|
|
|
logger = logging.getLogger("nexus.heartbeat_flush")
|
|
|
|
FLUSH_INTERVAL = 600 # 10 minutes (seconds)
|
|
REDIS_KEY_PREFIX = "heartbeat:"
|
|
REDIS_KEY_EXPIRE = 600 # 10 minutes TTL for Redis heartbeat keys
|
|
|
|
|
|
async def heartbeat_flush_loop():
|
|
"""Background task: every 10min, flush Redis heartbeat data to MySQL.
|
|
|
|
Flow:
|
|
1. Scan all heartbeat:* keys in Redis
|
|
2. For each key, extract server_id + heartbeat data
|
|
3. Batch UPDATE MySQL servers table (is_online, system_info, last_heartbeat, agent_version)
|
|
4. Log count of flushed servers
|
|
"""
|
|
logger.info("Heartbeat flush loop started (interval: 10min)")
|
|
while True:
|
|
await asyncio.sleep(FLUSH_INTERVAL)
|
|
|
|
redis = get_redis()
|
|
if redis is None:
|
|
logger.warning("Redis unavailable — skip heartbeat flush")
|
|
continue
|
|
|
|
# Scan all heartbeat:* keys
|
|
try:
|
|
keys = []
|
|
cursor = 0
|
|
while True:
|
|
cursor, batch = await redis.scan(cursor, match=f"{REDIS_KEY_PREFIX}*", count=100)
|
|
keys.extend(batch)
|
|
if cursor == 0:
|
|
break
|
|
except Exception as e:
|
|
logger.error(f"Redis scan failed: {e}")
|
|
continue
|
|
|
|
if not keys:
|
|
logger.info("No heartbeat keys in Redis — no agents reporting yet")
|
|
continue
|
|
|
|
flushed = 0
|
|
try:
|
|
async with AsyncSessionLocal() as session:
|
|
for key in keys:
|
|
try:
|
|
server_id = int(key.replace(REDIS_KEY_PREFIX, ""))
|
|
data = await redis.hgetall(key)
|
|
|
|
if not data:
|
|
continue
|
|
|
|
is_online = data.get("is_online") == "True"
|
|
system_info = data.get("system_info", "{}")
|
|
agent_version = data.get("agent_version", "")
|
|
|
|
# Parse last_heartbeat
|
|
last_hb_str = data.get("last_heartbeat", "")
|
|
try:
|
|
last_heartbeat = datetime.fromisoformat(last_hb_str)
|
|
except (ValueError, TypeError):
|
|
last_heartbeat = datetime.now(timezone.utc)
|
|
|
|
await session.execute(
|
|
update(Server)
|
|
.where(Server.id == server_id)
|
|
.values(
|
|
is_online=is_online,
|
|
system_info=system_info,
|
|
last_heartbeat=last_heartbeat,
|
|
agent_version=agent_version,
|
|
)
|
|
)
|
|
flushed += 1
|
|
except Exception as e:
|
|
logger.warning(f"Failed to flush server {key}: {e}")
|
|
# Don't rollback here — just skip this server and continue.
|
|
# A partial UPDATE on a single server won't corrupt others.
|
|
# The final commit() will persist all successful updates.
|
|
|
|
await session.commit()
|
|
logger.info(f"Heartbeat flush: {flushed}/{len(keys)} servers updated to MySQL")
|
|
except Exception as e:
|
|
logger.error(f"Heartbeat flush batch failed: {e}") |