feat: time drift detection between agent and central server
Agent (web/agent/agent.py): - Heartbeat now includes system_info.agent_time (UTC ISO string) Central (server/api/agent.py): - On each heartbeat, compare agent_time vs central datetime.now(utc) - Thresholds: warn >= 30s, crit >= 60s (affects TOTP/cron accuracy) - Stores time_drift_seconds + drift_level in Redis heartbeat key - Sends Telegram alert (5-min cooldown) when drift >= 60s critical API (server/api/servers.py): - List and detail endpoints expose time_drift_seconds + drift_level from Redis Frontend (web/app/servers.html): - driftBadge(): shows clock emoji badge next to server name - amber badge: >=30s (warn) - red badge: >=60s (crit) - tooltip shows exact drift and NTP check hint - no badge when drift < 30s or no data Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Your Name
@@ -159,6 +159,38 @@ async def receive_heartbeat(
|
||||
raise HTTPException(status_code=401, detail="Invalid API key for this server")
|
||||
|
||||
# ── 1. Write to Redis (real-time data) ──
|
||||
|
||||
# Time drift detection: compare agent_time with central server time
|
||||
TIME_DRIFT_WARN_SECONDS = 30 # ≥30s: warning (yellow)
|
||||
TIME_DRIFT_CRIT_SECONDS = 60 # ≥60s: critical (red, affects TOTP / cron)
|
||||
|
||||
time_drift_seconds: float = 0.0
|
||||
drift_level: str = "ok" # ok / warn / crit
|
||||
agent_time_str = system_info.get("agent_time", "") if system_info else ""
|
||||
if agent_time_str:
|
||||
try:
|
||||
agent_ts = datetime.fromisoformat(agent_time_str)
|
||||
central_ts = datetime.now(timezone.utc)
|
||||
# Normalise to aware for comparison
|
||||
if agent_ts.tzinfo is None:
|
||||
agent_ts = agent_ts.replace(tzinfo=timezone.utc)
|
||||
drift = abs((central_ts - agent_ts).total_seconds())
|
||||
time_drift_seconds = round(drift, 1)
|
||||
if drift >= TIME_DRIFT_CRIT_SECONDS:
|
||||
drift_level = "crit"
|
||||
logger.warning(
|
||||
"Clock drift CRITICAL: server %s drift=%.1fs (agent_time=%s)",
|
||||
server_id, drift, agent_time_str,
|
||||
)
|
||||
elif drift >= TIME_DRIFT_WARN_SECONDS:
|
||||
drift_level = "warn"
|
||||
logger.warning(
|
||||
"Clock drift WARNING: server %s drift=%.1fs",
|
||||
server_id, drift,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to parse agent_time for drift check: {e}")
|
||||
|
||||
redis = get_redis()
|
||||
try:
|
||||
redis_key = f"{REDIS_KEY_PREFIX}{server_id}"
|
||||
@@ -167,6 +199,8 @@ async def receive_heartbeat(
|
||||
"system_info": json.dumps(system_info),
|
||||
"last_heartbeat": datetime.now(timezone.utc).isoformat(),
|
||||
"agent_version": agent_version,
|
||||
"time_drift_seconds": str(time_drift_seconds),
|
||||
"drift_level": drift_level,
|
||||
})
|
||||
# Set TTL = flush interval × 1.5 (prevent stale online status after key expires)
|
||||
await redis.expire(redis_key, REDIS_KEY_EXPIRE)
|
||||
@@ -183,6 +217,21 @@ async def receive_heartbeat(
|
||||
except Exception:
|
||||
logger.debug(f"Failed to look up server name for server {server_id}")
|
||||
|
||||
# ── 1c. Time drift Telegram alert (critical only, 5-min cooldown like metric alerts) ──
|
||||
if drift_level == "crit":
|
||||
from server.api.websocket import _should_push_telegram
|
||||
drift_alert_key = f"alert:{server_id}:time_drift"
|
||||
if _should_push_telegram(drift_alert_key):
|
||||
try:
|
||||
from server.infrastructure.telegram import send_telegram_system_alert
|
||||
await send_telegram_system_alert(
|
||||
f"🕐 时钟偏差过大 — 服务器 #{server_id} ({server_name})\n"
|
||||
f"偏差 {time_drift_seconds}s（≥60s 影响 TOTP / cron 精度）\n"
|
||||
f"请检查 NTP 配置"
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failed to send drift alert via Telegram")
|
||||
|
||||
# ── 2. Alert detection ──
|
||||
alert_thresholds = {
|
||||
"cpu": settings.CPU_ALERT_THRESHOLD,
|
||||
|
||||
@@ -70,6 +70,12 @@ async def list_servers(
|
||||
pass # Keep MySQL value
|
||||
server_data["last_heartbeat"] = heartbeat.get("last_heartbeat", server_data.get("last_heartbeat"))
|
||||
server_data["agent_version"] = heartbeat.get("agent_version", server_data.get("agent_version", ""))
|
||||
# Time drift detection fields
|
||||
try:
|
||||
server_data["time_drift_seconds"] = float(heartbeat.get("time_drift_seconds", 0))
|
||||
except (ValueError, TypeError):
|
||||
server_data["time_drift_seconds"] = 0.0
|
||||
server_data["drift_level"] = heartbeat.get("drift_level", "ok")
|
||||
server_data["_source"] = "redis"
|
||||
except Exception as e:
|
||||
logger.warning(f"Redis read failed for server {server.id}: {e}")
|
||||
@@ -173,6 +179,11 @@ async def get_server(
|
||||
pass
|
||||
server_data["last_heartbeat"] = heartbeat.get("last_heartbeat", server_data.get("last_heartbeat"))
|
||||
server_data["agent_version"] = heartbeat.get("agent_version", server_data.get("agent_version", ""))
|
||||
try:
|
||||
server_data["time_drift_seconds"] = float(heartbeat.get("time_drift_seconds", 0))
|
||||
except (ValueError, TypeError):
|
||||
server_data["time_drift_seconds"] = 0.0
|
||||
server_data["drift_level"] = heartbeat.get("drift_level", "ok")
|
||||
server_data["_source"] = "redis"
|
||||
except Exception as e:
|
||||
logger.warning(f"Redis read failed for server {id}: {e}")
|
||||
|
||||
@@ -316,6 +316,8 @@ async def send_heartbeat():
|
||||
"disk_usage": disk,
|
||||
"hostname": platform.node(),
|
||||
"platform": platform.platform(),
|
||||
# agent_time: central uses this to detect clock drift
|
||||
"agent_time": datetime.now(timezone.utc).isoformat(),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
+12
-1
@@ -156,7 +156,7 @@
|
||||
tbody.innerHTML=servers.map(s=>`<tr class="hover:bg-slate-800/30 transition cursor-pointer ${selectedServerId===s.id?'bg-brand/5':''}" onclick="selectServer(${s.id})" id="row-${s.id}">
|
||||
<td class="px-4 py-3" onclick="event.stopPropagation()"><input type="checkbox" class="rounded border-slate-600 bg-slate-700 text-brand focus:ring-brand srv-chk" data-id="${s.id}" onchange="toggleSelect(${s.id},this.checked)" ${_selectedIds.has(s.id)?'checked':''}></td>
|
||||
<td class="px-4 py-3"><span class="inline-block w-2.5 h-2.5 rounded-full ${s.is_online?'bg-green-400':'bg-red-400'} ${s.is_online?'':'animate-pulse'}"></span></td>
|
||||
<td class="px-4 py-3 text-white font-medium">${esc(s.name)}</td>
|
||||
<td class="px-4 py-3 text-white font-medium">${esc(s.name)}${driftBadge(s)}</td>
|
||||
<td class="px-4 py-3 text-slate-400">${esc(s.domain)}:${s.port||22}</td>
|
||||
<td class="px-4 py-3 text-slate-400">${esc(s.category||'--')}</td>
|
||||
<td class="px-4 py-3 text-slate-400">${s.agent_version||'--'}</td>
|
||||
@@ -281,6 +281,17 @@
|
||||
hideServerModal();loadServers();
|
||||
if(selectedServerId&&id==selectedServerId)selectServer(parseInt(id));
|
||||
}
|
||||
function driftBadge(s){
|
||||
const level=s.drift_level;
|
||||
const drift=s.time_drift_seconds||0;
|
||||
if(!level||level==='ok'||!drift)return'';
|
||||
const isCrit=level==='crit';
|
||||
const color=isCrit?'bg-red-500/20 text-red-400 border-red-500/40':'bg-amber-500/20 text-amber-400 border-amber-500/40';
|
||||
const icon='🕐';
|
||||
const tip=`时钟偏差 ${drift}s${isCrit?' — 影响 TOTP/cron，请检查 NTP':' — 建议检查 NTP'}`;
|
||||
return ` <span class="ml-1.5 inline-flex items-center gap-0.5 px-1.5 py-0.5 rounded border ${color} text-[10px] font-medium align-middle cursor-default" title="${tip}">${icon} ${drift}s</span>`;
|
||||
}
|
||||
|
||||
function esc(s){if(!s)return'';const d=document.createElement('div');d.textContent=s;return d.innerHTML}
|
||||
|
||||
// ── Generate Agent API Key ──
|
||||
|
||||
Reference in New Issue
Block a user