feat: Phase 2 security audit fixes + script execution platform
Security (P0/P1/P2): - WebSSH: dedicated short-lived token, server_id binding, 4003 close code - Remove /api/agent/exec from control plane (RCE surface eliminated) - Global JWT middleware (JwtAuthMiddleware) with install-mode bypass - decrypt_value: failure returns None, never leaks ciphertext - Telegram: sanitize_external_message strips sensitive fields - Agent auth: startup enforces non-empty API key, compare_digest - Credentials: password_set bool flag, no plaintext in API responses - audit_log: writes admin_username + ip_address on all CUD ops - Install lock: all /api/install/* except GET /status return 403 post-install - WebSocket dedup: publish once, subscribers deliver locally Script execution platform: - script_jobs.py / script_job_callback.py: async batching and callback - script_execution_store.py / script_callback_rate.py: Redis-backed state - script_execution_flush.py: background flusher to MySQL - scripts.html / script-executions.html: full execution UI - agent_url.py: centralised URL builder Frontend: - All 13 pages migrated from CDN to /app/vendor/ (no external deps) - vendor/: alpinejs, tailwindcss-browser, xterm, xterm-addon-*, qrious - Dashboard WebSocket real-time alerts; 8h JWT session timeout Tests: - test_security_unit.py: 15 unit tests (JWT, sanitize, vendor, install 403) - test_api.py: env-configurable admin credentials Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Your Name
+82
-74
@@ -10,13 +10,15 @@ Heartbeat flow:
|
||||
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from server.api.dependencies import get_server_service
|
||||
from server.api.schemas import AgentHeartbeat, AgentExec
|
||||
from server.api.dependencies import get_db, get_server_service
|
||||
from server.api.schemas import AgentHeartbeat, AgentScriptCallback
|
||||
from server.api.websocket import broadcast_alert, broadcast_recovery
|
||||
from server.application.services.server_service import ServerService
|
||||
from server.config import settings
|
||||
@@ -40,12 +42,11 @@ def _verify_api_key(x_api_key: str = Header(...)):
|
||||
match, the request is still allowed through so the handler can verify
|
||||
the per-server key (which requires server_id from the payload).
|
||||
|
||||
For strict global-only verification, use _verify_global_api_key instead.
|
||||
"""
|
||||
if not x_api_key:
|
||||
raise HTTPException(status_code=401, detail="Missing API key")
|
||||
# If the global key matches, allow immediately
|
||||
if x_api_key == settings.API_KEY:
|
||||
if secrets.compare_digest(x_api_key, settings.API_KEY):
|
||||
return x_api_key
|
||||
# Non-matching key — allow through for per-server verification in handler.
|
||||
# The handler MUST call _verify_server_api_key() before processing.
|
||||
@@ -54,19 +55,6 @@ def _verify_api_key(x_api_key: str = Header(...)):
|
||||
return x_api_key
|
||||
|
||||
|
||||
def _verify_global_api_key(x_api_key: str = Header(...)):
|
||||
"""Strict API key verification — only accepts the global API key.
|
||||
|
||||
Used for security-sensitive endpoints like /exec where per-server
|
||||
key fallback is NOT acceptable (arbitrary command execution).
|
||||
"""
|
||||
if not x_api_key:
|
||||
raise HTTPException(status_code=401, detail="Missing API key")
|
||||
if x_api_key != settings.API_KEY:
|
||||
raise HTTPException(status_code=403, detail="Invalid API key")
|
||||
return x_api_key
|
||||
|
||||
|
||||
async def _verify_server_api_key(server_id: int, provided_key: str, service: ServerService) -> bool:
|
||||
"""Verify API key against per-server agent_api_key, falling back to global API key.
|
||||
|
||||
@@ -78,12 +66,21 @@ async def _verify_server_api_key(server_id: int, provided_key: str, service: Ser
|
||||
try:
|
||||
server = await service.get_server(server_id)
|
||||
if server and server.agent_api_key:
|
||||
return provided_key == server.agent_api_key
|
||||
return secrets.compare_digest(provided_key, server.agent_api_key)
|
||||
except Exception:
|
||||
pass
|
||||
logger.debug(f"Failed to look up server {server_id} for per-server API key check, falling back to global key")
|
||||
|
||||
# Fallback: global API key
|
||||
return provided_key == settings.API_KEY
|
||||
return secrets.compare_digest(provided_key, settings.API_KEY)
|
||||
|
||||
|
||||
def _threshold_for(thresholds: dict, metric: str) -> float:
|
||||
defaults = {
|
||||
"cpu": settings.CPU_ALERT_THRESHOLD,
|
||||
"mem": settings.MEM_ALERT_THRESHOLD,
|
||||
"disk": settings.DISK_ALERT_THRESHOLD,
|
||||
}
|
||||
return float(thresholds.get(metric, defaults.get(metric, 80)))
|
||||
|
||||
|
||||
def _detect_alerts(system_info: dict, thresholds: dict) -> list:
|
||||
@@ -97,11 +94,11 @@ def _detect_alerts(system_info: dict, thresholds: dict) -> list:
|
||||
mem = system_info.get("mem_usage") or system_info.get("mem")
|
||||
disk = system_info.get("disk_usage") or system_info.get("disk")
|
||||
|
||||
if cpu is not None and float(cpu) > thresholds.get("cpu", settings.CPU_ALERT_THRESHOLD):
|
||||
if cpu is not None and float(cpu) > _threshold_for(thresholds, "cpu"):
|
||||
alerts.append(("cpu", float(cpu)))
|
||||
if mem is not None and float(mem) > thresholds.get("mem", settings.MEM_ALERT_THRESHOLD):
|
||||
if mem is not None and float(mem) > _threshold_for(thresholds, "mem"):
|
||||
alerts.append(("mem", float(mem)))
|
||||
if disk is not None and float(disk) > thresholds.get("disk", settings.DISK_ALERT_THRESHOLD):
|
||||
if disk is not None and float(disk) > _threshold_for(thresholds, "disk"):
|
||||
alerts.append(("disk", float(disk)))
|
||||
|
||||
return alerts
|
||||
@@ -120,7 +117,7 @@ def _detect_recovery(system_info: dict, prev_alerts: set, thresholds: dict) -> l
|
||||
metric, prev_val = parts[0], float(parts[1])
|
||||
current = system_info.get(f"{metric}_usage") or system_info.get(metric)
|
||||
|
||||
if current is not None and float(current) < thresholds.get(metric, 80):
|
||||
if current is not None and float(current) < _threshold_for(thresholds, metric):
|
||||
recoveries.append((metric, float(current)))
|
||||
|
||||
return recoveries
|
||||
@@ -172,7 +169,7 @@ async def receive_heartbeat(
|
||||
if server:
|
||||
server_name = server.name
|
||||
except Exception:
|
||||
pass
|
||||
logger.debug(f"Failed to look up server name for server {server_id}")
|
||||
|
||||
# ── 2. Alert detection ──
|
||||
alert_thresholds = {
|
||||
@@ -193,7 +190,7 @@ async def receive_heartbeat(
|
||||
await redis.sadd(f"{REDIS_ALERT_KEY_PREFIX}{server_id}", f"{alert_type}:{alert_value}")
|
||||
await redis.expire(f"{REDIS_ALERT_KEY_PREFIX}{server_id}", 3600) # 1h TTL
|
||||
except Exception:
|
||||
pass # Non-critical
|
||||
logger.debug(f"Failed to track alert in Redis for server {server_id}")
|
||||
|
||||
# ── 3. Recovery detection ──
|
||||
if system_info:
|
||||
@@ -218,56 +215,67 @@ async def receive_heartbeat(
|
||||
|
||||
return {"status": "ok", "server_id": server_id}
|
||||
|
||||
|
||||
@router.post("/exec", response_model=dict)
|
||||
async def agent_exec(
|
||||
payload: AgentExec,
|
||||
api_key: str = Depends(_verify_global_api_key),
|
||||
@router.post("/script-callback", response_model=dict)
|
||||
async def script_job_callback(
|
||||
payload: AgentScriptCallback,
|
||||
request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Execute a shell command on this Agent server
|
||||
"""Receive long-task completion from child host (curl after nohup script exits).
|
||||
|
||||
This endpoint is called by Nexus to dispatch commands to remote Agents.
|
||||
Each Agent server runs its own instance of this endpoint.
|
||||
|
||||
SECURITY: Only the GLOBAL API key is accepted (no per-server key fallback).
|
||||
This endpoint executes arbitrary commands — strict authentication required.
|
||||
Authenticated by one-time job_id + secret stored in Redis at exec time.
|
||||
"""
|
||||
command = payload.command
|
||||
timeout = payload.timeout
|
||||
if payload.sudo:
|
||||
command = f"sudo {command}"
|
||||
from server.application.services.script_job_callback import apply_script_job_callback
|
||||
from server.api.websocket import broadcast_system_event
|
||||
from server.infrastructure.database.script_repo import ScriptExecutionRepositoryImpl
|
||||
from server.infrastructure.redis.script_callback_rate import check_script_callback_rate
|
||||
|
||||
import asyncio
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_shell(
|
||||
command,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
||||
exit_code = proc.returncode or 0
|
||||
client_ip = request.client.host if request.client else ""
|
||||
await check_script_callback_rate(payload.job_id, client_ip)
|
||||
|
||||
stdout_text = stdout.decode()[:10000]
|
||||
stderr_text = stderr.decode()[:10000]
|
||||
repo = ScriptExecutionRepositoryImpl(db)
|
||||
result = await apply_script_job_callback(
|
||||
repo,
|
||||
job_id=payload.job_id,
|
||||
server_id=payload.server_id,
|
||||
secret=payload.secret,
|
||||
exit_code=payload.exit_code,
|
||||
message=payload.message,
|
||||
log_tail=payload.log_tail,
|
||||
operator="agent",
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success" if exit_code == 0 else "failed",
|
||||
"stdout": stdout_text,
|
||||
"stderr": stderr_text,
|
||||
"exit_code": exit_code,
|
||||
}
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
return {
|
||||
"status": "timeout",
|
||||
"stdout": "",
|
||||
"stderr": f"Command timed out after {timeout}s",
|
||||
"exit_code": -1,
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"status": "error",
|
||||
"stdout": "",
|
||||
"stderr": str(e),
|
||||
"exit_code": -1,
|
||||
}
|
||||
if not result:
|
||||
raise HTTPException(status_code=404, detail="Invalid or expired job")
|
||||
|
||||
from server.infrastructure.database.audit_log_repo import AuditLogRepositoryImpl
|
||||
from server.domain.models import AuditLog
|
||||
|
||||
audit_repo = AuditLogRepositoryImpl(db)
|
||||
await audit_repo.create(AuditLog(
|
||||
admin_username="agent",
|
||||
action="script_job_callback",
|
||||
target_type="script_execution",
|
||||
target_id=result["execution_id"],
|
||||
detail=(
|
||||
f"server_id={payload.server_id} job_id={payload.job_id} "
|
||||
f"exit_code={payload.exit_code} status={result.get('status')}"
|
||||
),
|
||||
))
|
||||
|
||||
await broadcast_system_event(
|
||||
"script_job_done",
|
||||
f"脚本执行 #{result['execution_id']} 服务器 {result['server_id']} "
|
||||
f"{'成功' if result['exit_code'] == 0 else '失败'}",
|
||||
)
|
||||
logger.info(
|
||||
"Script job callback: execution_id=%s server_id=%s exit_code=%s",
|
||||
result["execution_id"],
|
||||
result["server_id"],
|
||||
result["exit_code"],
|
||||
)
|
||||
return {"status": "ok", **result}
|
||||
|
||||
|
||||
# Remote command execution lives on each managed host: web/agent/agent.py POST /exec
|
||||
# Do NOT expose shell exec on the Nexus control plane (was P0 RCE via global API key).
|
||||
Reference in New Issue
Block a user