Files
Nexus/server/main.py
T
Your Name f745714530 fix: allow Vuetify SPA index.html through AppAuthMiddleware
The new Vuetify SPA uses hash routing — all pages served from single
index.html. The old middleware only whitelisted individual HTML page
paths (login.html etc.), blocking the SPA entry point with 404.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-05-31 13:13:13 +08:00

600 lines
23 KiB
Python

"""Nexus — FastAPI Application Entry Point
Clean Architecture: Presentation Layer (API routes + middleware + lifespan)
Lifespan startup (when .env exists — normal mode):
1. Verify SECRET_KEY is set (fatal if missing)
2. init_db() — create tables
3. init_redis() — ADR-009 mandatory validation (exit if unavailable)
4. load_settings_from_db() — override mutable settings from MySQL
5. Start Redis Pub/Sub subscriber (ADR-010)
6. Launch background tasks (heartbeat flush, self-monitor)
Lifespan startup (no .env — install mode):
- Only the /api/install/ routes are functional
- All other API routes return 503 "System not configured"
- User completes the install wizard at /app/install.html
D2: install wizard (install.html + FastAPI API)
D7: DB session leak fix — middleware auto-manages session lifecycle per request.
"""
import asyncio
import logging
from contextlib import asynccontextmanager
from pathlib import Path
from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse
from starlette.exceptions import HTTPException as StarletteHTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from server.config import settings
from server.infrastructure.database.session import init_db, AsyncSessionLocal
# Detect install mode: no .env = first-time setup
# Evaluated at call time so install wizard writing .env is detected immediately
ROOT_DIR = Path(__file__).resolve().parent.parent
def is_install_mode() -> bool:
"""Check install mode dynamically (not at import time)."""
return not (ROOT_DIR / ".env").exists()
# Install wizard (no JWT required — runs before system is configured)
from server.api.install import router as install_router
from server.api.auth_jwt import JwtAuthMiddleware
from server.api.auth import REFRESH_COOKIE_NAME
# API routes (require JWT + full app init)
from server.api.servers import router as servers_router
from server.api.auth import router as auth_router
from server.api.agent import router as agent_router
from server.api.scripts import router as scripts_router
from server.api.settings import (
router as settings_router,
schedule_router,
preset_router,
ssh_key_preset_router,
audit_router,
retry_router,
alert_history_router,
)
from server.api.websocket import router as websocket_router
from server.api.health import router as health_router
from server.api.assets import router as assets_router
from server.api.webssh import router as webssh_router
from server.api.sync_v2 import router as sync_v2_router
from server.api.search import router as search_router
# Background tasks
from server.background.heartbeat_flush import heartbeat_flush_loop
from server.background.script_execution_flush import script_execution_flush_loop
from server.background.self_monitor import self_monitor_loop
from server.background.schedule_runner import schedule_runner_loop
from server.background.retry_runner import retry_runner_loop
from server.background.ip_allowlist_refresh import ip_allowlist_refresh_loop
logger = logging.getLogger("nexus")
# Track background tasks for clean shutdown
_background_tasks: list[asyncio.Task] = []
# Redis-based primary worker lock (prevents duplicate background tasks with multi-worker uvicorn)
_PRIMARY_LOCK_KEY = "nexus:primary_worker"
_PRIMARY_LOCK_TTL = 30 # seconds — must renew periodically
async def _acquire_primary_lock() -> bool:
"""Try to become the primary worker via Redis SETNX.
Only one uvicorn worker will hold this lock at a time.
The lock auto-expires after PRIMARY_LOCK_TTL seconds (safety net for crashes).
Primary worker renews the lock periodically via _renew_primary_lock().
"""
try:
from server.infrastructure.redis.client import get_redis
import os
redis = get_redis()
worker_id = str(os.getpid())
# SET with NX (only if not exists) + EX (TTL)
acquired = await redis.set(_PRIMARY_LOCK_KEY, worker_id, nx=True, ex=_PRIMARY_LOCK_TTL)
if acquired:
# Start lock renewal background task
asyncio.create_task(_renew_primary_lock(), name="primary_lock_renewal")
logger.info(f"Primary worker lock acquired (pid={worker_id})")
return True
return False
except Exception as e:
# If Redis fails, assume primary (single-worker mode)
logger.warning(f"Primary lock check failed, assuming primary: {e}")
return True
async def _renew_primary_lock():
"""Periodically renew the primary worker lock in Redis."""
import os
worker_id = str(os.getpid())
while True:
await asyncio.sleep(_PRIMARY_LOCK_TTL // 2) # Renew at half the TTL
try:
from server.infrastructure.redis.client import get_redis
redis = get_redis()
current = await redis.get(_PRIMARY_LOCK_KEY)
if current == worker_id:
await redis.expire(_PRIMARY_LOCK_KEY, _PRIMARY_LOCK_TTL)
else:
logger.warning("Primary worker lock lost — another worker took over; cancelling background tasks")
for task in _background_tasks:
task.cancel()
break
except Exception as e:
logger.error(f"Primary lock renewal failed: {e}")
# ── D7: DB Session Middleware (fixes session leak in 4 service factories) ──
class DbSessionMiddleware:
"""Pure ASGI middleware: auto-manage DB session lifecycle per HTTP request.
Problem: 4 service factories (get_server_service, get_script_service,
get_auth_service, get_sync_service) create AsyncSessionLocal() but never
close them — causing connection pool exhaustion (P0 bug).
Solution: Middleware opens session at request start, stores in scope["state"],
and closes it after response. Service factories read from request.state.db
instead of creating new sessions.
Pattern: Similar to Django's ATOMIC_REQUESTS — session lifetime = request lifetime.
NOTE: Uses pure ASGI (not BaseHTTPMiddleware) to avoid MissingGreenlet errors
with SQLAlchemy async. BaseHTTPMiddleware's call_next() runs the endpoint in
a separate task, which breaks SQLAlchemy's async greenlet context, causing
session.commit() and session.refresh() to fail.
"""
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
await self.app(scope, receive, send)
return
path = scope.get("path", "")
# Skip non-API routes (WebSocket, health, static files)
if not path.startswith("/api/"):
await self.app(scope, receive, send)
return
# Skip install API (it manages its own DB connections)
if path.startswith("/api/install/"):
await self.app(scope, receive, send)
return
# Skip in install mode (no global engine available)
if is_install_mode():
await self.app(scope, receive, send)
return
# Ensure scope has a state dict (for request.state.db sharing)
scope.setdefault("state", {})
# Open session for this request
async with AsyncSessionLocal() as session:
scope["state"]["db"] = session
try:
await self.app(scope, receive, send)
except Exception:
await session.rollback()
raise
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Application lifecycle: startup → background tasks → shutdown
Install mode (no .env): only install API + static files available.
Normal mode (.env exists): full initialization with DB, Redis, background tasks.
"""
# ── Install mode: skip everything ──
if is_install_mode():
logger.info("⚠ INSTALL MODE — .env not found. Visit /app/install.html to configure.")
yield
return
# ── Normal mode: full startup ──
if not settings.SECRET_KEY:
logger.error("SECRET_KEY is empty! Set it in .env or MySQL settings table.")
raise SystemExit("SECRET_KEY is required for Nexus to start.")
if not settings.API_KEY:
logger.error("API_KEY is empty! Set it in .env (generated by install wizard).")
raise SystemExit("API_KEY is required for Agent authentication and encryption fallback.")
if not settings.ENCRYPTION_KEY:
logger.error("ENCRYPTION_KEY is empty! Set it in .env (generated by install wizard).")
raise SystemExit("ENCRYPTION_KEY is required for credential encryption.")
# 1. Initialize database tables
await init_db()
logger.info("Database tables initialized")
# 1b. D8: Run data migrations (category → Node, backfill platform_id)
from server.infrastructure.database.migrations import run_migrations
await run_migrations()
# 2. Initialize Redis (ADR-009: mandatory — exits if unavailable)
from server.infrastructure.redis.client import init_redis, close_redis
await init_redis(app)
# 3. Load mutable settings from MySQL (override .env values)
async with AsyncSessionLocal() as session:
overridden = await settings.load_settings_from_db(session)
logger.info(f"Settings from DB: {overridden} overrides applied")
# Migrate api_base_url from .env to MySQL if missing (for existing installations)
await settings.ensure_api_base_url_in_db(session)
# 4. Start Redis Pub/Sub subscriber (ADR-010)
from server.api.websocket import start_redis_subscriber, stop_redis_subscriber
await start_redis_subscriber()
# 5. Launch background tasks (only on primary worker to avoid duplicate execution)
# When running with --workers N, each worker gets its own lifespan.
# Use Redis-based leader election to ensure only one worker runs background tasks.
is_primary = await _acquire_primary_lock()
if is_primary:
task_flush = asyncio.create_task(heartbeat_flush_loop(), name="heartbeat_flush")
task_script_flush = asyncio.create_task(
script_execution_flush_loop(), name="script_execution_flush"
)
task_monitor = asyncio.create_task(self_monitor_loop(), name="self_monitor")
task_schedule = asyncio.create_task(schedule_runner_loop(), name="schedule_runner")
task_retry = asyncio.create_task(retry_runner_loop(), name="retry_runner")
task_ip_refresh = asyncio.create_task(ip_allowlist_refresh_loop(), name="ip_allowlist_refresh")
_background_tasks.extend([
task_flush, task_script_flush, task_monitor, task_schedule, task_retry, task_ip_refresh,
])
logger.info("Primary worker — background tasks launched")
else:
logger.info("Secondary worker — background tasks skipped (primary worker handles them)")
# 6. Start asyncssh connection pool (W1)
from server.infrastructure.ssh.asyncssh_pool import ssh_pool
await ssh_pool.start()
logger.info(
f"{settings.SYSTEM_NAME} v6.0.0 started — "
f"background tasks: heartbeat_flush(10min), script_execution_flush(60s), "
f"self_monitor(30s), schedule_runner(60s), retry_runner(5min)"
)
yield
# ── Shutdown ──
# Cancel WebSocket heartbeat task (H11)
from server.api.websocket import manager as ws_manager
ws_manager.cancel_heartbeat()
for task in _background_tasks:
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
_background_tasks.clear()
# Stop Redis Pub/Sub subscriber
await stop_redis_subscriber()
# Close Redis connection
await close_redis()
# Stop asyncssh connection pool
from server.infrastructure.ssh.asyncssh_pool import ssh_pool
await ssh_pool.stop()
# Close shared Telegram httpx client
from server.infrastructure.telegram import close_client as close_telegram_client
await close_telegram_client()
logger.info(f"{settings.SYSTEM_NAME} shutting down — background tasks cancelled")
app = FastAPI(
title=settings.SYSTEM_NAME if not is_install_mode() else "Nexus Installer",
description=f"{settings.SYSTEM_TITLE}" if not is_install_mode() else "Nexus Installation Wizard",
version="6.0.0",
lifespan=lifespan,
)
# ── Install mode middleware: block non-install routes when not configured ──
class InstallModeMiddleware:
"""Pure ASGI middleware: in install mode, only /api/install/, /app/, /health are accessible.
All other API routes return 503 Service Unavailable.
NOTE: Uses pure ASGI (not BaseHTTPMiddleware) to avoid MissingGreenlet errors
with SQLAlchemy async.
"""
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
await self.app(scope, receive, send)
return
if is_install_mode():
path = scope.get("path", "")
# Redirect root to install wizard
if path == "/":
from fastapi.responses import RedirectResponse
response = RedirectResponse(url="/app/install.html")
await response(scope, receive, send)
return
# Allow: install API, static files, health check
if path.startswith("/api/install/") or path.startswith("/app/") or path == "/health":
await self.app(scope, receive, send)
return
if path.startswith("/ws/"):
from fastapi.responses import JSONResponse
response = JSONResponse(
status_code=503,
content={"detail": "安装模式下 WebSocket 不可用,请先完成安装"},
)
await response(scope, receive, send)
return
# Block everything else
from fastapi.responses import JSONResponse
response = JSONResponse(
status_code=503,
content={"detail": "系统尚未配置,请先访问 /app/install.html 完成安装"},
)
await response(scope, receive, send)
return
await self.app(scope, receive, send)
# Install mode middleware (must be first — before DB session middleware)
app.add_middleware(InstallModeMiddleware)
# Security headers middleware (pure ASGI — injects headers on every HTTP response)
class SecurityHeadersMiddleware:
"""Pure ASGI middleware: inject security-related response headers.
- X-Content-Type-Options: nosniff — prevents MIME-type sniffing
- X-Frame-Options: DENY — prevents clickjacking via iframes
- Referrer-Policy: strict-origin-when-cross-origin — limits referrer leakage
- Permissions-Policy: restricts browser features (camera, mic, geolocation)
NOTE: Uses pure ASGI (not BaseHTTPMiddleware) to avoid MissingGreenlet errors.
Intercepts ASGI response messages to inject headers before they reach the client.
"""
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
await self.app(scope, receive, send)
return
# Wrap send to inject security headers into the response
async def send_with_headers(message):
if message["type"] == "http.response.start":
headers = list(message.get("headers", []))
headers.append((b"x-content-type-options", b"nosniff"))
headers.append((b"x-frame-options", b"DENY"))
headers.append((b"referrer-policy", b"strict-origin-when-cross-origin"))
headers.append((b"permissions-policy", b"camera=(), microphone=(), geolocation=()"))
message["headers"] = headers
await send(message)
await self.app(scope, receive, send_with_headers)
# ── AppAuth: protect /app/ pages behind refresh-cookie auth ──
# Public /app/ paths that anyone can access without login
_APP_PUBLIC_PATHS = frozenset({
"/app/login.html",
"/app/install.html",
"/app/forgot-password.html",
"/app/", # Vuetify SPA entry (index.html)
"/app/index.html", # Vuetify SPA entry explicit
})
_APP_PUBLIC_PREFIXES = (
"/app/vendor/", # third-party JS/CSS
"/app/api.js", # shared API module (no auth on its own)
"/app/layout.js", # shared layout module
)
def _extract_cookie(scope: dict, name: str) -> str | None:
"""Extract a cookie value from ASGI scope headers."""
for header_name, header_value in scope.get("headers", []):
if header_name == b"cookie":
val = header_value.decode("latin-1", errors="replace")
for part in val.split("; "):
if part.startswith(f"{name}="):
return part[len(name) + 1:]
return None
def _is_app_public_path(path: str) -> bool:
"""Check if the /app/ path is publicly accessible (no login required)."""
if path in _APP_PUBLIC_PATHS:
return True
for prefix in _APP_PUBLIC_PREFIXES:
if path.startswith(prefix):
return True
# Allow non-HTML static assets (CSS, JS, images, fonts)
ext = path.rsplit(".", 1)[-1].lower() if "." in path else ""
if ext in ("css", "js", "svg", "png", "ico", "woff", "woff2"):
return True
return False
class AppAuthMiddleware:
"""Pure ASGI middleware: verify nexus_refresh cookie on /app/ HTML pages.
Without a valid refresh cookie → blank HTML 404 (don't leak that the app exists).
Whitelisted paths (login, install, vendor, static assets) are always allowed.
NOTE: Uses pure ASGI (not BaseHTTPMiddleware) to avoid MissingGreenlet errors.
"""
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
await self.app(scope, receive, send)
return
path = scope.get("path", "")
# Only protect /app/ HTML pages
if not path.startswith("/app/"):
await self.app(scope, receive, send)
return
if is_install_mode():
await self.app(scope, receive, send)
return
if _is_app_public_path(path):
await self.app(scope, receive, send)
return
# Check nexus_refresh cookie
token = _extract_cookie(scope, REFRESH_COOKIE_NAME)
if not token:
response = HTMLResponse(status_code=404)
await response(scope, receive, send)
return
# Refresh token format: "token:admin_id:token_version" (opaque, not JWT)
# Validate format only — full auth happens on /api/ via JwtAuthMiddleware
parts = token.rsplit(":", 2)
if len(parts) != 3:
response = HTMLResponse(status_code=404)
await response(scope, receive, send)
return
try:
admin_id = int(parts[1])
token_version = int(parts[2])
except (ValueError, TypeError):
response = HTMLResponse(status_code=404)
await response(scope, receive, send)
return
if admin_id <= 0 or token_version < 0:
response = HTMLResponse(status_code=404)
await response(scope, receive, send)
return
# Token format is valid — user has a session; let them through
await self.app(scope, receive, send)
app.add_middleware(SecurityHeadersMiddleware)
# AppAuth: protect /app/ HTML pages behind refresh-cookie (standalone, before JWT/DB/CORS)
app.add_middleware(AppAuthMiddleware)
# F2a-10 / F2d-02: global JWT gate for all /api/* (inside DbSession — uses request.state.db)
app.add_middleware(JwtAuthMiddleware)
# D7: DB session leak fix middleware (must be added BEFORE CORS so it wraps all requests)
app.add_middleware(DbSessionMiddleware)
# CORS — restrict to actual frontend origin (ECC security fix: was allow_origins=["*"])
app.add_middleware(
CORSMiddleware,
allow_origins=[o.strip() for o in settings.CORS_ORIGINS.split(",") if o.strip()] if settings.CORS_ORIGINS else ["http://localhost:8600"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Register all API routers
# D2: Install wizard (always registered — works in both modes)
app.include_router(install_router)
# Normal API routers (require full app init)
app.include_router(servers_router)
app.include_router(auth_router)
app.include_router(agent_router)
app.include_router(scripts_router)
app.include_router(settings_router)
app.include_router(schedule_router)
app.include_router(preset_router)
app.include_router(ssh_key_preset_router)
app.include_router(audit_router)
app.include_router(alert_history_router)
app.include_router(retry_router)
# WebSocket + Health check routers (new in v6.0)
app.include_router(websocket_router)
app.include_router(health_router)
# Asset management (Platform, Node, SSH Sessions, Command Logs)
app.include_router(assets_router)
# Web SSH Terminal
app.include_router(webssh_router)
# Sync Engine v2 (Step 4)
app.include_router(sync_v2_router)
# Global Search
app.include_router(search_router)
# ── Custom 404: return blank HTML to hide backend identity ──
@app.exception_handler(StarletteHTTPException)
async def custom_http_exception_handler(request: Request, exc: StarletteHTTPException):
"""Convert 404 JSON to blank HTML — don't leak backend (FastAPI) info."""
if exc.status_code == 404:
return HTMLResponse(status_code=404)
# Let other HTTP exceptions return normally (auth errors, validation errors, etc.)
from fastapi.responses import JSONResponse
return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail})
# ── Log 422 validation errors (production-grade: avoids silent 422 on misconfigured agents) ──
from fastapi.exceptions import RequestValidationError
@app.exception_handler(RequestValidationError)
async def validation_error_handler(request: Request, exc: RequestValidationError):
client_host = request.client.host if request.client else "unknown"
content_type = request.headers.get("content-type", "")
errors = exc.errors()
logger.warning(
"422 %s %s from %s content_type=%s errors=%s",
request.method, request.url.path, client_host, content_type, errors,
)
from fastapi.responses import JSONResponse
return JSONResponse(status_code=422, content={"detail": exc.errors()})
# ── Static files: serve web/app/ at /app/ (HTML pages + JS/CSS assets) ──
# In production, Nginx serves these directly; this is for dev/standalone mode.
WEB_APP_DIR = ROOT_DIR / "web" / "app"
if WEB_APP_DIR.is_dir():
app.mount("/app", StaticFiles(directory=str(WEB_APP_DIR), html=True), name="static_app")
# ── Agent static files: serve web/agent/ at /agent/ (install.sh, agent.py, etc.) ──
# Required by remote install: curl -fsSL {base_url}/agent/install.sh
WEB_AGENT_DIR = ROOT_DIR / "web" / "agent"
if WEB_AGENT_DIR.is_dir():
app.mount("/agent", StaticFiles(directory=str(WEB_AGENT_DIR)), name="static_agent")