Nexus Agent
b62f1039db
SSH origin checks cover auth contracts and once-schedule create/delete when external HTTPS is blocked; includes install script for probe credentials. Co-authored-by: Cursor <cursoragent@cursor.com>
438 lines
15 KiB
Python
438 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Production probe — public + authenticated API checks (no secrets printed).
|
|
|
|
Modes:
|
|
external — curl public BASE (may fail on some ISP routes)
|
|
origin — SSH to host, curl http://127.0.0.1:8600
|
|
auto — both; origin auth checks are required, external public is best-effort
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
|
|
@dataclass
|
|
class Config:
|
|
base_external: str
|
|
ssh_host: str
|
|
ssh_key: str | None
|
|
origin_url: str
|
|
admin_user: str
|
|
probe_password: str | None
|
|
probe_client_ip: str | None
|
|
timeout: int
|
|
|
|
|
|
def _load_secrets_env() -> None:
|
|
secrets = os.path.join(ROOT, "deploy", "nexus-1panel.secrets.sh")
|
|
if not os.path.isfile(secrets):
|
|
return
|
|
# Parse KEY=VAL / export KEY=VAL (no shell source — avoids arbitrary code)
|
|
with open(secrets, encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
if line.startswith("export "):
|
|
line = line[7:].strip()
|
|
if "=" not in line:
|
|
continue
|
|
key, _, val = line.partition("=")
|
|
key = key.strip()
|
|
val = val.strip().strip("'\"")
|
|
if key and key not in os.environ:
|
|
os.environ[key] = val
|
|
|
|
|
|
def build_config() -> Config:
|
|
_load_secrets_env()
|
|
pw = os.environ.get("NEXUS_TEST_ADMIN_PASSWORD", "").strip() or None
|
|
probe_ip = os.environ.get("NEXUS_PROBE_CLIENT_IP", "").strip() or None
|
|
return Config(
|
|
base_external=os.environ.get("NEXUS_PROBE_BASE", "https://api.synaglobal.vip").rstrip("/"),
|
|
ssh_host=os.environ.get("NEXUS_SSH", "nexus"),
|
|
ssh_key=os.environ.get("NEXUS_SSH_KEY") or None,
|
|
origin_url=os.environ.get("NEXUS_PROBE_ORIGIN", "http://127.0.0.1:8600").rstrip("/"),
|
|
admin_user=os.environ.get("NEXUS_TEST_ADMIN_USER", "admin"),
|
|
probe_password=pw,
|
|
probe_client_ip=probe_ip,
|
|
timeout=int(os.environ.get("NEXUS_PROBE_TIMEOUT", "20")),
|
|
)
|
|
|
|
|
|
def ssh_base(cfg: Config) -> list[str]:
|
|
cmd = ["ssh", "-o", "BatchMode=yes", "-o", f"ConnectTimeout={cfg.timeout}"]
|
|
if cfg.ssh_key:
|
|
cmd.extend(["-i", cfg.ssh_key])
|
|
cmd.append(cfg.ssh_host)
|
|
return cmd
|
|
|
|
|
|
def fetch_env_from_origin(cfg: Config, key: str) -> str | None:
|
|
remote = (
|
|
"sudo bash -c "
|
|
+ shlex.quote(
|
|
"for f in /opt/nexus/docker/.env.prod /opt/nexus/.env "
|
|
"/www/wwwroot/api.synaglobal.vip/.env; do "
|
|
'if [[ -f "$f" ]]; then '
|
|
f'grep -E "^{key}=" "$f" 2>/dev/null | head -1 | cut -d= -f2- | tr -d \'"\'; '
|
|
"exit 0; fi; done"
|
|
)
|
|
)
|
|
try:
|
|
r = subprocess.run(
|
|
ssh_base(cfg) + [remote],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=cfg.timeout + 5,
|
|
check=False,
|
|
)
|
|
except (subprocess.TimeoutExpired, OSError):
|
|
return None
|
|
out = (r.stdout or "").strip()
|
|
return out if out else None
|
|
|
|
|
|
def fetch_password_from_origin(cfg: Config) -> str | None:
|
|
if cfg.probe_password:
|
|
return cfg.probe_password
|
|
return fetch_env_from_origin(cfg, "NEXUS_TEST_ADMIN_PASSWORD")
|
|
|
|
|
|
def fetch_probe_client_ip(cfg: Config) -> str | None:
|
|
if cfg.probe_client_ip:
|
|
return cfg.probe_client_ip
|
|
from_env = fetch_env_from_origin(cfg, "NEXUS_PROBE_CLIENT_IP")
|
|
if from_env:
|
|
return from_env
|
|
# First IP from login_manual_ips (login allowlist) for origin probe via X-Real-IP
|
|
remote = (
|
|
"sudo docker exec nexus-prod-nexus-1 python3 -c "
|
|
+ shlex.quote(
|
|
"import asyncio\n"
|
|
"from server.infrastructure.database.session import AsyncSessionLocal\n"
|
|
"from sqlalchemy import text\n"
|
|
"async def main():\n"
|
|
" async with AsyncSessionLocal() as s:\n"
|
|
" for k in ('login_manual_ips','login_allowed_ips','login_subscription_ips'):\n"
|
|
" r = await s.execute(text('SELECT value FROM settings WHERE `key` = :k'), {'k': k})\n"
|
|
" row = r.first()\n"
|
|
" if row and row[0]:\n"
|
|
" for part in str(row[0]).split(','):\n"
|
|
" ip = part.strip().split('/')[0].strip()\n"
|
|
" if ip:\n"
|
|
" print(ip); return\n"
|
|
"asyncio.run(main())\n"
|
|
)
|
|
)
|
|
try:
|
|
r = subprocess.run(
|
|
ssh_base(cfg) + [remote],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=cfg.timeout + 15,
|
|
check=False,
|
|
)
|
|
except (subprocess.TimeoutExpired, OSError):
|
|
return None
|
|
out = (r.stdout or "").strip().splitlines()
|
|
return out[0].strip() if out else None
|
|
|
|
|
|
def http_request(
|
|
url: str,
|
|
*,
|
|
method: str = "GET",
|
|
headers: dict[str, str] | None = None,
|
|
body: dict[str, Any] | None = None,
|
|
timeout: int = 20,
|
|
) -> tuple[int, str]:
|
|
data = None
|
|
req_headers = dict(headers or {})
|
|
if body is not None:
|
|
data = json.dumps(body).encode("utf-8")
|
|
req_headers.setdefault("Content-Type", "application/json")
|
|
req = urllib.request.Request(url, data=data, headers=req_headers, method=method)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
return resp.status, resp.read().decode("utf-8", errors="replace")
|
|
except urllib.error.HTTPError as e:
|
|
payload = e.read().decode("utf-8", errors="replace")
|
|
return e.code, payload
|
|
|
|
|
|
def origin_curl(
|
|
cfg: Config,
|
|
path: str,
|
|
*,
|
|
method: str = "GET",
|
|
token: str | None = None,
|
|
body: dict[str, Any] | None = None,
|
|
client_ip: str | None = None,
|
|
) -> tuple[int, str]:
|
|
url = f"{cfg.origin_url}{path}"
|
|
parts = [
|
|
"curl",
|
|
"-sS",
|
|
"--max-time",
|
|
str(cfg.timeout),
|
|
"-w",
|
|
"\\n__HTTP_CODE__%{http_code}",
|
|
"-X",
|
|
method,
|
|
]
|
|
if client_ip:
|
|
parts.extend(
|
|
["-H", f"X-Real-IP: {client_ip}", "-H", f"X-Forwarded-For: {client_ip}"]
|
|
)
|
|
if token:
|
|
parts.extend(["-H", f"Authorization: Bearer {token}"])
|
|
if body is not None:
|
|
parts.extend(["-H", "Content-Type: application/json", "-d", json.dumps(body)])
|
|
parts.append(url)
|
|
remote = " ".join(shlex.quote(p) for p in parts)
|
|
r = subprocess.run(
|
|
ssh_base(cfg) + [remote],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=cfg.timeout + 10,
|
|
check=False,
|
|
)
|
|
if r.returncode != 0 and not r.stdout:
|
|
raise RuntimeError(f"origin curl failed: {(r.stderr or '').strip() or r.returncode}")
|
|
out = r.stdout or ""
|
|
if "\n__HTTP_CODE__" not in out:
|
|
raise RuntimeError(f"origin curl bad output for {path}")
|
|
text, _, code_s = out.rpartition("\n__HTTP_CODE__")
|
|
return int(code_s), text
|
|
|
|
|
|
def check_public_external(cfg: Config) -> tuple[bool, str]:
|
|
try:
|
|
code, body = http_request(f"{cfg.base_external}/health", timeout=cfg.timeout)
|
|
if code == 200 and body.strip() == "ok":
|
|
return True, "✓ external /health → ok"
|
|
return False, f"FAIL external /health HTTP {code}: {body[:80]}"
|
|
except Exception as e:
|
|
return False, f"WARN external /health unreachable ({e}) — use origin path or VPN"
|
|
|
|
|
|
def check_public_external_spa(cfg: Config) -> tuple[bool, str]:
|
|
try:
|
|
code, _ = http_request(f"{cfg.base_external}/app/", timeout=cfg.timeout)
|
|
if code == 200:
|
|
return True, "✓ external /app/ → HTTP 200"
|
|
return False, f"FAIL external /app/ HTTP {code}"
|
|
except Exception as e:
|
|
return False, f"WARN external /app/ unreachable ({e})"
|
|
|
|
|
|
def check_public_origin(cfg: Config) -> tuple[bool, str]:
|
|
try:
|
|
code, body = origin_curl(cfg, "/health")
|
|
if code == 200 and body.strip() == "ok":
|
|
return True, "✓ origin /health → ok"
|
|
return False, f"FAIL origin /health HTTP {code}: {body[:80]}"
|
|
except Exception as e:
|
|
return False, f"FAIL origin /health ({e})"
|
|
|
|
|
|
def login_origin(cfg: Config, password: str, client_ip: str | None) -> str:
|
|
code, body = origin_curl(
|
|
cfg,
|
|
"/api/auth/login",
|
|
method="POST",
|
|
body={"username": cfg.admin_user, "password": password},
|
|
client_ip=client_ip,
|
|
)
|
|
if code != 200:
|
|
raise RuntimeError(f"login HTTP {code}: {body[:200]}")
|
|
data = json.loads(body)
|
|
token = data.get("access_token") or ""
|
|
if not token:
|
|
raise RuntimeError("login missing access_token")
|
|
return token
|
|
|
|
|
|
def assert_json(code: int, body: str, path: str) -> dict[str, Any]:
|
|
if code != 200:
|
|
raise RuntimeError(f"{path} HTTP {code}: {body[:200]}")
|
|
return json.loads(body)
|
|
|
|
|
|
def run_auth_checks(cfg: Config, token: str, client_ip: str | None) -> list[str]:
|
|
lines: list[str] = []
|
|
ip = client_ip
|
|
code, body = origin_curl(cfg, "/health/detail", token=token, client_ip=ip)
|
|
detail = assert_json(code, body, "/health/detail")
|
|
checks = detail.get("checks") or {}
|
|
for key in ("mysql", "redis"):
|
|
if checks.get(key) != "ok":
|
|
raise RuntimeError(f"/health/detail checks.{key}={checks.get(key)!r}")
|
|
lines.append("✓ /health/detail mysql+redis ok")
|
|
|
|
code, body = origin_curl(cfg, "/api/alert-history/stats", token=token, client_ip=ip)
|
|
stats = assert_json(code, body, "/api/alert-history/stats")
|
|
for key in ("today", "active", "recovered", "top_server"):
|
|
if key not in stats:
|
|
raise RuntimeError(f"alert stats missing {key}")
|
|
lines.append("✓ /api/alert-history/stats contract OK")
|
|
|
|
code, body = origin_curl(
|
|
cfg, "/api/alert-history/?page=1&per_page=50", token=token, client_ip=ip
|
|
)
|
|
alerts = assert_json(code, body, "/api/alert-history/")
|
|
if "items" not in alerts or "total" not in alerts:
|
|
raise RuntimeError("alert-history missing items/total")
|
|
if alerts.get("per_page") not in (50, None) and len(alerts["items"]) > 50:
|
|
raise RuntimeError(f"alert per_page mismatch: {alerts.get('per_page')}")
|
|
lines.append(
|
|
f"✓ alert-history per_page=50 items={len(alerts['items'])} total={alerts['total']}"
|
|
)
|
|
|
|
for path in (
|
|
"/api/assets/command-logs?page=1&per_page=5",
|
|
"/api/assets/ssh-sessions?page=1&per_page=5",
|
|
):
|
|
code, body = origin_curl(cfg, path, token=token, client_ip=ip)
|
|
data = assert_json(code, body, path)
|
|
if "items" not in data or "total" not in data:
|
|
raise RuntimeError(f"{path} missing pagination")
|
|
lines.append(f" {path}: items={len(data['items'])} total={data['total']}")
|
|
lines.append("✓ paginated command/session APIs OK")
|
|
|
|
code, body = origin_curl(cfg, "/api/schedules/", token=token, client_ip=ip)
|
|
schedules = assert_json(code, body, "/api/schedules/")
|
|
if not isinstance(schedules, list):
|
|
raise RuntimeError("/api/schedules/ not a list")
|
|
lines.append(f"✓ /api/schedules/ count={len(schedules)}")
|
|
|
|
code, body = origin_curl(cfg, "/api/servers/?per_page=1", token=token, client_ip=ip)
|
|
servers = assert_json(code, body, "/api/servers/")
|
|
items = servers.get("items") or []
|
|
if not items:
|
|
raise RuntimeError("no servers for schedule probe")
|
|
sid = items[0]["id"]
|
|
|
|
probe_name = "probe-once-health-check"
|
|
create_body = {
|
|
"name": probe_name,
|
|
"schedule_type": "push",
|
|
"run_mode": "once",
|
|
"cron_expr": "0 10 * * *",
|
|
"fire_at": "2026-12-01T10:00:00.000Z",
|
|
"source_path": "/tmp/nexus_probe",
|
|
"server_ids": f"[{sid}]",
|
|
"enabled": False,
|
|
}
|
|
code, body = origin_curl(
|
|
cfg,
|
|
"/api/schedules/",
|
|
method="POST",
|
|
token=token,
|
|
body=create_body,
|
|
client_ip=ip,
|
|
)
|
|
if code != 201:
|
|
raise RuntimeError(f"once schedule create HTTP {code}: {body[:300]}")
|
|
created = json.loads(body)
|
|
rid = created.get("id")
|
|
if not rid:
|
|
raise RuntimeError("schedule create missing id")
|
|
lines.append(f"✓ once push schedule create HTTP 201 id={rid}")
|
|
|
|
d_code, _ = origin_curl(
|
|
cfg, f"/api/schedules/{rid}", method="DELETE", token=token, client_ip=ip
|
|
)
|
|
if d_code not in (200, 204):
|
|
raise RuntimeError(f"schedule delete HTTP {d_code}")
|
|
lines.append(f"✓ probe schedule deleted id={rid}")
|
|
|
|
return lines
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Nexus production probe")
|
|
parser.add_argument(
|
|
"--mode",
|
|
choices=("auto", "external", "origin"),
|
|
default="auto",
|
|
help="auto: external best-effort + origin required",
|
|
)
|
|
args = parser.parse_args()
|
|
cfg = build_config()
|
|
|
|
print("=== Public (external) ===")
|
|
ext_public_ok = True
|
|
if args.mode in ("auto", "external"):
|
|
ok, msg = check_public_external(cfg)
|
|
print(msg)
|
|
if not ok:
|
|
ext_public_ok = False
|
|
ok2, msg2 = check_public_external_spa(cfg)
|
|
print(msg2)
|
|
if not ok2:
|
|
ext_public_ok = False
|
|
try:
|
|
code, _ = http_request(f"{cfg.base_external}/app/install.html", timeout=cfg.timeout)
|
|
print(f" /app/install.html → HTTP {code} (404 expected when locked)")
|
|
except Exception:
|
|
print(" /app/install.html → skip (external unreachable)")
|
|
else:
|
|
print(" (skipped)")
|
|
|
|
print("=== Public (origin via SSH) ===")
|
|
if args.mode in ("auto", "origin"):
|
|
ok, msg = check_public_origin(cfg)
|
|
print(msg)
|
|
if not ok:
|
|
print("=== FAILED: origin unreachable ===")
|
|
return 1
|
|
else:
|
|
print(" (skipped)")
|
|
|
|
print("=== Authenticated (origin via SSH) ===")
|
|
pw = fetch_password_from_origin(cfg)
|
|
if not pw:
|
|
print("SKIP authenticated checks (NEXUS_TEST_ADMIN_PASSWORD not on server)")
|
|
print(" → run: NEXUS_TEST_ADMIN_PASSWORD='…' bash scripts/prod_probe_install.sh")
|
|
if args.mode == "auto" and not ext_public_ok:
|
|
print("=== PASSED (origin only; external path blocked) ===")
|
|
return 0
|
|
if args.mode == "origin":
|
|
return 0
|
|
return 0
|
|
|
|
client_ip = fetch_probe_client_ip(cfg)
|
|
if not client_ip:
|
|
print("SKIP authenticated checks (no NEXUS_PROBE_CLIENT_IP / login allowlist IP)")
|
|
print(" → set NEXUS_PROBE_CLIENT_IP in .env.prod to a whitelisted login IP")
|
|
return 0
|
|
|
|
try:
|
|
token = login_origin(cfg, pw, client_ip)
|
|
print(f"✓ login OK (probe client IP {client_ip})")
|
|
for line in run_auth_checks(cfg, token, client_ip):
|
|
print(line)
|
|
except Exception as e:
|
|
print(f"FAIL authenticated: {e}")
|
|
return 1
|
|
|
|
print("=== All production probe checks passed ===")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|