release: nexus btpanel session fix and app-v2
This commit is contained in:
Codex Release Bot
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Batch SSH diagnostics for Nexus Agent on sub-servers (run from central / dev with .env).
|
||||
|
||||
Collects per server:
|
||||
- Central: Redis heartbeat, MySQL last_heartbeat, agent_api_key_set
|
||||
- Remote (SSH): systemctl, redacted config.json, recent 401/heartbeat errors, /health curl
|
||||
|
||||
Usage:
|
||||
python scripts/agent_remote_diagnose.py --id 95
|
||||
python scripts/agent_remote_diagnose.py --name 颜又文
|
||||
python scripts/agent_remote_diagnose.py --offline-only --limit 20
|
||||
python scripts/agent_remote_diagnose.py --ids 95,12,33
|
||||
|
||||
Requires .env with DATABASE_URL + REDIS_URL (or NEXUS_*). SSH creds from servers table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from server.application.agent_diagnose_service import ( # noqa: E402
|
||||
REDIS_KEY_PREFIX,
|
||||
central_status,
|
||||
diagnose_server_record,
|
||||
has_agent_monitoring,
|
||||
)
|
||||
|
||||
|
||||
def _load_dotenv() -> None:
|
||||
env_path = ROOT / ".env"
|
||||
if not env_path.is_file():
|
||||
return
|
||||
for line in env_path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
key, value = key.strip(), value.strip().strip("\"'")
|
||||
if key.startswith("NEXUS_") and key[6:] not in os.environ:
|
||||
os.environ[key[6:]] = value
|
||||
elif key and key not in os.environ:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
async def _fetch_servers(
|
||||
session: Any,
|
||||
*,
|
||||
server_id: int | None,
|
||||
server_ids: list[int] | None,
|
||||
name_like: str | None,
|
||||
offline_only: bool,
|
||||
limit: int,
|
||||
) -> list[Any]:
|
||||
from sqlalchemy import or_, select
|
||||
|
||||
from server.domain.models import Server
|
||||
from server.infrastructure.redis.client import get_redis
|
||||
|
||||
q = select(Server).order_by(Server.id)
|
||||
if server_id is not None:
|
||||
q = q.where(Server.id == server_id)
|
||||
elif server_ids:
|
||||
q = q.where(Server.id.in_(server_ids))
|
||||
if name_like:
|
||||
q = q.where(or_(Server.name.like(f"%{name_like}%"), Server.domain.like(f"%{name_like}%")))
|
||||
|
||||
result = await session.execute(q)
|
||||
servers = list(result.scalars().all())
|
||||
|
||||
if offline_only:
|
||||
redis = get_redis()
|
||||
filtered = []
|
||||
for s in servers:
|
||||
if not has_agent_monitoring(s):
|
||||
continue
|
||||
hb = await redis.hgetall(f"{REDIS_KEY_PREFIX}{s.id}")
|
||||
if central_status(True, hb or None) == "offline":
|
||||
filtered.append(s)
|
||||
servers = filtered
|
||||
|
||||
if limit > 0:
|
||||
servers = servers[:limit]
|
||||
return servers
|
||||
|
||||
|
||||
async def run_diagnose(args: argparse.Namespace) -> list[dict[str, Any]]:
|
||||
from server.config import settings
|
||||
from server.infrastructure.database.session import AsyncSessionLocal, init_db
|
||||
from server.infrastructure.redis.client import close_redis, get_redis, init_redis
|
||||
|
||||
await init_db()
|
||||
await init_redis(settings.REDIS_URL)
|
||||
redis = get_redis()
|
||||
|
||||
ids: list[int] | None = None
|
||||
if args.ids:
|
||||
ids = [int(x.strip()) for x in args.ids.split(",") if x.strip()]
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
servers = await _fetch_servers(
|
||||
session,
|
||||
server_id=args.id,
|
||||
server_ids=ids,
|
||||
name_like=args.name,
|
||||
offline_only=args.offline_only,
|
||||
limit=args.limit,
|
||||
)
|
||||
|
||||
sem = asyncio.Semaphore(max(1, args.concurrency))
|
||||
|
||||
async def one(server: Any) -> dict[str, Any]:
|
||||
async with sem:
|
||||
hb = await redis.hgetall(f"{REDIS_KEY_PREFIX}{server.id}")
|
||||
row = await diagnose_server_record(
|
||||
server,
|
||||
hb or None,
|
||||
ssh=not args.central_only,
|
||||
timeout=args.timeout,
|
||||
)
|
||||
return {
|
||||
"id": row["server_id"],
|
||||
"name": row["server_name"],
|
||||
"domain": row.get("server_domain", ""),
|
||||
"central_status": row["central_status"],
|
||||
"has_agent": row["has_agent"],
|
||||
"agent_version_db": row["agent_version_db"],
|
||||
"last_heartbeat_db": row["last_heartbeat_db"],
|
||||
"last_heartbeat_redis": row["last_heartbeat_redis"],
|
||||
"remote": row.get("remote"),
|
||||
}
|
||||
|
||||
try:
|
||||
return list(await asyncio.gather(*(one(s) for s in servers)))
|
||||
finally:
|
||||
await close_redis()
|
||||
|
||||
|
||||
def _format_report(rows: list[dict[str, Any]], *, verbose: bool) -> str:
|
||||
lines: list[str] = []
|
||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
lines.append(f"# Nexus Agent remote diagnose @ {ts}")
|
||||
lines.append(f"servers: {len(rows)}")
|
||||
lines.append("")
|
||||
|
||||
for row in rows:
|
||||
lines.append(f"## [{row['id']}] {row['name']} ({row['domain']})")
|
||||
lines.append(
|
||||
f"- central: **{row['central_status']}** | "
|
||||
f"redis_hb={row['last_heartbeat_redis'] or '—'} | "
|
||||
f"mysql_hb={row['last_heartbeat_db'] or '—'} | "
|
||||
f"agent_db={row['agent_version_db'] or '—'}"
|
||||
)
|
||||
remote = row.get("remote")
|
||||
if remote:
|
||||
if remote.get("error"):
|
||||
lines.append(f"- SSH: **FAIL** — {remote['error']}")
|
||||
elif not remote.get("ssh_ok"):
|
||||
lines.append(
|
||||
f"- SSH: exit {remote.get('exit_code')} stderr={remote.get('stderr', '')[:200]}"
|
||||
)
|
||||
if verbose or remote.get("stdout"):
|
||||
body = remote.get("log_errors") or remote.get("systemctl_detail") or "(empty)"
|
||||
lines.append("```")
|
||||
lines.append(str(body)[:4000])
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description="SSH batch diagnostics for Nexus Agent on sub-servers")
|
||||
p.add_argument("--id", type=int, help="Single server id")
|
||||
p.add_argument("--ids", help="Comma-separated server ids")
|
||||
p.add_argument("--name", help="Filter name/domain substring (SQL LIKE)")
|
||||
p.add_argument("--offline-only", action="store_true", help="Only servers with Agent but no Redis heartbeat")
|
||||
p.add_argument("--limit", type=int, default=50, help="Max servers (default 50)")
|
||||
p.add_argument("--concurrency", type=int, default=5, help="Parallel SSH probes (default 5)")
|
||||
p.add_argument("--timeout", type=int, default=45, help="SSH command timeout seconds")
|
||||
p.add_argument("--central-only", action="store_true", help="Skip SSH; only Redis/MySQL snapshot")
|
||||
p.add_argument("-o", "--output", help="Write report markdown to file")
|
||||
p.add_argument("-q", "--quiet", action="store_true", help="Only print summary lines")
|
||||
return p
|
||||
|
||||
|
||||
def main() -> int:
|
||||
_load_dotenv()
|
||||
args = build_parser().parse_args()
|
||||
if not any([args.id, args.ids, args.name, args.offline_only]):
|
||||
print("Specify --id, --ids, --name, or --offline-only", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
rows = asyncio.run(run_diagnose(args))
|
||||
if not rows:
|
||||
print("No matching servers.")
|
||||
return 0
|
||||
|
||||
report = _format_report(rows, verbose=not args.quiet)
|
||||
if args.output:
|
||||
Path(args.output).write_text(report, encoding="utf-8")
|
||||
print(f"Wrote {args.output} ({len(rows)} servers)")
|
||||
else:
|
||||
print(report)
|
||||
|
||||
offline = sum(1 for r in rows if r["central_status"] == "offline")
|
||||
ssh_fail = sum(
|
||||
1 for r in rows if r.get("remote") and (r["remote"].get("error") or not r["remote"].get("ssh_ok"))
|
||||
)
|
||||
print(f"Summary: total={len(rows)} central_offline={offline} ssh_fail={ssh_fail}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user