Codex Release Bot
179 lines
6.6 KiB
Python
179 lines
6.6 KiB
Python
"""Nexus Agent remote diagnosis — central Redis/MySQL snapshot + optional SSH probe."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
from typing import Any
|
||
|
||
REDIS_KEY_PREFIX = "heartbeat:"
|
||
|
||
REMOTE_AGENT_DIAG_SHELL = r"""
|
||
INSTALL_DIR=/opt/nexus-agent
|
||
echo "### systemctl ###"
|
||
(systemctl is-active nexus-agent 2>/dev/null || echo inactive)
|
||
systemctl show nexus-agent -p ActiveState,SubState,ActiveEnterTimestamp --value 2>/dev/null | tr '\n' ' '
|
||
echo
|
||
echo "### config_redacted ###"
|
||
if [ -f "$INSTALL_DIR/config.json" ]; then
|
||
python3 -c "import json; c=json.load(open('$INSTALL_DIR/config.json')); k=c.get('api_key',''); c['api_key']=((k[:8]+'...') if k else ''); print(json.dumps(c, ensure_ascii=False))"
|
||
else
|
||
echo missing_config
|
||
fi
|
||
echo "### log_errors ###"
|
||
for f in /var/log/nexus-agent.log "$INSTALL_DIR/agent.log"; do
|
||
if [ -f "$f" ]; then
|
||
echo "--- $f ---"
|
||
tail -40 "$f" | grep -E '401|Heartbeat rejected|Stopping heartbeat|Heartbeat error|discarded' || echo "(no matching lines)"
|
||
fi
|
||
done
|
||
if command -v journalctl >/dev/null 2>&1; then
|
||
echo "--- journalctl ---"
|
||
journalctl -u nexus-agent --no-pager -n 30 2>/dev/null | grep -E '401|Heartbeat rejected|Stopping heartbeat|Heartbeat error|discarded' || echo "(no matching lines)"
|
||
fi
|
||
echo "### central_health ###"
|
||
CU=$(python3 -c "import json; c=json.load(open('$INSTALL_DIR/config.json')); print((c.get('central') or {}).get('url','') or c.get('central_url',''))" 2>/dev/null || true)
|
||
if [ -n "$CU" ]; then
|
||
curl -fsS -o /dev/null -w "http_code=%{http_code}\n" "${CU%/}/health" --connect-timeout 8 2>&1 || echo curl_failed
|
||
else
|
||
echo no_central_url
|
||
fi
|
||
""".strip()
|
||
|
||
|
||
def has_agent_monitoring(server: Any) -> bool:
|
||
return bool((server.agent_api_key or "").strip() or (server.agent_version or "").strip())
|
||
|
||
|
||
def central_status(has_agent: bool, heartbeat: dict[str, str] | None) -> str:
|
||
if heartbeat:
|
||
return "online" if heartbeat.get("is_online") == "True" else "offline"
|
||
if has_agent:
|
||
return "offline"
|
||
return "unknown"
|
||
|
||
|
||
async def probe_remote_agent(server: Any, *, timeout: int = 45) -> dict[str, Any]:
|
||
from server.application.server_batch_common import sudo_wrap
|
||
from server.infrastructure.ssh.asyncssh_pool import exec_ssh_command
|
||
|
||
ssh_user = (server.username or "root").strip()
|
||
cmd = sudo_wrap(REMOTE_AGENT_DIAG_SHELL, ssh_user)
|
||
try:
|
||
r = await exec_ssh_command(server, cmd, timeout=timeout)
|
||
stdout = (r.get("stdout") or "").strip()
|
||
parsed = parse_remote_probe(stdout)
|
||
return {
|
||
"ssh_ok": r.get("exit_code") == 0,
|
||
"exit_code": r.get("exit_code"),
|
||
"stderr": (r.get("stderr") or "").strip()[:500],
|
||
"error": "",
|
||
**parsed,
|
||
}
|
||
except Exception as exc:
|
||
return {
|
||
"ssh_ok": False,
|
||
"exit_code": -1,
|
||
"stderr": "",
|
||
"error": str(exc)[:500],
|
||
"systemctl_active": None,
|
||
"systemctl_detail": "",
|
||
"config": None,
|
||
"log_errors": "",
|
||
"central_health_http_code": None,
|
||
}
|
||
|
||
|
||
def parse_remote_probe(stdout: str) -> dict[str, Any]:
|
||
sections: dict[str, str] = {}
|
||
current = ""
|
||
buf: list[str] = []
|
||
for line in stdout.splitlines():
|
||
if line.startswith("### ") and line.endswith(" ###"):
|
||
if current:
|
||
sections[current] = "\n".join(buf).strip()
|
||
current = line[4:-4]
|
||
buf = []
|
||
else:
|
||
buf.append(line)
|
||
if current:
|
||
sections[current] = "\n".join(buf).strip()
|
||
|
||
systemctl_lines = [ln for ln in sections.get("systemctl", "").splitlines() if ln.strip()]
|
||
systemctl_active = systemctl_lines[0].strip() if systemctl_lines else ""
|
||
systemctl_detail = systemctl_lines[1].strip() if len(systemctl_lines) > 1 else ""
|
||
|
||
config_raw = sections.get("config_redacted", "")
|
||
config: dict[str, Any] | None = None
|
||
if config_raw and config_raw != "missing_config":
|
||
try:
|
||
config = json.loads(config_raw)
|
||
except json.JSONDecodeError:
|
||
config = {"raw": config_raw[:500]}
|
||
|
||
health_raw = sections.get("central_health", "")
|
||
http_code: int | None = None
|
||
m = re.search(r"http_code=(\d+)", health_raw)
|
||
if m:
|
||
http_code = int(m.group(1))
|
||
|
||
return {
|
||
"systemctl_active": systemctl_active or None,
|
||
"systemctl_detail": systemctl_detail,
|
||
"config": config,
|
||
"log_errors": sections.get("log_errors", ""),
|
||
"central_health_http_code": http_code,
|
||
}
|
||
|
||
|
||
def build_diagnose_errors(row: dict[str, Any]) -> list[str]:
|
||
errors: list[str] = []
|
||
if row.get("has_agent") and row.get("central_status") == "offline":
|
||
errors.append("中心侧显示 Agent 离线（Redis 无新鲜心跳）")
|
||
remote = row.get("remote") or {}
|
||
if remote.get("error"):
|
||
errors.append(f"SSH 连接失败: {remote['error']}")
|
||
elif remote and not remote.get("ssh_ok"):
|
||
stderr = (remote.get("stderr") or "").strip()
|
||
errors.append(f"SSH 命令异常（退出码 {remote.get('exit_code')}）{': ' + stderr if stderr else ''}")
|
||
active = remote.get("systemctl_active")
|
||
if active and active != "active":
|
||
errors.append(f"子机 nexus-agent 服务状态: {active}")
|
||
log_errors = (remote.get("log_errors") or "").strip()
|
||
if log_errors and "(no matching lines)" not in log_errors:
|
||
snippet = log_errors.replace("\n", " | ")[:400]
|
||
errors.append(f"近期 Agent 错误日志: {snippet}")
|
||
code = remote.get("central_health_http_code")
|
||
if code is not None and code != 200:
|
||
errors.append(f"子机访问中心 /health 返回 HTTP {code}")
|
||
elif remote.get("config") and code is None and "no_central_url" not in (remote.get("log_errors") or ""):
|
||
if remote.get("ssh_ok") and not remote.get("config"):
|
||
pass
|
||
return errors
|
||
|
||
|
||
async def diagnose_server_record(
|
||
server: Any,
|
||
heartbeat: dict[str, str] | None,
|
||
*,
|
||
ssh: bool = True,
|
||
timeout: int = 45,
|
||
) -> dict[str, Any]:
|
||
has_agent = has_agent_monitoring(server)
|
||
status = central_status(has_agent, heartbeat)
|
||
row: dict[str, Any] = {
|
||
"server_id": server.id,
|
||
"server_name": server.name,
|
||
"server_domain": server.domain,
|
||
"central_status": status,
|
||
"has_agent": has_agent,
|
||
"agent_version_db": (server.agent_version or "").strip(),
|
||
"last_heartbeat_db": str(server.last_heartbeat or ""),
|
||
"last_heartbeat_redis": (heartbeat or {}).get("last_heartbeat", ""),
|
||
"redis_online": (heartbeat or {}).get("is_online") == "True",
|
||
}
|
||
if ssh:
|
||
row["remote"] = await probe_remote_agent(server, timeout=timeout)
|
||
row["errors"] = build_diagnose_errors(row)
|
||
return row
|