86 lines
2.9 KiB
Python
86 lines
2.9 KiB
Python
|
"""SSH-based remote health / metrics probe (no Agent HTTP port required)."""
|
||
|
|
|
||
|
|
import json
|
||
|
|
import logging
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
from server.domain.models import Server
|
||
|
|
from server.infrastructure.ssh.asyncssh_pool import exec_ssh_command
|
||
|
|
|
||
|
|
logger = logging.getLogger("nexus.remote_probe")
|
||
|
|
|
||
|
|
# Emits one JSON line: {"status":"healthy","system_info":{...}}
|
||
|
|
SSH_HEALTH_METRICS_CMD = r"""python3 -c "
|
||
|
|
import json, platform
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
info = {
|
||
|
|
'hostname': platform.node(),
|
||
|
|
'platform': platform.platform(),
|
||
|
|
'agent_time': datetime.now(timezone.utc).isoformat(),
|
||
|
|
'probe': 'ssh',
|
||
|
|
}
|
||
|
|
try:
|
||
|
|
import psutil
|
||
|
|
info['cpu_usage'] = psutil.cpu_percent(interval=0.3)
|
||
|
|
info['mem_usage'] = psutil.virtual_memory().percent
|
||
|
|
info['disk_usage'] = round(psutil.disk_usage('/').percent, 1)
|
||
|
|
except Exception as exc:
|
||
|
|
info['metrics_error'] = str(exc)[:200]
|
||
|
|
print(json.dumps({'status': 'healthy', 'system_info': info}))
|
||
|
|
" 2>/dev/null || python3.12 -c "
|
||
|
|
import json, platform
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
info = {
|
||
|
|
'hostname': platform.node(),
|
||
|
|
'platform': platform.platform(),
|
||
|
|
'agent_time': datetime.now(timezone.utc).isoformat(),
|
||
|
|
'probe': 'ssh',
|
||
|
|
}
|
||
|
|
try:
|
||
|
|
import psutil
|
||
|
|
info['cpu_usage'] = psutil.cpu_percent(interval=0.3)
|
||
|
|
info['mem_usage'] = psutil.virtual_memory().percent
|
||
|
|
info['disk_usage'] = round(psutil.disk_usage('/').percent, 1)
|
||
|
|
except Exception as exc:
|
||
|
|
info['metrics_error'] = str(exc)[:200]
|
||
|
|
print(json.dumps({'status': 'healthy', 'system_info': info}))
|
||
|
|
" 2>/dev/null || echo '{"status":"healthy","system_info":{"hostname":"'"$(hostname)"'","probe":"ssh","ssh_only":true}}'
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
def _parse_health_stdout(stdout: str) -> dict[str, Any] | None:
|
||
|
|
for line in reversed(stdout.strip().splitlines()):
|
||
|
|
line = line.strip()
|
||
|
|
if not line.startswith("{"):
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
data = json.loads(line)
|
||
|
|
if isinstance(data, dict) and data.get("status"):
|
||
|
|
return data
|
||
|
|
except json.JSONDecodeError:
|
||
|
|
continue
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
async def ssh_health_probe(server: Server, timeout: int = 15) -> dict[str, Any]:
|
||
|
|
"""Run metrics probe over SSH; returns {status, system_info?, error?, channel}."""
|
||
|
|
result = await exec_ssh_command(server, SSH_HEALTH_METRICS_CMD, timeout=timeout)
|
||
|
|
if result.get("exit_code", -1) != 0:
|
||
|
|
return {
|
||
|
|
"status": "offline",
|
||
|
|
"error": (result.get("stderr") or result.get("stdout") or "SSH probe failed")[:300],
|
||
|
|
"channel": "ssh",
|
||
|
|
}
|
||
|
|
parsed = _parse_health_stdout(result.get("stdout", ""))
|
||
|
|
if not parsed:
|
||
|
|
return {
|
||
|
|
"status": "online",
|
||
|
|
"system_info": {"probe": "ssh", "raw": result.get("stdout", "")[:500]},
|
||
|
|
"channel": "ssh",
|
||
|
|
}
|
||
|
|
return {
|
||
|
|
"status": "online" if parsed.get("status") == "healthy" else "offline",
|
||
|
|
"system_info": parsed,
|
||
|
|
"channel": "ssh",
|
||
|
|
}
|