Nexus Agent
adee25deb6
Use chr(34) inside the bash-wrapped python -c probe so monitor slots work again after hardware detail rollout.
371 lines
12 KiB
Python
371 lines
12 KiB
Python
"""SSH-based remote health / metrics probe (no Agent HTTP port required)."""
|
|
|
|
import json
|
|
from typing import Any
|
|
|
|
from server.domain.models import Server
|
|
from server.infrastructure.ssh.asyncssh_pool import exec_ssh_command
|
|
|
|
from server.utils.watch_probe_errors import watch_probe_error_zh
|
|
|
|
# Emits one JSON line: {"status":"healthy","system_info":{...}}
|
|
SSH_HEALTH_METRICS_CMD = r"""python3 -c "
|
|
import json, platform
|
|
from datetime import datetime, timezone
|
|
info = {
|
|
'hostname': platform.node(),
|
|
'platform': platform.platform(),
|
|
'agent_time': datetime.now(timezone.utc).isoformat(),
|
|
'probe': 'ssh',
|
|
}
|
|
try:
|
|
import psutil
|
|
info['cpu_usage'] = psutil.cpu_percent(interval=0.3)
|
|
info['mem_usage'] = psutil.virtual_memory().percent
|
|
info['disk_usage'] = round(psutil.disk_usage('/').percent, 1)
|
|
except Exception as exc:
|
|
info['metrics_error'] = str(exc)[:200]
|
|
print(json.dumps({'status': 'healthy', 'system_info': info}))
|
|
" 2>/dev/null || python3.12 -c "
|
|
import json, platform
|
|
from datetime import datetime, timezone
|
|
info = {
|
|
'hostname': platform.node(),
|
|
'platform': platform.platform(),
|
|
'agent_time': datetime.now(timezone.utc).isoformat(),
|
|
'probe': 'ssh',
|
|
}
|
|
try:
|
|
import psutil
|
|
info['cpu_usage'] = psutil.cpu_percent(interval=0.3)
|
|
info['mem_usage'] = psutil.virtual_memory().percent
|
|
info['disk_usage'] = round(psutil.disk_usage('/').percent, 1)
|
|
except Exception as exc:
|
|
info['metrics_error'] = str(exc)[:200]
|
|
print(json.dumps({'status': 'healthy', 'system_info': info}))
|
|
" 2>/dev/null || echo '{"status":"healthy","system_info":{"hostname":"'"$(hostname)"'","probe":"ssh","ssh_only":true}}'
|
|
"""
|
|
|
|
|
|
SSH_WATCH_METRICS_CMD = r"""python3 -c "
|
|
import json
|
|
try:
|
|
import psutil
|
|
except ImportError:
|
|
print(json.dumps({'status':'error','error':'psutil missing'}))
|
|
raise SystemExit(0)
|
|
try:
|
|
load = list(psutil.getloadavg())
|
|
except (AttributeError, OSError, NotImplementedError):
|
|
load = [0.0, 0.0, 0.0]
|
|
try:
|
|
net = psutil.net_io_counters()
|
|
except Exception:
|
|
net = None
|
|
try:
|
|
disk_io = psutil.disk_io_counters()
|
|
except Exception:
|
|
disk_io = None
|
|
try:
|
|
import time, platform
|
|
cpu = psutil.cpu_percent(interval=0.3)
|
|
vm = psutil.virtual_memory()
|
|
du = psutil.disk_usage('/')
|
|
swap = psutil.swap_memory()
|
|
mem = vm.percent
|
|
disk = round(du.percent, 1)
|
|
cpu_cores = psutil.cpu_count(logical=True) or 1
|
|
cpu_phys = psutil.cpu_count(logical=False) or cpu_cores
|
|
mem_total_gb = round(vm.total / (1024**3), 1)
|
|
mem_used_gb = round(vm.used / (1024**3), 1)
|
|
mem_avail_gb = round(vm.available / (1024**3), 1)
|
|
disk_total_gb = round(du.total / (1024**3), 1)
|
|
disk_used_gb = round(du.used / (1024**3), 1)
|
|
disk_free_gb = round(du.free / (1024**3), 1)
|
|
swap_total_gb = round(swap.total / (1024**3), 1)
|
|
swap_used_gb = round(swap.used / (1024**3), 1)
|
|
swap_pct = round(swap.percent, 1)
|
|
uptime_seconds = int(time.time() - psutil.boot_time())
|
|
cpu_model = (platform.processor() or '').strip()[:120]
|
|
if not cpu_model:
|
|
try:
|
|
for _line in open('/proc/cpuinfo'):
|
|
if _line.lower().startswith('model name'):
|
|
cpu_model = _line.split(':', 1)[1].strip()[:120]
|
|
break
|
|
except Exception:
|
|
pass
|
|
os_release = ''
|
|
try:
|
|
for _line in open('/etc/os-release'):
|
|
if _line.startswith('PRETTY_NAME='):
|
|
os_release = _line.split('=', 1)[1].strip().strip(chr(34))[:120]
|
|
break
|
|
except Exception:
|
|
pass
|
|
except Exception as exc:
|
|
print(json.dumps({'status':'error','error': str(exc)[:200]}))
|
|
raise SystemExit(0)
|
|
out = {
|
|
'status': 'ok',
|
|
'cpu_usage': cpu,
|
|
'mem_usage': mem,
|
|
'disk_usage': disk,
|
|
'cpu_cores': cpu_cores,
|
|
'cpu_cores_physical': cpu_phys,
|
|
'cpu_model': cpu_model,
|
|
'memory_total_gb': mem_total_gb,
|
|
'memory_used_gb': mem_used_gb,
|
|
'memory_available_gb': mem_avail_gb,
|
|
'disk_total_gb': disk_total_gb,
|
|
'disk_used_gb': disk_used_gb,
|
|
'disk_free_gb': disk_free_gb,
|
|
'swap_total_gb': swap_total_gb,
|
|
'swap_used_gb': swap_used_gb,
|
|
'swap_pct': swap_pct,
|
|
'os_release': os_release,
|
|
'uptime_seconds': uptime_seconds,
|
|
'disk_mount': '/',
|
|
'load_avg': load,
|
|
'net_io': {
|
|
'bytes_sent': int(net.bytes_sent) if net else 0,
|
|
'bytes_recv': int(net.bytes_recv) if net else 0,
|
|
},
|
|
'disk_io': {
|
|
'read_bytes': int(disk_io.read_bytes) if disk_io else 0,
|
|
'write_bytes': int(disk_io.write_bytes) if disk_io else 0,
|
|
},
|
|
}
|
|
print(json.dumps(out))
|
|
" 2>/dev/null || python3.12 -c "
|
|
import json
|
|
try:
|
|
import psutil
|
|
except ImportError:
|
|
print(json.dumps({'status':'error','error':'psutil missing'}))
|
|
raise SystemExit(0)
|
|
try:
|
|
load = list(psutil.getloadavg())
|
|
except (AttributeError, OSError, NotImplementedError):
|
|
load = [0.0, 0.0, 0.0]
|
|
try:
|
|
net = psutil.net_io_counters()
|
|
except Exception:
|
|
net = None
|
|
try:
|
|
disk_io = psutil.disk_io_counters()
|
|
except Exception:
|
|
disk_io = None
|
|
try:
|
|
import time, platform
|
|
cpu = psutil.cpu_percent(interval=0.3)
|
|
vm = psutil.virtual_memory()
|
|
du = psutil.disk_usage('/')
|
|
swap = psutil.swap_memory()
|
|
mem = vm.percent
|
|
disk = round(du.percent, 1)
|
|
cpu_cores = psutil.cpu_count(logical=True) or 1
|
|
cpu_phys = psutil.cpu_count(logical=False) or cpu_cores
|
|
mem_total_gb = round(vm.total / (1024**3), 1)
|
|
mem_used_gb = round(vm.used / (1024**3), 1)
|
|
mem_avail_gb = round(vm.available / (1024**3), 1)
|
|
disk_total_gb = round(du.total / (1024**3), 1)
|
|
disk_used_gb = round(du.used / (1024**3), 1)
|
|
disk_free_gb = round(du.free / (1024**3), 1)
|
|
swap_total_gb = round(swap.total / (1024**3), 1)
|
|
swap_used_gb = round(swap.used / (1024**3), 1)
|
|
swap_pct = round(swap.percent, 1)
|
|
uptime_seconds = int(time.time() - psutil.boot_time())
|
|
cpu_model = (platform.processor() or '').strip()[:120]
|
|
if not cpu_model:
|
|
try:
|
|
for _line in open('/proc/cpuinfo'):
|
|
if _line.lower().startswith('model name'):
|
|
cpu_model = _line.split(':', 1)[1].strip()[:120]
|
|
break
|
|
except Exception:
|
|
pass
|
|
os_release = ''
|
|
try:
|
|
for _line in open('/etc/os-release'):
|
|
if _line.startswith('PRETTY_NAME='):
|
|
os_release = _line.split('=', 1)[1].strip().strip(chr(34))[:120]
|
|
break
|
|
except Exception:
|
|
pass
|
|
except Exception as exc:
|
|
print(json.dumps({'status':'error','error': str(exc)[:200]}))
|
|
raise SystemExit(0)
|
|
out = {
|
|
'status': 'ok',
|
|
'cpu_usage': cpu,
|
|
'mem_usage': mem,
|
|
'disk_usage': disk,
|
|
'cpu_cores': cpu_cores,
|
|
'cpu_cores_physical': cpu_phys,
|
|
'cpu_model': cpu_model,
|
|
'memory_total_gb': mem_total_gb,
|
|
'memory_used_gb': mem_used_gb,
|
|
'memory_available_gb': mem_avail_gb,
|
|
'disk_total_gb': disk_total_gb,
|
|
'disk_used_gb': disk_used_gb,
|
|
'disk_free_gb': disk_free_gb,
|
|
'swap_total_gb': swap_total_gb,
|
|
'swap_used_gb': swap_used_gb,
|
|
'swap_pct': swap_pct,
|
|
'os_release': os_release,
|
|
'uptime_seconds': uptime_seconds,
|
|
'disk_mount': '/',
|
|
'load_avg': load,
|
|
'net_io': {
|
|
'bytes_sent': int(net.bytes_sent) if net else 0,
|
|
'bytes_recv': int(net.bytes_recv) if net else 0,
|
|
},
|
|
'disk_io': {
|
|
'read_bytes': int(disk_io.read_bytes) if disk_io else 0,
|
|
'write_bytes': int(disk_io.write_bytes) if disk_io else 0,
|
|
},
|
|
}
|
|
print(json.dumps(out))
|
|
" 2>/dev/null || echo '{"status":"error","error":"ssh_probe_failed"}'
|
|
"""
|
|
|
|
SSH_WATCH_PROCESSES_CMD = r"""python3 -c "
|
|
import json
|
|
try:
|
|
import psutil
|
|
except ImportError:
|
|
print(json.dumps({'status':'error','error':'psutil missing'}))
|
|
raise SystemExit(0)
|
|
procs = []
|
|
for p in psutil.process_iter(['pid','name','cpu_percent','memory_percent']):
|
|
try:
|
|
i = p.info
|
|
procs.append({
|
|
'pid': i.get('pid'),
|
|
'name': (i.get('name') or '')[:80],
|
|
'cpu_pct': round(float(i.get('cpu_percent') or 0), 1),
|
|
'mem_pct': round(float(i.get('memory_percent') or 0), 1),
|
|
})
|
|
except Exception:
|
|
pass
|
|
procs.sort(key=lambda x: x.get('cpu_pct') or 0, reverse=True)
|
|
print(json.dumps({'status':'ok','top_processes': procs[:5]}))
|
|
" 2>/dev/null || echo '{"status":"error","error":"ssh_probe_failed"}'
|
|
"""
|
|
|
|
|
|
def _parse_json_stdout(stdout: str) -> dict[str, Any] | None:
|
|
text = (stdout or "").strip()
|
|
if not text:
|
|
return None
|
|
candidates: list[str] = []
|
|
for line in reversed(text.splitlines()):
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
start = line.find("{")
|
|
if start >= 0:
|
|
candidates.append(line[start:])
|
|
elif line.startswith("{"):
|
|
candidates.append(line)
|
|
if text.startswith("{"):
|
|
candidates.append(text)
|
|
seen: set[str] = set()
|
|
for chunk in candidates:
|
|
if chunk in seen:
|
|
continue
|
|
seen.add(chunk)
|
|
try:
|
|
data = json.loads(chunk)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if isinstance(data, dict) and data.get("status"):
|
|
return data
|
|
return None
|
|
|
|
|
|
async def ssh_health_probe(server: Server, timeout: int = 15) -> dict[str, Any]:
|
|
"""Run metrics probe over SSH; returns {status, system_info?, error?, channel}."""
|
|
result = await exec_ssh_command(server, SSH_HEALTH_METRICS_CMD, timeout=timeout)
|
|
if result.get("exit_code", -1) != 0:
|
|
return {
|
|
"status": "offline",
|
|
"error": (result.get("stderr") or result.get("stdout") or "SSH probe failed")[:300],
|
|
"channel": "ssh",
|
|
}
|
|
parsed = _parse_json_stdout(result.get("stdout", ""))
|
|
if not parsed:
|
|
return {
|
|
"status": "online",
|
|
"system_info": {"probe": "ssh", "raw": result.get("stdout", "")[:500]},
|
|
"channel": "ssh",
|
|
}
|
|
return {
|
|
"status": "online" if parsed.get("status") == "healthy" else "offline",
|
|
"system_info": parsed,
|
|
"channel": "ssh",
|
|
}
|
|
|
|
|
|
async def ssh_watch_probe(server: Server, timeout: int = 8) -> dict[str, Any]:
|
|
"""Full watch metrics over SSH (CPU/mem/disk/net/disk-io)."""
|
|
import time
|
|
|
|
started = time.monotonic()
|
|
result = await exec_ssh_command(server, SSH_WATCH_METRICS_CMD, timeout=timeout)
|
|
duration_ms = int((time.monotonic() - started) * 1000)
|
|
if result.get("exit_code", -1) != 0:
|
|
err_raw = (result.get("stderr") or result.get("stdout") or "SSH watch probe failed")[:300]
|
|
if "timed out" in err_raw.lower() or duration_ms >= timeout * 1000 - 50:
|
|
return {
|
|
"ok": False,
|
|
"probe_status": "ssh_timeout",
|
|
"error": watch_probe_error_zh(err_raw, status="ssh_timeout"),
|
|
"duration_ms": duration_ms,
|
|
}
|
|
if "auth" in err_raw.lower() or "permission denied" in err_raw.lower():
|
|
return {
|
|
"ok": False,
|
|
"probe_status": "ssh_auth",
|
|
"error": watch_probe_error_zh(err_raw, status="ssh_auth"),
|
|
"duration_ms": duration_ms,
|
|
}
|
|
return {
|
|
"ok": False,
|
|
"probe_status": "offline",
|
|
"error": watch_probe_error_zh(err_raw, status="offline"),
|
|
"duration_ms": duration_ms,
|
|
}
|
|
|
|
parsed = _parse_json_stdout(result.get("stdout", ""))
|
|
if not parsed or parsed.get("status") != "ok":
|
|
raw = (result.get("stdout") or "").strip().replace("\n", " ")[:200]
|
|
err_key = (parsed or {}).get("error") or "parse_error"
|
|
err_combined = f"parse_error: {raw}" if err_key == "parse_error" and raw else str(err_key)
|
|
return {
|
|
"ok": False,
|
|
"probe_status": "parse_error",
|
|
"error": watch_probe_error_zh(err_combined, status="parse_error"),
|
|
"duration_ms": duration_ms,
|
|
}
|
|
parsed["duration_ms"] = duration_ms
|
|
return {"ok": True, "payload": parsed, "duration_ms": duration_ms}
|
|
|
|
|
|
async def ssh_watch_processes(server: Server, timeout: int = 8) -> list[dict[str, Any]] | None:
|
|
"""Fetch top-5 processes over SSH (10s cadence during watch)."""
|
|
result = await exec_ssh_command(server, SSH_WATCH_PROCESSES_CMD, timeout=timeout)
|
|
if result.get("exit_code", -1) != 0:
|
|
return None
|
|
parsed = _parse_json_stdout(result.get("stdout", ""))
|
|
if not parsed or parsed.get("status") != "ok":
|
|
return None
|
|
from server.utils.watch_metrics import sanitize_processes
|
|
|
|
return sanitize_processes(parsed.get("top_processes"))
|
|
|
|
|
|
def _parse_health_stdout(stdout: str) -> dict[str, Any] | None:
|
|
return _parse_json_stdout(stdout)
|