"""SSH-based remote health / metrics probe (no Agent HTTP port required).""" import json from typing import Any from server.domain.models import Server from server.infrastructure.ssh.asyncssh_pool import exec_ssh_command from server.utils.watch_probe_errors import watch_probe_error_zh # Emits one JSON line: {"status":"healthy","system_info":{...}} SSH_HEALTH_METRICS_CMD = r"""python3 -c " import json, platform from datetime import datetime, timezone info = { 'hostname': platform.node(), 'platform': platform.platform(), 'agent_time': datetime.now(timezone.utc).isoformat(), 'probe': 'ssh', } try: import psutil info['cpu_usage'] = psutil.cpu_percent(interval=0.3) info['mem_usage'] = psutil.virtual_memory().percent info['disk_usage'] = round(psutil.disk_usage('/').percent, 1) except Exception as exc: info['metrics_error'] = str(exc)[:200] print(json.dumps({'status': 'healthy', 'system_info': info})) " 2>/dev/null || python3.12 -c " import json, platform from datetime import datetime, timezone info = { 'hostname': platform.node(), 'platform': platform.platform(), 'agent_time': datetime.now(timezone.utc).isoformat(), 'probe': 'ssh', } try: import psutil info['cpu_usage'] = psutil.cpu_percent(interval=0.3) info['mem_usage'] = psutil.virtual_memory().percent info['disk_usage'] = round(psutil.disk_usage('/').percent, 1) except Exception as exc: info['metrics_error'] = str(exc)[:200] print(json.dumps({'status': 'healthy', 'system_info': info})) " 2>/dev/null || echo '{"status":"healthy","system_info":{"hostname":"'"$(hostname)"'","probe":"ssh","ssh_only":true}}' """ SSH_WATCH_METRICS_CMD = r"""python3 -c " import json try: import psutil except ImportError: print(json.dumps({'status':'error','error':'psutil missing'})) raise SystemExit(0) try: load = list(psutil.getloadavg()) except (AttributeError, OSError, NotImplementedError): load = [0.0, 0.0, 0.0] try: net = psutil.net_io_counters() except Exception: net = None try: disk_io = psutil.disk_io_counters() except Exception: disk_io = None try: cpu = psutil.cpu_percent(interval=0.3) mem = psutil.virtual_memory().percent disk = round(psutil.disk_usage('/').percent, 1) except Exception as exc: print(json.dumps({'status':'error','error': str(exc)[:200]})) raise SystemExit(0) out = { 'status': 'ok', 'cpu_usage': cpu, 'mem_usage': mem, 'disk_usage': disk, 'disk_mount': '/', 'load_avg': load, 'net_io': { 'bytes_sent': int(net.bytes_sent) if net else 0, 'bytes_recv': int(net.bytes_recv) if net else 0, }, 'disk_io': { 'read_bytes': int(disk_io.read_bytes) if disk_io else 0, 'write_bytes': int(disk_io.write_bytes) if disk_io else 0, }, } print(json.dumps(out)) " 2>/dev/null || python3.12 -c " import json try: import psutil except ImportError: print(json.dumps({'status':'error','error':'psutil missing'})) raise SystemExit(0) try: load = list(psutil.getloadavg()) except (AttributeError, OSError, NotImplementedError): load = [0.0, 0.0, 0.0] try: net = psutil.net_io_counters() except Exception: net = None try: disk_io = psutil.disk_io_counters() except Exception: disk_io = None try: cpu = psutil.cpu_percent(interval=0.3) mem = psutil.virtual_memory().percent disk = round(psutil.disk_usage('/').percent, 1) except Exception as exc: print(json.dumps({'status':'error','error': str(exc)[:200]})) raise SystemExit(0) out = { 'status': 'ok', 'cpu_usage': cpu, 'mem_usage': mem, 'disk_usage': disk, 'disk_mount': '/', 'load_avg': load, 'net_io': { 'bytes_sent': int(net.bytes_sent) if net else 0, 'bytes_recv': int(net.bytes_recv) if net else 0, }, 'disk_io': { 'read_bytes': int(disk_io.read_bytes) if disk_io else 0, 'write_bytes': int(disk_io.write_bytes) if disk_io else 0, }, } print(json.dumps(out)) " 2>/dev/null || echo '{"status":"error","error":"ssh_probe_failed"}' """ SSH_WATCH_PROCESSES_CMD = r"""python3 -c " import json try: import psutil except ImportError: print(json.dumps({'status':'error','error':'psutil missing'})) raise SystemExit(0) procs = [] for p in psutil.process_iter(['pid','name','cpu_percent','memory_percent']): try: i = p.info procs.append({ 'pid': i.get('pid'), 'name': (i.get('name') or '')[:80], 'cpu_pct': round(float(i.get('cpu_percent') or 0), 1), 'mem_pct': round(float(i.get('memory_percent') or 0), 1), }) except Exception: pass procs.sort(key=lambda x: x.get('cpu_pct') or 0, reverse=True) print(json.dumps({'status':'ok','top_processes': procs[:5]})) " 2>/dev/null || echo '{"status":"error","error":"ssh_probe_failed"}' """ def _parse_json_stdout(stdout: str) -> dict[str, Any] | None: text = (stdout or "").strip() if not text: return None candidates: list[str] = [] for line in reversed(text.splitlines()): line = line.strip() if not line: continue start = line.find("{") if start >= 0: candidates.append(line[start:]) elif line.startswith("{"): candidates.append(line) if text.startswith("{"): candidates.append(text) seen: set[str] = set() for chunk in candidates: if chunk in seen: continue seen.add(chunk) try: data = json.loads(chunk) except json.JSONDecodeError: continue if isinstance(data, dict) and data.get("status"): return data return None async def ssh_health_probe(server: Server, timeout: int = 15) -> dict[str, Any]: """Run metrics probe over SSH; returns {status, system_info?, error?, channel}.""" result = await exec_ssh_command(server, SSH_HEALTH_METRICS_CMD, timeout=timeout) if result.get("exit_code", -1) != 0: return { "status": "offline", "error": (result.get("stderr") or result.get("stdout") or "SSH probe failed")[:300], "channel": "ssh", } parsed = _parse_json_stdout(result.get("stdout", "")) if not parsed: return { "status": "online", "system_info": {"probe": "ssh", "raw": result.get("stdout", "")[:500]}, "channel": "ssh", } return { "status": "online" if parsed.get("status") == "healthy" else "offline", "system_info": parsed, "channel": "ssh", } async def ssh_watch_probe(server: Server, timeout: int = 8) -> dict[str, Any]: """Full watch metrics over SSH (CPU/mem/disk/net/disk-io).""" import time started = time.monotonic() result = await exec_ssh_command(server, SSH_WATCH_METRICS_CMD, timeout=timeout) duration_ms = int((time.monotonic() - started) * 1000) if result.get("exit_code", -1) != 0: err_raw = (result.get("stderr") or result.get("stdout") or "SSH watch probe failed")[:300] if "timed out" in err_raw.lower() or duration_ms >= timeout * 1000 - 50: return { "ok": False, "probe_status": "ssh_timeout", "error": watch_probe_error_zh(err_raw, status="ssh_timeout"), "duration_ms": duration_ms, } if "auth" in err_raw.lower() or "permission denied" in err_raw.lower(): return { "ok": False, "probe_status": "ssh_auth", "error": watch_probe_error_zh(err_raw, status="ssh_auth"), "duration_ms": duration_ms, } return { "ok": False, "probe_status": "offline", "error": watch_probe_error_zh(err_raw, status="offline"), "duration_ms": duration_ms, } parsed = _parse_json_stdout(result.get("stdout", "")) if not parsed or parsed.get("status") != "ok": raw = (result.get("stdout") or "").strip().replace("\n", " ")[:200] err_key = (parsed or {}).get("error") or "parse_error" err_combined = f"parse_error: {raw}" if err_key == "parse_error" and raw else str(err_key) return { "ok": False, "probe_status": "parse_error", "error": watch_probe_error_zh(err_combined, status="parse_error"), "duration_ms": duration_ms, } parsed["duration_ms"] = duration_ms return {"ok": True, "payload": parsed, "duration_ms": duration_ms} async def ssh_watch_processes(server: Server, timeout: int = 8) -> list[dict[str, Any]] | None: """Fetch top-5 processes over SSH (10s cadence during watch).""" result = await exec_ssh_command(server, SSH_WATCH_PROCESSES_CMD, timeout=timeout) if result.get("exit_code", -1) != 0: return None parsed = _parse_json_stdout(result.get("stdout", "")) if not parsed or parsed.get("status") != "ok": return None from server.utils.watch_metrics import sanitize_processes return sanitize_processes(parsed.get("top_processes")) def _parse_health_stdout(stdout: str) -> dict[str, Any] | None: return _parse_json_stdout(stdout)