release: nexus btpanel session fix and app-v2
This commit is contained in:
Codex Release Bot
@@ -0,0 +1,84 @@
|
||||
"""Shared Agent install/upgrade paths and shell fragments (aligned with web/agent/install.sh)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from server.application.server_batch_common import agent_files_curl_cmds, sudo_wrap
|
||||
|
||||
# Must match web/agent/install.sh
|
||||
AGENT_INSTALL_DIR = "/opt/nexus-agent"
|
||||
AGENT_PIP_PACKAGES = (
|
||||
"fastapi==0.115.6",
|
||||
"uvicorn==0.34.0",
|
||||
"httpx==0.28.1",
|
||||
"psutil",
|
||||
"python-multipart==0.0.19",
|
||||
)
|
||||
|
||||
|
||||
def agent_venv_python(install_dir: str = AGENT_INSTALL_DIR) -> str:
|
||||
return f"{install_dir}/.venv/bin/python"
|
||||
|
||||
|
||||
def agent_pip_install_cmd(venv_python: str | None = None) -> str:
|
||||
py = venv_python or agent_venv_python()
|
||||
pkgs = " ".join(AGENT_PIP_PACKAGES)
|
||||
return f"{py} -m pip install -q {pkgs}"
|
||||
|
||||
|
||||
def agent_python_version_check_cmd(venv_python: str | None = None) -> str:
|
||||
py = venv_python or agent_venv_python()
|
||||
return (
|
||||
f"{py} -c '"
|
||||
"import sys; v=sys.version_info; "
|
||||
"print(f\"py_ok_{v.major}.{v.minor}.{v.micro}\") "
|
||||
"if v.major==3 and v.minor>=10 else sys.exit(1)'"
|
||||
)
|
||||
|
||||
|
||||
def agent_rsync_install_cmd() -> str:
|
||||
return (
|
||||
"(command -v rsync >/dev/null 2>&1 || "
|
||||
"(apt-get update -qq && apt-get install -y -qq rsync) || "
|
||||
"(yum install -y -q rsync) || (dnf install -y -q rsync))"
|
||||
)
|
||||
|
||||
|
||||
def agent_kill_port_cmd(agent_port: int) -> str:
|
||||
return f"fuser -k {int(agent_port)}/tcp 2>/dev/null || true; "
|
||||
|
||||
|
||||
def build_agent_upgrade_cmd(
|
||||
*,
|
||||
base_url: str,
|
||||
ssh_user: str,
|
||||
install_dir: str = AGENT_INSTALL_DIR,
|
||||
agent_port: int = 8601,
|
||||
) -> str:
|
||||
"""Shell pipeline: rsync → py check → pip → backup → download → restart → verify."""
|
||||
venv_python = agent_venv_python(install_dir)
|
||||
systemctl_prefix = "sudo " if ssh_user != "root" else ""
|
||||
return sudo_wrap(
|
||||
f"{agent_rsync_install_cmd()} "
|
||||
f"&& {agent_python_version_check_cmd(venv_python)} "
|
||||
f"&& {agent_pip_install_cmd(venv_python)} "
|
||||
f"&& cp {install_dir}/agent.py {install_dir}/agent.py.bak "
|
||||
f"&& {agent_files_curl_cmds(base_url, install_dir)} "
|
||||
f"&& {agent_kill_port_cmd(agent_port)}{systemctl_prefix}systemctl restart nexus-agent "
|
||||
f"&& sleep 2 "
|
||||
f"&& {systemctl_prefix}systemctl is-active nexus-agent "
|
||||
f"&& echo 'upgrade_ok'",
|
||||
ssh_user,
|
||||
)
|
||||
|
||||
|
||||
def build_agent_upgrade_rollback_cmd(
|
||||
*,
|
||||
ssh_user: str,
|
||||
install_dir: str = AGENT_INSTALL_DIR,
|
||||
) -> str:
|
||||
systemctl_prefix = "sudo " if ssh_user != "root" else ""
|
||||
return sudo_wrap(
|
||||
f"cp {install_dir}/agent.py.bak {install_dir}/agent.py "
|
||||
f"&& {systemctl_prefix}systemctl restart nexus-agent",
|
||||
ssh_user,
|
||||
)
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Central Nexus Agent version (SSOT) and semver-style comparison."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
_AGENT_PY = Path(__file__).resolve().parents[2] / "web" / "agent" / "agent.py"
|
||||
_AGENT_VERSION_CONST_RE = re.compile(r'^AGENT_VERSION\s*=\s*"([^"]+)"', re.MULTILINE)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_central_agent_version() -> str:
|
||||
"""Read AGENT_VERSION constant from web/agent/agent.py (heartbeat SSOT)."""
|
||||
text = _AGENT_PY.read_text(encoding="utf-8")
|
||||
match = _AGENT_VERSION_CONST_RE.search(text)
|
||||
if not match:
|
||||
raise RuntimeError(f"AGENT_VERSION not found in {_AGENT_PY}")
|
||||
return match.group(1)
|
||||
|
||||
|
||||
def version_tuple(version: str) -> tuple[int, ...]:
|
||||
"""Parse dotted numeric prefix, e.g. 2.0.0 -> (2, 0, 0)."""
|
||||
version = (version or "").strip()
|
||||
if not version:
|
||||
return ()
|
||||
parts: list[int] = []
|
||||
for segment in re.split(r"[.-]", version):
|
||||
if segment.isdigit():
|
||||
parts.append(int(segment))
|
||||
else:
|
||||
break
|
||||
return tuple(parts)
|
||||
|
||||
|
||||
def is_version_lower(remote: str, central: str) -> bool:
|
||||
"""True when remote is strictly older than central."""
|
||||
remote_t = version_tuple(remote)
|
||||
central_t = version_tuple(central)
|
||||
if not remote_t:
|
||||
return True
|
||||
if not central_t:
|
||||
return False
|
||||
return remote_t < central_t
|
||||
|
||||
|
||||
def is_version_at_least(remote: str, central: str) -> bool:
|
||||
"""True when remote >= central (equal counts as skip)."""
|
||||
remote_t = version_tuple(remote)
|
||||
central_t = version_tuple(central)
|
||||
if not remote_t:
|
||||
return False
|
||||
if not central_t:
|
||||
return bool(remote_t)
|
||||
return remote_t >= central_t
|
||||
|
||||
|
||||
def agent_is_installed(
|
||||
*,
|
||||
agent_version_db: str | None,
|
||||
heartbeat: dict[str, str] | None = None,
|
||||
) -> bool:
|
||||
"""True when Agent has reported a version or heartbeat (not merely a DB API key)."""
|
||||
if heartbeat:
|
||||
if (heartbeat.get("agent_version") or "").strip():
|
||||
return True
|
||||
if heartbeat.get("last_heartbeat"):
|
||||
return True
|
||||
return bool((agent_version_db or "").strip())
|
||||
|
||||
|
||||
def compute_agent_guidance(
|
||||
*,
|
||||
agent_version_db: str | None,
|
||||
heartbeat: dict[str, str] | None = None,
|
||||
central: str | None = None,
|
||||
) -> dict[str, str | None]:
|
||||
"""UI hints: install / upgrade / offline / ok."""
|
||||
central_v = central or get_central_agent_version()
|
||||
live_ver = ""
|
||||
if heartbeat:
|
||||
live_ver = (heartbeat.get("agent_version") or "").strip()
|
||||
if not live_ver:
|
||||
live_ver = (agent_version_db or "").strip()
|
||||
|
||||
has_heartbeat = bool(heartbeat and heartbeat.get("last_heartbeat"))
|
||||
installed = agent_is_installed(agent_version_db=agent_version_db, heartbeat=heartbeat)
|
||||
|
||||
base: dict[str, str | None] = {
|
||||
"central_agent_version": central_v,
|
||||
"agent_action": "ok",
|
||||
"agent_action_message": None,
|
||||
}
|
||||
|
||||
if not installed:
|
||||
return {
|
||||
**base,
|
||||
"agent_action": "install",
|
||||
"agent_action_message": (
|
||||
f"尚未安装 Agent（主站版本 {central_v}）。"
|
||||
"安装后可自动上报 CPU/内存/磁盘与在线状态；未安装时仅能通过 SSH 手动探活。"
|
||||
),
|
||||
}
|
||||
|
||||
if live_ver and is_version_lower(live_ver, central_v):
|
||||
return {
|
||||
**base,
|
||||
"agent_action": "upgrade",
|
||||
"agent_action_message": (
|
||||
f"当前 Agent {live_ver}，低于主站 {central_v}，建议升级以获取最新心跳与探针能力。"
|
||||
),
|
||||
}
|
||||
|
||||
if not has_heartbeat:
|
||||
return {
|
||||
**base,
|
||||
"agent_action": "offline",
|
||||
"agent_action_message": (
|
||||
"Agent 已登记但暂无心跳，请检查子机 nexus-agent 服务或点击「诊断」。"
|
||||
),
|
||||
}
|
||||
|
||||
return base
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Shared metric threshold helpers for Agent heartbeat and watch probes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from server.config import settings
|
||||
|
||||
logger = logging.getLogger("nexus.alert_metrics")
|
||||
|
||||
REDIS_ALERT_KEY_PREFIX = "alerts:"
|
||||
|
||||
|
||||
def threshold_for(thresholds: dict, metric: str) -> float:
|
||||
defaults = {
|
||||
"cpu": settings.CPU_ALERT_THRESHOLD,
|
||||
"mem": settings.MEM_ALERT_THRESHOLD,
|
||||
"disk": settings.DISK_ALERT_THRESHOLD,
|
||||
}
|
||||
return float(thresholds.get(metric, defaults.get(metric, 80)))
|
||||
|
||||
|
||||
def safe_float(value, name: str) -> Optional[float]:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (ValueError, TypeError):
|
||||
logger.warning(f"Agent sent non-numeric {name}: {value!r:.50} — skipped")
|
||||
return None
|
||||
|
||||
|
||||
def detect_alerts(system_info: dict, thresholds: dict) -> list:
|
||||
alerts = []
|
||||
cpu = safe_float(system_info.get("cpu_usage") or system_info.get("cpu"), "cpu")
|
||||
mem = safe_float(system_info.get("mem_usage") or system_info.get("mem"), "mem")
|
||||
disk = safe_float(system_info.get("disk_usage") or system_info.get("disk"), "disk")
|
||||
|
||||
if cpu is not None and cpu > threshold_for(thresholds, "cpu"):
|
||||
alerts.append(("cpu", cpu))
|
||||
if mem is not None and mem > threshold_for(thresholds, "mem"):
|
||||
alerts.append(("mem", mem))
|
||||
if disk is not None and disk > threshold_for(thresholds, "disk"):
|
||||
alerts.append(("disk", disk))
|
||||
|
||||
return alerts
|
||||
|
||||
|
||||
def alert_metric_key(entry: str) -> str | None:
|
||||
if not entry:
|
||||
return None
|
||||
metric = entry.split(":", 1)[0].strip()
|
||||
return metric or None
|
||||
|
||||
|
||||
def detect_recovery(system_info: dict, prev_alerts: set, thresholds: dict) -> list:
|
||||
recoveries: list[tuple[str, float]] = []
|
||||
seen_metrics: set[str] = set()
|
||||
for prev in prev_alerts:
|
||||
metric = alert_metric_key(prev)
|
||||
if not metric or metric in seen_metrics:
|
||||
continue
|
||||
current = safe_float(
|
||||
system_info.get(f"{metric}_usage") or system_info.get(metric), metric
|
||||
)
|
||||
if current is not None and current < threshold_for(thresholds, metric):
|
||||
seen_metrics.add(metric)
|
||||
recoveries.append((metric, current))
|
||||
|
||||
return recoveries
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Consecutive over-threshold heartbeats before metric alerts fire (C2 debounce)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from server.utils.alert_metrics import (
|
||||
REDIS_ALERT_KEY_PREFIX,
|
||||
alert_metric_key as _alert_metric_key,
|
||||
detect_recovery as _detect_recovery,
|
||||
safe_float as _safe_float,
|
||||
threshold_for as _threshold_for,
|
||||
)
|
||||
from server.api.websocket import broadcast_alert, broadcast_recovery
|
||||
from server.config import settings
|
||||
|
||||
logger = logging.getLogger("nexus.alert_streak")
|
||||
|
||||
REDIS_STREAK_PREFIX = "alert_streak:"
|
||||
STREAK_TTL_SECONDS = 3600
|
||||
METRICS = ("cpu", "mem", "disk")
|
||||
|
||||
|
||||
def streak_required() -> int:
|
||||
"""Minimum consecutive over-threshold samples before alerting (default 3)."""
|
||||
return max(1, int(getattr(settings, "ALERT_STREAK_REQUIRED", 3)))
|
||||
|
||||
|
||||
def _metric_value(system_info: dict[str, Any], metric: str) -> float | None:
|
||||
return _safe_float(
|
||||
system_info.get(f"{metric}_usage") or system_info.get(metric),
|
||||
metric,
|
||||
)
|
||||
|
||||
|
||||
async def record_over_threshold(redis, server_id: int, metric: str) -> int:
|
||||
key = f"{REDIS_STREAK_PREFIX}{server_id}:{metric}"
|
||||
count = int(await redis.incr(key))
|
||||
await redis.expire(key, STREAK_TTL_SECONDS)
|
||||
return count
|
||||
|
||||
|
||||
async def reset_streak(redis, server_id: int, metric: str) -> None:
|
||||
await redis.delete(f"{REDIS_STREAK_PREFIX}{server_id}:{metric}")
|
||||
|
||||
|
||||
async def process_metric_alerts(
|
||||
*,
|
||||
redis,
|
||||
server_id: int,
|
||||
server_name: str,
|
||||
system_info: dict[str, Any],
|
||||
thresholds: dict[str, float],
|
||||
log_prefix: str = "Alert",
|
||||
) -> None:
|
||||
"""Evaluate CPU/mem/disk with streak gate; broadcast alert/recovery."""
|
||||
if not system_info:
|
||||
return
|
||||
|
||||
required = streak_required()
|
||||
alert_key = f"{REDIS_ALERT_KEY_PREFIX}{server_id}"
|
||||
|
||||
try:
|
||||
prev_alerts = await redis.smembers(alert_key)
|
||||
except Exception:
|
||||
logger.debug("Failed to read active alerts for server %s", server_id, exc_info=True)
|
||||
prev_alerts = set()
|
||||
|
||||
active_metrics = {
|
||||
m for m in (_alert_metric_key(x) for x in prev_alerts) if m
|
||||
}
|
||||
|
||||
for metric in METRICS:
|
||||
value = _metric_value(system_info, metric)
|
||||
if value is None:
|
||||
continue
|
||||
threshold = _threshold_for(thresholds, metric)
|
||||
|
||||
if value > threshold:
|
||||
try:
|
||||
streak = await record_over_threshold(redis, server_id, metric)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Failed to record alert streak for server %s %s",
|
||||
server_id,
|
||||
metric,
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
|
||||
if streak >= required and metric not in active_metrics:
|
||||
logger.warning(
|
||||
"%s: server %s %s=%s%% (streak %s/%s)",
|
||||
log_prefix,
|
||||
server_id,
|
||||
metric,
|
||||
value,
|
||||
streak,
|
||||
required,
|
||||
)
|
||||
await broadcast_alert(server_id, metric, value, server_name)
|
||||
try:
|
||||
await redis.sadd(alert_key, metric)
|
||||
await redis.expire(alert_key, 3600)
|
||||
active_metrics.add(metric)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Failed to track alert in Redis for server %s",
|
||||
server_id,
|
||||
exc_info=True,
|
||||
)
|
||||
elif streak < required:
|
||||
logger.debug(
|
||||
"%s streak %s/%s: server %s %s=%s%% (suppressed)",
|
||||
log_prefix,
|
||||
streak,
|
||||
required,
|
||||
server_id,
|
||||
metric,
|
||||
value,
|
||||
)
|
||||
else:
|
||||
try:
|
||||
await reset_streak(redis, server_id, metric)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Failed to reset alert streak for server %s %s",
|
||||
server_id,
|
||||
metric,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if not prev_alerts:
|
||||
return
|
||||
|
||||
recoveries = _detect_recovery(system_info, prev_alerts, thresholds)
|
||||
for metric, value in recoveries:
|
||||
logger.info("Recovery: server %s %s=%s%%", server_id, metric, value)
|
||||
await broadcast_recovery(server_id, metric, value, server_name)
|
||||
try:
|
||||
await reset_streak(redis, server_id, metric)
|
||||
for prev in prev_alerts:
|
||||
if _alert_metric_key(prev) == metric:
|
||||
await redis.srem(alert_key, prev)
|
||||
except Exception as e:
|
||||
logger.error("Recovery cleanup failed for server %s: %s", server_id, e)
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Resolve real client IP behind reverse proxy (1Panel / Nginx / Docker)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
|
||||
from starlette.requests import Request
|
||||
|
||||
# Upstream hop is a private/loopback/docker address → trust forwarded headers.
|
||||
_TRUSTED_PROXY_NETWORKS: tuple[ipaddress.IPv4Network | ipaddress.IPv6Network, ...] = (
|
||||
ipaddress.ip_network("127.0.0.0/8"),
|
||||
ipaddress.ip_network("10.0.0.0/8"),
|
||||
ipaddress.ip_network("172.16.0.0/12"),
|
||||
ipaddress.ip_network("192.168.0.0/16"),
|
||||
ipaddress.ip_network("::1/128"),
|
||||
ipaddress.ip_network("fc00::/7"),
|
||||
)
|
||||
|
||||
|
||||
def _parse_ip(value: str) -> str | None:
|
||||
text = (value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
# X-Real-IP may rarely contain comma-separated values
|
||||
candidate = text.split(",")[0].strip()
|
||||
try:
|
||||
ipaddress.ip_address(candidate)
|
||||
except ValueError:
|
||||
return None
|
||||
return candidate
|
||||
|
||||
|
||||
def _is_trusted_proxy_hop(ip: str) -> bool:
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
except ValueError:
|
||||
return False
|
||||
return any(addr in net for net in _TRUSTED_PROXY_NETWORKS)
|
||||
|
||||
|
||||
def _ip_from_forwarded_headers(request: Request) -> str | None:
|
||||
"""X-Real-IP first; X-Forwarded-For last (Nginx appends $remote_addr)."""
|
||||
x_real = _parse_ip(request.headers.get("X-Real-IP", ""))
|
||||
if x_real:
|
||||
return x_real
|
||||
|
||||
xff = (request.headers.get("X-Forwarded-For") or "").strip()
|
||||
if not xff:
|
||||
return None
|
||||
parts = [p.strip() for p in xff.split(",") if p.strip()]
|
||||
if not parts:
|
||||
return None
|
||||
return _parse_ip(parts[-1])
|
||||
|
||||
|
||||
def get_client_ip(request: Request) -> str:
|
||||
"""Return client IP for auth, audit, and rate limits.
|
||||
|
||||
When the TCP peer is a trusted reverse proxy (Docker bridge, loopback, LAN),
|
||||
use X-Real-IP / X-Forwarded-For. Direct public connections ignore those
|
||||
headers to prevent spoofing.
|
||||
"""
|
||||
direct = request.client.host if request.client else ""
|
||||
if direct and _is_trusted_proxy_hop(direct):
|
||||
forwarded = _ip_from_forwarded_headers(request)
|
||||
if forwarded:
|
||||
return forwarded
|
||||
return direct or "unknown"
|
||||
@@ -0,0 +1,79 @@
|
||||
"""User-visible timestamps — always Asia/Shanghai (北京时间)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
_CALENDAR_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
||||
|
||||
BEIJING_TZ = ZoneInfo("Asia/Shanghai")
|
||||
OPERATOR_TZ = BEIJING_TZ
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def ensure_utc(dt: datetime) -> datetime:
|
||||
if dt.tzinfo is None:
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def to_naive_utc(dt: datetime) -> datetime:
|
||||
return ensure_utc(dt).replace(tzinfo=None)
|
||||
|
||||
|
||||
def to_beijing(dt: datetime) -> datetime:
|
||||
"""Convert aware or naive-UTC datetime to Beijing wall clock."""
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(BEIJING_TZ)
|
||||
|
||||
|
||||
def format_beijing(dt: datetime | None = None, *, with_label: bool = True) -> str:
|
||||
"""Format datetime for operators (default: now in 北京时间)."""
|
||||
when = to_beijing(dt or datetime.now(timezone.utc))
|
||||
text = when.strftime("%Y-%m-%d %H:%M:%S")
|
||||
return f"{text} 北京时间" if with_label else text
|
||||
|
||||
|
||||
def format_beijing_now() -> str:
|
||||
return format_beijing(with_label=True)
|
||||
|
||||
|
||||
def operator_wall(dt: datetime) -> tuple[int, int, int, int, int]:
|
||||
"""Cron wall-clock components in operator TZ: minute, hour, day, month, dow (0=Sunday)."""
|
||||
bj = to_beijing(ensure_utc(dt))
|
||||
return (
|
||||
bj.minute,
|
||||
bj.hour,
|
||||
bj.day,
|
||||
bj.month,
|
||||
(bj.weekday() + 1) % 7,
|
||||
)
|
||||
|
||||
|
||||
def parse_operator_calendar_date(raw: str) -> str:
|
||||
"""Validate YYYY-MM-DD for operator (Beijing) calendar day filters."""
|
||||
text = (raw or "").strip()[:10]
|
||||
if not _CALENDAR_DATE_RE.match(text):
|
||||
raise ValueError(f"日期格式须为 YYYY-MM-DD，收到: {raw!r}")
|
||||
try:
|
||||
datetime.fromisoformat(text)
|
||||
except ValueError as e:
|
||||
raise ValueError(f"日期格式须为 YYYY-MM-DD，收到: {raw!r}") from e
|
||||
return text
|
||||
|
||||
|
||||
def beijing_day_range_utc(date_yyyy_mm_dd: str) -> tuple[datetime, datetime]:
|
||||
"""Inclusive Beijing calendar day → naive UTC (start, end) for MySQL filters."""
|
||||
base = datetime.fromisoformat(parse_operator_calendar_date(date_yyyy_mm_dd))
|
||||
start_bj = base.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=BEIJING_TZ)
|
||||
end_bj = base.replace(hour=23, minute=59, second=59, microsecond=999999, tzinfo=BEIJING_TZ)
|
||||
return (
|
||||
start_bj.astimezone(timezone.utc).replace(tzinfo=None),
|
||||
end_bj.astimezone(timezone.utc).replace(tzinfo=None),
|
||||
)
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Safety policy for recursive remote chmod."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# Paths that must never receive chmod -R / chown -R via the UI.
|
||||
#
|
||||
# ``script_service`` is the explicit administrator shell feature and may run
|
||||
# arbitrary commands by design. File-manager permission changes are safer UI
|
||||
# actions, so recursive chmod/chown should not be able to accidentally rewrite
|
||||
# operating-system trees.
|
||||
FORBIDDEN_RECURSIVE_CHMOD_PATHS = frozenset({
|
||||
"/",
|
||||
"/bin",
|
||||
"/boot",
|
||||
"/dev",
|
||||
"/etc",
|
||||
"/lib",
|
||||
"/lib64",
|
||||
"/proc",
|
||||
"/run",
|
||||
"/sbin",
|
||||
"/sys",
|
||||
"/usr",
|
||||
"/var",
|
||||
})
|
||||
|
||||
# Block descendants of critical system trees too. ``/var`` is handled more
|
||||
# narrowly so common web roots like ``/var/www`` can still be managed, while
|
||||
# state/log/package-manager trees remain protected.
|
||||
FORBIDDEN_RECURSIVE_CHMOD_PREFIXES = (
|
||||
"/bin/",
|
||||
"/boot/",
|
||||
"/dev/",
|
||||
"/etc/",
|
||||
"/lib/",
|
||||
"/lib64/",
|
||||
"/proc/",
|
||||
"/run/",
|
||||
"/sbin/",
|
||||
"/sys/",
|
||||
"/usr/",
|
||||
"/var/cache/",
|
||||
"/var/lib/",
|
||||
"/var/log/",
|
||||
"/var/run/",
|
||||
"/var/spool/",
|
||||
)
|
||||
|
||||
RECURSIVE_CHMOD_TIMEOUT_SEC = 300
|
||||
SINGLE_CHMOD_TIMEOUT_SEC = 15
|
||||
|
||||
|
||||
def assert_recursive_chmod_allowed(remote_path: str) -> None:
|
||||
"""Raise ValueError if recursive chmod must be rejected for this path."""
|
||||
if remote_path in FORBIDDEN_RECURSIVE_CHMOD_PATHS or remote_path.startswith(
|
||||
FORBIDDEN_RECURSIVE_CHMOD_PREFIXES
|
||||
):
|
||||
raise ValueError(f"禁止对系统路径递归修改权限: {remote_path}")
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Server-level file-management elevation policy (stored in ``servers.extra_attrs``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import StrEnum
|
||||
|
||||
from server.domain.models import Server
|
||||
|
||||
FILES_ELEVATION_ATTR = "files_elevation"
|
||||
VALID_FILES_ELEVATIONS = frozenset({"off", "auto_sudo", "always_sudo"})
|
||||
DEFAULT_FILES_ELEVATION = "auto_sudo"
|
||||
SUDOERS_DOC_PATH = "docs/deploy/nexus-files-sudoers.example"
|
||||
|
||||
|
||||
class FilesElevation(StrEnum):
|
||||
OFF = "off"
|
||||
AUTO = "auto_sudo"
|
||||
ALWAYS = "always_sudo"
|
||||
|
||||
|
||||
def normalize_files_elevation(value: object | None) -> FilesElevation:
|
||||
"""Return elevation mode; unknown values fall back to AUTO."""
|
||||
if value is None or value == "":
|
||||
return FilesElevation.AUTO
|
||||
raw = str(value).strip().lower()
|
||||
if raw in VALID_FILES_ELEVATIONS:
|
||||
return FilesElevation(raw)
|
||||
return FilesElevation.AUTO
|
||||
|
||||
|
||||
def get_files_elevation(server: Server) -> FilesElevation:
|
||||
attrs = server.extra_attrs if isinstance(server.extra_attrs, dict) else {}
|
||||
return normalize_files_elevation(attrs.get(FILES_ELEVATION_ATTR))
|
||||
|
||||
|
||||
def merge_files_elevation_into_extra_attrs(
|
||||
extra_attrs: dict | None,
|
||||
files_elevation: str | None,
|
||||
) -> dict | None:
|
||||
if files_elevation is None:
|
||||
return extra_attrs
|
||||
out = dict(extra_attrs or {})
|
||||
mode = normalize_files_elevation(files_elevation)
|
||||
out[FILES_ELEVATION_ATTR] = mode.value
|
||||
return out
|
||||
|
||||
|
||||
def apply_files_elevation_payload(data: dict) -> None:
|
||||
"""Pop ``files_elevation`` from API payload and merge into ``extra_attrs``."""
|
||||
fe = data.pop("files_elevation", None)
|
||||
if fe is not None:
|
||||
data["extra_attrs"] = merge_files_elevation_into_extra_attrs(
|
||||
data.get("extra_attrs"),
|
||||
fe,
|
||||
)
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Post-upload permission fixup for file manager (reuse rsync push defaults)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import shlex
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from server.application.services.sync_engine_v2 import _RSYNC_CHMOD_RE, _RSYNC_CHOWN_RE
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from server.domain.models import Server
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_upload_dir_mode(chmod_setting: str) -> str | None:
|
||||
"""Extract octal directory mode from rsync-style ``D755,F755``."""
|
||||
raw = (chmod_setting or "").strip()
|
||||
if not raw or not _RSYNC_CHMOD_RE.fullmatch(raw):
|
||||
return None
|
||||
m = re.search(r"D([0-7]{3,4})", raw, re.IGNORECASE)
|
||||
if m:
|
||||
return m.group(1)
|
||||
return parse_upload_file_mode(raw)
|
||||
|
||||
|
||||
def parse_upload_file_mode(chmod_setting: str) -> str | None:
|
||||
"""Extract octal file mode from rsync-style ``D755,F755`` or plain ``644``."""
|
||||
raw = (chmod_setting or "").strip()
|
||||
if not raw or not _RSYNC_CHMOD_RE.fullmatch(raw):
|
||||
return None
|
||||
m = re.search(r"F([0-7]{3,4})", raw, re.IGNORECASE)
|
||||
if m:
|
||||
return m.group(1)
|
||||
if re.fullmatch(r"[0-7]{3,4}", raw):
|
||||
return raw
|
||||
for part in reversed(raw.split(",")):
|
||||
piece = part.strip()
|
||||
if len(piece) > 1 and piece[0] in "Ff" and re.fullmatch(r"[0-7]{3,4}", piece[1:]):
|
||||
return piece[1:]
|
||||
return None
|
||||
|
||||
|
||||
def parse_upload_chown(chown_setting: str) -> tuple[str, str] | None:
|
||||
"""Return ``(owner, group)`` when setting is valid; else None."""
|
||||
raw = (chown_setting or "").strip()
|
||||
if not raw or not _RSYNC_CHOWN_RE.fullmatch(raw):
|
||||
return None
|
||||
if ":" in raw:
|
||||
owner, group = raw.split(":", 1)
|
||||
return owner.strip(), group.strip()
|
||||
return raw, raw
|
||||
|
||||
|
||||
def upload_permission_plan() -> tuple[tuple[str, str] | None, str | None]:
|
||||
"""Read ``NEXUS_RSYNC_PUSH_CHOWN`` / ``NEXUS_RSYNC_PUSH_CHMOD`` for uploads."""
|
||||
from server.config import settings
|
||||
|
||||
return (
|
||||
parse_upload_chown(settings.RSYNC_PUSH_CHOWN or ""),
|
||||
parse_upload_file_mode(settings.RSYNC_PUSH_CHMOD or ""),
|
||||
)
|
||||
|
||||
|
||||
async def apply_upload_file_permissions(server: Server, remote_path: str) -> dict:
|
||||
"""chown/chmod uploaded file; mirrors rsync push defaults (www:www, F755)."""
|
||||
from server.infrastructure.ssh.remote_shell import exec_ssh_command_with_fallback
|
||||
|
||||
chown_pair, file_mode = upload_permission_plan()
|
||||
if not chown_pair and not file_mode:
|
||||
return {"applied": False, "skipped": True}
|
||||
|
||||
path_q = shlex.quote(remote_path)
|
||||
elevated = False
|
||||
errors: list[str] = []
|
||||
|
||||
if chown_pair:
|
||||
owner_spec = shlex.quote(f"{chown_pair[0]}:{chown_pair[1]}")
|
||||
result = await exec_ssh_command_with_fallback(
|
||||
server, f"chown {owner_spec} {path_q}", timeout=30,
|
||||
)
|
||||
elevated = bool(result.get("elevated"))
|
||||
if result.get("exit_code") != 0:
|
||||
err = ((result.get("stderr") or "") + (result.get("stdout") or "")).strip()[:200]
|
||||
errors.append(err or "chown 失败")
|
||||
|
||||
if file_mode and not errors:
|
||||
result = await exec_ssh_command_with_fallback(
|
||||
server, f"chmod {file_mode} {path_q}", timeout=30,
|
||||
)
|
||||
elevated = elevated or bool(result.get("elevated"))
|
||||
if result.get("exit_code") != 0:
|
||||
err = ((result.get("stderr") or "") + (result.get("stdout") or "")).strip()[:200]
|
||||
errors.append(err or "chmod 失败")
|
||||
|
||||
if errors:
|
||||
logger.warning(
|
||||
"Upload permission fixup failed for %s on server %s: %s",
|
||||
remote_path,
|
||||
getattr(server, "id", "?"),
|
||||
"; ".join(errors),
|
||||
)
|
||||
|
||||
owner, group = chown_pair if chown_pair else (None, None)
|
||||
return {
|
||||
"applied": not errors and bool(chown_pair or file_mode),
|
||||
"skipped": False,
|
||||
"owner": owner,
|
||||
"group": group,
|
||||
"mode": file_mode,
|
||||
"elevated": elevated,
|
||||
"error": "; ".join(errors) if errors else None,
|
||||
}
|
||||
|
||||
|
||||
async def apply_transfer_permissions(
|
||||
server: Server,
|
||||
remote_path: str,
|
||||
*,
|
||||
recursive: bool = True,
|
||||
) -> dict:
|
||||
"""chown/chmod after cross-server relay or package deliver (default www:www, D755/F755)."""
|
||||
from server.config import settings
|
||||
from server.infrastructure.ssh.remote_shell import exec_ssh_command_with_fallback
|
||||
|
||||
chown_pair, file_mode = upload_permission_plan()
|
||||
dir_mode = parse_upload_dir_mode(settings.RSYNC_PUSH_CHMOD or "")
|
||||
if not chown_pair and not file_mode:
|
||||
return {"applied": False, "skipped": True, "path": remote_path}
|
||||
|
||||
path_q = shlex.quote(remote_path)
|
||||
elevated = False
|
||||
errors: list[str] = []
|
||||
|
||||
if chown_pair:
|
||||
owner_spec = shlex.quote(f"{chown_pair[0]}:{chown_pair[1]}")
|
||||
chown_flag = "-R " if recursive else ""
|
||||
result = await exec_ssh_command_with_fallback(
|
||||
server,
|
||||
f"chown {chown_flag}{owner_spec} {path_q}",
|
||||
timeout=120,
|
||||
)
|
||||
elevated = bool(result.get("elevated"))
|
||||
if result.get("exit_code") != 0:
|
||||
err = ((result.get("stderr") or "") + (result.get("stdout") or "")).strip()[:200]
|
||||
errors.append(err or "chown 失败")
|
||||
|
||||
if file_mode and not errors:
|
||||
if recursive and dir_mode and dir_mode != file_mode:
|
||||
for kind, mode in (("d", dir_mode), ("f", file_mode)):
|
||||
result = await exec_ssh_command_with_fallback(
|
||||
server,
|
||||
f"find {path_q} -type {kind} -exec chmod {shlex.quote(mode)} {{}} +",
|
||||
timeout=120,
|
||||
)
|
||||
elevated = elevated or bool(result.get("elevated"))
|
||||
if result.get("exit_code") != 0:
|
||||
err = ((result.get("stderr") or "") + (result.get("stdout") or "")).strip()[:200]
|
||||
errors.append(err or f"chmod {kind} 失败")
|
||||
break
|
||||
else:
|
||||
mode = file_mode
|
||||
chmod_flag = "-R " if recursive else ""
|
||||
result = await exec_ssh_command_with_fallback(
|
||||
server,
|
||||
f"chmod {chmod_flag}{shlex.quote(mode)} {path_q}",
|
||||
timeout=120,
|
||||
)
|
||||
elevated = elevated or bool(result.get("elevated"))
|
||||
if result.get("exit_code") != 0:
|
||||
err = ((result.get("stderr") or "") + (result.get("stdout") or "")).strip()[:200]
|
||||
errors.append(err or "chmod 失败")
|
||||
|
||||
if errors:
|
||||
logger.warning(
|
||||
"Transfer permission fixup failed for %s on server %s: %s",
|
||||
remote_path,
|
||||
getattr(server, "id", "?"),
|
||||
"; ".join(errors),
|
||||
)
|
||||
|
||||
owner, group = chown_pair if chown_pair else (None, None)
|
||||
return {
|
||||
"applied": not errors and bool(chown_pair or file_mode),
|
||||
"skipped": False,
|
||||
"path": remote_path,
|
||||
"owner": owner,
|
||||
"group": group,
|
||||
"dir_mode": dir_mode,
|
||||
"file_mode": file_mode,
|
||||
"elevated": elevated,
|
||||
"error": "; ".join(errors) if errors else None,
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Attach alert_logs to fleet metric sample windows (10min flush alignment)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from server.utils.display_time import to_naive_utc
|
||||
|
||||
ALERT_TYPE_LABELS = {
|
||||
"cpu": "CPU",
|
||||
"mem": "内存",
|
||||
"disk": "磁盘",
|
||||
"time_drift": "时钟漂移",
|
||||
"system": "系统",
|
||||
}
|
||||
|
||||
|
||||
def _parse_naive_utc(value: datetime | str | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return to_naive_utc(value)
|
||||
raw = str(value).strip()
|
||||
if not raw:
|
||||
return None
|
||||
normalized = raw.replace(" ", "T") if " " in raw and "T" not in raw else raw
|
||||
if normalized.endswith("Z"):
|
||||
normalized = normalized[:-1] + "+00:00"
|
||||
try:
|
||||
dt = datetime.fromisoformat(normalized)
|
||||
except ValueError:
|
||||
return None
|
||||
return to_naive_utc(dt)
|
||||
|
||||
|
||||
def _alert_to_dict(alert: Any) -> dict[str, Any]:
|
||||
atype = str(getattr(alert, "alert_type", "") or "")
|
||||
return {
|
||||
"server_name": getattr(alert, "server_name", None) or "",
|
||||
"server_id": getattr(alert, "server_id", None),
|
||||
"alert_type": atype,
|
||||
"alert_type_label": ALERT_TYPE_LABELS.get(atype, atype),
|
||||
"value": getattr(alert, "value", None) or "",
|
||||
"is_recovery": bool(getattr(alert, "is_recovery", False)),
|
||||
"created_at": str(getattr(alert, "created_at", "") or ""),
|
||||
}
|
||||
|
||||
|
||||
def attach_alerts_to_fleet_points(
|
||||
points: list[dict[str, Any]],
|
||||
alerts: list[Any],
|
||||
*,
|
||||
interval_minutes: int = 10,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Map each sample to alerts in (previous_sample, current_sample] window."""
|
||||
if not points:
|
||||
return []
|
||||
|
||||
parsed: list[tuple[datetime | None, dict[str, Any]]] = []
|
||||
for p in points:
|
||||
parsed.append((_parse_naive_utc(p.get("recorded_at")), p))
|
||||
|
||||
alert_rows: list[tuple[datetime, dict[str, Any]]] = []
|
||||
for a in alerts:
|
||||
adt = _parse_naive_utc(getattr(a, "created_at", None))
|
||||
if adt is not None:
|
||||
alert_rows.append((adt, _alert_to_dict(a)))
|
||||
|
||||
window = timedelta(minutes=max(1, interval_minutes))
|
||||
enriched: list[dict[str, Any]] = []
|
||||
|
||||
for i, (pt_dt, p) in enumerate(parsed):
|
||||
if pt_dt is None:
|
||||
enriched.append({**p, "alerts": [], "alert_count": 0})
|
||||
continue
|
||||
|
||||
prev_dt = parsed[i - 1][0] if i > 0 else None
|
||||
start = prev_dt if prev_dt is not None else pt_dt - window
|
||||
bucket = [
|
||||
item
|
||||
for adt, item in alert_rows
|
||||
if start < adt <= pt_dt
|
||||
]
|
||||
enriched.append({**p, "alerts": bucket, "alert_count": len(bucket)})
|
||||
|
||||
return enriched
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Fleet-wide metric aggregation from Agent heartbeat payloads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _safe_pct(value: Any) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
num = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if num < 0 or num > 100:
|
||||
return None
|
||||
return num
|
||||
|
||||
|
||||
def _metric_from_system_info(system_info: dict[str, Any], primary: str, fallback: str) -> float | None:
|
||||
return _safe_pct(system_info.get(primary) if system_info.get(primary) is not None else system_info.get(fallback))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FleetMetricAggregate:
|
||||
online_count: int
|
||||
sample_count: int
|
||||
cpu_avg: int | None
|
||||
mem_avg: int | None
|
||||
disk_avg: int | None
|
||||
|
||||
|
||||
def aggregate_fleet_metrics(heartbeats: list[dict[str, Any]]) -> FleetMetricAggregate:
|
||||
"""Aggregate Redis heartbeat hash rows (is_online + system_info JSON string)."""
|
||||
online_count = 0
|
||||
cpu_vals: list[float] = []
|
||||
mem_vals: list[float] = []
|
||||
disk_vals: list[float] = []
|
||||
|
||||
for row in heartbeats:
|
||||
if row.get("is_online") in (True, "True"):
|
||||
online_count += 1
|
||||
raw = row.get("system_info") or "{}"
|
||||
try:
|
||||
info = json.loads(raw) if isinstance(raw, str) else (raw or {})
|
||||
except json.JSONDecodeError:
|
||||
info = {}
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
cpu = _metric_from_system_info(info, "cpu_usage", "cpu")
|
||||
mem = _metric_from_system_info(info, "mem_usage", "mem")
|
||||
disk = _metric_from_system_info(info, "disk_usage", "disk")
|
||||
if cpu is not None:
|
||||
cpu_vals.append(cpu)
|
||||
if mem is not None:
|
||||
mem_vals.append(mem)
|
||||
if disk is not None:
|
||||
disk_vals.append(disk)
|
||||
|
||||
sample_count = max(len(cpu_vals), len(mem_vals), len(disk_vals))
|
||||
|
||||
def _avg(vals: list[float]) -> int | None:
|
||||
if not vals:
|
||||
return None
|
||||
return round(sum(vals) / len(vals))
|
||||
|
||||
return FleetMetricAggregate(
|
||||
online_count=online_count,
|
||||
sample_count=sample_count,
|
||||
cpu_avg=_avg(cpu_vals),
|
||||
mem_avg=_avg(mem_vals),
|
||||
disk_avg=_avg(disk_vals),
|
||||
)
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Translate HTTPException detail strings to Chinese for admin-facing API responses."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
# Exact English literals from server/api (and shared services)
|
||||
_DETAIL_EXACT: dict[str, str] = {
|
||||
"Not found": "不存在",
|
||||
"Server not found": "服务器不存在",
|
||||
"Platform not found": "平台不存在",
|
||||
"Node not found": "节点不存在",
|
||||
"Admin not found": "管理员不存在",
|
||||
"Setting not found": "设置项不存在",
|
||||
"API_KEY not found": "API_KEY 不存在",
|
||||
"Schedule not found": "调度任务不存在",
|
||||
"Preset not found": "凭据预设不存在",
|
||||
"SSH key preset not found": "SSH 密钥预设不存在",
|
||||
"telegram_bot_token not found": "telegram_bot_token 不存在",
|
||||
"Retry job not found": "重试任务不存在",
|
||||
"Credential not found": "凭据不存在",
|
||||
"Execution not found": "执行记录不存在",
|
||||
"Script not found": "脚本不存在",
|
||||
"Login failed": "登录失败",
|
||||
"Missing refresh token": "缺少刷新令牌",
|
||||
"Invalid refresh token": "刷新令牌无效",
|
||||
"Missing Authorization header": "缺少 Authorization 请求头",
|
||||
"Invalid or expired token": "令牌无效或已过期",
|
||||
"Can only setup TOTP for yourself": "只能为自己设置 TOTP",
|
||||
"Can only enable TOTP for yourself": "只能为自己启用 TOTP",
|
||||
"Can only disable TOTP for yourself": "只能为自己禁用 TOTP",
|
||||
"Setup failed": "设置失败",
|
||||
"Enable failed": "启用失败",
|
||||
"Disable failed": "禁用失败",
|
||||
"Server has no domain configured": "服务器未配置域名",
|
||||
"Missing API key": "缺少 API Key",
|
||||
"Invalid API key for this server": "该服务器的 API Key 无效",
|
||||
"Invalid or expired job": "任务无效或已过期",
|
||||
"new_path required for rename": "重命名需要指定 new_path",
|
||||
"Invalid operation": "无效操作",
|
||||
"server_id required": "server_id 必填",
|
||||
"Too many script callback requests": "脚本回调请求过于频繁",
|
||||
}
|
||||
|
||||
# AuthService ``reason`` codes → user-facing Chinese (when ``message`` absent)
|
||||
_AUTH_REASON_ZH: dict[str, str] = {
|
||||
"admin_not_found": "管理员不存在",
|
||||
"no_secret": "请先完成 TOTP 设置",
|
||||
"invalid_totp": "TOTP 验证码错误",
|
||||
"invalid_password": "当前密码错误",
|
||||
"totp_required": "需要 TOTP 验证码",
|
||||
"invalid_token": "令牌无效或已过期",
|
||||
"invalid_credentials": "用户名或密码错误",
|
||||
"account_locked": "登录尝试过多，请 15 分钟后重试",
|
||||
"account_disabled": "账户已被禁用",
|
||||
"ip_blocked": "拒绝访问",
|
||||
"service_unavailable": "认证服务暂不可用，请稍后重试",
|
||||
}
|
||||
|
||||
_DETAIL_PATTERNS: list[tuple[re.Pattern[str], str]] = [
|
||||
(
|
||||
re.compile(r"^Command failed \(exit (\d+)\)$"),
|
||||
r"命令失败（退出码 \1）",
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"^Setting '([^']+)' is immutable and cannot be modified via API$"
|
||||
),
|
||||
r"设置项「\1」不可通过 API 修改",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _looks_chinese(text: str) -> bool:
|
||||
return any("\u4e00" <= ch <= "\u9fff" for ch in text)
|
||||
|
||||
|
||||
def translate_http_detail_str(msg: str) -> str:
|
||||
"""Return Chinese detail when a known translation exists."""
|
||||
if not msg or _looks_chinese(msg):
|
||||
return msg
|
||||
if msg in _DETAIL_EXACT:
|
||||
return _DETAIL_EXACT[msg]
|
||||
for pattern, repl in _DETAIL_PATTERNS:
|
||||
if pattern.fullmatch(msg):
|
||||
return pattern.sub(repl, msg)
|
||||
return msg
|
||||
|
||||
|
||||
def auth_failure_detail(result: dict[str, Any], *, fallback: str = "操作失败") -> str:
|
||||
"""Map AuthService failure dict to Chinese ``HTTPException.detail``."""
|
||||
msg = result.get("message")
|
||||
if isinstance(msg, str) and msg.strip():
|
||||
return translate_http_detail_str(msg)
|
||||
reason = str(result.get("reason") or "")
|
||||
if reason in _AUTH_REASON_ZH:
|
||||
return _AUTH_REASON_ZH[reason]
|
||||
return fallback if _looks_chinese(fallback) else translate_http_detail_str(fallback)
|
||||
|
||||
|
||||
def translate_http_detail(detail: Any) -> Any:
|
||||
"""Translate ``HTTPException.detail`` (str | list | dict) for JSON responses."""
|
||||
if isinstance(detail, str):
|
||||
return translate_http_detail_str(detail)
|
||||
if isinstance(detail, list):
|
||||
return [translate_http_detail(item) for item in detail]
|
||||
if isinstance(detail, dict):
|
||||
out: dict[Any, Any] = {}
|
||||
for key, value in detail.items():
|
||||
if key in ("msg", "message") and isinstance(value, str):
|
||||
out[key] = translate_http_detail_str(value)
|
||||
else:
|
||||
out[key] = value
|
||||
return out
|
||||
return detail
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Login IP allowlist helpers (shared by auth API precheck and AuthService.login)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
|
||||
from server.config import settings
|
||||
|
||||
_TRUTHY = frozenset({"true", "1", "yes", "on"})
|
||||
|
||||
|
||||
def is_allowlist_enforced() -> bool:
|
||||
"""Whether LOGIN_ALLOWLIST_ENABLED is on."""
|
||||
val = (getattr(settings, "LOGIN_ALLOWLIST_ENABLED", "false") or "false").lower()
|
||||
return val in _TRUTHY
|
||||
|
||||
|
||||
def get_allowed_login_ips() -> list[str]:
|
||||
"""Combined allowlist used by login checks.
|
||||
|
||||
Prefer LOGIN_ALLOWED_IPS — rebuilt on each subscription refresh and broadcast
|
||||
to all workers via Redis. Recomputing subscription + manual locally can diverge
|
||||
when workers receive partial or out-of-order Pub/Sub updates.
|
||||
"""
|
||||
combined_raw = (getattr(settings, "LOGIN_ALLOWED_IPS", "") or "").strip()
|
||||
if combined_raw:
|
||||
return [ip.strip() for ip in combined_raw.split(",") if ip.strip()]
|
||||
|
||||
sub_ips_raw = (getattr(settings, "LOGIN_SUBSCRIPTION_IPS", "") or "").strip()
|
||||
manual_ips_raw = (getattr(settings, "LOGIN_MANUAL_IPS", "") or "").strip()
|
||||
all_ips_raw = ",".join(filter(None, [sub_ips_raw, manual_ips_raw]))
|
||||
return [ip.strip() for ip in all_ips_raw.split(",") if ip.strip()]
|
||||
|
||||
|
||||
def ip_in_allowlist(client_ip: str, allowed: list[str]) -> bool:
|
||||
"""Check if client_ip matches any entry in the allowlist."""
|
||||
try:
|
||||
client_addr = ipaddress.ip_address(client_ip)
|
||||
for entry in allowed:
|
||||
entry = entry.strip()
|
||||
try:
|
||||
if "/" in entry:
|
||||
if client_addr in ipaddress.ip_network(entry, strict=False):
|
||||
return True
|
||||
elif client_addr == ipaddress.ip_address(entry):
|
||||
return True
|
||||
except ValueError:
|
||||
if client_ip == entry:
|
||||
return True
|
||||
except ValueError:
|
||||
return client_ip in allowed
|
||||
return False
|
||||
|
||||
|
||||
def check_login_ip(client_ip: str) -> tuple[bool, bool, str | None]:
|
||||
"""Evaluate login IP access.
|
||||
|
||||
Returns:
|
||||
(allowlist_enabled, allowed, message)
|
||||
- allowlist off: (False, True, None)
|
||||
- allowlist on, empty list: (True, True, None) — nothing to enforce
|
||||
- allowlist on, IP ok: (True, True, None)
|
||||
- allowlist on, IP blocked: (True, False, message)
|
||||
"""
|
||||
if not is_allowlist_enforced():
|
||||
return False, True, None
|
||||
|
||||
if not client_ip or client_ip in ("", "unknown"):
|
||||
return True, True, None
|
||||
|
||||
allowed_ips = get_allowed_login_ips()
|
||||
if not allowed_ips:
|
||||
return True, True, None
|
||||
|
||||
if ip_in_allowlist(client_ip, allowed_ips):
|
||||
return True, True, None
|
||||
|
||||
return (
|
||||
True,
|
||||
False,
|
||||
f"当前 IP ({client_ip}) 不在登录白名单中，请检查代理或联系管理员",
|
||||
)
|
||||
|
||||
|
||||
def is_whitelisted_login_ip(client_ip: str) -> bool:
|
||||
"""True when allowlist is enforced, non-empty, and client IP matches."""
|
||||
if not is_allowlist_enforced():
|
||||
return False
|
||||
if not client_ip or client_ip in ("", "unknown"):
|
||||
return False
|
||||
allowed_ips = get_allowed_login_ips()
|
||||
return bool(allowed_ips) and ip_in_allowlist(client_ip, allowed_ips)
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Detect primary nginx site domain on remote Baota/generic hosts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shlex
|
||||
|
||||
|
||||
def build_nginx_domain_detect_command(*, target_hint: str = "") -> str:
|
||||
"""Build remote shell snippet that prints one site hostname to stdout.
|
||||
|
||||
Lookup order:
|
||||
1. Baota vhost basename ``/www/server/panel/vhost/nginx/{domain}.conf``
|
||||
2. ``server_name`` in Baota vhosts and generic nginx dirs
|
||||
Prefer *target_hint* when it matches a candidate.
|
||||
"""
|
||||
hint = (target_hint or "").strip().lower()
|
||||
qhint = shlex.quote(hint) if hint else "''"
|
||||
return (
|
||||
f"TARGET_HINT={qhint}; "
|
||||
"_is_domain() { "
|
||||
'local h="${1,,}"; '
|
||||
'[[ "$h" == *.* ]] || return 1; '
|
||||
'[[ "$h" =~ ^[0-9]+(\\.[0-9]+){3}$ ]] && return 1; '
|
||||
'case "$h" in default|0.default|0.site_total|phpmyadmin|localhost|127.0.0.1|_*|*.local) return 1 ;; esac; '
|
||||
"return 0; "
|
||||
"}; "
|
||||
"_pick() { "
|
||||
'local best=""; '
|
||||
"for h in \"$@\"; do "
|
||||
'[ -n "$h" ] || continue; '
|
||||
"_is_domain \"$h\" || continue; "
|
||||
'if [ -n "$TARGET_HINT" ] && [ "$h" = "$TARGET_HINT" ]; then echo "$h"; return 0; fi; '
|
||||
'if [ -z "$best" ] || [[ "$h" < "$best" ]]; then best="$h"; fi; '
|
||||
"done; "
|
||||
'[ -n "$best" ] && echo "$best"; '
|
||||
"}; "
|
||||
"cands=(); "
|
||||
'BT="/www/server/panel/vhost/nginx"; '
|
||||
'if [ -d "$BT" ]; then '
|
||||
'for f in "$BT"/*.conf; do '
|
||||
'[ -f "$f" ] || continue; '
|
||||
'base=$(basename "$f" .conf); '
|
||||
"_is_domain \"$base\" && cands+=(\"$base\"); "
|
||||
"done; "
|
||||
"fi; "
|
||||
'if [ ${#cands[@]} -gt 0 ]; then _pick "${cands[@]}"; exit 0; fi; '
|
||||
"_names_from() { "
|
||||
'grep -E "^[[:space:]]*server_name[[:space:]]" "$1" 2>/dev/null '
|
||||
"| head -1 | sed 's/.*server_name[[:space:]]\\+//;s/;//;s/\\$//g'; "
|
||||
"}; "
|
||||
'for dir in "$BT" /etc/nginx/sites-enabled /etc/nginx/conf.d; do '
|
||||
'[ -d "$dir" ] || continue; '
|
||||
'for f in "$dir"/*; do '
|
||||
'[ -f "$f" ] || continue; '
|
||||
'raw=$(_names_from "$f"); '
|
||||
'for token in $raw; do '
|
||||
'token="${token,,}"; '
|
||||
"_is_domain \"$token\" && cands+=(\"$token\"); "
|
||||
"done; "
|
||||
"done; "
|
||||
"done; "
|
||||
'_pick "${cands[@]}"'
|
||||
)
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Notification toggle helpers — shared by Telegram and alert dispatch."""
|
||||
|
||||
from server.config import settings
|
||||
|
||||
_DISABLED = frozenset({"false", "0", "no", "off"})
|
||||
|
||||
# alert_type (cpu/mem/disk) → Settings attribute
|
||||
_RESOURCE_ALERT_ATTR: dict[str, str] = {
|
||||
"cpu": "NOTIFY_ALERT_CPU",
|
||||
"mem": "NOTIFY_ALERT_MEM",
|
||||
"disk": "NOTIFY_ALERT_DISK",
|
||||
}
|
||||
|
||||
|
||||
def notify_enabled(setting_value: str | bool | None) -> bool:
|
||||
"""Return True when a notify_* setting is enabled."""
|
||||
if setting_value is None:
|
||||
return True
|
||||
if isinstance(setting_value, bool):
|
||||
return setting_value
|
||||
return setting_value.strip().lower() not in _DISABLED
|
||||
|
||||
|
||||
def resource_alert_notify_enabled(alert_type: str) -> bool:
|
||||
"""Check whether Telegram should fire for a resource alert type."""
|
||||
attr = _RESOURCE_ALERT_ATTR.get(alert_type.lower(), "NOTIFY_ALERT_CPU")
|
||||
return notify_enabled(getattr(settings, attr, "true"))
|
||||
|
||||
|
||||
def recovery_notify_enabled() -> bool:
|
||||
"""Global toggle for resource metric recovery Telegram."""
|
||||
return notify_enabled(settings.NOTIFY_RECOVERY)
|
||||
|
||||
|
||||
def resource_recovery_notify_enabled(metric: str) -> bool:
|
||||
"""Telegram recovery requires global NOTIFY_RECOVERY and per-metric alert toggle."""
|
||||
if not recovery_notify_enabled():
|
||||
return False
|
||||
return resource_alert_notify_enabled(metric)
|
||||
|
||||
|
||||
def notify_attr_enabled(attr_name: str, default: str = "true") -> bool:
|
||||
"""Check a named NOTIFY_* settings attribute."""
|
||||
return notify_enabled(getattr(settings, attr_name, default))
|
||||
|
||||
|
||||
def offline_alert_notify_enabled() -> bool:
|
||||
"""Telegram toggle for server offline (Agent heartbeat lost) alerts."""
|
||||
return notify_attr_enabled("NOTIFY_ALERT_OFFLINE")
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Linux/POSIX path helpers for remote SSH paths and Nexus-host Unix paths.
|
||||
|
||||
Use this module whenever manipulating path *strings* for Linux targets.
|
||||
Do not use ``os.path`` for remote paths — on Windows dev hosts ``os.path.join``
|
||||
produces backslashes and breaks SSH/SFTP commands.
|
||||
|
||||
For real filesystem I/O on the Nexus host (``open``, ``os.scandir``), keep
|
||||
``os.path.realpath`` / ``os.walk`` after normalizing with :func:`to_posix`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import posixpath
|
||||
|
||||
UPLOAD_STAGING_PREFIX = "/tmp/nexus_upload_"
|
||||
|
||||
# Baota default web root — push/sync fallback when server.target_path is empty.
|
||||
DEFAULT_PUSH_TARGET_PATH = "/www/wwwroot"
|
||||
|
||||
FORBIDDEN_NEXUS_SOURCE_PREFIXES = (
|
||||
"/etc",
|
||||
"/root",
|
||||
"/home",
|
||||
"/var",
|
||||
"/boot",
|
||||
"/dev",
|
||||
"/proc",
|
||||
"/sys",
|
||||
"/run",
|
||||
"/srv",
|
||||
)
|
||||
|
||||
|
||||
class PosixPathError(ValueError):
|
||||
"""Invalid or unsafe POSIX path."""
|
||||
|
||||
|
||||
def to_posix(path: str) -> str:
|
||||
"""Normalize slashes to forward slash (does not resolve ``..``)."""
|
||||
return (path or "").strip().replace("\\", "/")
|
||||
|
||||
|
||||
def reject_windows_separators(path: str, *, field: str = "path") -> None:
|
||||
if "\\" in path or "\0" in path:
|
||||
raise PosixPathError(f"{field} 必须使用 Linux 路径（不允许反斜杠）")
|
||||
|
||||
|
||||
def normalize_remote_abs_path(path: str) -> str:
|
||||
"""Validate and normalize a remote absolute path (SSH/SFTP)."""
|
||||
raw_in = (path or "").strip()
|
||||
reject_windows_separators(raw_in, field="path")
|
||||
raw = to_posix(raw_in)
|
||||
if not raw.startswith("/"):
|
||||
raise PosixPathError("路径必须为绝对路径")
|
||||
parts = [p for p in raw.split("/") if p and p != "."]
|
||||
if ".." in parts:
|
||||
raise PosixPathError("路径不允许包含 '..'")
|
||||
normalized = "/" + "/".join(parts) if parts else "/"
|
||||
if len(normalized) > 1 and normalized.endswith("/"):
|
||||
normalized = normalized.rstrip("/")
|
||||
return normalized or "/"
|
||||
|
||||
|
||||
def posix_join(*parts: str) -> str:
|
||||
"""Join path segments using POSIX rules (safe on any dev OS)."""
|
||||
if not parts:
|
||||
return "/"
|
||||
acc = to_posix(parts[0])
|
||||
for part in parts[1:]:
|
||||
seg = to_posix(part)
|
||||
if not seg or seg == "/":
|
||||
continue
|
||||
acc = posixpath.join(acc, seg.lstrip("/"))
|
||||
return posixpath.normpath(acc) or "/"
|
||||
|
||||
|
||||
def posix_basename(path: str) -> str:
|
||||
return posixpath.basename(to_posix(path.rstrip("/")))
|
||||
|
||||
|
||||
def posix_dirname(path: str) -> str:
|
||||
parent = posixpath.dirname(to_posix(path.rstrip("/")))
|
||||
return parent or "/"
|
||||
|
||||
|
||||
def remote_join(directory: str, name: str) -> str:
|
||||
"""Join remote directory + file name (both logical Linux paths)."""
|
||||
base = normalize_remote_abs_path(directory)
|
||||
segment = to_posix(name).strip("/")
|
||||
reject_windows_separators(segment, field="name")
|
||||
if ".." in segment.split("/"):
|
||||
raise PosixPathError("名称不能包含 '..'")
|
||||
if not segment:
|
||||
return base
|
||||
return posix_join(base, segment)
|
||||
|
||||
|
||||
def resolve_nexus_host_path(path: str) -> str:
|
||||
"""Resolve a path on the Nexus host filesystem (Linux deployment)."""
|
||||
posix = to_posix(path)
|
||||
reject_windows_separators(posix, field="path")
|
||||
return os.path.realpath(posix)
|
||||
|
||||
|
||||
def resolve_nexus_push_source_path(path: str) -> str:
|
||||
"""Resolve and validate a Nexus host push source directory."""
|
||||
stripped = path.strip()
|
||||
if ".." in to_posix(stripped).split("/"):
|
||||
raise PosixPathError("路径不允许包含 '..'")
|
||||
real_path = resolve_nexus_host_path(stripped)
|
||||
for prefix in FORBIDDEN_NEXUS_SOURCE_PREFIXES:
|
||||
if real_path == prefix or real_path.startswith(prefix + "/"):
|
||||
raise PosixPathError(f"不允许使用系统路径: {prefix}")
|
||||
if not os.path.isdir(real_path):
|
||||
raise PosixPathError(f"源路径不存在: {real_path}")
|
||||
return real_path
|
||||
|
||||
|
||||
def ensure_under_nexus_upload(path: str) -> str:
|
||||
"""Resolve *path* and ensure it stays under ``/tmp/nexus_upload_*``."""
|
||||
real = resolve_nexus_host_path(path)
|
||||
if not to_posix(real).startswith(UPLOAD_STAGING_PREFIX):
|
||||
raise PosixPathError("仅允许访问上传临时目录")
|
||||
return real
|
||||
|
||||
|
||||
def assert_zip_member_safe(extract_dir: str, member_name: str) -> str:
|
||||
"""Zip-slip check; returns normalized member path under *extract_dir*."""
|
||||
base = normalize_remote_abs_path(extract_dir)
|
||||
member = to_posix(member_name).lstrip("/")
|
||||
if not member or member == ".":
|
||||
return base
|
||||
if member == ".." or member.startswith("../") or "/../" in f"/{member}/":
|
||||
raise PosixPathError(f"ZIP 包含非法路径: {member_name}")
|
||||
candidate = posixpath.normpath(posixpath.join(base, member))
|
||||
if candidate != base and not candidate.startswith(base + "/"):
|
||||
raise PosixPathError(f"ZIP 包含非法路径: {member_name}")
|
||||
return candidate
|
||||
|
||||
|
||||
def is_path_under_prefix(resolved: str, prefix: str) -> bool:
|
||||
"""True if *resolved* is *prefix* or a child (POSIX string compare after norm)."""
|
||||
r = normalize_remote_abs_path(to_posix(resolved))
|
||||
p = normalize_remote_abs_path(to_posix(prefix))
|
||||
return r == p or r.startswith(p + "/")
|
||||
|
||||
|
||||
def normalize_remote_dest(path: str | None, *, default: str = DEFAULT_PUSH_TARGET_PATH) -> str:
|
||||
"""Normalize remote target directory (push/rsync/diagnose); uses *default* when empty."""
|
||||
raw = (path or "").strip() or default
|
||||
return normalize_remote_abs_path(raw)
|
||||
|
||||
|
||||
def resolve_push_target_path(
|
||||
server_target_path: str | None,
|
||||
override: str | None = None,
|
||||
) -> str:
|
||||
"""Resolve rsync push destination: override → server.target_path → ``/www/wwwroot``.
|
||||
|
||||
Independent of :func:`is_unset_target_path`: empty or placeholder ``/www/wwwroot`` in DB
|
||||
still counts as「未设路径」in UI, but push may target ``/www/wwwroot`` (Baota web root).
|
||||
"""
|
||||
if override is not None and str(override).strip():
|
||||
return normalize_remote_abs_path(str(override).strip())
|
||||
raw = (server_target_path or "").strip()
|
||||
if raw:
|
||||
return normalize_remote_abs_path(raw)
|
||||
return DEFAULT_PUSH_TARGET_PATH
|
||||
|
||||
|
||||
def is_unset_target_path(path: str | None) -> bool:
|
||||
"""True when push target path is empty or still the BT default placeholder.
|
||||
|
||||
``/www/wwwroot/`` (trailing slash) is *not* unset — only exact ``/www/wwwroot`` after trim.
|
||||
"""
|
||||
p = (path or "").strip()
|
||||
return p == "" or p == "/www/wwwroot"
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Install psutil on managed servers via SSH (system Python for watch probe)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from server.application.server_batch_common import sudo_wrap
|
||||
from server.domain.models import Server
|
||||
from server.infrastructure.ssh.asyncssh_pool import exec_ssh_command
|
||||
|
||||
# pip --user first (no root); package managers and system pip via run_root (sudo when non-root).
|
||||
PSUTIL_INSTALL_SHELL = r"""
|
||||
run_root() {
|
||||
if [ "$(id -u)" -eq 0 ]; then
|
||||
"$@"
|
||||
else
|
||||
sudo "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
verify_psutil() {
|
||||
for py in python3 python3.12 python3.11 python3.10; do
|
||||
if command -v "$py" >/dev/null 2>&1 && "$py" -c "import psutil" 2>/dev/null; then
|
||||
echo psutil_ok
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
verify_psutil && exit 0
|
||||
|
||||
for py in python3 python3.12 python3.11 python3.10; do
|
||||
command -v "$py" >/dev/null 2>&1 || continue
|
||||
if "$py" -m pip install --user -q psutil 2>/dev/null; then
|
||||
verify_psutil && exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
if run_root apt-get update -qq \
|
||||
&& run_root env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3-psutil; then
|
||||
verify_psutil && exit 0
|
||||
fi
|
||||
fi
|
||||
if command -v dnf >/dev/null 2>&1; then
|
||||
if run_root dnf install -y -q python3-psutil; then
|
||||
verify_psutil && exit 0
|
||||
fi
|
||||
fi
|
||||
if command -v yum >/dev/null 2>&1; then
|
||||
if run_root yum install -y -q python3-psutil; then
|
||||
verify_psutil && exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
for py in python3 python3.12 python3.11 python3.10; do
|
||||
command -v "$py" >/dev/null 2>&1 || continue
|
||||
if run_root "$py" -m pip install -q psutil 2>/dev/null; then
|
||||
verify_psutil && exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
echo "ERROR: psutil install failed — pip/apt unavailable or permission denied" >&2
|
||||
exit 1
|
||||
"""
|
||||
|
||||
_APT_PERM_RE = re.compile(
|
||||
r"lock file|permission denied|unable to lock|not allowed to execute",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def _psutil_install_error_zh(stderr: str, stdout: str) -> str:
|
||||
text = f"{stderr}\n{stdout}"
|
||||
if _APT_PERM_RE.search(text):
|
||||
return (
|
||||
"安装 psutil 需要 root 或免密 sudo（apt/pip）。"
|
||||
"请改用 root SSH，或在子机配置免密 sudo 后重试；也可手动执行："
|
||||
"sudo apt-get install -y python3-psutil"
|
||||
)
|
||||
if "sudo requires a password" in text.lower():
|
||||
return "sudo 需要密码，请配置免密 sudo 或使用 root SSH 登录"
|
||||
line = ""
|
||||
for chunk in (stderr, stdout):
|
||||
for raw in reversed(chunk.splitlines()):
|
||||
raw = raw.strip()
|
||||
if raw and not raw.startswith("W:"):
|
||||
line = raw[:300]
|
||||
break
|
||||
if line:
|
||||
break
|
||||
return line or "psutil 安装失败"
|
||||
|
||||
|
||||
def build_psutil_install_cmd(*, ssh_user: str) -> str:
|
||||
return sudo_wrap(PSUTIL_INSTALL_SHELL.strip(), ssh_user)
|
||||
|
||||
|
||||
async def install_psutil_via_ssh(server: Server, *, timeout: int = 120) -> dict[str, str | int | bool]:
|
||||
"""Run remote install; returns success, message, stdout/stderr."""
|
||||
ssh_user = (server.username or "root").strip()
|
||||
cmd = build_psutil_install_cmd(ssh_user=ssh_user)
|
||||
result = await exec_ssh_command(server, cmd, timeout=timeout)
|
||||
stdout = (result.get("stdout") or "").strip()
|
||||
stderr = (result.get("stderr") or "").strip()
|
||||
raw_exit = result.get("exit_code")
|
||||
exit_code = int(raw_exit) if raw_exit is not None else 1
|
||||
if exit_code == 0 and "psutil_ok" in stdout:
|
||||
return {
|
||||
"success": True,
|
||||
"exit_code": 0,
|
||||
"stdout": stdout[:2000],
|
||||
"stderr": stderr[:500],
|
||||
"message": "psutil 已就绪",
|
||||
}
|
||||
return {
|
||||
"success": False,
|
||||
"exit_code": exit_code,
|
||||
"stdout": stdout[:2000],
|
||||
"stderr": stderr[:500],
|
||||
"message": _psutil_install_error_zh(stderr, stdout),
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Rsync remote receiver elevation — mirrors ``files_elevation`` for file push.
|
||||
|
||||
Policy (same as file manager): **root** SSH → no sudo; **non-root** with default
|
||||
``auto_sudo`` (or ``always_sudo``) → ``sudo -n rsync`` on the receiver.
|
||||
Only ``files_elevation=off`` disables elevation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from server.domain.models import Server
|
||||
from server.utils.files_elevation import FilesElevation, get_files_elevation
|
||||
|
||||
# Fixed remote receiver command (no user input); requires NOPASSWD rsync in sudoers.
|
||||
RSYNC_SUDO_RECEIVER_PATH = "sudo -n rsync"
|
||||
|
||||
|
||||
def is_root_ssh_user(server: Server) -> bool:
|
||||
ssh_user = (server.username or "root").strip() or "root"
|
||||
return ssh_user == "root"
|
||||
|
||||
|
||||
def rsync_receiver_path_for_push(server: Server) -> str | None:
|
||||
"""Return ``--rsync-path`` for non-root servers when elevation is enabled."""
|
||||
if is_root_ssh_user(server):
|
||||
return None
|
||||
if get_files_elevation(server) == FilesElevation.OFF:
|
||||
return None
|
||||
return RSYNC_SUDO_RECEIVER_PATH
|
||||
|
||||
|
||||
def rsync_elevation_log_hint(server: Server) -> str | None:
|
||||
"""Short tag for sync_logs.push_permission."""
|
||||
if is_root_ssh_user(server):
|
||||
return None
|
||||
if get_files_elevation(server) == FilesElevation.OFF:
|
||||
return None
|
||||
return "rsync=sudo"
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Schedule cycle ↔ 5-field cron (Baota-compatible)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, TypedDict
|
||||
|
||||
|
||||
class ScheduleCycleConfig(TypedDict, total=False):
|
||||
type: str
|
||||
minute: int
|
||||
hour: int
|
||||
interval: int
|
||||
week: int
|
||||
monthDay: int
|
||||
customCron: str
|
||||
|
||||
|
||||
def _clamp_int(n: int | float, lo: int, hi: int) -> int:
|
||||
try:
|
||||
v = int(n)
|
||||
except (TypeError, ValueError):
|
||||
return lo
|
||||
return max(lo, min(hi, v))
|
||||
|
||||
|
||||
SCHEDULE_MIN_INTERVAL_MINUTES = 1
|
||||
|
||||
|
||||
def build_cron_from_cycle(config: dict[str, Any]) -> str:
|
||||
kind = config.get("type", "day")
|
||||
if kind == "custom":
|
||||
raw = str(config.get("customCron") or "").strip()
|
||||
if not raw:
|
||||
raise ValueError("自定义 cron 不能为空")
|
||||
parts = raw.split()
|
||||
if len(parts) != 5:
|
||||
raise ValueError("cron 表达式必须为 5 个字段")
|
||||
return raw
|
||||
|
||||
minute = _clamp_int(config.get("minute", 0), 0, 59)
|
||||
hour = _clamp_int(config.get("hour", 0), 0, 23)
|
||||
|
||||
if kind == "day":
|
||||
return f"{minute} {hour} * * *"
|
||||
if kind == "day-n":
|
||||
n = _clamp_int(config.get("interval", 1), 1, 365)
|
||||
return f"{minute} {hour} */{n} * *"
|
||||
if kind == "hour":
|
||||
return f"{minute} * * * *"
|
||||
if kind == "hour-n":
|
||||
n = _clamp_int(config.get("interval", 1), 1, 23)
|
||||
return f"{minute} */{n} * * *"
|
||||
if kind == "minute-n":
|
||||
n = _clamp_int(config.get("interval", 1), SCHEDULE_MIN_INTERVAL_MINUTES, 59)
|
||||
return f"*/{n} * * * *"
|
||||
if kind == "week":
|
||||
w = _clamp_int(config.get("week", 0), 0, 6)
|
||||
return f"{minute} {hour} * * {w}"
|
||||
if kind == "month":
|
||||
dom = _clamp_int(config.get("monthDay", 1), 1, 31)
|
||||
return f"{minute} {hour} {dom} * *"
|
||||
raise ValueError(f"未知的周期类型: {kind}")
|
||||
|
||||
|
||||
def _parse_step_or_fixed(field: str) -> tuple[str, int | None]:
|
||||
if field == "*":
|
||||
return "star", None
|
||||
if field.startswith("*/"):
|
||||
step = int(field[2:])
|
||||
return "step", step
|
||||
return "fixed", int(field)
|
||||
|
||||
|
||||
def parse_cron_to_cycle(cron: str) -> dict[str, Any]:
|
||||
base: dict[str, Any] = {
|
||||
"type": "day",
|
||||
"minute": 0,
|
||||
"hour": 2,
|
||||
"interval": 2,
|
||||
"week": 1,
|
||||
"monthDay": 1,
|
||||
"customCron": "",
|
||||
}
|
||||
expr = (cron or "").strip()
|
||||
if not expr:
|
||||
return base
|
||||
|
||||
parts = expr.split()
|
||||
if len(parts) != 5:
|
||||
return {**base, "type": "custom", "customCron": expr}
|
||||
|
||||
min_f, hour_f, dom_f, mon_f, dow_f = parts
|
||||
if mon_f != "*":
|
||||
return {**base, "type": "custom", "customCron": expr}
|
||||
|
||||
min_k, min_v = _parse_step_or_fixed(min_f)
|
||||
hour_k, hour_v = _parse_step_or_fixed(hour_f)
|
||||
dom_k, dom_v = _parse_step_or_fixed(dom_f)
|
||||
dow_k, dow_v = _parse_step_or_fixed(dow_f)
|
||||
|
||||
if min_k == "step" and hour_k == "star" and dom_k == "star" and dow_k == "star":
|
||||
return {**base, "type": "minute-n", "interval": min_v}
|
||||
if min_k == "fixed" and hour_k == "star" and dom_k == "star" and dow_k == "star":
|
||||
return {**base, "type": "hour", "minute": min_v}
|
||||
if min_k == "fixed" and hour_k == "step" and dom_k == "star" and dow_k == "star":
|
||||
return {**base, "type": "hour-n", "minute": min_v, "interval": hour_v}
|
||||
if min_k == "fixed" and hour_k == "fixed" and dom_k == "star" and dow_k == "fixed":
|
||||
w = 0 if dow_f == "7" else dow_v
|
||||
return {**base, "type": "week", "minute": min_v, "hour": hour_v, "week": w}
|
||||
if min_k == "fixed" and hour_k == "fixed" and dom_k == "fixed" and dow_k == "star":
|
||||
return {**base, "type": "month", "minute": min_v, "hour": hour_v, "monthDay": dom_v}
|
||||
if min_k == "fixed" and hour_k == "fixed" and dom_k == "step" and dow_k == "star":
|
||||
return {**base, "type": "day-n", "minute": min_v, "hour": hour_v, "interval": dom_v}
|
||||
if min_k == "fixed" and hour_k == "fixed" and dom_k == "star" and dow_k == "star":
|
||||
return {**base, "type": "day", "minute": min_v, "hour": hour_v}
|
||||
|
||||
return {**base, "type": "custom", "customCron": expr}
|
||||
|
||||
|
||||
_WEEK_LABELS = ["周日", "周一", "周二", "周三", "周四", "周五", "周六"]
|
||||
|
||||
|
||||
def format_cycle_label(cron_or_config: str | dict[str, Any]) -> str:
|
||||
config = parse_cron_to_cycle(cron_or_config) if isinstance(cron_or_config, str) else cron_or_config
|
||||
kind = config.get("type", "day")
|
||||
if kind == "custom":
|
||||
cc = config.get("customCron") or ""
|
||||
return f"自定义: {cc}" if cc else "—"
|
||||
|
||||
minute = int(config.get("minute", 0))
|
||||
hour = int(config.get("hour", 0))
|
||||
time_s = f"{hour:02d}:{minute:02d}"
|
||||
|
||||
if kind == "day":
|
||||
return f"当天 {time_s}"
|
||||
if kind == "day-n":
|
||||
return f"每 {config.get('interval', 1)} 天 {time_s}"
|
||||
if kind == "hour":
|
||||
return f"每小时 第 {minute} 分"
|
||||
if kind == "hour-n":
|
||||
return f"每 {config.get('interval', 1)} 小时 第 {minute} 分"
|
||||
if kind == "minute-n":
|
||||
return f"每 {config.get('interval', 1)} 分钟"
|
||||
if kind == "week":
|
||||
w = int(config.get("week", 0))
|
||||
wd = _WEEK_LABELS[w] if 0 <= w < 7 else f"周{w}"
|
||||
return f"每周{wd} {time_s}"
|
||||
if kind == "month":
|
||||
return f"每月 {config.get('monthDay', 1)} 日 {time_s}"
|
||||
return "—"
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Resolve public/private IPs for server alert Telegram messages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("nexus.server_alert_ips")
|
||||
|
||||
|
||||
def is_public_ip(value: str | None) -> bool:
|
||||
text = (value or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
try:
|
||||
addr = ipaddress.ip_address(text)
|
||||
except ValueError:
|
||||
return False
|
||||
return not (
|
||||
addr.is_private
|
||||
or addr.is_loopback
|
||||
or addr.is_link_local
|
||||
or addr.is_reserved
|
||||
)
|
||||
|
||||
|
||||
def is_private_ip(value: str | None) -> bool:
|
||||
text = (value or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
try:
|
||||
addr = ipaddress.ip_address(text)
|
||||
except ValueError:
|
||||
return False
|
||||
return addr.is_private or addr.is_loopback or addr.is_link_local
|
||||
|
||||
|
||||
def classify_domain_ip(domain: str | None) -> tuple[str | None, str | None]:
|
||||
"""Return (public_ip, private_ip) when domain is a literal IP."""
|
||||
text = (domain or "").strip()
|
||||
if not text:
|
||||
return None, None
|
||||
try:
|
||||
ipaddress.ip_address(text)
|
||||
except ValueError:
|
||||
return None, None
|
||||
if is_public_ip(text):
|
||||
return text, None
|
||||
if is_private_ip(text):
|
||||
return None, text
|
||||
return None, None
|
||||
|
||||
|
||||
def merge_alert_ips(
|
||||
*,
|
||||
domain: str | None = None,
|
||||
extra_attrs: dict[str, Any] | None = None,
|
||||
system_info: dict[str, Any] | None = None,
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Pick public/private IPs for Telegram display.
|
||||
|
||||
外网 SSOT：servers.domain 为公网 IP 时优先（运维登记连接地址，如 120.79.11.13）。
|
||||
内网：Agent 上报 private_ip 优先，否则 domain 为私网 IP 时使用。
|
||||
Agent ipify 仅作 domain/extra 均无公网时的兜底。
|
||||
"""
|
||||
public_ip: str | None = None
|
||||
private_ip: str | None = None
|
||||
|
||||
info = system_info if isinstance(system_info, dict) else {}
|
||||
attrs = extra_attrs if isinstance(extra_attrs, dict) else {}
|
||||
dom_pub, dom_priv = classify_domain_ip(domain)
|
||||
|
||||
if dom_pub:
|
||||
public_ip = dom_pub
|
||||
|
||||
if not public_ip:
|
||||
for key in ("public_ip", "wan_ip", "external_ip"):
|
||||
raw = attrs.get(key)
|
||||
if raw and is_public_ip(str(raw)):
|
||||
public_ip = str(raw).strip()
|
||||
break
|
||||
|
||||
if not public_ip:
|
||||
for key in ("public_ip", "wan_ip", "external_ip"):
|
||||
raw = info.get(key)
|
||||
if raw and is_public_ip(str(raw)):
|
||||
public_ip = str(raw).strip()
|
||||
break
|
||||
|
||||
for key in ("private_ip", "lan_ip", "internal_ip"):
|
||||
raw = info.get(key)
|
||||
if raw and is_private_ip(str(raw)):
|
||||
private_ip = str(raw).strip()
|
||||
break
|
||||
|
||||
if not private_ip and dom_priv:
|
||||
private_ip = dom_priv
|
||||
|
||||
return public_ip, private_ip
|
||||
|
||||
|
||||
def format_alert_ip_line(public_ip: str | None, private_ip: str | None) -> str:
|
||||
pub = (public_ip or "").strip() or "—"
|
||||
priv = (private_ip or "").strip() or "—"
|
||||
return f"IP地址：{pub}(外) {priv}(内)"
|
||||
|
||||
|
||||
async def resolve_alert_ips_for_server(server_id: int) -> tuple[str | None, str | None]:
|
||||
"""Load server row + Redis heartbeat system_info for Telegram IP lines."""
|
||||
from server.infrastructure.database.server_repo import ServerRepositoryImpl
|
||||
from server.infrastructure.database.session import AsyncSessionLocal
|
||||
from server.infrastructure.redis.client import get_redis
|
||||
|
||||
domain: str | None = None
|
||||
extra_attrs: dict[str, Any] | None = None
|
||||
system_info: dict[str, Any] | None = None
|
||||
|
||||
try:
|
||||
async with AsyncSessionLocal() as session:
|
||||
server = await ServerRepositoryImpl(session).get_by_id(server_id)
|
||||
if server:
|
||||
domain = server.domain
|
||||
extra_attrs = server.extra_attrs if isinstance(server.extra_attrs, dict) else None
|
||||
except Exception as e:
|
||||
logger.warning("resolve_alert_ips DB read failed server_id=%s: %s", server_id, e)
|
||||
|
||||
try:
|
||||
redis = get_redis()
|
||||
raw = await redis.hget(f"heartbeat:{server_id}", "system_info")
|
||||
if raw:
|
||||
parsed = json.loads(raw)
|
||||
if isinstance(parsed, dict):
|
||||
system_info = parsed
|
||||
except Exception as e:
|
||||
logger.debug("resolve_alert_ips Redis read failed server_id=%s: %s", server_id, e)
|
||||
|
||||
return merge_alert_ips(domain=domain, extra_attrs=extra_attrs, system_info=system_info)
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Per-server metric parsing and offline segment merging."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from server.utils.fleet_metrics import _metric_from_system_info
|
||||
|
||||
|
||||
def parse_system_info_metrics(system_info: dict[str, Any] | str | None) -> tuple[int | None, int | None]:
|
||||
"""Extract CPU/mem integer percentages from Agent system_info."""
|
||||
if system_info is None:
|
||||
return None, None
|
||||
info: dict[str, Any]
|
||||
if isinstance(system_info, str):
|
||||
try:
|
||||
parsed = json.loads(system_info)
|
||||
except json.JSONDecodeError:
|
||||
return None, None
|
||||
info = parsed if isinstance(parsed, dict) else {}
|
||||
elif isinstance(system_info, dict):
|
||||
info = system_info
|
||||
else:
|
||||
return None, None
|
||||
|
||||
cpu = _metric_from_system_info(info, "cpu_usage", "cpu")
|
||||
mem = _metric_from_system_info(info, "mem_usage", "mem")
|
||||
cpu_i = round(cpu) if cpu is not None else None
|
||||
mem_i = round(mem) if mem is not None else None
|
||||
return cpu_i, mem_i
|
||||
|
||||
|
||||
def sample_from_redis_heartbeat(data: dict[str, Any]) -> tuple[bool, int | None, int | None]:
|
||||
"""Map Redis heartbeat hash to (is_online, cpu_pct, mem_pct) for one flush sample."""
|
||||
is_online = data.get("is_online") == "True"
|
||||
if not is_online:
|
||||
return False, None, None
|
||||
cpu, mem = parse_system_info_metrics(data.get("system_info"))
|
||||
return True, cpu, mem
|
||||
|
||||
|
||||
def merge_offline_segments(points: list[dict[str, Any]]) -> list[dict[str, str]]:
|
||||
"""Merge consecutive is_online=false points into {from, to} segments (recorded_at strings)."""
|
||||
if not points:
|
||||
return []
|
||||
|
||||
segments: list[dict[str, str]] = []
|
||||
seg_start: str | None = None
|
||||
prev_at: str | None = None
|
||||
|
||||
for point in points:
|
||||
recorded_at = str(point.get("recorded_at") or "")
|
||||
if not recorded_at:
|
||||
continue
|
||||
is_online = bool(point.get("is_online"))
|
||||
if not is_online:
|
||||
if seg_start is None:
|
||||
seg_start = recorded_at
|
||||
elif seg_start is not None and prev_at is not None:
|
||||
segments.append({"from": seg_start, "to": prev_at})
|
||||
seg_start = None
|
||||
prev_at = recorded_at
|
||||
|
||||
if seg_start is not None and prev_at is not None:
|
||||
segments.append({"from": seg_start, "to": prev_at})
|
||||
|
||||
return segments
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Server list search history — normalize and merge rules (DB payload shape)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
MAX_SERVER_SEARCH_HISTORY = 12
|
||||
MAX_PUSH_SEARCH_HISTORY = 5
|
||||
MAX_TERMINAL_SEARCH_HISTORY = 10
|
||||
MAX_SERVER_SEARCH_QUERY_LEN = 200
|
||||
SERVERS_SEARCH_CONTEXT = "servers_search"
|
||||
PUSH_SEARCH_CONTEXT = "push_search"
|
||||
TERMINAL_SEARCH_CONTEXT = "terminal_search"
|
||||
|
||||
|
||||
def normalize_search_query(query: str | None) -> str:
|
||||
return (query or "").strip()[:MAX_SERVER_SEARCH_QUERY_LEN]
|
||||
|
||||
|
||||
def empty_search_state() -> dict:
|
||||
return {"history": [], "last": None}
|
||||
|
||||
|
||||
def parse_search_state(raw: dict | None, *, max_history: int = MAX_SERVER_SEARCH_HISTORY) -> dict:
|
||||
if not raw:
|
||||
return empty_search_state()
|
||||
history_raw = raw.get("history")
|
||||
history: list[str] = []
|
||||
if isinstance(history_raw, list):
|
||||
for item in history_raw:
|
||||
if not isinstance(item, str):
|
||||
continue
|
||||
q = normalize_search_query(item)
|
||||
if q and q not in history:
|
||||
history.append(q)
|
||||
if len(history) >= max_history:
|
||||
break
|
||||
last_raw = raw.get("last")
|
||||
last = normalize_search_query(last_raw) if isinstance(last_raw, str) else ""
|
||||
last = last or None
|
||||
return {"history": history, "last": last}
|
||||
|
||||
|
||||
def apply_search_record(state: dict, query: str, *, max_history: int = MAX_SERVER_SEARCH_HISTORY) -> dict:
|
||||
q = normalize_search_query(query)
|
||||
history = list(state.get("history") or [])
|
||||
if not q:
|
||||
return {"history": history, "last": None}
|
||||
history = [q] + [x for x in history if x != q][: max_history - 1]
|
||||
return {"history": history, "last": q}
|
||||
|
||||
|
||||
def apply_search_replace(
|
||||
*,
|
||||
history: list[str],
|
||||
last: str | None = None,
|
||||
max_history: int = MAX_SERVER_SEARCH_HISTORY,
|
||||
) -> dict:
|
||||
cleaned: list[str] = []
|
||||
for item in history:
|
||||
q = normalize_search_query(item)
|
||||
if q and q not in cleaned:
|
||||
cleaned.append(q)
|
||||
if len(cleaned) >= max_history:
|
||||
break
|
||||
last_norm = normalize_search_query(last) if last else ""
|
||||
last_val = last_norm or (cleaned[0] if cleaned else None)
|
||||
return {"history": cleaned, "last": last_val}
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Resolve display site hostname from server fields (mirrors frontend serverSiteUrl.ts)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
_IP_RE = re.compile(r"^\d{1,3}(\.\d{1,3}){3}$")
|
||||
_CJK_RE = re.compile(r"[\u4e00-\u9fff]")
|
||||
_ASCII_HOST_RE = re.compile(
|
||||
r"^[a-z0-9](?:[a-z0-9-]*\.)+[a-z]{2,63}$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_SKIP_TARGET_SEGMENTS = frozenset({"shop", "crmeb", "public", "me", "crmweb"})
|
||||
|
||||
|
||||
def is_ip_host(value: str | None) -> bool:
|
||||
"""True when *value* is a literal IPv4/IPv6 address (SSH domain field)."""
|
||||
text = (value or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
try:
|
||||
ipaddress.ip_address(text.split("/")[0])
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _host_from_string(raw: str) -> str | None:
|
||||
trimmed = raw.strip()
|
||||
if not trimmed:
|
||||
return None
|
||||
# 备注（中文 tonex 名等）勿当站点域名
|
||||
if _CJK_RE.search(trimmed):
|
||||
return None
|
||||
if trimmed.startswith(("http://", "https://")):
|
||||
try:
|
||||
host = urlparse(trimmed).hostname
|
||||
except Exception:
|
||||
return None
|
||||
if host and not is_ip_host(host) and _ASCII_HOST_RE.match(host):
|
||||
return host.lower()
|
||||
return None
|
||||
host = trimmed.replace("http://", "").replace("https://", "")
|
||||
host = host.split("/")[0].split(":")[0].strip()
|
||||
if not host or is_ip_host(host) or not _ASCII_HOST_RE.match(host):
|
||||
return None
|
||||
return host.lower()
|
||||
|
||||
|
||||
def parse_site_host_from_target_path(target_path: str | None) -> str | None:
|
||||
p = (target_path or "").strip().rstrip("/")
|
||||
if not p:
|
||||
return None
|
||||
parts = [seg for seg in p.split("/") if seg]
|
||||
root_idx = parts.index("wwwroot") if "wwwroot" in parts else -1
|
||||
candidates: list[str] = []
|
||||
if root_idx >= 0:
|
||||
candidates.extend(parts[root_idx + 1 :])
|
||||
elif parts:
|
||||
candidates.append(parts[-1])
|
||||
for seg in reversed(candidates):
|
||||
if seg.lower() in _SKIP_TARGET_SEGMENTS:
|
||||
continue
|
||||
if _IP_RE.match(seg):
|
||||
continue
|
||||
if "." in seg:
|
||||
return seg.lower()
|
||||
return None
|
||||
|
||||
|
||||
def resolve_server_site_host(
|
||||
*,
|
||||
description: str | None = None,
|
||||
target_path: str | None = None,
|
||||
extra_attrs: dict[str, Any] | None = None,
|
||||
) -> str | None:
|
||||
extra = extra_attrs if isinstance(extra_attrs, dict) else {}
|
||||
for key in ("site_url", "site_domain", "website"):
|
||||
val = extra.get(key)
|
||||
if isinstance(val, str):
|
||||
host = _host_from_string(val)
|
||||
if host:
|
||||
return host
|
||||
if description and description.strip():
|
||||
host = _host_from_string(description)
|
||||
if host:
|
||||
return host
|
||||
return parse_site_host_from_target_path(target_path)
|
||||
|
||||
|
||||
def resolve_server_site_host_from_server(server: Any) -> str | None:
|
||||
return resolve_server_site_host(
|
||||
description=getattr(server, "description", None),
|
||||
target_path=getattr(server, "target_path", None),
|
||||
extra_attrs=getattr(server, "extra_attrs", None),
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Validate SSH private key PEM and optionally derive OpenSSH public key."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncssh
|
||||
|
||||
|
||||
def normalize_ssh_private_key_pem(private_key: str) -> str:
|
||||
"""Strip outer whitespace; PEM body newlines preserved."""
|
||||
return (private_key or "").strip()
|
||||
|
||||
|
||||
def prepare_ssh_key_preset_fields(
|
||||
private_key: str,
|
||||
public_key: str | None = None,
|
||||
) -> tuple[str, str | None]:
|
||||
"""
|
||||
Validate PEM private key (RSA / OPENSSH / PKCS#8) and return (private, public).
|
||||
|
||||
If public_key is empty, derive OpenSSH public key from private (asyncssh).
|
||||
"""
|
||||
pem = normalize_ssh_private_key_pem(private_key)
|
||||
if not pem:
|
||||
raise ValueError("私钥不能为空")
|
||||
if "BEGIN" not in pem or "PRIVATE KEY" not in pem:
|
||||
raise ValueError(
|
||||
"私钥须为 PEM 格式（整文件粘贴，含 -----BEGIN … PRIVATE KEY----- 首尾行）"
|
||||
)
|
||||
try:
|
||||
key = asyncssh.import_private_key(pem)
|
||||
except Exception as exc:
|
||||
raise ValueError("私钥格式无效，无法解析；请确认是未加密的 PEM 私钥") from exc
|
||||
|
||||
pub = (public_key or "").strip() or None
|
||||
if not pub:
|
||||
exported = key.export_public_key("openssh")
|
||||
pub = exported.decode() if isinstance(exported, bytes) else exported
|
||||
return pem, pub
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Translate rsync/SSH push error text to Chinese for sync_logs and API responses."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# Order matters: more specific patterns first.
|
||||
_PATTERNS: list[tuple[re.Pattern[str], str]] = [
|
||||
(re.compile(r"rsync:\s*connection unexpectedly closed", re.I), "rsync：连接意外关闭"),
|
||||
(re.compile(r"rsync error:\s*some files/attrs were not transferred", re.I), "rsync 错误：部分文件或属性未能传输"),
|
||||
(re.compile(r"rsync error:\s*error in rsync protocol data stream", re.I), "rsync 错误：协议数据流异常"),
|
||||
(re.compile(r"rsync error:\s*", re.I), "rsync 错误："),
|
||||
(re.compile(r"rsync:\s*", re.I), "rsync："),
|
||||
(re.compile(r"Host key is not trusted for host", re.I), "SSH 主机密钥未信任（未知主机）"),
|
||||
(re.compile(r"Host key verification failed", re.I), "SSH 主机密钥验证失败"),
|
||||
(re.compile(r"Permission denied for user .+ on host", re.I), "SSH 认证失败：用户名或密码/密钥不正确"),
|
||||
(re.compile(r"Login failed for user", re.I), "SSH 登录失败：用户名或密码/密钥不正确"),
|
||||
(re.compile(r"Key exchange failed", re.I), "SSH 密钥交换失败"),
|
||||
(re.compile(r"Disconnect reason:", re.I), "SSH 连接断开："),
|
||||
(re.compile(r"OpenSSH protocol error", re.I), "SSH 协议错误"),
|
||||
(re.compile(r"Connection lost", re.I), "SSH 连接丢失"),
|
||||
(re.compile(r"Connection closed", re.I), "SSH 连接已关闭"),
|
||||
(re.compile(r"Could not resolve hostname", re.I), "无法解析主机名"),
|
||||
(re.compile(r"Permission denied,\s*please try again", re.I), "权限被拒绝，请重试"),
|
||||
(re.compile(r"Permission denied\s*\(publickey", re.I), "公钥认证失败，权限被拒绝"),
|
||||
(re.compile(r"Permission denied", re.I), "权限被拒绝"),
|
||||
(re.compile(r"No such file or directory", re.I), "文件或目录不存在"),
|
||||
(re.compile(r"Connection refused", re.I), "连接被拒绝"),
|
||||
(re.compile(r"Connection timed out", re.I), "连接超时"),
|
||||
(re.compile(r"Connection reset by peer", re.I), "连接被对端重置"),
|
||||
(re.compile(r"Network is unreachable", re.I), "网络不可达"),
|
||||
(re.compile(r"No route to host", re.I), "无法路由到主机"),
|
||||
(re.compile(r"No space left on device", re.I), "磁盘空间不足"),
|
||||
(re.compile(r"Read-only file system", re.I), "文件系统为只读"),
|
||||
(re.compile(r"Operation timed out", re.I), "操作超时"),
|
||||
(re.compile(r"Operation not permitted", re.I), "操作不允许"),
|
||||
(re.compile(r"Authentication failed", re.I), "认证失败"),
|
||||
(re.compile(r"Broken pipe", re.I), "连接中断（管道破裂）"),
|
||||
(re.compile(r"Protocol error", re.I), "协议错误"),
|
||||
(re.compile(r"Too many open files", re.I), "打开的文件过多"),
|
||||
(re.compile(r"Name or service not known", re.I), "无法解析主机名或服务名"),
|
||||
(re.compile(r"ssh:\s*connect to host", re.I), "ssh：无法连接到主机"),
|
||||
(re.compile(r"ssh:\s*", re.I), "ssh："),
|
||||
(re.compile(r"Command failed\s*\(exit\s*(\d+)\)", re.I), r"命令失败（退出码 \1）"),
|
||||
(re.compile(r"rsync exited with code\s*(\d+)", re.I), r"rsync 退出码 \1"),
|
||||
(re.compile(r"\bexit code\s*(\d+)", re.I), r"退出码 \1"),
|
||||
(re.compile(r"Server not found", re.I), "服务器不存在"),
|
||||
(re.compile(r"failed to set times on", re.I), "无法设置文件时间戳："),
|
||||
(re.compile(r"sender", re.I), "发送端"),
|
||||
(re.compile(r"receiver", re.I), "接收端"),
|
||||
]
|
||||
|
||||
_CJK_RE = re.compile(r"[\u4e00-\u9fff\u3400-\u4dbf]")
|
||||
|
||||
|
||||
def _is_primarily_chinese(text: str) -> bool:
|
||||
cjk = len(_CJK_RE.findall(text))
|
||||
if cjk == 0:
|
||||
return False
|
||||
# Heuristic: any CJK and CJK chars >= 20% of non-whitespace length
|
||||
compact = re.sub(r"\s+", "", text)
|
||||
if not compact:
|
||||
return False
|
||||
return cjk / len(compact) >= 0.2
|
||||
|
||||
|
||||
def translate_ssh_error_message(message: str | None) -> str | None:
|
||||
"""Return a Chinese-localized SSH/rsync error string, or the original if already Chinese."""
|
||||
return translate_sync_error_message(message)
|
||||
|
||||
|
||||
def translate_sync_error_message(message: str | None) -> str | None:
|
||||
"""Return a Chinese-localized push/sync error string, or the original if already Chinese."""
|
||||
if message is None:
|
||||
return None
|
||||
text = message.strip()
|
||||
if not text:
|
||||
return message
|
||||
if _is_primarily_chinese(text):
|
||||
return message
|
||||
|
||||
result = text
|
||||
for pattern, repl in _PATTERNS:
|
||||
result = pattern.sub(repl, result)
|
||||
return result
|
||||
@@ -0,0 +1,51 @@
|
||||
"""UTF-8 text I/O and EOL helpers (Windows dev → Linux deploy safe).
|
||||
|
||||
Use for config files (.env, .json, shell scripts) written on the Nexus host or
|
||||
by the install wizard — never rely on platform-default encodings or CRLF from
|
||||
the OS.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
EolKind = Literal["LF", "CRLF", "CR"]
|
||||
|
||||
|
||||
def normalize_text_eol(text: str) -> str:
|
||||
"""Normalize any common EOL to LF for internal processing."""
|
||||
return text.replace("\r\n", "\n").replace("\r", "\n")
|
||||
|
||||
|
||||
def detect_text_eol(text: str) -> EolKind:
|
||||
"""Detect dominant line ending style (for API / editor round-trip)."""
|
||||
if "\r\n" in text:
|
||||
return "CRLF"
|
||||
if "\r" in text:
|
||||
return "CR"
|
||||
return "LF"
|
||||
|
||||
|
||||
def read_utf8_text(path: Path) -> str:
|
||||
"""Read UTF-8 text; strip BOM; do not alter EOL (for round-trip edits)."""
|
||||
return path.read_text(encoding="utf-8-sig")
|
||||
|
||||
|
||||
def read_utf8_lf(path: Path) -> str:
|
||||
"""Read UTF-8 and normalize line endings to LF."""
|
||||
return normalize_text_eol(read_utf8_text(path))
|
||||
|
||||
|
||||
def write_utf8_lf(path: Path, content: str) -> None:
|
||||
"""Write UTF-8 with LF newlines only (safe on Windows dev, required on Ubuntu)."""
|
||||
path.write_text(normalize_text_eol(content), encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
def write_utf8_lf_lines(path: Path, lines: list[str]) -> None:
|
||||
"""Write lines joined with LF (no trailing newline unless last line is empty)."""
|
||||
body = "\n".join(lines)
|
||||
if body and not body.endswith("\n"):
|
||||
write_utf8_lf(path, body + "\n")
|
||||
else:
|
||||
write_utf8_lf(path, body)
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Parse GNU `ls -la` permission strings and listing lines."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
||||
_TZ_OFFSET = re.compile(r"^[+-]\d{4}$")
|
||||
|
||||
|
||||
def modified_from_ls_parts(parts: list[str]) -> str:
|
||||
"""Build display timestamp from ``ls`` fields (without filename)."""
|
||||
if len(parts) < 6:
|
||||
return ""
|
||||
if len(parts) >= 7 and _ISO_DATE.match(parts[5]):
|
||||
time_part = parts[6].split(".", 1)[0] if parts[6] else ""
|
||||
return f"{parts[5]} {time_part}".strip()
|
||||
if len(parts) >= 9:
|
||||
return " ".join(parts[5:8])
|
||||
return " ".join(parts[5:])
|
||||
|
||||
|
||||
def parse_ls_la_line(line: str) -> dict | None:
|
||||
"""Parse one ``ls -la`` line; returns None for total/skip lines.
|
||||
|
||||
Handles two timestamp formats produced by different ``ls`` implementations:
|
||||
- Standard (3-token date): ``perms nlinks owner group size Mon DD HH:MM name`` → 9+ fields
|
||||
- GNU long-iso (2-token): ``perms nlinks owner group size YYYY-MM-DD HH:MM name`` → 8+ fields
|
||||
|
||||
With ``--time-style=long-iso`` the date collapses from 3 tokens to 2, so
|
||||
the total token count drops from 9 to 8. Directories and regular files were
|
||||
silently dropped (only symlinks survived because their names contain
|
||||
``" -> target"`` which pushed the count back to 9).
|
||||
"""
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("total "):
|
||||
return None
|
||||
|
||||
# Split into at most 9 pieces so filenames with spaces are kept intact.
|
||||
parts = stripped.split(None, 8)
|
||||
if len(parts) < 8:
|
||||
return None
|
||||
|
||||
perms = parts[0]
|
||||
is_dir = perms.startswith("d")
|
||||
is_link = perms.startswith("l")
|
||||
|
||||
# Detect format by checking whether field[5] is an ISO date (YYYY-MM-DD).
|
||||
# long-iso: [perms nlinks owner group size DATE TIME name] → name at index 7
|
||||
# standard: [perms nlinks owner group size MON DAY TIME name] → name at index 8
|
||||
if _ISO_DATE.match(parts[5]):
|
||||
# long-iso: DATE TIME [TZ] name — GNU ls may append +0800 before filename
|
||||
if len(parts) >= 9 and _TZ_OFFSET.match(parts[7]):
|
||||
name = parts[8]
|
||||
elif len(parts) >= 8:
|
||||
name = parts[7]
|
||||
else:
|
||||
return None
|
||||
modified = f"{parts[5]} {parts[6].split('.', 1)[0]}"
|
||||
else:
|
||||
# standard format — need at least 9 tokens for the name at index 8
|
||||
if len(parts) < 9:
|
||||
return None
|
||||
name = parts[8]
|
||||
modified = modified_from_ls_parts(parts)
|
||||
|
||||
link_target = None
|
||||
if is_link and " -> " in name:
|
||||
name, link_target = name.split(" -> ", 1)
|
||||
if name in (".", ".."):
|
||||
return None
|
||||
|
||||
return {
|
||||
"name": name,
|
||||
"is_dir": is_dir,
|
||||
"is_link": is_link,
|
||||
"link_target": link_target,
|
||||
"size": parts[4],
|
||||
"perms": perms,
|
||||
"owner": parts[2],
|
||||
"group": parts[3] if len(parts) > 3 else "",
|
||||
"modified": modified,
|
||||
}
|
||||
|
||||
|
||||
def perms_to_mode_octal(perms: str) -> str | None:
|
||||
"""Convert ``drwxr-xr-x`` / ``-rw-r--r--`` to ``755`` / ``644``."""
|
||||
if not perms or len(perms) < 10:
|
||||
return None
|
||||
bits = perms[1:10]
|
||||
if len(bits) != 9:
|
||||
return None
|
||||
|
||||
def _tri(chunk: str) -> int:
|
||||
value = 0
|
||||
if chunk[0] == "r":
|
||||
value += 4
|
||||
if chunk[1] == "w":
|
||||
value += 2
|
||||
if chunk[2] in "xXsStT":
|
||||
value += 1
|
||||
return value
|
||||
|
||||
return f"{_tri(bits[0:3])}{_tri(bits[3:6])}{_tri(bits[6:9])}"
|
||||
|
||||
|
||||
def format_owner_label(owner: str, group: str | None = None) -> str:
|
||||
"""Primary list label: ``www`` or ``www:nogroup`` when group differs."""
|
||||
o = (owner or "").strip()
|
||||
g = (group or "").strip()
|
||||
if not o:
|
||||
return "—"
|
||||
if not g or g == o:
|
||||
return o
|
||||
return f"{o}:{g}"
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Translate Pydantic / FastAPI validation errors to Chinese for API 422 responses."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
# Pydantic v2 default messages (exact match)
|
||||
_MSG_EXACT: dict[str, str] = {
|
||||
"Field required": "必填",
|
||||
"Input should be a valid string": "必须是字符串",
|
||||
"Input should be a valid integer": "必须是整数",
|
||||
"Input should be a valid number": "必须是数字",
|
||||
"Input should be a valid boolean": "必须是布尔值",
|
||||
"Input should be a valid dictionary": "必须是对象",
|
||||
"Input should be a valid list": "必须是数组",
|
||||
"JSON decode error": "JSON 格式无效",
|
||||
"Input should be a valid dictionary or object to extract fields from": "请求体必须是 JSON 对象",
|
||||
"Input should be a valid UUID": "必须是有效的 UUID",
|
||||
"Input should be a valid datetime": "必须是有效的日期时间",
|
||||
"Input should be a valid date": "必须是有效的日期",
|
||||
"Input should be a valid time": "必须是有效的时间",
|
||||
"Input should be a valid url": "必须是有效的 URL",
|
||||
"Input should be a valid email address": "必须是有效的邮箱",
|
||||
}
|
||||
|
||||
# Regex patterns for parameterized Pydantic messages (legacy / string fields)
|
||||
_MSG_PATTERNS: list[tuple[re.Pattern[str], str]] = [
|
||||
(re.compile(r"^String should have at least (\d+) characters?$"), r"至少需要 \1 个字符"),
|
||||
(re.compile(r"^String should have at most (\d+) characters?$"), r"最多 \1 个字符"),
|
||||
(re.compile(r"^String should match pattern '(.+)'$"), r"格式不符合要求"),
|
||||
(re.compile(r"^Input should be greater than or equal to (\d+)"), r"不能小于 \1"),
|
||||
(re.compile(r"^Input should be less than or equal to (\d+)"), r"不能大于 \1"),
|
||||
(re.compile(r"^Input should be greater than (\d+)"), r"必须大于 \1"),
|
||||
(re.compile(r"^Input should be less than (\d+)"), r"必须小于 \1"),
|
||||
(re.compile(r"^List should have at least (\d+) items?(?: .*)?$"), r"至少需要 \1 项"),
|
||||
(re.compile(r"^List should have at most (\d+) items?(?: after validation, not (\d+))?$"), r"最多 \1 项"),
|
||||
]
|
||||
|
||||
|
||||
def translate_validation_msg(msg: str, err_type: str = "", ctx: dict[str, Any] | None = None) -> str:
|
||||
"""Return Chinese message when a known translation exists."""
|
||||
ctx = ctx or {}
|
||||
if err_type == "too_long" and ctx.get("field_type") == "List":
|
||||
max_len = ctx.get("max_length")
|
||||
actual = ctx.get("actual_length")
|
||||
if max_len is not None and actual is not None:
|
||||
return f"最多 {max_len} 项，当前 {actual} 项"
|
||||
if max_len is not None:
|
||||
return f"最多 {max_len} 项"
|
||||
if err_type == "too_short" and ctx.get("field_type") == "List":
|
||||
min_len = ctx.get("min_length")
|
||||
actual = ctx.get("actual_length")
|
||||
if min_len is not None and actual is not None:
|
||||
return f"至少需要 {min_len} 项，当前 {actual} 项"
|
||||
if min_len is not None:
|
||||
return f"至少需要 {min_len} 项"
|
||||
if err_type == "missing":
|
||||
return "必填"
|
||||
if not msg:
|
||||
return msg
|
||||
if msg in _MSG_EXACT:
|
||||
return _MSG_EXACT[msg]
|
||||
for pattern, repl in _MSG_PATTERNS:
|
||||
if pattern.match(msg):
|
||||
out = pattern.sub(repl, msg)
|
||||
# too_long list with "not N" suffix — append actual count when present
|
||||
m = re.search(r"not (\d+)$", msg)
|
||||
if m and "最多" in out and "当前" not in out:
|
||||
return f"{out}，当前 {m.group(1)} 项"
|
||||
return out
|
||||
if msg.startswith("Value error, "):
|
||||
return msg[len("Value error, "):]
|
||||
if err_type == "string_type":
|
||||
return "必须是字符串"
|
||||
if err_type in ("int_type", "int_parsing"):
|
||||
return "必须是整数"
|
||||
if err_type in ("float_type", "float_parsing"):
|
||||
return "必须是数字"
|
||||
if err_type == "bool_parsing":
|
||||
return "必须是 true 或 false"
|
||||
if err_type == "less_than_equal" and ctx.get("le") is not None:
|
||||
return f"不能大于 {ctx['le']}"
|
||||
if err_type == "greater_than_equal" and ctx.get("ge") is not None:
|
||||
return f"不能小于 {ctx['ge']}"
|
||||
if err_type == "less_than" and ctx.get("lt") is not None:
|
||||
return f"必须小于 {ctx['lt']}"
|
||||
if err_type == "greater_than" and ctx.get("gt") is not None:
|
||||
return f"必须大于 {ctx['gt']}"
|
||||
return msg
|
||||
|
||||
|
||||
_LOC_PREFIX_SKIP = frozenset({"query", "body", "path", "header", "cookie"})
|
||||
|
||||
_FIELD_ZH: dict[str, str] = {
|
||||
"per_page": "每页条数",
|
||||
"page": "页码",
|
||||
"limit": "条数",
|
||||
"offset": "偏移量",
|
||||
"server_ids": "服务器",
|
||||
"source_path": "源路径",
|
||||
"target_path": "目标路径",
|
||||
"sync_mode": "同步模式",
|
||||
"batch_id": "批次 ID",
|
||||
"batch_size": "批次大小",
|
||||
"concurrency": "并发数",
|
||||
"value": "值",
|
||||
"username": "用户名",
|
||||
"password": "密码",
|
||||
"name": "名称",
|
||||
"cron_expr": "Cron 表达式",
|
||||
"fire_at": "执行时间",
|
||||
}
|
||||
|
||||
|
||||
def translate_loc_zh(loc: Any) -> str:
|
||||
"""Human-readable Chinese field path (no query/body prefixes)."""
|
||||
if not isinstance(loc, (list, tuple)):
|
||||
return "参数"
|
||||
parts: list[str] = []
|
||||
for piece in loc:
|
||||
if isinstance(piece, str) and piece in _LOC_PREFIX_SKIP:
|
||||
continue
|
||||
if isinstance(piece, int):
|
||||
parts.append(f"第 {piece + 1} 项")
|
||||
continue
|
||||
key = str(piece)
|
||||
parts.append(_FIELD_ZH.get(key, key))
|
||||
return ".".join(parts) if parts else "参数"
|
||||
|
||||
|
||||
def translate_validation_errors(errors: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Clone Pydantic error dicts with translated ``msg`` and ``loc_zh``."""
|
||||
translated: list[dict[str, Any]] = []
|
||||
for err in errors:
|
||||
ctx = err.get("ctx") if isinstance(err.get("ctx"), dict) else None
|
||||
item = {k: v for k, v in err.items() if k != "ctx"}
|
||||
item["msg"] = translate_validation_msg(
|
||||
str(err.get("msg", "")),
|
||||
str(err.get("type", "")),
|
||||
ctx,
|
||||
)
|
||||
item["loc_zh"] = translate_loc_zh(err.get("loc"))
|
||||
translated.append(item)
|
||||
return translated
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Watch probe alert linkage — reuse agent threshold detection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from server.config import settings
|
||||
from server.infrastructure.redis.client import get_redis
|
||||
from server.utils.alert_streak import process_metric_alerts
|
||||
from server.utils.watch_metrics import PROBE_OK, WatchProbeSample
|
||||
|
||||
logger = logging.getLogger("nexus.watch_alerts")
|
||||
|
||||
|
||||
def _sample_to_system_info(sample: WatchProbeSample) -> dict:
|
||||
return {
|
||||
"cpu_usage": sample.cpu_pct,
|
||||
"mem_usage": sample.mem_pct,
|
||||
"disk_usage": sample.disk_pct,
|
||||
}
|
||||
|
||||
|
||||
async def process_watch_probe_alerts(
|
||||
*,
|
||||
server_id: int,
|
||||
server_name: str,
|
||||
sample: WatchProbeSample,
|
||||
) -> None:
|
||||
"""Fire alert/recovery broadcasts when pinned-server probe exceeds thresholds."""
|
||||
if sample.probe_status != PROBE_OK:
|
||||
return
|
||||
|
||||
thresholds = {
|
||||
"cpu": settings.CPU_ALERT_THRESHOLD,
|
||||
"mem": settings.MEM_ALERT_THRESHOLD,
|
||||
"disk": settings.DISK_ALERT_THRESHOLD,
|
||||
}
|
||||
|
||||
await process_metric_alerts(
|
||||
redis=get_redis(),
|
||||
server_id=server_id,
|
||||
server_name=server_name,
|
||||
system_info=_sample_to_system_info(sample),
|
||||
thresholds=thresholds,
|
||||
log_prefix="Watch alert",
|
||||
)
|
||||
@@ -0,0 +1,562 @@
|
||||
"""Watch slot probe — rate calculation and metric normalization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from server.utils.fleet_metrics import _metric_from_system_info, _safe_pct
|
||||
|
||||
WATCH_MAX_SLOTS = 4
|
||||
WATCH_PIN_TTL_DEFAULT_MINUTES = 30
|
||||
WATCH_PIN_TTL_ALLOWED_MINUTES = frozenset({30, 60, 120, 480, 1440})
|
||||
# 兼容旧 API / 列名
|
||||
WATCH_PIN_TTL_HOURS = 8
|
||||
WATCH_PIN_TTL_EXTENDED_HOURS = 24
|
||||
|
||||
|
||||
def normalize_watch_ttl_minutes(minutes: int) -> int:
|
||||
"""监测窗口：30分 / 1h / 2h / 8h / 24h（默认 30 分）。"""
|
||||
if minutes in WATCH_PIN_TTL_ALLOWED_MINUTES:
|
||||
return minutes
|
||||
# 历史：按小时传入
|
||||
legacy_map = {8: 480, 24: 1440, 1: 60, 2: 120}
|
||||
if minutes in legacy_map:
|
||||
return legacy_map[minutes]
|
||||
return WATCH_PIN_TTL_DEFAULT_MINUTES
|
||||
|
||||
|
||||
def normalize_watch_ttl_hours(hours: int) -> int:
|
||||
"""Deprecated — use normalize_watch_ttl_minutes(hours * 60) for hour values."""
|
||||
return normalize_watch_ttl_minutes(hours * 60 if hours in (1, 2, 8, 24) else hours)
|
||||
|
||||
|
||||
def resolve_pin_ttl_minutes(*, ttl_minutes: int | None, ttl_hours: int | None) -> int:
|
||||
if ttl_minutes and int(ttl_minutes) in WATCH_PIN_TTL_ALLOWED_MINUTES:
|
||||
return int(ttl_minutes)
|
||||
if ttl_hours in (8, 24):
|
||||
return int(ttl_hours) * 60
|
||||
return WATCH_PIN_TTL_DEFAULT_MINUTES
|
||||
WATCH_PROBE_INTERVAL_SEC = 5
|
||||
WATCH_REDIS_FRESH_SEC = 90
|
||||
WATCH_AGENT_FRESH_SEC = 12 # 2× 5s watch interval + slack
|
||||
WATCH_AGENT_INTERVAL_SEC = 5
|
||||
WATCH_PROBE_TIMEOUT_SEC = 8
|
||||
|
||||
PROBE_OK = "ok"
|
||||
PROBE_SSH_TIMEOUT = "ssh_timeout"
|
||||
PROBE_SSH_AUTH = "ssh_auth"
|
||||
PROBE_NO_CRED = "no_cred"
|
||||
PROBE_REDIS_STALE = "redis_stale"
|
||||
PROBE_PARSE_ERROR = "parse_error"
|
||||
PROBE_OFFLINE = "offline"
|
||||
|
||||
|
||||
@dataclass
|
||||
class WatchProbeSample:
|
||||
probe_status: str
|
||||
source: str
|
||||
is_online: bool = False
|
||||
duration_ms: int = 0
|
||||
error: str | None = None
|
||||
cpu_pct: int | None = None
|
||||
mem_pct: int | None = None
|
||||
disk_pct: int | None = None
|
||||
cpu_cores: int | None = None
|
||||
cpu_cores_physical: int | None = None
|
||||
cpu_model: str | None = None
|
||||
memory_total_gb: float | None = None
|
||||
memory_used_gb: float | None = None
|
||||
memory_available_gb: float | None = None
|
||||
disk_total_gb: float | None = None
|
||||
disk_used_gb: float | None = None
|
||||
disk_free_gb: float | None = None
|
||||
swap_total_gb: float | None = None
|
||||
swap_used_gb: float | None = None
|
||||
swap_pct: float | None = None
|
||||
os_release: str | None = None
|
||||
uptime_seconds: int | None = None
|
||||
disk_mount: str = "/"
|
||||
load_1: float | None = None
|
||||
load_5: float | None = None
|
||||
load_15: float | None = None
|
||||
process_running: int | None = None
|
||||
process_total: int | None = None
|
||||
per_cpu_pct: list[float] | None = None
|
||||
mem_free_mb: int | None = None
|
||||
mem_shared_mb: int | None = None
|
||||
mem_buffers_mb: int | None = None
|
||||
mem_cached_mb: int | None = None
|
||||
net_bytes_sent: int | None = None
|
||||
net_bytes_recv: int | None = None
|
||||
disk_read_bytes: int | None = None
|
||||
disk_write_bytes: int | None = None
|
||||
net_up_bps: int | None = None
|
||||
net_down_bps: int | None = None
|
||||
disk_read_bps: int | None = None
|
||||
disk_write_bps: int | None = None
|
||||
processes_json: list[dict[str, Any]] | dict[str, Any] | None = None
|
||||
|
||||
_LIVE_HARDWARE_KEYS = (
|
||||
"cpu_cores",
|
||||
"cpu_cores_physical",
|
||||
"cpu_model",
|
||||
"memory_total_gb",
|
||||
"memory_used_gb",
|
||||
"memory_available_gb",
|
||||
"disk_total_gb",
|
||||
"disk_used_gb",
|
||||
"disk_free_gb",
|
||||
"disk_mount",
|
||||
"swap_total_gb",
|
||||
"swap_used_gb",
|
||||
"swap_pct",
|
||||
"os_release",
|
||||
"uptime_seconds",
|
||||
"process_running",
|
||||
"process_total",
|
||||
"per_cpu_pct",
|
||||
"mem_free_mb",
|
||||
"mem_shared_mb",
|
||||
"mem_buffers_mb",
|
||||
"mem_cached_mb",
|
||||
)
|
||||
|
||||
def to_live_dict(self, *, server_id: int) -> dict[str, Any]:
|
||||
out: dict[str, Any] = {
|
||||
"server_id": server_id,
|
||||
"probe_status": self.probe_status,
|
||||
"source": self.source,
|
||||
"is_online": self.is_online,
|
||||
"cpu_pct": self.cpu_pct,
|
||||
"mem_pct": self.mem_pct,
|
||||
"disk_pct": self.disk_pct,
|
||||
"load_1": self.load_1,
|
||||
"load_5": self.load_5,
|
||||
"load_15": self.load_15,
|
||||
"error": self.error,
|
||||
}
|
||||
for key in self._LIVE_HARDWARE_KEYS:
|
||||
val = getattr(self, key, None)
|
||||
if val is not None and val != "":
|
||||
out[key] = val
|
||||
if self.processes_json:
|
||||
out["processes"] = self.processes_json
|
||||
return out
|
||||
|
||||
|
||||
def redis_sample_has_basic_metrics(sample: WatchProbeSample | None) -> bool:
|
||||
"""True when Agent heartbeat has CPU or memory — IO optional."""
|
||||
if sample is None or sample.probe_status != PROBE_OK:
|
||||
return False
|
||||
return sample.cpu_pct is not None or sample.mem_pct is not None
|
||||
|
||||
|
||||
def parse_heartbeat_system_info(heartbeat: dict[str, str]) -> dict[str, Any]:
|
||||
raw = heartbeat.get("system_info") or "{}"
|
||||
try:
|
||||
info = json.loads(raw) if isinstance(raw, str) else (raw or {})
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return info if isinstance(info, dict) else {}
|
||||
|
||||
|
||||
def heartbeat_watch_mode_active(heartbeat: dict[str, str]) -> bool:
|
||||
"""True when Agent explicitly reports watch_mode in the last heartbeat payload."""
|
||||
return bool(parse_heartbeat_system_info(heartbeat).get("watch_mode"))
|
||||
|
||||
|
||||
def agent_watch_heartbeat_fresh(heartbeat: dict[str, str]) -> bool:
|
||||
"""Stricter freshness for 5s Agent watch payloads (not 90s fleet heartbeat)."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
if heartbeat.get("is_online") != "True":
|
||||
return False
|
||||
ts = heartbeat.get("last_heartbeat") or ""
|
||||
if not ts:
|
||||
return False
|
||||
try:
|
||||
agent_ts = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
if agent_ts.tzinfo is None:
|
||||
agent_ts = agent_ts.replace(tzinfo=timezone.utc)
|
||||
age = (datetime.now(timezone.utc) - agent_ts).total_seconds()
|
||||
return age <= WATCH_AGENT_FRESH_SEC
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def agent_watch_sample_usable(
|
||||
heartbeat: dict[str, str] | None,
|
||||
sample: WatchProbeSample | None,
|
||||
) -> bool:
|
||||
"""Agent 5s path: fresh heartbeat + watch_mode + basic CPU/mem metrics."""
|
||||
if not heartbeat or sample is None:
|
||||
return False
|
||||
if not agent_watch_heartbeat_fresh(heartbeat):
|
||||
return False
|
||||
if not heartbeat_watch_mode_active(heartbeat):
|
||||
return False
|
||||
return redis_sample_has_basic_metrics(sample)
|
||||
|
||||
|
||||
def sanitize_processes(raw: Any) -> list[dict[str, Any]] | None:
|
||||
if not isinstance(raw, list):
|
||||
return None
|
||||
out: list[dict[str, Any]] = []
|
||||
for item in raw[:5]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
name = str(item.get("name") or "")[:80]
|
||||
out.append(
|
||||
{
|
||||
"pid": item.get("pid"),
|
||||
"name": name,
|
||||
"cpu_pct": item.get("cpu_pct"),
|
||||
"mem_pct": item.get("mem_pct"),
|
||||
}
|
||||
)
|
||||
return out or None
|
||||
|
||||
|
||||
def sanitize_process_bundle(raw: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
if not isinstance(raw, dict):
|
||||
return None
|
||||
top_cpu = sanitize_processes(raw.get("top_cpu")) or []
|
||||
top_mem = sanitize_processes(raw.get("top_mem")) or []
|
||||
legacy = sanitize_processes(raw.get("top_processes"))
|
||||
if legacy and not top_cpu:
|
||||
top_cpu = legacy
|
||||
if not top_cpu and not top_mem:
|
||||
return None
|
||||
return {"top_cpu": top_cpu, "top_mem": top_mem}
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RateState:
|
||||
net_bytes_sent: int | None = None
|
||||
net_bytes_recv: int | None = None
|
||||
disk_read_bytes: int | None = None
|
||||
disk_write_bytes: int | None = None
|
||||
recorded_at_ts: float | None = None
|
||||
|
||||
|
||||
def server_can_be_watched(server: Any) -> tuple[bool, str]:
|
||||
"""Return whether server has Agent or SSH credentials for watch probes."""
|
||||
if getattr(server, "agent_version", None):
|
||||
return True, ""
|
||||
if getattr(server, "password", None) or getattr(server, "ssh_key_private", None):
|
||||
return True, ""
|
||||
if getattr(server, "ssh_key_configured", None) or getattr(server, "preset_id", None):
|
||||
return True, ""
|
||||
if getattr(server, "ssh_key_preset_id", None):
|
||||
return True, ""
|
||||
return False, "需要已安装 Agent 或配置 SSH 凭据"
|
||||
|
||||
|
||||
def _int_pct(value: Any) -> int | None:
|
||||
pct = _safe_pct(value)
|
||||
return round(pct) if pct is not None else None
|
||||
|
||||
|
||||
def _positive_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
num = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return num if num >= 0 else None
|
||||
|
||||
|
||||
def _cpu_cores(value: Any) -> int | None:
|
||||
n = _positive_int(value)
|
||||
return n if n and n > 0 else None
|
||||
|
||||
|
||||
def _float_gb(value: Any) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
num = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if num < 0:
|
||||
return None
|
||||
return round(num, 1)
|
||||
|
||||
|
||||
def _short_text(value: Any, max_len: int = 120) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
return text[:max_len]
|
||||
|
||||
|
||||
def _parse_hardware_fields(data: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"cpu_cores": _cpu_cores(data.get("cpu_cores")),
|
||||
"cpu_cores_physical": _cpu_cores(data.get("cpu_cores_physical")),
|
||||
"cpu_model": _short_text(data.get("cpu_model")),
|
||||
"memory_total_gb": _float_gb(data.get("memory_total_gb")),
|
||||
"memory_used_gb": _float_gb(data.get("memory_used_gb")),
|
||||
"memory_available_gb": _float_gb(data.get("memory_available_gb")),
|
||||
"disk_total_gb": _float_gb(data.get("disk_total_gb")),
|
||||
"disk_used_gb": _float_gb(data.get("disk_used_gb")),
|
||||
"disk_free_gb": _float_gb(data.get("disk_free_gb")),
|
||||
"swap_total_gb": _float_gb(data.get("swap_total_gb")),
|
||||
"swap_used_gb": _float_gb(data.get("swap_used_gb")),
|
||||
"swap_pct": _float_gb(data.get("swap_pct")),
|
||||
"os_release": _short_text(data.get("os_release")),
|
||||
"uptime_seconds": _positive_int(data.get("uptime_seconds")),
|
||||
"process_running": _positive_int(data.get("process_running")),
|
||||
"process_total": _positive_int(data.get("process_total")),
|
||||
"per_cpu_pct": _parse_per_cpu_pct(data.get("per_cpu_pct")),
|
||||
"mem_free_mb": _positive_int(data.get("mem_free_mb")),
|
||||
"mem_shared_mb": _positive_int(data.get("mem_shared_mb")),
|
||||
"mem_buffers_mb": _positive_int(data.get("mem_buffers_mb")),
|
||||
"mem_cached_mb": _positive_int(data.get("mem_cached_mb")),
|
||||
}
|
||||
|
||||
|
||||
def _parse_per_cpu_pct(raw: Any) -> list[float] | None:
|
||||
if not isinstance(raw, (list, tuple)):
|
||||
return None
|
||||
out: list[float] = []
|
||||
for item in raw[:128]:
|
||||
try:
|
||||
out.append(round(float(item), 1))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return out or None
|
||||
|
||||
|
||||
def fill_hardware_gaps(sample: WatchProbeSample) -> None:
|
||||
"""Derive used/free GB from totals + % when probe omitted absolute values."""
|
||||
if sample.memory_used_gb is None and sample.memory_total_gb is not None and sample.mem_pct is not None:
|
||||
sample.memory_used_gb = round(sample.memory_total_gb * sample.mem_pct / 100, 1)
|
||||
if sample.memory_available_gb is None and sample.memory_total_gb is not None and sample.memory_used_gb is not None:
|
||||
sample.memory_available_gb = round(max(0.0, sample.memory_total_gb - sample.memory_used_gb), 1)
|
||||
if sample.disk_used_gb is None and sample.disk_total_gb is not None and sample.disk_pct is not None:
|
||||
sample.disk_used_gb = round(sample.disk_total_gb * sample.disk_pct / 100, 1)
|
||||
if sample.disk_free_gb is None and sample.disk_total_gb is not None and sample.disk_used_gb is not None:
|
||||
sample.disk_free_gb = round(max(0.0, sample.disk_total_gb - sample.disk_used_gb), 1)
|
||||
|
||||
|
||||
def _pick_hardware_field(ssh_sample: WatchProbeSample, redis_sample: WatchProbeSample, field: str) -> Any:
|
||||
sv = getattr(ssh_sample, field, None)
|
||||
rv = getattr(redis_sample, field, None)
|
||||
return sv if sv is not None and sv != "" else rv
|
||||
|
||||
|
||||
def compute_rates(
|
||||
prev: _RateState | None,
|
||||
*,
|
||||
net_bytes_sent: int | None,
|
||||
net_bytes_recv: int | None,
|
||||
disk_read_bytes: int | None,
|
||||
disk_write_bytes: int | None,
|
||||
now_ts: float,
|
||||
) -> tuple[int | None, int | None, int | None, int | None]:
|
||||
if prev is None or prev.recorded_at_ts is None:
|
||||
return None, None, None, None
|
||||
dt = now_ts - prev.recorded_at_ts
|
||||
if dt <= 0:
|
||||
return None, None, None, None
|
||||
|
||||
def _rate(curr: int | None, old: int | None) -> int | None:
|
||||
if curr is None or old is None:
|
||||
return None
|
||||
delta = curr - old
|
||||
if delta < 0:
|
||||
return None
|
||||
return int(delta / dt)
|
||||
|
||||
return (
|
||||
_rate(net_bytes_sent, prev.net_bytes_sent),
|
||||
_rate(net_bytes_recv, prev.net_bytes_recv),
|
||||
_rate(disk_read_bytes, prev.disk_read_bytes),
|
||||
_rate(disk_write_bytes, prev.disk_write_bytes),
|
||||
)
|
||||
|
||||
|
||||
def apply_rates_to_sample(sample: WatchProbeSample, prev: _RateState | None, now_ts: float) -> _RateState:
|
||||
up, down, dr, dw = compute_rates(
|
||||
prev,
|
||||
net_bytes_sent=sample.net_bytes_sent,
|
||||
net_bytes_recv=sample.net_bytes_recv,
|
||||
disk_read_bytes=sample.disk_read_bytes,
|
||||
disk_write_bytes=sample.disk_write_bytes,
|
||||
now_ts=now_ts,
|
||||
)
|
||||
sample.net_up_bps = up
|
||||
sample.net_down_bps = down
|
||||
sample.disk_read_bps = dr
|
||||
sample.disk_write_bps = dw
|
||||
return _RateState(
|
||||
net_bytes_sent=sample.net_bytes_sent,
|
||||
net_bytes_recv=sample.net_bytes_recv,
|
||||
disk_read_bytes=sample.disk_read_bytes,
|
||||
disk_write_bytes=sample.disk_write_bytes,
|
||||
recorded_at_ts=now_ts,
|
||||
)
|
||||
|
||||
|
||||
def parse_ssh_watch_payload(data: dict[str, Any]) -> WatchProbeSample:
|
||||
load = data.get("load_avg") or data.get("load") or []
|
||||
if isinstance(load, (list, tuple)) and len(load) >= 3:
|
||||
load_1, load_5, load_15 = float(load[0]), float(load[1]), float(load[2])
|
||||
else:
|
||||
load_1 = float(data["load_1"]) if data.get("load_1") is not None else None
|
||||
load_5 = float(data["load_5"]) if data.get("load_5") is not None else None
|
||||
load_15 = float(data["load_15"]) if data.get("load_15") is not None else None
|
||||
|
||||
processes = sanitize_process_bundle(data) if "top_cpu" in data or "top_mem" in data else None
|
||||
if processes is None:
|
||||
legacy = sanitize_processes(data.get("top_processes"))
|
||||
if legacy:
|
||||
processes = {"top_cpu": legacy, "top_mem": []}
|
||||
|
||||
sample = WatchProbeSample(
|
||||
probe_status=PROBE_OK,
|
||||
source="ssh",
|
||||
is_online=True,
|
||||
cpu_pct=_int_pct(data.get("cpu_usage")),
|
||||
mem_pct=_int_pct(data.get("mem_usage")),
|
||||
disk_pct=_int_pct(data.get("disk_usage")),
|
||||
disk_mount=str(data.get("disk_mount") or "/"),
|
||||
load_1=load_1,
|
||||
load_5=load_5,
|
||||
load_15=load_15,
|
||||
processes_json=processes,
|
||||
**_parse_hardware_fields(data),
|
||||
)
|
||||
fill_hardware_gaps(sample)
|
||||
return sample
|
||||
|
||||
|
||||
def sample_needs_load_supplement(sample: WatchProbeSample) -> bool:
|
||||
"""True when 1/5/15 load averages are missing from Agent-only path."""
|
||||
if sample.probe_status != PROBE_OK:
|
||||
return False
|
||||
return sample.load_1 is None or sample.load_5 is None or sample.load_15 is None
|
||||
|
||||
|
||||
def apply_load_avg_to_sample(sample: WatchProbeSample, load_avg: Any) -> WatchProbeSample:
|
||||
"""Fill load_1/5/15 on an existing sample (mutates in place)."""
|
||||
if not isinstance(load_avg, (list, tuple)) or len(load_avg) < 3:
|
||||
return sample
|
||||
if sample.load_1 is None:
|
||||
sample.load_1 = float(load_avg[0])
|
||||
if sample.load_5 is None:
|
||||
sample.load_5 = float(load_avg[1])
|
||||
if sample.load_15 is None:
|
||||
sample.load_15 = float(load_avg[2])
|
||||
return sample
|
||||
|
||||
|
||||
def parse_redis_watch_payload(heartbeat: dict[str, str]) -> WatchProbeSample | None:
|
||||
if heartbeat.get("is_online") != "True":
|
||||
return None
|
||||
raw = heartbeat.get("system_info") or "{}"
|
||||
try:
|
||||
info = json.loads(raw) if isinstance(raw, str) else (raw or {})
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if not isinstance(info, dict):
|
||||
return None
|
||||
|
||||
cpu = _metric_from_system_info(info, "cpu_usage", "cpu")
|
||||
mem = _metric_from_system_info(info, "mem_usage", "mem")
|
||||
disk = _metric_from_system_info(info, "disk_usage", "disk")
|
||||
load = info.get("load_avg") or []
|
||||
|
||||
load_1 = load_5 = load_15 = None
|
||||
if isinstance(load, (list, tuple)) and len(load) >= 3:
|
||||
load_1, load_5, load_15 = float(load[0]), float(load[1]), float(load[2])
|
||||
|
||||
sample = WatchProbeSample(
|
||||
probe_status=PROBE_OK,
|
||||
source="redis",
|
||||
is_online=True,
|
||||
cpu_pct=round(cpu) if cpu is not None else None,
|
||||
mem_pct=round(mem) if mem is not None else None,
|
||||
disk_pct=round(disk) if disk is not None else None,
|
||||
load_1=load_1,
|
||||
load_5=load_5,
|
||||
load_15=load_15,
|
||||
**_parse_hardware_fields(info),
|
||||
)
|
||||
if cpu is None and mem is None:
|
||||
return None
|
||||
fill_hardware_gaps(sample)
|
||||
return sample
|
||||
|
||||
|
||||
def merge_redis_and_ssh(redis_sample: WatchProbeSample | None, ssh_sample: WatchProbeSample) -> WatchProbeSample:
|
||||
if redis_sample is None:
|
||||
return ssh_sample
|
||||
if ssh_sample.probe_status != PROBE_OK:
|
||||
return redis_sample
|
||||
hw = {field: _pick_hardware_field(ssh_sample, redis_sample, field) for field in WatchProbeSample._LIVE_HARDWARE_KEYS}
|
||||
|
||||
def _pick_ssh_first(ssh_val, redis_val):
|
||||
return ssh_val if ssh_val is not None else redis_val
|
||||
|
||||
out = WatchProbeSample(
|
||||
probe_status=PROBE_OK,
|
||||
source="mixed",
|
||||
is_online=True,
|
||||
duration_ms=ssh_sample.duration_ms,
|
||||
# 5s Agent watch 优先；SSH 仅兜底或无 watch_mode 时
|
||||
cpu_pct=_pick_ssh_first(ssh_sample.cpu_pct, redis_sample.cpu_pct),
|
||||
mem_pct=_pick_ssh_first(ssh_sample.mem_pct, redis_sample.mem_pct),
|
||||
disk_pct=_pick_ssh_first(ssh_sample.disk_pct, redis_sample.disk_pct),
|
||||
disk_mount=str(hw.get("disk_mount") or ssh_sample.disk_mount or redis_sample.disk_mount or "/"),
|
||||
load_1=_pick_ssh_first(ssh_sample.load_1, redis_sample.load_1),
|
||||
load_5=_pick_ssh_first(ssh_sample.load_5, redis_sample.load_5),
|
||||
load_15=_pick_ssh_first(ssh_sample.load_15, redis_sample.load_15),
|
||||
cpu_cores=hw.get("cpu_cores"),
|
||||
cpu_cores_physical=hw.get("cpu_cores_physical"),
|
||||
cpu_model=hw.get("cpu_model"),
|
||||
memory_total_gb=hw.get("memory_total_gb"),
|
||||
memory_used_gb=hw.get("memory_used_gb"),
|
||||
memory_available_gb=hw.get("memory_available_gb"),
|
||||
disk_total_gb=hw.get("disk_total_gb"),
|
||||
disk_used_gb=hw.get("disk_used_gb"),
|
||||
disk_free_gb=hw.get("disk_free_gb"),
|
||||
swap_total_gb=hw.get("swap_total_gb"),
|
||||
swap_used_gb=hw.get("swap_used_gb"),
|
||||
swap_pct=hw.get("swap_pct"),
|
||||
os_release=hw.get("os_release"),
|
||||
uptime_seconds=hw.get("uptime_seconds"),
|
||||
process_running=hw.get("process_running"),
|
||||
process_total=hw.get("process_total"),
|
||||
per_cpu_pct=hw.get("per_cpu_pct"),
|
||||
mem_free_mb=hw.get("mem_free_mb"),
|
||||
mem_shared_mb=hw.get("mem_shared_mb"),
|
||||
mem_buffers_mb=hw.get("mem_buffers_mb"),
|
||||
mem_cached_mb=hw.get("mem_cached_mb"),
|
||||
processes_json=ssh_sample.processes_json or redis_sample.processes_json,
|
||||
)
|
||||
fill_hardware_gaps(out)
|
||||
return out
|
||||
|
||||
|
||||
def failure_sample(
|
||||
*,
|
||||
status: str,
|
||||
source: str,
|
||||
error: str,
|
||||
duration_ms: int = 0,
|
||||
is_online: bool = False,
|
||||
) -> WatchProbeSample:
|
||||
return WatchProbeSample(
|
||||
probe_status=status,
|
||||
source=source,
|
||||
is_online=is_online,
|
||||
duration_ms=duration_ms,
|
||||
error=error[:255],
|
||||
)
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Chinese user-facing messages for watch probe failures."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
_EXACT: dict[str, str] = {
|
||||
"parse_error": "探针返回数据无法解析",
|
||||
"psutil missing": "子机未安装 psutil",
|
||||
"ssh_probe_failed": "SSH 探针脚本执行失败",
|
||||
"SSH watch probe failed": "SSH 监测探针失败",
|
||||
"SSH probe failed": "SSH 探针失败",
|
||||
"timeout": "连接超时",
|
||||
"探针失败": "探针失败",
|
||||
"无 Agent 且 SSH 探针失败": "无 Agent 且 SSH 探针失败",
|
||||
"服务器不存在": "服务器不存在",
|
||||
}
|
||||
|
||||
|
||||
def watch_probe_error_zh(error: str | None, *, status: str | None = None) -> str | None:
|
||||
"""Map probe error text to Chinese for UI / audit display."""
|
||||
if not error:
|
||||
if status == "parse_error":
|
||||
return "探针返回数据无法解析"
|
||||
if status == "ssh_timeout":
|
||||
return "SSH 连接超时"
|
||||
if status == "ssh_auth":
|
||||
return "SSH 认证失败"
|
||||
if status == "no_cred":
|
||||
return "需要 Agent 或 SSH 凭据"
|
||||
if status == "redis_stale":
|
||||
return "Agent 心跳已过期"
|
||||
if status == "offline":
|
||||
return "目标服务器离线或不可达"
|
||||
return None
|
||||
|
||||
text = error.strip()
|
||||
if text in _EXACT:
|
||||
return _EXACT[text]
|
||||
|
||||
lower = text.lower()
|
||||
if lower.startswith("parse_error"):
|
||||
detail = text.split(":", 1)[1].strip() if ":" in text else ""
|
||||
if detail in _EXACT:
|
||||
return f"探针返回数据无法解析：{_EXACT[detail]}"
|
||||
if detail:
|
||||
return f"探针返回数据无法解析：{detail}"
|
||||
return "探针返回数据无法解析"
|
||||
|
||||
if "timed out" in lower or lower.endswith("timeout"):
|
||||
return f"SSH 连接超时：{text}"
|
||||
if "permission denied" in lower or "authentication failed" in lower:
|
||||
return f"SSH 认证失败：{text}"
|
||||
if "psutil missing" in lower:
|
||||
return "子机未安装 psutil"
|
||||
if "ssh_probe_failed" in lower or "ssh watch probe failed" in lower:
|
||||
return "SSH 探针脚本执行失败"
|
||||
|
||||
# Already Chinese or mixed — return as-is if contains CJK
|
||||
if any("\u4e00" <= ch <= "\u9fff" for ch in text):
|
||||
return text
|
||||
|
||||
return f"探针异常：{text}"
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Watch slot Redis snapshots — cleared when no active 5s monitoring remains."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from server.infrastructure.redis.client import get_redis
|
||||
|
||||
REDIS_WATCH_LIVE_PREFIX = "watch:live:"
|
||||
REDIS_WATCH_PROC_PREFIX = "watch:proc:"
|
||||
REDIS_WATCH_LAST_PREFIX = "watch:last:"
|
||||
|
||||
|
||||
async def clear_watch_redis_snapshot(server_id: int) -> None:
|
||||
"""Remove 5s watch sliding window; Agent continues normal 60s heartbeat."""
|
||||
redis = get_redis()
|
||||
await redis.delete(
|
||||
f"{REDIS_WATCH_LIVE_PREFIX}{server_id}",
|
||||
f"{REDIS_WATCH_PROC_PREFIX}{server_id}",
|
||||
f"{REDIS_WATCH_LAST_PREFIX}{server_id}",
|
||||
)
|
||||
Reference in New Issue
Block a user