release: nexus btpanel session fix and app-v2
This commit is contained in:
Codex Release Bot
@@ -0,0 +1,7 @@
|
||||
"""Nexus — Background Tasks Package
|
||||
"""
|
||||
|
||||
from server.background.heartbeat_flush import heartbeat_flush_loop
|
||||
from server.background.self_monitor import self_monitor_loop
|
||||
|
||||
__all__ = ["heartbeat_flush_loop", "self_monitor_loop"]
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Nightly cron: auto-install Agent on servers without agent_version."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from server.application.services.agent_install_schedule import (
|
||||
_cron_expr,
|
||||
_enabled,
|
||||
run_scheduled_agent_install,
|
||||
)
|
||||
from server.background.schedule_runner import _cron_match
|
||||
|
||||
logger = logging.getLogger("nexus.agent_install_loop")
|
||||
|
||||
CHECK_INTERVAL_SECONDS = 60
|
||||
|
||||
|
||||
async def agent_install_loop() -> None:
|
||||
"""Primary worker: fire scheduled install-agent when cron matches (Beijing wall clock)."""
|
||||
while True:
|
||||
try:
|
||||
if _enabled():
|
||||
now = datetime.now(timezone.utc)
|
||||
if _cron_match(_cron_expr(), now):
|
||||
result = await run_scheduled_agent_install()
|
||||
if result.get("action") == "started":
|
||||
logger.info(
|
||||
"Agent install schedule: job_id=%s servers=%s",
|
||||
result.get("job_id"),
|
||||
result.get("server_count"),
|
||||
)
|
||||
elif result.get("reason") not in ("already_ran_today",):
|
||||
logger.debug("Agent install schedule: %s", result)
|
||||
await asyncio.sleep(CHECK_INTERVAL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Agent install schedule loop error")
|
||||
await asyncio.sleep(CHECK_INTERVAL_SECONDS)
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Periodic purge of alert_logs older than ALERT_LOG_RETENTION_DAYS (7 days)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from server.infrastructure.database.alert_log_repo import (
|
||||
ALERT_LOG_RETENTION_DAYS,
|
||||
AlertLogRepositoryImpl,
|
||||
)
|
||||
from server.infrastructure.database.session import AsyncSessionLocal
|
||||
|
||||
logger = logging.getLogger("nexus.alert_log_purge")
|
||||
|
||||
PURGE_INTERVAL_SECONDS = 24 * 3600
|
||||
|
||||
|
||||
async def purge_stale_alert_logs() -> int:
|
||||
"""Delete alert_logs rows older than retention window. Returns rows removed."""
|
||||
async with AsyncSessionLocal() as session:
|
||||
repo = AlertLogRepositoryImpl(session)
|
||||
return await repo.purge_older_than(retention_days=ALERT_LOG_RETENTION_DAYS)
|
||||
|
||||
|
||||
async def alert_log_purge_loop() -> None:
|
||||
"""Run daily on primary worker; first purge runs immediately at startup."""
|
||||
while True:
|
||||
try:
|
||||
count = await purge_stale_alert_logs()
|
||||
if count:
|
||||
logger.info(
|
||||
"Alert log purge: removed %s row(s) older than %s days",
|
||||
count,
|
||||
ALERT_LOG_RETENTION_DAYS,
|
||||
)
|
||||
await asyncio.sleep(PURGE_INTERVAL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Alert log purge loop error")
|
||||
await asyncio.sleep(PURGE_INTERVAL_SECONDS)
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Periodic purge of audit_logs older than AUDIT_LOG_RETENTION_DAYS (7 days)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from server.infrastructure.database.audit_log_repo import (
|
||||
AUDIT_LOG_RETENTION_DAYS,
|
||||
AuditLogRepositoryImpl,
|
||||
)
|
||||
from server.infrastructure.database.session import AsyncSessionLocal
|
||||
|
||||
logger = logging.getLogger("nexus.audit_log_purge")
|
||||
|
||||
PURGE_INTERVAL_SECONDS = 24 * 3600
|
||||
|
||||
|
||||
async def purge_stale_audit_logs() -> int:
|
||||
"""Delete audit_logs rows older than retention window. Returns rows removed."""
|
||||
async with AsyncSessionLocal() as session:
|
||||
repo = AuditLogRepositoryImpl(session)
|
||||
return await repo.purge_older_than(retention_days=AUDIT_LOG_RETENTION_DAYS)
|
||||
|
||||
|
||||
async def audit_log_purge_loop() -> None:
|
||||
"""Run daily on primary worker; first purge runs immediately at startup."""
|
||||
while True:
|
||||
try:
|
||||
count = await purge_stale_audit_logs()
|
||||
if count:
|
||||
logger.info(
|
||||
"Audit log purge: removed %s row(s) older than %s days",
|
||||
count,
|
||||
AUDIT_LOG_RETENTION_DAYS,
|
||||
)
|
||||
await asyncio.sleep(PURGE_INTERVAL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Audit log purge loop error")
|
||||
await asyncio.sleep(PURGE_INTERVAL_SECONDS)
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Primary-worker loop: retry Baota SSH API bootstrap every 5 minutes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from server.application.services.btpanel_bootstrap_schedule import run_bootstrap_tick
|
||||
from server.config import settings
|
||||
|
||||
logger = logging.getLogger("nexus.bt_panel_bootstrap_loop")
|
||||
|
||||
|
||||
def _interval_seconds() -> int:
|
||||
raw = getattr(settings, "BT_PANEL_BOOTSTRAP_INTERVAL", 300)
|
||||
try:
|
||||
n = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
n = 300
|
||||
return max(60, n)
|
||||
|
||||
|
||||
async def bt_panel_bootstrap_loop() -> None:
|
||||
interval = _interval_seconds()
|
||||
logger.info("bt_panel_bootstrap_loop started interval=%ss", interval)
|
||||
while True:
|
||||
try:
|
||||
count = await run_bootstrap_tick()
|
||||
if count:
|
||||
logger.info("bt_panel_bootstrap_loop succeeded=%s", count)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("bt_panel_bootstrap_loop tick failed")
|
||||
await asyncio.sleep(interval)
|
||||
@@ -0,0 +1,200 @@
|
||||
"""Nexus — Heartbeat Flush Background Task
|
||||
Every 10 minutes: batch-flush Redis heartbeat data to MySQL servers table.
|
||||
MySQL only stores historical snapshots; frontend reads live data from Redis.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from server.infrastructure.database.session import AsyncSessionLocal
|
||||
from server.infrastructure.redis.client import get_redis
|
||||
from server.domain.models import Server
|
||||
from server.infrastructure.database.fleet_metric_repo import FleetMetricRepositoryImpl
|
||||
from server.infrastructure.database.server_metric_repo import ServerMetricRepositoryImpl
|
||||
from server.utils.fleet_metrics import aggregate_fleet_metrics
|
||||
from server.utils.server_metrics import sample_from_redis_heartbeat
|
||||
|
||||
logger = logging.getLogger("nexus.heartbeat_flush")
|
||||
|
||||
FLUSH_INTERVAL = 600 # 10 minutes (seconds)
|
||||
REDIS_KEY_PREFIX = "heartbeat:"
|
||||
# Redis TTL for live keys: server.api.agent.REDIS_KEY_EXPIRE (= FLUSH_INTERVAL * 1.5)
|
||||
|
||||
|
||||
async def _write_server_metric_samples(
|
||||
session: AsyncSession,
|
||||
redis_by_id: dict[int, dict],
|
||||
recorded_at: datetime,
|
||||
) -> int:
|
||||
"""Insert per-server CPU/mem samples for all agent-installed servers."""
|
||||
result = await session.execute(
|
||||
select(Server.id).where(
|
||||
Server.agent_version.isnot(None),
|
||||
Server.agent_version != "",
|
||||
)
|
||||
)
|
||||
agent_ids = [row[0] for row in result.all()]
|
||||
if not agent_ids:
|
||||
return 0
|
||||
|
||||
repo = ServerMetricRepositoryImpl(session)
|
||||
written = 0
|
||||
for server_id in agent_ids:
|
||||
data = redis_by_id.get(server_id)
|
||||
if data:
|
||||
is_online, cpu_pct, mem_pct = sample_from_redis_heartbeat(data)
|
||||
else:
|
||||
is_online, cpu_pct, mem_pct = False, None, None
|
||||
await repo.insert_sample(
|
||||
server_id=server_id,
|
||||
recorded_at=recorded_at,
|
||||
is_online=is_online,
|
||||
cpu_pct=cpu_pct,
|
||||
mem_pct=mem_pct,
|
||||
)
|
||||
written += 1
|
||||
return written
|
||||
|
||||
|
||||
async def _flush_redis_keys(session: AsyncSession, keys: list[str], redis) -> tuple[int, int, list[dict], dict[int, dict]]:
|
||||
"""Update servers from Redis keys; return fleet rows and id→heartbeat map."""
|
||||
flushed = 0
|
||||
skipped = 0
|
||||
heartbeat_rows: list[dict] = []
|
||||
redis_by_id: dict[int, dict] = {}
|
||||
|
||||
for key in keys:
|
||||
try:
|
||||
server_id = int(key.replace(REDIS_KEY_PREFIX, ""))
|
||||
data = await redis.hgetall(key)
|
||||
|
||||
if not data:
|
||||
continue
|
||||
|
||||
row = dict(data)
|
||||
heartbeat_rows.append(row)
|
||||
redis_by_id[server_id] = row
|
||||
is_online = data.get("is_online") == "True"
|
||||
system_info = data.get("system_info", "{}")
|
||||
agent_version = data.get("agent_version", "")
|
||||
|
||||
last_hb_str = data.get("last_heartbeat", "")
|
||||
try:
|
||||
last_heartbeat = datetime.fromisoformat(last_hb_str)
|
||||
if last_heartbeat.tzinfo is None:
|
||||
last_heartbeat = last_heartbeat.replace(tzinfo=timezone.utc)
|
||||
except (ValueError, TypeError):
|
||||
last_heartbeat = datetime.now(timezone.utc)
|
||||
|
||||
result = await session.execute(
|
||||
update(Server)
|
||||
.where(Server.id == server_id)
|
||||
.values(
|
||||
is_online=is_online,
|
||||
system_info=system_info,
|
||||
last_heartbeat=last_heartbeat,
|
||||
agent_version=agent_version,
|
||||
)
|
||||
)
|
||||
if (result.rowcount or 0) > 0:
|
||||
flushed += 1
|
||||
else:
|
||||
skipped += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to flush server {key}: {e}")
|
||||
skipped += 1
|
||||
|
||||
return flushed, skipped, heartbeat_rows, redis_by_id
|
||||
|
||||
|
||||
async def _commit_flush_metrics(
|
||||
session: AsyncSession,
|
||||
heartbeat_rows: list[dict],
|
||||
redis_by_id: dict[int, dict],
|
||||
recorded_at: datetime,
|
||||
) -> None:
|
||||
fleet_repo = FleetMetricRepositoryImpl(session)
|
||||
if heartbeat_rows:
|
||||
agg = aggregate_fleet_metrics(heartbeat_rows)
|
||||
await fleet_repo.insert_sample(
|
||||
recorded_at=recorded_at,
|
||||
online_count=agg.online_count,
|
||||
sample_count=agg.sample_count,
|
||||
cpu_avg=agg.cpu_avg,
|
||||
mem_avg=agg.mem_avg,
|
||||
disk_avg=agg.disk_avg,
|
||||
)
|
||||
purged_fleet = await fleet_repo.purge_older_than()
|
||||
|
||||
server_samples = await _write_server_metric_samples(session, redis_by_id, recorded_at)
|
||||
server_repo = ServerMetricRepositoryImpl(session)
|
||||
purged_server = await server_repo.purge_older_than()
|
||||
|
||||
await session.commit()
|
||||
if purged_fleet:
|
||||
logger.info("Fleet metric purge: removed %s rows older than retention", purged_fleet)
|
||||
if purged_server:
|
||||
logger.info("Server metric purge: removed %s rows older than retention", purged_server)
|
||||
if server_samples:
|
||||
logger.info("Server metric samples: wrote %s rows", server_samples)
|
||||
|
||||
|
||||
async def heartbeat_flush_loop():
|
||||
"""Background task: every 10min, flush Redis heartbeat data to MySQL.
|
||||
|
||||
Flow:
|
||||
1. Scan all heartbeat:* keys in Redis
|
||||
2. For each key, extract server_id + heartbeat data
|
||||
3. Batch UPDATE MySQL servers table (is_online, system_info, last_heartbeat, agent_version)
|
||||
4. Write fleet + per-server metric samples; purge retention
|
||||
"""
|
||||
logger.info("Heartbeat flush loop started (interval: 10min)")
|
||||
while True:
|
||||
await asyncio.sleep(FLUSH_INTERVAL)
|
||||
|
||||
redis = get_redis()
|
||||
recorded_at = datetime.now(timezone.utc)
|
||||
|
||||
try:
|
||||
keys: list[str] = []
|
||||
cursor = 0
|
||||
while True:
|
||||
cursor, batch = await redis.scan(cursor, match=f"{REDIS_KEY_PREFIX}*", count=100)
|
||||
keys.extend(batch)
|
||||
if cursor == 0:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Redis scan failed: {e}")
|
||||
continue
|
||||
|
||||
if not keys:
|
||||
logger.info("No heartbeat keys in Redis — writing offline server metric samples only")
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
await _commit_flush_metrics(session, [], {}, recorded_at)
|
||||
except Exception as e:
|
||||
logger.error(f"Heartbeat flush commit failed (no Redis keys): {e}")
|
||||
await session.rollback()
|
||||
continue
|
||||
|
||||
flushed = 0
|
||||
skipped = 0
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
flushed, skipped, heartbeat_rows, redis_by_id = await _flush_redis_keys(
|
||||
session, keys, redis
|
||||
)
|
||||
await _commit_flush_metrics(session, heartbeat_rows, redis_by_id, recorded_at)
|
||||
except Exception as e:
|
||||
logger.error(f"Heartbeat flush commit failed: {e}")
|
||||
await session.rollback()
|
||||
continue
|
||||
|
||||
logger.info(
|
||||
f"Heartbeat flush: {flushed}/{len(keys)} servers updated to MySQL"
|
||||
+ (f", {skipped} failed" if skipped else "")
|
||||
)
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Background task: refresh login subscription IPs every 2 hours.
|
||||
|
||||
Data model:
|
||||
login_subscription_url — URL to fetch (configurable in settings)
|
||||
login_subscription_ips — IPs from last fetch (completely replaced each time)
|
||||
login_manual_ips — manually added (never touched by auto-refresh)
|
||||
login_allowed_ips — combined: subscription + manual (used by auth check)
|
||||
login_allowlist_enabled — "true" / "false" master switch
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
logger = logging.getLogger("nexus.ip_allowlist_refresh")
|
||||
|
||||
REFRESH_INTERVAL = 7200 # 2 hours
|
||||
_last_refresh_at: str = ""
|
||||
_last_subscription_count: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RefreshResult:
|
||||
ok: bool
|
||||
subscription_ips: tuple[str, ...] = ()
|
||||
last_refresh: str = ""
|
||||
subscription_count: int = 0
|
||||
error: str | None = None
|
||||
|
||||
|
||||
async def ip_allowlist_refresh_loop():
|
||||
"""Every 2 hours, re-fetch subscription IPs and rebuild combined allowlist."""
|
||||
logger.info("IP allowlist refresh loop started (interval=%sh)", REFRESH_INTERVAL // 3600)
|
||||
# Initial refresh shortly after startup
|
||||
await asyncio.sleep(5)
|
||||
await do_refresh()
|
||||
while True:
|
||||
await asyncio.sleep(REFRESH_INTERVAL)
|
||||
await do_refresh()
|
||||
|
||||
|
||||
async def do_refresh() -> RefreshResult:
|
||||
"""Fetch subscription, replace subscription IPs, rebuild combined list."""
|
||||
return await _do_refresh()
|
||||
|
||||
|
||||
async def _do_refresh() -> RefreshResult:
|
||||
"""Fetch subscription, replace subscription IPs, rebuild combined list."""
|
||||
global _last_refresh_at, _last_subscription_count
|
||||
from server.config import settings
|
||||
from server.infrastructure.subscription_parser import parse_subscription_content
|
||||
|
||||
sub_url = (settings.LOGIN_SUBSCRIPTION_URL or "").strip()
|
||||
if not sub_url:
|
||||
return RefreshResult(ok=True)
|
||||
|
||||
logger.info("Refreshing subscription IPs from: %s", sub_url[:80])
|
||||
try:
|
||||
import httpx
|
||||
async with httpx.AsyncClient(timeout=20.0) as client:
|
||||
resp = await client.get(sub_url, follow_redirects=True)
|
||||
if resp.status_code != 200:
|
||||
logger.warning("Subscription returned HTTP %s", resp.status_code)
|
||||
return RefreshResult(ok=False, error=f"订阅链接返回 HTTP {resp.status_code}")
|
||||
new_sub_ips = parse_subscription_content(resp.text)
|
||||
except Exception as e:
|
||||
logger.error("Subscription fetch failed: %s", e)
|
||||
return RefreshResult(ok=False, error=f"拉取订阅失败: {e}")
|
||||
|
||||
# Manual IPs (always preserved)
|
||||
manual_raw = (settings.LOGIN_MANUAL_IPS or "").strip()
|
||||
manual_ips = [ip.strip() for ip in manual_raw.split(",") if ip.strip()]
|
||||
|
||||
# Combined list: subscription first, then manual (dedup)
|
||||
seen: set[str] = set()
|
||||
combined: list[str] = []
|
||||
for ip in new_sub_ips + manual_ips:
|
||||
if ip not in seen:
|
||||
seen.add(ip)
|
||||
combined.append(ip)
|
||||
|
||||
sub_val = ",".join(new_sub_ips)
|
||||
combined_val = ",".join(combined)
|
||||
|
||||
try:
|
||||
from server.infrastructure.database.session import AsyncSessionLocal
|
||||
from server.infrastructure.database.setting_repo import SettingRepositoryImpl
|
||||
from server.infrastructure.settings_broadcast import publish_setting_changes
|
||||
|
||||
from server.utils.display_time import format_beijing
|
||||
|
||||
refresh_at = format_beijing(with_label=False)
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
repo = SettingRepositoryImpl(session)
|
||||
await repo.set("login_subscription_ips", sub_val)
|
||||
await repo.set("login_allowed_ips", combined_val)
|
||||
await repo.set("login_subscription_last_refresh", refresh_at)
|
||||
|
||||
settings.LOGIN_SUBSCRIPTION_IPS = sub_val
|
||||
settings.LOGIN_ALLOWED_IPS = combined_val
|
||||
settings.LOGIN_SUBSCRIPTION_LAST_REFRESH = refresh_at
|
||||
|
||||
await publish_setting_changes({
|
||||
"login_subscription_ips": sub_val,
|
||||
"login_allowed_ips": combined_val,
|
||||
"login_subscription_last_refresh": refresh_at,
|
||||
})
|
||||
|
||||
_last_refresh_at = refresh_at
|
||||
_last_subscription_count = len(new_sub_ips)
|
||||
logger.info(
|
||||
"Subscription IPs refreshed: %d → combined %d (+ %d manual)",
|
||||
len(new_sub_ips), len(combined), len(manual_ips),
|
||||
)
|
||||
return RefreshResult(
|
||||
ok=True,
|
||||
subscription_ips=tuple(new_sub_ips),
|
||||
last_refresh=_last_refresh_at,
|
||||
subscription_count=len(new_sub_ips),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Failed to save subscription IPs: %s", e)
|
||||
return RefreshResult(ok=False, error=f"保存订阅 IP 失败: {e}")
|
||||
|
||||
|
||||
def get_last_refresh_time() -> str:
|
||||
if _last_refresh_at:
|
||||
return _last_refresh_at
|
||||
from server.config import settings
|
||||
return (getattr(settings, "LOGIN_SUBSCRIPTION_LAST_REFRESH", "") or "").strip()
|
||||
|
||||
|
||||
def get_subscription_count() -> int:
|
||||
return _last_subscription_count
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Nightly cron: auto-detect site domain for IP-only servers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from server.application.services.ip_domain_detect_schedule import (
|
||||
_cron_expr,
|
||||
_enabled,
|
||||
run_scheduled_ip_domain_detect,
|
||||
)
|
||||
from server.background.schedule_runner import _cron_match
|
||||
|
||||
logger = logging.getLogger("nexus.ip_domain_detect_loop")
|
||||
|
||||
CHECK_INTERVAL_SECONDS = 60
|
||||
|
||||
|
||||
async def ip_domain_detect_loop() -> None:
|
||||
"""Primary worker: fire scheduled detect-domain when cron matches (Beijing wall clock)."""
|
||||
while True:
|
||||
try:
|
||||
if _enabled():
|
||||
now = datetime.now(timezone.utc)
|
||||
if _cron_match(_cron_expr(), now):
|
||||
result = await run_scheduled_ip_domain_detect()
|
||||
if result.get("action") == "started":
|
||||
logger.info(
|
||||
"IP domain detect schedule: job_id=%s servers=%s",
|
||||
result.get("job_id"),
|
||||
result.get("server_count"),
|
||||
)
|
||||
elif result.get("reason") not in ("already_ran_today",):
|
||||
logger.debug("IP domain detect schedule: %s", result)
|
||||
await asyncio.sleep(CHECK_INTERVAL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("IP domain detect loop error")
|
||||
await asyncio.sleep(CHECK_INTERVAL_SECONDS)
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Daily cron: SSH preflight on offline servers; optional Telegram offline report."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from server.application.services.offline_daily_report_schedule import (
|
||||
_cron_expr,
|
||||
_enabled,
|
||||
_health_check_enabled,
|
||||
run_scheduled_offline_daily_report,
|
||||
run_scheduled_offline_preflight,
|
||||
)
|
||||
from server.background.schedule_runner import _cron_match
|
||||
|
||||
logger = logging.getLogger("nexus.offline_daily_report_loop")
|
||||
|
||||
CHECK_INTERVAL_SECONDS = 60
|
||||
|
||||
|
||||
async def offline_daily_report_loop() -> None:
|
||||
"""Primary worker: SSH preflight and/or Telegram report when cron matches (Beijing)."""
|
||||
while True:
|
||||
try:
|
||||
now = datetime.now(timezone.utc)
|
||||
if _cron_match(_cron_expr(), now):
|
||||
preflight_stats: dict[str, int] | None = None
|
||||
if _health_check_enabled():
|
||||
preflight_result = await run_scheduled_offline_preflight()
|
||||
if preflight_result.get("action") == "checked":
|
||||
preflight_stats = preflight_result.get("preflight")
|
||||
logger.info(
|
||||
"Offline SSH preflight: checked=%s online=%s offline=%s",
|
||||
preflight_stats.get("checked") if preflight_stats else 0,
|
||||
preflight_stats.get("online") if preflight_stats else 0,
|
||||
preflight_stats.get("offline") if preflight_stats else 0,
|
||||
)
|
||||
elif preflight_result.get("reason") not in ("already_ran_today",):
|
||||
logger.debug("Offline SSH preflight: %s", preflight_result)
|
||||
|
||||
if _enabled():
|
||||
result = await run_scheduled_offline_daily_report(
|
||||
preflight=preflight_stats,
|
||||
)
|
||||
if result.get("action") == "sent":
|
||||
logger.info(
|
||||
"Offline daily report: monitored=%s offline=%s",
|
||||
result.get("monitored"),
|
||||
result.get("offline"),
|
||||
)
|
||||
elif result.get("reason") not in ("already_ran_today",):
|
||||
logger.debug("Offline daily report: %s", result)
|
||||
|
||||
await asyncio.sleep(CHECK_INTERVAL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Offline daily report loop error")
|
||||
await asyncio.sleep(CHECK_INTERVAL_SECONDS)
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Nexus — Retry Runner Background Task
|
||||
Every 5 minutes: check pending PushRetryJobs, retry failed pushes with exponential backoff.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
from server.infrastructure.database.session import AsyncSessionLocal
|
||||
from server.domain.models import PushRetryJob
|
||||
|
||||
logger = logging.getLogger("nexus.retry_runner")
|
||||
|
||||
RETRY_CHECK_INTERVAL = 300 # 5 minutes
|
||||
RETRY_BACKOFF_BASE = 60 # seconds — doubles each retry
|
||||
|
||||
|
||||
def _backoff_seconds(retry_count: int) -> int:
|
||||
return RETRY_BACKOFF_BASE * (2 ** min(max(retry_count - 1, 0), 6))
|
||||
|
||||
|
||||
def apply_push_retry_outcome(
|
||||
job: PushRetryJob,
|
||||
sync_result: dict,
|
||||
*,
|
||||
now: datetime,
|
||||
) -> None:
|
||||
"""Update retry job fields from a sync_files() result (retry_count already incremented)."""
|
||||
sync_error = sync_result.get("error")
|
||||
sync_total = int(sync_result.get("total") or 0)
|
||||
sync_completed = int(sync_result.get("completed") or 0)
|
||||
sync_failed = int(sync_result.get("failed") or 0)
|
||||
sync_cancelled = int(sync_result.get("cancelled") or 0)
|
||||
|
||||
if sync_error or sync_total == 0:
|
||||
job.last_error = (sync_error or "推送源不可用或无目标服务器")[:500]
|
||||
if job.retry_count >= job.max_retries:
|
||||
job.status = "failed"
|
||||
else:
|
||||
job.status = "pending"
|
||||
job.next_retry_at = now + timedelta(seconds=_backoff_seconds(job.retry_count))
|
||||
return
|
||||
|
||||
if sync_failed == 0 and sync_completed > 0:
|
||||
job.status = "completed"
|
||||
return
|
||||
|
||||
if sync_cancelled > 0 and sync_failed == 0 and sync_completed == 0:
|
||||
job.status = "failed"
|
||||
job.last_error = "推送已取消"
|
||||
return
|
||||
|
||||
if job.retry_count >= job.max_retries:
|
||||
job.status = "failed"
|
||||
job.last_error = f"Retry exhausted after {job.retry_count} attempts"
|
||||
return
|
||||
|
||||
job.status = "pending"
|
||||
job.next_retry_at = now + timedelta(seconds=_backoff_seconds(job.retry_count))
|
||||
job.last_error = f"Retry {job.retry_count} failed"
|
||||
|
||||
|
||||
async def _claim_next_retry_job(session, now: datetime) -> PushRetryJob | None:
|
||||
"""Claim one due retry job with row lock (SKIP LOCKED)."""
|
||||
from sqlalchemy import select
|
||||
|
||||
result = await session.execute(
|
||||
select(PushRetryJob)
|
||||
.where(
|
||||
PushRetryJob.status == "pending",
|
||||
PushRetryJob.retry_count < PushRetryJob.max_retries,
|
||||
PushRetryJob.next_retry_at <= now,
|
||||
)
|
||||
.limit(1)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
job = result.scalar_one_or_none()
|
||||
if not job:
|
||||
return None
|
||||
job.status = "running"
|
||||
await session.commit()
|
||||
return job
|
||||
|
||||
|
||||
async def retry_runner_loop():
|
||||
"""Background task: every 5min, retry pending retry jobs that are due."""
|
||||
logger.info("Retry runner loop started (interval: 5min)")
|
||||
while True:
|
||||
await asyncio.sleep(RETRY_CHECK_INTERVAL)
|
||||
|
||||
try:
|
||||
async with AsyncSessionLocal() as session:
|
||||
now = datetime.now(timezone.utc)
|
||||
retried = 0
|
||||
|
||||
while True:
|
||||
job = await _claim_next_retry_job(session, now)
|
||||
if not job:
|
||||
break
|
||||
|
||||
job_id = job.id
|
||||
try:
|
||||
from server.application.services.sync_engine_v2 import SyncEngineV2
|
||||
from server.infrastructure.database.server_repo import ServerRepositoryImpl
|
||||
from server.infrastructure.database.sync_log_repo import SyncLogRepositoryImpl
|
||||
from server.infrastructure.database.audit_log_repo import AuditLogRepositoryImpl
|
||||
from server.infrastructure.database.push_schedule_repo import PushRetryJobRepositoryImpl
|
||||
|
||||
engine = SyncEngineV2(
|
||||
server_repo=ServerRepositoryImpl(session),
|
||||
sync_log_repo=SyncLogRepositoryImpl(session),
|
||||
audit_repo=AuditLogRepositoryImpl(session),
|
||||
retry_repo=PushRetryJobRepositoryImpl(session),
|
||||
)
|
||||
|
||||
sync_result = await engine.sync_files(
|
||||
server_ids=[job.server_id],
|
||||
source_path=job.source_path,
|
||||
target_path=job.target_path,
|
||||
trigger_type="retry",
|
||||
operator=job.operator or "retry_runner",
|
||||
)
|
||||
|
||||
job.retry_count += 1
|
||||
apply_push_retry_outcome(job, sync_result, now=now)
|
||||
|
||||
if job.status == "completed":
|
||||
logger.info(
|
||||
f"Retry job {job_id}: success after {job.retry_count} attempts"
|
||||
)
|
||||
elif job.status == "failed":
|
||||
logger.warning(
|
||||
f"Retry job {job_id}: failed — {job.last_error}"
|
||||
)
|
||||
else:
|
||||
backoff = int((job.next_retry_at - now).total_seconds())
|
||||
logger.warning(
|
||||
f"Retry job {job_id}: attempt {job.retry_count} pending, "
|
||||
f"next retry in {backoff}s — {job.last_error}"
|
||||
)
|
||||
|
||||
await session.commit()
|
||||
retried += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Retry job {job_id} execution error: {e}")
|
||||
await session.rollback()
|
||||
job = await session.get(PushRetryJob, job_id)
|
||||
if not job:
|
||||
continue
|
||||
job.retry_count += 1
|
||||
job.last_error = str(e)[:500]
|
||||
if job.retry_count >= job.max_retries:
|
||||
job.status = "failed"
|
||||
logger.warning(
|
||||
f"Retry job {job_id}: max retries reached (exception), marking failed"
|
||||
)
|
||||
else:
|
||||
job.status = "pending"
|
||||
job.next_retry_at = now + timedelta(
|
||||
seconds=_backoff_seconds(job.retry_count)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
if retried:
|
||||
logger.info(f"Retry runner: {retried} jobs processed")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Retry runner check failed: {e}")
|
||||
@@ -0,0 +1,292 @@
|
||||
"""Nexus — Schedule Runner Background Task
|
||||
Every 60 seconds: check PushSchedule cron expressions, trigger due schedules.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from server.infrastructure.database.session import AsyncSessionLocal
|
||||
from server.domain.models import PushSchedule
|
||||
from server.utils.display_time import ensure_utc, operator_wall, to_beijing, to_naive_utc
|
||||
|
||||
logger = logging.getLogger("nexus.schedule_runner")
|
||||
|
||||
SCHEDULE_CHECK_INTERVAL = 60 # seconds
|
||||
|
||||
|
||||
def _cron_match(cron_expr: str, at_utc: datetime) -> bool:
|
||||
"""Match 5-field cron against operator (Beijing) wall clock at UTC instant *at_utc*."""
|
||||
parts = cron_expr.strip().split()
|
||||
if len(parts) != 5:
|
||||
return False
|
||||
|
||||
minute, hour, day, month, dow = operator_wall(at_utc)
|
||||
fields = [minute, hour, day, month, dow]
|
||||
|
||||
for current_val, field_expr in zip(fields, parts):
|
||||
if not _field_match(field_expr, current_val):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def rollback_schedule_after_failed_run(schedule, prev_last_run_at, run_mode: str) -> None:
|
||||
"""Restore schedule so cron/once can fire again after execution failure."""
|
||||
schedule.last_run_at = prev_last_run_at
|
||||
if run_mode == "once":
|
||||
schedule.enabled = True
|
||||
|
||||
|
||||
def _field_match(expr: str, value: int) -> bool:
|
||||
"""Match a single cron field expression against a value.
|
||||
|
||||
Supports: * (any), */N (step), specific values, comma-separated lists,
|
||||
and range (N-M). Each comma-separated part is tried independently;
|
||||
the function returns True as soon as any part matches.
|
||||
"""
|
||||
for part in expr.split(","):
|
||||
if part == "*":
|
||||
return True
|
||||
if part.startswith("*/"):
|
||||
step = int(part[2:])
|
||||
if step == 0:
|
||||
continue # */0 is invalid; skip rather than ZeroDivisionError
|
||||
if value % step == 0:
|
||||
return True
|
||||
continue
|
||||
if "-" in part:
|
||||
low, high = part.split("-", 1)
|
||||
if int(low) <= value <= int(high):
|
||||
return True
|
||||
else:
|
||||
# cron: 7 also means Sunday (=0)
|
||||
if value == int(part) or (int(part) == 7 and value == 0):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _compute_next_cron_run(cron_expr: str, after_utc: datetime) -> datetime | None:
|
||||
"""Return next UTC naive instant strictly after *after_utc* matching *cron_expr* (Beijing wall)."""
|
||||
after = ensure_utc(after_utc)
|
||||
candidate_bj = to_beijing(after).replace(second=0, microsecond=0) + timedelta(minutes=1)
|
||||
limit_bj = to_beijing(after) + timedelta(days=366)
|
||||
while candidate_bj <= limit_bj:
|
||||
at_utc = candidate_bj.astimezone(timezone.utc)
|
||||
if _cron_match(cron_expr, at_utc):
|
||||
return to_naive_utc(at_utc)
|
||||
candidate_bj += timedelta(minutes=1)
|
||||
return None
|
||||
|
||||
|
||||
def compute_schedule_next_run(schedule: PushSchedule, now: datetime | None = None) -> datetime | None:
|
||||
"""Compute next scheduled run time for API display."""
|
||||
if not schedule.enabled:
|
||||
return None
|
||||
now = now or datetime.now(timezone.utc)
|
||||
run_mode = getattr(schedule, "run_mode", None) or "cron"
|
||||
|
||||
if run_mode == "once":
|
||||
fire_at = getattr(schedule, "fire_at", None)
|
||||
if not fire_at or schedule.last_run_at:
|
||||
return None
|
||||
now_naive = now.replace(tzinfo=None)
|
||||
fire_naive = fire_at.replace(tzinfo=None) if fire_at.tzinfo else fire_at
|
||||
if fire_naive <= now_naive:
|
||||
return None
|
||||
return fire_at
|
||||
|
||||
cron_expr = schedule.cron_expr
|
||||
if not cron_expr:
|
||||
return None
|
||||
return _compute_next_cron_run(cron_expr.strip(), now)
|
||||
|
||||
|
||||
def _schedule_is_due(schedule: PushSchedule, now: datetime) -> bool:
|
||||
"""Return True if *schedule* should fire at *now* (cron or once)."""
|
||||
run_mode = getattr(schedule, "run_mode", None) or "cron"
|
||||
|
||||
if run_mode == "once":
|
||||
fire_at = getattr(schedule, "fire_at", None)
|
||||
if not fire_at:
|
||||
return False
|
||||
now_naive = now.replace(tzinfo=None)
|
||||
fire_naive = fire_at.replace(tzinfo=None) if fire_at.tzinfo else fire_at
|
||||
if fire_naive > now_naive:
|
||||
return False
|
||||
if schedule.last_run_at:
|
||||
return False
|
||||
return True
|
||||
|
||||
cron_expr = getattr(schedule, "cron_expr", None)
|
||||
if not cron_expr or not _cron_match(cron_expr, ensure_utc(now)):
|
||||
return False
|
||||
if schedule.last_run_at:
|
||||
last_run = schedule.last_run_at
|
||||
now_naive = now.replace(tzinfo=None)
|
||||
if last_run.tzinfo is not None:
|
||||
last_run = last_run.replace(tzinfo=None)
|
||||
seconds_since = (now_naive - last_run).total_seconds()
|
||||
if seconds_since < SCHEDULE_CHECK_INTERVAL:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
async def schedule_runner_loop():
|
||||
"""Background task: check schedules every 60s, trigger due cron jobs."""
|
||||
logger.info("Schedule runner loop started (interval: 60s)")
|
||||
while True:
|
||||
await asyncio.sleep(SCHEDULE_CHECK_INTERVAL)
|
||||
|
||||
try:
|
||||
async with AsyncSessionLocal() as session:
|
||||
from sqlalchemy import select
|
||||
id_result = await session.execute(
|
||||
select(PushSchedule.id).where(PushSchedule.enabled == True)
|
||||
)
|
||||
schedule_ids = list(id_result.scalars().all())
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
triggered = 0
|
||||
|
||||
for sched_id in schedule_ids:
|
||||
try:
|
||||
lock_result = await session.execute(
|
||||
select(PushSchedule)
|
||||
.where(
|
||||
PushSchedule.id == sched_id,
|
||||
PushSchedule.enabled == True,
|
||||
)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
schedule = lock_result.scalar_one_or_none()
|
||||
if not schedule:
|
||||
continue
|
||||
|
||||
if not _schedule_is_due(schedule, now):
|
||||
await session.rollback()
|
||||
continue
|
||||
|
||||
run_mode = getattr(schedule, "run_mode", None) or "cron"
|
||||
|
||||
# Parse server_ids from JSON
|
||||
try:
|
||||
server_ids = json.loads(schedule.server_ids)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
logger.error(f"Schedule {schedule.id}: invalid server_ids JSON")
|
||||
await session.rollback()
|
||||
continue
|
||||
|
||||
if not server_ids:
|
||||
await session.rollback()
|
||||
continue
|
||||
|
||||
# Mark as running BEFORE execution to prevent double-trigger
|
||||
prev_last_run_at = schedule.last_run_at
|
||||
schedule.last_run_at = to_naive_utc(now)
|
||||
if run_mode == "once":
|
||||
schedule.enabled = False
|
||||
await session.commit()
|
||||
|
||||
schedule_type = getattr(schedule, "schedule_type", "push") or "push"
|
||||
execution_ok = True
|
||||
|
||||
try:
|
||||
if schedule_type == "script":
|
||||
# ── Script execution schedule ──
|
||||
from server.application.services.script_service import ScriptService
|
||||
from server.infrastructure.database.script_repo import (
|
||||
ScriptRepositoryImpl, ScriptExecutionRepositoryImpl,
|
||||
)
|
||||
from server.infrastructure.database.db_credential_repo import DbCredentialRepositoryImpl
|
||||
from server.infrastructure.database.server_repo import ServerRepositoryImpl
|
||||
from server.infrastructure.database.audit_log_repo import AuditLogRepositoryImpl
|
||||
|
||||
cmd = getattr(schedule, 'script_content', None) or ''
|
||||
script_id = getattr(schedule, 'script_id', None)
|
||||
|
||||
# If no inline command, load from script library
|
||||
if not cmd and script_id:
|
||||
script = await ScriptRepositoryImpl(session).get_by_id(script_id)
|
||||
if script:
|
||||
cmd = script.content or ''
|
||||
|
||||
if not cmd:
|
||||
logger.error(f"Schedule '{schedule.name}': no command or script content, skipping")
|
||||
execution_ok = False
|
||||
else:
|
||||
svc = ScriptService(
|
||||
script_repo=ScriptRepositoryImpl(session),
|
||||
execution_repo=ScriptExecutionRepositoryImpl(session),
|
||||
credential_repo=DbCredentialRepositoryImpl(session),
|
||||
server_repo=ServerRepositoryImpl(session),
|
||||
audit_repo=AuditLogRepositoryImpl(session),
|
||||
)
|
||||
timeout = getattr(schedule, 'exec_timeout', None) or 60
|
||||
long_task = bool(getattr(schedule, 'long_task', False))
|
||||
execution = await svc.execute_command(
|
||||
command=cmd,
|
||||
server_ids=server_ids,
|
||||
script_id=script_id,
|
||||
timeout=timeout,
|
||||
operator=f"schedule:{schedule.name}",
|
||||
long_task=long_task,
|
||||
)
|
||||
logger.info(
|
||||
f"Schedule '{schedule.name}' (script) triggered: "
|
||||
f"execution_id={execution.id} on {len(server_ids)} servers"
|
||||
)
|
||||
else:
|
||||
# ── File push schedule (original behaviour) ──
|
||||
from server.application.services.sync_engine_v2 import SyncEngineV2
|
||||
from server.infrastructure.database.server_repo import ServerRepositoryImpl
|
||||
from server.infrastructure.database.sync_log_repo import SyncLogRepositoryImpl
|
||||
from server.infrastructure.database.audit_log_repo import AuditLogRepositoryImpl
|
||||
from server.infrastructure.database.push_schedule_repo import PushRetryJobRepositoryImpl
|
||||
|
||||
engine = SyncEngineV2(
|
||||
server_repo=ServerRepositoryImpl(session),
|
||||
sync_log_repo=SyncLogRepositoryImpl(session),
|
||||
audit_repo=AuditLogRepositoryImpl(session),
|
||||
retry_repo=PushRetryJobRepositoryImpl(session),
|
||||
)
|
||||
|
||||
result = await engine.sync_files(
|
||||
server_ids=server_ids,
|
||||
source_path=schedule.source_path or '/tmp',
|
||||
target_path=getattr(schedule, "target_path", None) or None,
|
||||
sync_mode=getattr(schedule, "sync_mode", None) or "incremental",
|
||||
trigger_type="schedule",
|
||||
operator=f"schedule:{schedule.name}",
|
||||
)
|
||||
logger.info(
|
||||
f"Schedule '{schedule.name}' (push) triggered: "
|
||||
f"{result['completed']}/{result['total']} succeeded"
|
||||
)
|
||||
if result.get("error"):
|
||||
execution_ok = False
|
||||
elif result.get("total", 0) > 0 and result.get("completed", 0) == 0:
|
||||
execution_ok = False
|
||||
except Exception as exec_err:
|
||||
logger.error(f"Schedule {schedule.id} execution failed: {exec_err}")
|
||||
execution_ok = False
|
||||
|
||||
if not execution_ok:
|
||||
fresh = await session.get(PushSchedule, sched_id)
|
||||
if fresh:
|
||||
rollback_schedule_after_failed_run(
|
||||
fresh, prev_last_run_at, run_mode
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
triggered += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Schedule {sched_id} setup/parse failed: {e}")
|
||||
await session.rollback()
|
||||
|
||||
if triggered:
|
||||
logger.info(f"Schedule runner: {triggered} schedules triggered")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Schedule runner check failed: {e}")
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Batch-flush script execution live state from Redis to MySQL (every 60s, up to 50 rows/cycle)."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from server.domain.models import AuditLog
|
||||
from server.infrastructure.database.audit_log_repo import AuditLogRepositoryImpl
|
||||
from server.infrastructure.database.script_repo import ScriptExecutionRepositoryImpl
|
||||
from server.infrastructure.database.session import AsyncSessionLocal
|
||||
from server.infrastructure.redis.script_execution_store import (
|
||||
FLUSH_INTERVAL_SECONDS,
|
||||
detect_stuck_executions,
|
||||
flush_pending_batch,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("nexus.script_execution_flush")
|
||||
|
||||
|
||||
async def script_execution_flush_loop():
|
||||
"""Every minute: flush queued script execution snapshots to MySQL."""
|
||||
logger.info("Script execution flush loop started (interval=%ss)", FLUSH_INTERVAL_SECONDS)
|
||||
while True:
|
||||
await asyncio.sleep(FLUSH_INTERVAL_SECONDS)
|
||||
try:
|
||||
async with AsyncSessionLocal() as session:
|
||||
repo = ScriptExecutionRepositoryImpl(session)
|
||||
audit_repo = AuditLogRepositoryImpl(session)
|
||||
|
||||
async def _audit(action, target_type, target_id, operator, _repo=audit_repo): # noqa: B023
|
||||
await _repo.create(AuditLog(
|
||||
admin_username=operator or "system",
|
||||
action=action,
|
||||
target_type=target_type,
|
||||
target_id=target_id,
|
||||
detail=f"执行ID={target_id}",
|
||||
))
|
||||
|
||||
stuck = await detect_stuck_executions(repo, _audit)
|
||||
count = await flush_pending_batch(repo)
|
||||
if stuck:
|
||||
logger.info("Script execution: auto-marked %s stuck", stuck)
|
||||
if count:
|
||||
logger.info("Script execution flush: %s record(s) written to MySQL", count)
|
||||
except Exception as e:
|
||||
logger.error("Script execution flush loop error: %s", e)
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Nexus — Python Self-Monitor Background Task
|
||||
Every 30 seconds: check Redis, MySQL, WebSocket connections.
|
||||
If critical service fails → send Telegram system alert.
|
||||
If service recovers → send Telegram recovery notification.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from server.infrastructure.database.session import AsyncSessionLocal
|
||||
from server.infrastructure.redis.client import get_redis
|
||||
from server.infrastructure.telegram import send_telegram_system_alert
|
||||
from server.api.websocket import manager as ws_manager, sync_manager as ws_sync_manager
|
||||
from server.config import settings
|
||||
|
||||
logger = logging.getLogger("nexus.self_monitor")
|
||||
|
||||
MONITOR_INTERVAL = 30 # seconds
|
||||
_TELEGRAM_COOLDOWN = 300 # 5 minutes — same alert won't re-push Telegram
|
||||
|
||||
# Track previous state for recovery detection
|
||||
_prev_redis_ok = True
|
||||
_prev_mysql_ok = True
|
||||
# Track last Telegram push time per alert key (prevent spam)
|
||||
_last_telegram_time: dict = {}
|
||||
|
||||
|
||||
async def self_monitor_loop():
|
||||
"""Background task: every 30s, self-check critical services.
|
||||
|
||||
Checks:
|
||||
1. Redis connection — if lost, send Telegram warning; if recovered, send recovery
|
||||
2. MySQL connection — if lost, send Telegram warning; if recovered, send recovery
|
||||
3. WebSocket client count (via ConnectionManager)
|
||||
"""
|
||||
global _prev_redis_ok, _prev_mysql_ok
|
||||
|
||||
def _should_send(key: str) -> bool:
|
||||
"""Rate-limit Telegram pushes to 1 per 5 minutes per alert key."""
|
||||
now = time.monotonic()
|
||||
last = _last_telegram_time.get(key, 0)
|
||||
if now - last < _TELEGRAM_COOLDOWN:
|
||||
return False
|
||||
_last_telegram_time[key] = now
|
||||
return True
|
||||
|
||||
logger.info("Self-monitor loop started (interval: 30s)")
|
||||
while True:
|
||||
await asyncio.sleep(MONITOR_INTERVAL)
|
||||
|
||||
# ── Redis check ──
|
||||
redis_ok = True
|
||||
try:
|
||||
redis = get_redis()
|
||||
await redis.ping()
|
||||
except Exception:
|
||||
redis_ok = False
|
||||
logger.warning("Redis health check failed", exc_info=True)
|
||||
try:
|
||||
if settings.REDIS_URL and _should_send("NOTIFY_SYSTEM_REDIS"):
|
||||
await send_telegram_system_alert(
|
||||
summary="Redis 连接失败，无法 ping 通",
|
||||
title="Redis 连接异常",
|
||||
notify_key="NOTIFY_SYSTEM_REDIS",
|
||||
icon="🔴",
|
||||
details=[
|
||||
"检测来源：Layer 2 self_monitor（30 秒周期）",
|
||||
"影响范围：子机实时心跳、在线状态、Redis 缓存读写可能中断",
|
||||
"数据后果：前端仪表盘实时数据可能停滞，历史数据仍可从 MySQL 读取",
|
||||
],
|
||||
action="请检查 1Panel Redis 容器状态与 NEXUS_REDIS_URL 配置",
|
||||
)
|
||||
except Exception:
|
||||
logger.error("Failed to send Redis alert via Telegram")
|
||||
|
||||
# Redis recovery notification
|
||||
if redis_ok and not _prev_redis_ok:
|
||||
try:
|
||||
await send_telegram_system_alert(
|
||||
summary="Redis 连接已恢复，ping 检测通过",
|
||||
title="Redis 连接恢复",
|
||||
notify_key="NOTIFY_SYSTEM_REDIS",
|
||||
icon="🟢",
|
||||
details=[
|
||||
"检测来源：Layer 2 self_monitor（30 秒周期）",
|
||||
"当前状态：Redis 可正常读写，实时心跳应恢复更新",
|
||||
],
|
||||
)
|
||||
_last_telegram_time.pop("NOTIFY_SYSTEM_REDIS", None) # clear cooldown
|
||||
except Exception:
|
||||
logger.error("Failed to send Redis recovery via Telegram")
|
||||
logger.info("Redis connection recovered")
|
||||
_prev_redis_ok = redis_ok
|
||||
|
||||
# ── MySQL check ──
|
||||
mysql_ok = True
|
||||
try:
|
||||
async with AsyncSessionLocal() as session:
|
||||
await session.execute(text("SELECT 1"))
|
||||
except Exception:
|
||||
mysql_ok = False
|
||||
logger.warning("MySQL health check failed", exc_info=True)
|
||||
try:
|
||||
if _should_send("NOTIFY_SYSTEM_MYSQL"):
|
||||
await send_telegram_system_alert(
|
||||
summary="MySQL 连接失败，SELECT 1 探测未通过",
|
||||
title="MySQL 数据库异常",
|
||||
notify_key="NOTIFY_SYSTEM_MYSQL",
|
||||
icon="🔴",
|
||||
details=[
|
||||
"检测来源：Layer 2 self_monitor（30 秒周期）",
|
||||
"影响范围：设置、子机列表、审计、推送任务等所有 DB 读写",
|
||||
"严重程度：平台核心功能可能不可用",
|
||||
],
|
||||
action="请检查 1Panel MySQL 容器、磁盘空间与 DATABASE_URL 配置",
|
||||
)
|
||||
except Exception:
|
||||
logger.error("Failed to send MySQL alert via Telegram", exc_info=True)
|
||||
|
||||
# MySQL recovery notification
|
||||
if mysql_ok and not _prev_mysql_ok:
|
||||
try:
|
||||
await send_telegram_system_alert(
|
||||
summary="MySQL 连接已恢复，SELECT 1 探测通过",
|
||||
title="MySQL 数据库恢复",
|
||||
notify_key="NOTIFY_SYSTEM_MYSQL",
|
||||
icon="🟢",
|
||||
details=[
|
||||
"检测来源：Layer 2 self_monitor（30 秒周期）",
|
||||
"当前状态：数据库读写应已恢复正常",
|
||||
],
|
||||
)
|
||||
_last_telegram_time.pop("NOTIFY_SYSTEM_MYSQL", None) # clear cooldown
|
||||
except Exception:
|
||||
logger.error("Failed to send MySQL recovery via Telegram")
|
||||
logger.info("MySQL connection recovered")
|
||||
_prev_mysql_ok = mysql_ok
|
||||
|
||||
# ── Log status ──
|
||||
logger.debug(
|
||||
f"Self-monitor: redis={'ok' if redis_ok else 'error'}, "
|
||||
f"mysql={'ok' if mysql_ok else 'error'}, "
|
||||
f"ws_alert_clients={ws_manager.client_count}, "
|
||||
f"ws_sync_clients={ws_sync_manager.client_count}"
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Background reconciliation for server batch jobs — stuck detection (60s interval)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from server.application.services.server_batch_service import detect_stuck_batch_jobs
|
||||
from server.infrastructure.redis.server_batch_store import RECONCILE_INTERVAL_SECONDS
|
||||
|
||||
logger = logging.getLogger("nexus.server_batch_reconcile")
|
||||
|
||||
|
||||
async def server_batch_reconcile_loop() -> None:
|
||||
"""Every minute: auto-finalize batch jobs with no Redis progress for 1h."""
|
||||
logger.info(
|
||||
"Server batch reconcile loop started (interval=%ss)",
|
||||
RECONCILE_INTERVAL_SECONDS,
|
||||
)
|
||||
while True:
|
||||
await asyncio.sleep(RECONCILE_INTERVAL_SECONDS)
|
||||
try:
|
||||
stuck = await detect_stuck_batch_jobs()
|
||||
if stuck:
|
||||
logger.info("Server batch reconcile: %s stuck job(s) finalized", stuck)
|
||||
except Exception as e:
|
||||
logger.error("Server batch reconcile loop error: %s", e)
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Nexus — Server offline alert background task.
|
||||
|
||||
Every 30 seconds: detect online→offline transitions among Agent-monitored servers
|
||||
and push one Telegram alert per transition (edge-triggered).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from server.application.server_batch_common import batch_server_display_name
|
||||
from server.application.server_connectivity import (
|
||||
REDIS_KEY_PREFIX,
|
||||
heartbeat_indicates_online,
|
||||
monitored_server_filter,
|
||||
)
|
||||
from server.domain.models import Server
|
||||
from server.infrastructure.database.session import AsyncSessionLocal
|
||||
from server.infrastructure.redis.client import get_redis
|
||||
|
||||
logger = logging.getLogger("nexus.server_offline_monitor")
|
||||
|
||||
MONITOR_INTERVAL = 30 # seconds — align with self_monitor
|
||||
_BATCH_SIZE = 200
|
||||
|
||||
# server_id → last known online (None = cold start, no alert)
|
||||
_prev_online: dict[int, bool] = {}
|
||||
|
||||
|
||||
async def _load_monitored_servers() -> list[tuple[int, str]]:
|
||||
async with AsyncSessionLocal() as session:
|
||||
result = await session.execute(
|
||||
select(Server.id, Server.name, Server.domain, Server.description).where(
|
||||
monitored_server_filter()
|
||||
)
|
||||
)
|
||||
return [
|
||||
(
|
||||
int(row[0]),
|
||||
batch_server_display_name(
|
||||
Server(id=int(row[0]), name=row[1], domain=row[2], description=row[3])
|
||||
),
|
||||
)
|
||||
for row in result.all()
|
||||
]
|
||||
|
||||
|
||||
async def _check_offline_transitions() -> int:
|
||||
"""Scan monitored servers; return count of new offline alerts fired."""
|
||||
servers = await _load_monitored_servers()
|
||||
if not servers:
|
||||
return 0
|
||||
|
||||
redis = get_redis()
|
||||
alerts_sent = 0
|
||||
|
||||
for i in range(0, len(servers), _BATCH_SIZE):
|
||||
chunk = servers[i : i + _BATCH_SIZE]
|
||||
pipe = redis.pipeline()
|
||||
for server_id, _ in chunk:
|
||||
pipe.hgetall(f"{REDIS_KEY_PREFIX}{server_id}")
|
||||
heartbeats = await pipe.execute()
|
||||
|
||||
for (server_id, server_name), hb in zip(chunk, heartbeats):
|
||||
currently_online = heartbeat_indicates_online(hb)
|
||||
prev_online = _prev_online.get(server_id)
|
||||
|
||||
if prev_online is None:
|
||||
_prev_online[server_id] = currently_online
|
||||
continue
|
||||
|
||||
if prev_online and not currently_online:
|
||||
from server.api.websocket import broadcast_offline_alert
|
||||
|
||||
last_hb = (hb or {}).get("last_heartbeat", "")
|
||||
await broadcast_offline_alert(server_id, server_name, last_hb)
|
||||
alerts_sent += 1
|
||||
logger.warning("Server offline alert: id=%s name=%s", server_id, server_name)
|
||||
|
||||
_prev_online[server_id] = currently_online
|
||||
|
||||
return alerts_sent
|
||||
|
||||
|
||||
async def server_offline_monitor_loop():
|
||||
"""Background task: edge-detect server offline and notify Telegram once per event."""
|
||||
logger.info("Server offline monitor started (interval: %ss)", MONITOR_INTERVAL)
|
||||
while True:
|
||||
await asyncio.sleep(MONITOR_INTERVAL)
|
||||
try:
|
||||
sent = await _check_offline_transitions()
|
||||
if sent:
|
||||
logger.info("Server offline monitor: %s alert(s) sent", sent)
|
||||
except Exception:
|
||||
logger.error("Server offline monitor tick failed", exc_info=True)
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Periodic purge of sync_logs older than SYNC_LOG_RETENTION_DAYS (design: 30 days)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from server.infrastructure.database.session import AsyncSessionLocal
|
||||
from server.infrastructure.database.sync_log_repo import (
|
||||
SYNC_LOG_RETENTION_DAYS,
|
||||
SyncLogRepositoryImpl,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("nexus.sync_log_purge")
|
||||
|
||||
PURGE_INTERVAL_SECONDS = 24 * 3600
|
||||
|
||||
|
||||
async def purge_stale_sync_logs() -> int:
|
||||
"""Delete sync_logs rows older than retention window. Returns rows removed."""
|
||||
async with AsyncSessionLocal() as session:
|
||||
repo = SyncLogRepositoryImpl(session)
|
||||
return await repo.purge_older_than(retention_days=SYNC_LOG_RETENTION_DAYS)
|
||||
|
||||
|
||||
async def sync_log_purge_loop() -> None:
|
||||
"""Run daily on primary worker; also invoked once at API startup."""
|
||||
while True:
|
||||
try:
|
||||
count = await purge_stale_sync_logs()
|
||||
if count:
|
||||
logger.info(
|
||||
"Sync log purge: removed %s row(s) older than %s days",
|
||||
count,
|
||||
SYNC_LOG_RETENTION_DAYS,
|
||||
)
|
||||
await asyncio.sleep(PURGE_INTERVAL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Sync log purge loop error")
|
||||
await asyncio.sleep(PURGE_INTERVAL_SECONDS)
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Background file push — API returns immediately; progress via WebSocket."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
|
||||
logger = logging.getLogger("nexus.sync_push_runner")
|
||||
|
||||
|
||||
async def run_sync_files_background(
|
||||
*,
|
||||
server_ids: List[int],
|
||||
source_path: str,
|
||||
target_path: Optional[str],
|
||||
sync_mode: str,
|
||||
operator: str,
|
||||
batch_size: int,
|
||||
concurrency: int,
|
||||
batch_id: str,
|
||||
) -> None:
|
||||
"""Execute sync_files in an isolated DB session (not request-scoped)."""
|
||||
from server.application.services.sync_engine_v2 import SyncEngineV2
|
||||
from server.infrastructure.database.audit_log_repo import AuditLogRepositoryImpl
|
||||
from server.infrastructure.database.push_schedule_repo import PushRetryJobRepositoryImpl
|
||||
from server.infrastructure.database.server_repo import ServerRepositoryImpl
|
||||
from server.infrastructure.database.session import AsyncSessionLocal
|
||||
from server.infrastructure.database.sync_log_repo import SyncLogRepositoryImpl
|
||||
|
||||
try:
|
||||
async with AsyncSessionLocal() as session:
|
||||
engine = SyncEngineV2(
|
||||
server_repo=ServerRepositoryImpl(session),
|
||||
sync_log_repo=SyncLogRepositoryImpl(session),
|
||||
audit_repo=AuditLogRepositoryImpl(session),
|
||||
retry_repo=PushRetryJobRepositoryImpl(session),
|
||||
)
|
||||
await engine.sync_files(
|
||||
server_ids=server_ids,
|
||||
source_path=source_path,
|
||||
target_path=target_path,
|
||||
sync_mode=sync_mode,
|
||||
operator=operator,
|
||||
batch_size=batch_size,
|
||||
concurrency=concurrency,
|
||||
batch_id=batch_id,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Background sync_files failed: batch_id=%s", batch_id)
|
||||
|
||||
|
||||
def schedule_sync_files(**kwargs) -> None:
|
||||
"""Fire-and-forget push task on the running event loop."""
|
||||
import asyncio
|
||||
|
||||
asyncio.create_task(
|
||||
run_sync_files_background(**kwargs),
|
||||
name=f"sync_push:{kwargs.get('batch_id', '')}",
|
||||
)
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Nightly cron: auto-detect target_path for servers still on BT default."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from server.application.services.unset_path_detect_schedule import (
|
||||
_cron_expr,
|
||||
_enabled,
|
||||
run_scheduled_unset_path_detect,
|
||||
)
|
||||
from server.background.schedule_runner import _cron_match
|
||||
|
||||
logger = logging.getLogger("nexus.unset_path_detect_loop")
|
||||
|
||||
CHECK_INTERVAL_SECONDS = 60
|
||||
|
||||
|
||||
async def unset_path_detect_loop() -> None:
|
||||
"""Primary worker: fire scheduled detect-path when cron matches (Beijing wall clock)."""
|
||||
while True:
|
||||
try:
|
||||
if _enabled():
|
||||
now = datetime.now(timezone.utc)
|
||||
if _cron_match(_cron_expr(), now):
|
||||
result = await run_scheduled_unset_path_detect()
|
||||
if result.get("action") == "started":
|
||||
logger.info(
|
||||
"Unset path detect schedule: job_id=%s servers=%s",
|
||||
result.get("job_id"),
|
||||
result.get("server_count"),
|
||||
)
|
||||
elif result.get("reason") not in ("already_ran_today",):
|
||||
logger.debug("Unset path detect schedule: %s", result)
|
||||
await asyncio.sleep(CHECK_INTERVAL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Unset path detect loop error")
|
||||
await asyncio.sleep(CHECK_INTERVAL_SECONDS)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Periodic cleanup of stale /tmp/nexus_upload_* ZIP extract directories.
|
||||
|
||||
Removes dirs older than MAX_AGE_HOURS that were left behind after failed/cancelled
|
||||
pushes or abandoned uploads. Successful manual pushes are already cleaned in sync_engine_v2.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
import glob as glob_module
|
||||
|
||||
from server.utils.posix_paths import UPLOAD_STAGING_PREFIX
|
||||
|
||||
logger = logging.getLogger("nexus.upload_staging_cleanup")
|
||||
|
||||
LOOP_INTERVAL_SECONDS = 3600
|
||||
MAX_AGE_SECONDS = 24 * 3600
|
||||
|
||||
|
||||
def _cleanup_stale_upload_dirs() -> int:
|
||||
"""Remove upload staging dirs not modified within MAX_AGE_SECONDS. Returns count removed."""
|
||||
now = time.time()
|
||||
removed = 0
|
||||
pattern = f"{UPLOAD_STAGING_PREFIX}*"
|
||||
for path in glob_module.glob(pattern):
|
||||
if not os.path.isdir(path):
|
||||
continue
|
||||
try:
|
||||
real = os.path.realpath(path)
|
||||
if not real.startswith(UPLOAD_STAGING_PREFIX):
|
||||
logger.warning("Skip upload cleanup (path escape): %s", path)
|
||||
continue
|
||||
age = now - os.path.getmtime(real)
|
||||
if age < MAX_AGE_SECONDS:
|
||||
continue
|
||||
shutil.rmtree(real, ignore_errors=True)
|
||||
if not os.path.exists(real):
|
||||
removed += 1
|
||||
logger.info("Removed stale upload staging dir (age %.0fh): %s", age / 3600, real)
|
||||
except OSError as exc:
|
||||
logger.warning("Upload staging cleanup skip %s: %s", path, exc)
|
||||
return removed
|
||||
|
||||
|
||||
async def upload_staging_cleanup_loop() -> None:
|
||||
"""Run hourly on primary worker."""
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(LOOP_INTERVAL_SECONDS)
|
||||
count = await asyncio.to_thread(_cleanup_stale_upload_dirs)
|
||||
if count:
|
||||
logger.info("Upload staging cleanup cycle removed %d director(ies)", count)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("Upload staging cleanup loop error")
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Background watch probe loop — 5s sampling for pinned servers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from server.domain.models import Server
|
||||
from server.infrastructure.database.session import AsyncSessionLocal
|
||||
from server.infrastructure.database.server_repo import ServerRepositoryImpl
|
||||
from server.infrastructure.database.watch_repo import WatchRepositoryImpl
|
||||
from server.infrastructure.redis.client import get_redis
|
||||
from server.infrastructure.ssh.remote_probe import ssh_watch_probe, ssh_watch_processes, ssh_load_avg_probe
|
||||
from server.utils.watch_alerts import process_watch_probe_alerts
|
||||
from server.utils.watch_probe_errors import watch_probe_error_zh
|
||||
from server.utils.watch_state import clear_watch_redis_snapshot
|
||||
from server.utils.watch_metrics import (
|
||||
WATCH_PROBE_INTERVAL_SEC,
|
||||
WATCH_PROBE_TIMEOUT_SEC,
|
||||
WATCH_REDIS_FRESH_SEC,
|
||||
WatchProbeSample,
|
||||
agent_watch_sample_usable,
|
||||
failure_sample,
|
||||
merge_redis_and_ssh,
|
||||
parse_redis_watch_payload,
|
||||
parse_ssh_watch_payload,
|
||||
fill_hardware_gaps,
|
||||
apply_load_avg_to_sample,
|
||||
sample_needs_load_supplement,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("nexus.watch_probe")
|
||||
|
||||
REDIS_HEARTBEAT_PREFIX = "heartbeat:"
|
||||
REDIS_WATCH_LIVE_PREFIX = "watch:live:"
|
||||
REDIS_WATCH_PROC_PREFIX = "watch:proc:"
|
||||
WATCH_LIVE_MAX_POINTS = 360
|
||||
WATCH_RECORDS_RETENTION_DAYS = 7
|
||||
_process_cycle: dict[int, int] = defaultdict(int)
|
||||
|
||||
|
||||
async def _supplement_load_if_needed(server: Server, sample: WatchProbeSample) -> WatchProbeSample:
|
||||
"""Agent heartbeat omits load_avg; lightweight SSH when full probe was skipped."""
|
||||
if not sample_needs_load_supplement(sample):
|
||||
return sample
|
||||
load_result = await ssh_load_avg_probe(server, timeout=5)
|
||||
if load_result.get("ok"):
|
||||
payload = load_result.get("payload") or {}
|
||||
apply_load_avg_to_sample(sample, payload.get("load_avg"))
|
||||
return sample
|
||||
|
||||
|
||||
def _utc_naive_now() -> datetime:
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
|
||||
def _heartbeat_fresh(heartbeat: dict[str, str]) -> bool:
|
||||
if not heartbeat or heartbeat.get("is_online") != "True":
|
||||
return False
|
||||
ts = heartbeat.get("last_heartbeat") or ""
|
||||
if not ts:
|
||||
return False
|
||||
try:
|
||||
agent_ts = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
if agent_ts.tzinfo is None:
|
||||
agent_ts = agent_ts.replace(tzinfo=timezone.utc)
|
||||
age = (datetime.now(timezone.utc) - agent_ts).total_seconds()
|
||||
return age <= WATCH_REDIS_FRESH_SEC
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
async def _append_live_point(redis, server_id: int, point: dict[str, Any]) -> None:
|
||||
key = f"{REDIS_WATCH_LIVE_PREFIX}{server_id}"
|
||||
await redis.lpush(key, json.dumps(point))
|
||||
await redis.ltrim(key, 0, WATCH_LIVE_MAX_POINTS - 1)
|
||||
await redis.expire(key, 3600)
|
||||
|
||||
|
||||
async def _save_processes(redis, server_id: int, processes: list[dict[str, Any]] | None) -> None:
|
||||
if not processes:
|
||||
return
|
||||
await redis.set(f"{REDIS_WATCH_PROC_PREFIX}{server_id}", json.dumps(processes), ex=60)
|
||||
|
||||
|
||||
async def probe_server_metrics(
|
||||
server: Server,
|
||||
heartbeat: dict[str, str] | None,
|
||||
*,
|
||||
fetch_processes: bool = False,
|
||||
) -> WatchProbeSample:
|
||||
redis_sample: WatchProbeSample | None = None
|
||||
if heartbeat and _heartbeat_fresh(heartbeat):
|
||||
redis_sample = parse_redis_watch_payload(heartbeat)
|
||||
|
||||
if agent_watch_sample_usable(heartbeat, redis_sample) and redis_sample is not None:
|
||||
sample = redis_sample
|
||||
sample.source = "agent"
|
||||
sample.is_online = True
|
||||
if fetch_processes:
|
||||
procs = await ssh_watch_processes(server, timeout=WATCH_PROBE_TIMEOUT_SEC)
|
||||
if procs:
|
||||
sample.processes_json = procs
|
||||
fill_hardware_gaps(sample)
|
||||
return sample
|
||||
|
||||
# Agent 未进 watch 模式或心跳过期 → SSH psutil（与宝塔同源）
|
||||
ssh_result = await ssh_watch_probe(server, timeout=WATCH_PROBE_TIMEOUT_SEC)
|
||||
if ssh_result.get("ok"):
|
||||
ssh_sample = parse_ssh_watch_payload(ssh_result["payload"])
|
||||
ssh_sample.duration_ms = int(ssh_result.get("duration_ms") or 0)
|
||||
sample = merge_redis_and_ssh(redis_sample, ssh_sample) if redis_sample else ssh_sample
|
||||
elif redis_sample is not None:
|
||||
redis_sample.duration_ms = int(ssh_result.get("duration_ms") or 0)
|
||||
sample = await _supplement_load_if_needed(server, redis_sample)
|
||||
else:
|
||||
status = ssh_result.get("probe_status") or "offline"
|
||||
if not heartbeat and status == "offline":
|
||||
return failure_sample(
|
||||
status="no_cred",
|
||||
source="ssh",
|
||||
error=watch_probe_error_zh(
|
||||
ssh_result.get("error") or "无 Agent 且 SSH 探针失败",
|
||||
status="no_cred",
|
||||
) or "无 Agent 且 SSH 探针失败",
|
||||
duration_ms=int(ssh_result.get("duration_ms") or 0),
|
||||
)
|
||||
return failure_sample(
|
||||
status=status,
|
||||
source="ssh",
|
||||
error=watch_probe_error_zh(ssh_result.get("error") or "探针失败", status=status) or "探针失败",
|
||||
duration_ms=int(ssh_result.get("duration_ms") or 0),
|
||||
)
|
||||
|
||||
if fetch_processes and sample.probe_status == "ok":
|
||||
procs = await ssh_watch_processes(server, timeout=WATCH_PROBE_TIMEOUT_SEC)
|
||||
if procs:
|
||||
sample.processes_json = procs
|
||||
|
||||
if sample.probe_status == "ok":
|
||||
fill_hardware_gaps(sample)
|
||||
|
||||
return sample
|
||||
|
||||
|
||||
async def _probe_server_with_pins(
|
||||
*,
|
||||
watch_repo: WatchRepositoryImpl,
|
||||
server_repo: ServerRepositoryImpl,
|
||||
redis,
|
||||
server_id: int,
|
||||
pin_rows: list[dict[str, Any]],
|
||||
now: datetime,
|
||||
fetch_processes: bool,
|
||||
) -> None:
|
||||
"""Probe one server, persist Redis snapshot, WS push, and DB records for all its pins."""
|
||||
server = await server_repo.get_by_id(server_id)
|
||||
if not server:
|
||||
sample = failure_sample(status="offline", source="ssh", error="服务器不存在")
|
||||
server_name = f"server-{server_id}"
|
||||
else:
|
||||
server_name = server.name or f"server-{server_id}"
|
||||
heartbeat = await redis.hgetall(f"{REDIS_HEARTBEAT_PREFIX}{server_id}")
|
||||
sample = await probe_server_metrics(
|
||||
server,
|
||||
heartbeat or None,
|
||||
fetch_processes=fetch_processes,
|
||||
)
|
||||
|
||||
live = sample.to_live_dict(server_id=server_id)
|
||||
live["ts"] = now.isoformat()
|
||||
await _append_live_point(redis, server_id, live)
|
||||
if sample.processes_json:
|
||||
await _save_processes(redis, server_id, sample.processes_json)
|
||||
|
||||
if sample.probe_status == "ok" and server:
|
||||
try:
|
||||
await process_watch_probe_alerts(
|
||||
server_id=server_id,
|
||||
server_name=server_name,
|
||||
sample=sample,
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("watch alert processing failed", exc_info=True)
|
||||
|
||||
admin_ids = [r["admin_id"] for r in pin_rows]
|
||||
try:
|
||||
from server.api.websocket import publish_watch_metrics
|
||||
|
||||
await publish_watch_metrics(
|
||||
{
|
||||
"type": "watch_metrics",
|
||||
"server_id": server_id,
|
||||
"admin_ids": admin_ids,
|
||||
"metrics": live,
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("watch WS publish failed", exc_info=True)
|
||||
|
||||
for row in pin_rows:
|
||||
await watch_repo.insert_probe_record(
|
||||
session_id=row["session_id"],
|
||||
server_id=server_id,
|
||||
admin_id=row["admin_id"],
|
||||
recorded_at=now,
|
||||
source=sample.source,
|
||||
probe_status=sample.probe_status,
|
||||
duration_ms=sample.duration_ms,
|
||||
is_online=sample.is_online,
|
||||
cpu_pct=sample.cpu_pct,
|
||||
mem_pct=sample.mem_pct,
|
||||
disk_pct=sample.disk_pct,
|
||||
disk_mount=sample.disk_mount,
|
||||
load_1=sample.load_1,
|
||||
load_5=sample.load_5,
|
||||
load_15=sample.load_15,
|
||||
processes_json=sample.processes_json,
|
||||
error=sample.error,
|
||||
)
|
||||
|
||||
|
||||
async def trigger_immediate_watch_probe(server_id: int) -> None:
|
||||
"""Eager probe after pin add or monitoring resume — fills Redis before list_slots."""
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
watch_repo = WatchRepositoryImpl(db)
|
||||
pin_rows = [
|
||||
p
|
||||
for p in await watch_repo.list_active_pins_all(monitoring_only=True)
|
||||
if p["server_id"] == server_id
|
||||
]
|
||||
if not pin_rows:
|
||||
return
|
||||
server_repo = ServerRepositoryImpl(db)
|
||||
redis = get_redis()
|
||||
now = _utc_naive_now()
|
||||
_process_cycle[server_id] += 1
|
||||
fetch_procs = _process_cycle[server_id] % 2 == 0
|
||||
await _probe_server_with_pins(
|
||||
watch_repo=watch_repo,
|
||||
server_repo=server_repo,
|
||||
redis=redis,
|
||||
server_id=server_id,
|
||||
pin_rows=pin_rows,
|
||||
now=now,
|
||||
fetch_processes=fetch_procs,
|
||||
)
|
||||
await db.commit()
|
||||
except Exception:
|
||||
logger.exception("immediate watch probe failed for server %s", server_id)
|
||||
|
||||
|
||||
async def _run_probe_cycle() -> None:
|
||||
async with AsyncSessionLocal() as db:
|
||||
watch_repo = WatchRepositoryImpl(db)
|
||||
server_repo = ServerRepositoryImpl(db)
|
||||
expired_servers = await watch_repo.expire_due_pins()
|
||||
for server_id in expired_servers:
|
||||
if await watch_repo.count_monitoring_pins_for_server(server_id) == 0:
|
||||
await clear_watch_redis_snapshot(server_id)
|
||||
pins = await watch_repo.list_active_pins_all(monitoring_only=True)
|
||||
if not pins:
|
||||
await db.commit()
|
||||
return
|
||||
|
||||
by_server: dict[int, list[dict[str, Any]]] = defaultdict(list)
|
||||
for p in pins:
|
||||
by_server[p["server_id"]].append(p)
|
||||
|
||||
redis = get_redis()
|
||||
now = _utc_naive_now()
|
||||
|
||||
for server_id, pin_rows in by_server.items():
|
||||
_process_cycle[server_id] += 1
|
||||
fetch_procs = _process_cycle[server_id] % 2 == 0
|
||||
await _probe_server_with_pins(
|
||||
watch_repo=watch_repo,
|
||||
server_repo=server_repo,
|
||||
redis=redis,
|
||||
server_id=server_id,
|
||||
pin_rows=pin_rows,
|
||||
now=now,
|
||||
fetch_processes=fetch_procs,
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def _purge_old_records() -> None:
|
||||
cutoff = _utc_naive_now() - timedelta(days=WATCH_RECORDS_RETENTION_DAYS)
|
||||
async with AsyncSessionLocal() as db:
|
||||
repo = WatchRepositoryImpl(db)
|
||||
deleted = await repo.purge_records_before(cutoff)
|
||||
await db.commit()
|
||||
if deleted:
|
||||
logger.info("Purged %s watch_probe_records older than %s days", deleted, WATCH_RECORDS_RETENTION_DAYS)
|
||||
|
||||
|
||||
async def watch_probe_loop() -> None:
|
||||
"""Run watch probes every 5 seconds (primary worker only)."""
|
||||
purge_counter = 0
|
||||
while True:
|
||||
try:
|
||||
await _run_probe_cycle()
|
||||
purge_counter += 1
|
||||
if purge_counter >= 720: # ~1h at 5s
|
||||
purge_counter = 0
|
||||
await _purge_old_records()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception("watch_probe cycle failed")
|
||||
await asyncio.sleep(WATCH_PROBE_INTERVAL_SEC)
|
||||
Reference in New Issue
Block a user