r
6859846b7c
- bootstrap 失败恢复 SSH 临时登录回退 - ssh_bootstrap 真安装检测与 config 目录自愈 - axs Gitea 发布源切换为 https://axs.kuma1xn.vip
329 lines
12 KiB
Python
329 lines
12 KiB
Python
"""Watch pin API integration tests."""
|
|
|
|
from datetime import datetime, timezone
|
|
from unittest.mock import AsyncMock
|
|
|
|
import bcrypt
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from server.domain.models import Admin, Server
|
|
from server.utils.watch_metrics import WATCH_MAX_SLOTS
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_watch_redis(monkeypatch):
|
|
mock_redis = AsyncMock()
|
|
mock_redis.lindex = AsyncMock(return_value=None)
|
|
mock_redis.lrange = AsyncMock(return_value=[])
|
|
mock_redis.get = AsyncMock(return_value=None)
|
|
mock_redis.hgetall = AsyncMock(return_value={})
|
|
mock_redis.delete = AsyncMock(return_value=1)
|
|
monkeypatch.setattr("server.application.services.watch_service.get_redis", lambda: mock_redis)
|
|
monkeypatch.setattr("server.utils.watch_state.get_redis", lambda: mock_redis)
|
|
|
|
async def noop_immediate_probe(_server_id: int) -> None:
|
|
return None
|
|
|
|
monkeypatch.setattr(
|
|
"server.application.services.watch_service.trigger_immediate_watch_probe",
|
|
noop_immediate_probe,
|
|
)
|
|
return mock_redis
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watch_pin_crud(db_session, test_admin, mock_watch_redis):
|
|
from server.application.services.watch_service import SlotsFullError, WatchService
|
|
|
|
for i in range(WATCH_MAX_SLOTS):
|
|
s = Server(name=f"watch-{i}", domain=f"w{i}.example.com", port=22, agent_version="1.0")
|
|
db_session.add(s)
|
|
await db_session.commit()
|
|
servers = (await db_session.execute(select(Server))).scalars().all()
|
|
|
|
svc = WatchService(db_session)
|
|
admin = test_admin["admin"]
|
|
|
|
r0 = await svc.pin_server(admin=admin, server_id=servers[0].id)
|
|
assert len(r0["slots"]) == 4
|
|
assert r0["slots"][0]["empty"] is False
|
|
|
|
await svc.pin_server(admin=admin, server_id=servers[0].id)
|
|
|
|
for s in servers[1:]:
|
|
await svc.pin_server(admin=admin, server_id=s.id)
|
|
|
|
with pytest.raises(SlotsFullError):
|
|
extra = Server(name="extra", domain="extra.example.com", port=22, agent_version="1.0")
|
|
db_session.add(extra)
|
|
await db_session.flush()
|
|
await svc.pin_server(admin=admin, server_id=extra.id)
|
|
|
|
await svc.remove_slot(admin=admin, slot_index=0)
|
|
listing = await svc.list_slots(admin.id)
|
|
assert listing["slots"][0]["empty"] is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watch_probe_record_on_failure(db_session):
|
|
from server.infrastructure.database.watch_repo import WatchRepositoryImpl
|
|
from server.utils.watch_metrics import failure_sample
|
|
|
|
repo = WatchRepositoryImpl(db_session)
|
|
admin = Admin(username="watch_probe_admin", password_hash=bcrypt.hashpw(b"x", bcrypt.gensalt()).decode(), is_active=True)
|
|
db_session.add(admin)
|
|
server = Server(name="probe-fail", domain="pf.example.com", port=22)
|
|
db_session.add(server)
|
|
await db_session.flush()
|
|
|
|
created = await repo.create_pin(admin_id=admin.id, server_id=server.id, slot_index=0)
|
|
sample = failure_sample(status="ssh_timeout", source="ssh", error="timeout")
|
|
row = await repo.insert_probe_record(
|
|
session_id=created["session_id"],
|
|
server_id=server.id,
|
|
admin_id=admin.id,
|
|
recorded_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
|
source=sample.source,
|
|
probe_status=sample.probe_status,
|
|
duration_ms=8000,
|
|
is_online=False,
|
|
cpu_pct=None,
|
|
mem_pct=None,
|
|
disk_pct=None,
|
|
error=sample.error,
|
|
)
|
|
await db_session.commit()
|
|
assert row.probe_status == "ssh_timeout"
|
|
assert row.cpu_pct is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watch_pin_monitoring_toggle(db_session, test_admin, mock_watch_redis):
|
|
from server.application.services.watch_service import WatchService
|
|
|
|
server = Server(name="watch-toggle", domain="wt.example.com", port=22, agent_version="1.0")
|
|
db_session.add(server)
|
|
await db_session.commit()
|
|
|
|
svc = WatchService(db_session)
|
|
admin = test_admin["admin"]
|
|
|
|
await svc.pin_server(admin=admin, server_id=server.id)
|
|
listing = await svc.list_slots(admin.id)
|
|
assert listing["slots"][0]["monitoring_enabled"] is True
|
|
|
|
paused = await svc.set_slot_monitoring(
|
|
admin=admin,
|
|
slot_index=0,
|
|
monitoring_enabled=False,
|
|
)
|
|
assert paused["slots"][0]["monitoring_enabled"] is False
|
|
assert paused["slots"][0]["remaining_sec"] is None
|
|
|
|
resumed = await svc.set_slot_monitoring(
|
|
admin=admin,
|
|
slot_index=0,
|
|
monitoring_enabled=True,
|
|
)
|
|
assert resumed["slots"][0]["monitoring_enabled"] is True
|
|
assert resumed["slots"][0]["remaining_sec"] is not None
|
|
assert resumed["slots"][0]["remaining_sec"] > 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watch_resume_eager_probe_fills_metrics(
|
|
db_session, test_admin, mock_watch_redis, monkeypatch,
|
|
):
|
|
"""开启监测后立即探针，list_slots 应带上 metrics（不等 5s 周期）。"""
|
|
import json
|
|
|
|
from server.application.services.watch_service import WatchService
|
|
|
|
server = Server(name="watch-eager", domain="we.example.com", port=22, agent_version="1.0")
|
|
db_session.add(server)
|
|
await db_session.commit()
|
|
|
|
async def fake_immediate_probe(server_id: int) -> None:
|
|
live = json.dumps({"probe_status": "ok", "cpu_pct": 9.5, "server_id": server_id})
|
|
mock_watch_redis.lindex = AsyncMock(return_value=live)
|
|
|
|
monkeypatch.setattr(
|
|
"server.application.services.watch_service.trigger_immediate_watch_probe",
|
|
fake_immediate_probe,
|
|
)
|
|
|
|
svc = WatchService(db_session)
|
|
admin = test_admin["admin"]
|
|
await svc.pin_server(admin=admin, server_id=server.id)
|
|
await svc.set_slot_monitoring(admin=admin, slot_index=0, monitoring_enabled=False)
|
|
|
|
resumed = await svc.set_slot_monitoring(
|
|
admin=admin,
|
|
slot_index=0,
|
|
monitoring_enabled=True,
|
|
)
|
|
metrics = resumed["slots"][0]["metrics"]
|
|
assert metrics is not None
|
|
assert metrics["probe_status"] == "ok"
|
|
assert metrics["cpu_pct"] == 9.5
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watch_pin_ttl_minutes(db_session, test_admin, mock_watch_redis):
|
|
from server.application.services.watch_service import WatchService
|
|
from server.utils.watch_metrics import WATCH_PIN_TTL_DEFAULT_MINUTES
|
|
|
|
server = Server(name="watch-ttl", domain="wt2.example.com", port=22, agent_version="1.0")
|
|
db_session.add(server)
|
|
await db_session.commit()
|
|
|
|
svc = WatchService(db_session)
|
|
admin = test_admin["admin"]
|
|
|
|
await svc.pin_server(admin=admin, server_id=server.id)
|
|
listing = await svc.list_slots(admin.id)
|
|
assert listing["slots"][0]["ttl_minutes"] == WATCH_PIN_TTL_DEFAULT_MINUTES
|
|
assert listing["slots"][0]["remaining_sec"] <= 30 * 60
|
|
|
|
extended = await svc.set_slot_ttl(
|
|
admin=admin,
|
|
slot_index=0,
|
|
ttl_minutes=1440,
|
|
)
|
|
assert extended["slots"][0]["ttl_minutes"] == 1440
|
|
assert extended["slots"][0]["remaining_sec"] >= 1440 * 60 - 5
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watch_pin_ttl_expiry_list_syncs_before_display(db_session, test_admin, mock_watch_redis):
|
|
"""TTL 到期后、探针循环未跑 expire 前，list_slots 应同步为暂停态而非空槽。"""
|
|
from datetime import timedelta
|
|
|
|
from server.infrastructure.database.watch_repo import WatchRepositoryImpl, _utc_naive_now
|
|
from server.application.services.watch_service import WatchService
|
|
|
|
server = Server(name="watch-expire-list", domain="wel.example.com", port=22, agent_version="1.0")
|
|
db_session.add(server)
|
|
await db_session.commit()
|
|
|
|
svc = WatchService(db_session)
|
|
admin = test_admin["admin"]
|
|
await svc.pin_server(admin=admin, server_id=server.id)
|
|
|
|
repo = WatchRepositoryImpl(db_session)
|
|
pin = await repo.get_pin_by_slot(admin.id, 0)
|
|
assert pin is not None
|
|
pin.expires_at = _utc_naive_now() - timedelta(seconds=1)
|
|
await db_session.commit()
|
|
|
|
# 不手动调用 expire_due_pins — 模拟探针循环尚未处理的窗口
|
|
listing = await svc.list_slots(admin.id)
|
|
assert listing["slots"][0]["empty"] is False
|
|
assert listing["slots"][0]["monitoring_enabled"] is False
|
|
assert listing["slots"][0]["remaining_sec"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watch_pin_ttl_expiry_pin_server_no_duplicate(db_session, test_admin, mock_watch_redis):
|
|
"""到期 ghost pin 不应导致同槽 create_pin 唯一约束冲突。"""
|
|
from datetime import timedelta
|
|
|
|
from server.infrastructure.database.watch_repo import WatchRepositoryImpl, _utc_naive_now
|
|
from server.application.services.watch_service import WatchService
|
|
|
|
server_a = Server(name="watch-exp-a", domain="wea.example.com", port=22, agent_version="1.0")
|
|
server_b = Server(name="watch-exp-b", domain="web.example.com", port=22, agent_version="1.0")
|
|
db_session.add_all([server_a, server_b])
|
|
await db_session.commit()
|
|
|
|
svc = WatchService(db_session)
|
|
admin = test_admin["admin"]
|
|
await svc.pin_server(admin=admin, server_id=server_a.id, slot_index=0)
|
|
|
|
repo = WatchRepositoryImpl(db_session)
|
|
pin = await repo.get_pin_by_slot(admin.id, 0)
|
|
assert pin is not None
|
|
pin.expires_at = _utc_naive_now() - timedelta(seconds=1)
|
|
await db_session.commit()
|
|
|
|
# 替换槽位服务器 — 应先 finalize 再 replace，不能 IntegrityError
|
|
result = await svc.pin_server(admin=admin, server_id=server_b.id, slot_index=0)
|
|
assert result["slots"][0]["empty"] is False
|
|
assert result["slots"][0]["server_id"] == server_b.id
|
|
assert result["slots"][0]["monitoring_enabled"] is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watch_install_psutil_requires_pin(db_session, test_admin, mock_watch_redis):
|
|
from server.application.services.watch_service import WatchService
|
|
|
|
server = Server(name="watch-psutil", domain="wp.example.com", port=22, agent_version="1.0")
|
|
db_session.add(server)
|
|
await db_session.commit()
|
|
|
|
svc = WatchService(db_session)
|
|
admin = test_admin["admin"]
|
|
|
|
with pytest.raises(ValueError, match="不在当前监测槽"):
|
|
await svc.install_psutil_ssh(admin=admin, server_id=server.id)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watch_install_psutil_success(db_session, test_admin, mock_watch_redis, monkeypatch):
|
|
from server.application.services.watch_service import WatchService
|
|
|
|
server = Server(name="watch-psutil2", domain="wp2.example.com", port=22, agent_version="1.0")
|
|
db_session.add(server)
|
|
await db_session.commit()
|
|
|
|
svc = WatchService(db_session)
|
|
admin = test_admin["admin"]
|
|
await svc.pin_server(admin=admin, server_id=server.id)
|
|
|
|
async def _fake_install(_server):
|
|
return {"success": True, "message": "psutil 已就绪"}
|
|
|
|
monkeypatch.setattr(
|
|
"server.utils.psutil_install.install_psutil_via_ssh",
|
|
_fake_install,
|
|
)
|
|
|
|
result = await svc.install_psutil_ssh(admin=admin, server_id=server.id)
|
|
assert result["success"] is True
|
|
assert result["message"] == "psutil 已就绪"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_watch_pin_ttl_expiry_pauses_not_deletes(db_session, test_admin, mock_watch_redis):
|
|
from datetime import timedelta
|
|
|
|
from server.infrastructure.database.watch_repo import WatchRepositoryImpl, _utc_naive_now
|
|
from server.application.services.watch_service import WatchService
|
|
|
|
server = Server(name="watch-expire", domain="we.example.com", port=22, agent_version="1.0")
|
|
db_session.add(server)
|
|
await db_session.commit()
|
|
|
|
svc = WatchService(db_session)
|
|
admin = test_admin["admin"]
|
|
await svc.pin_server(admin=admin, server_id=server.id)
|
|
|
|
repo = WatchRepositoryImpl(db_session)
|
|
pin = await repo.get_pin_by_slot(admin.id, 0)
|
|
assert pin is not None
|
|
pin.expires_at = _utc_naive_now() - timedelta(seconds=1)
|
|
await db_session.commit()
|
|
|
|
expired = await repo.expire_due_pins()
|
|
assert server.id in expired
|
|
await db_session.commit()
|
|
|
|
still = await repo.get_pin_by_slot(admin.id, 0)
|
|
assert still is not None
|
|
assert still.monitoring_enabled is False
|
|
|
|
listing = await svc.list_slots(admin.id)
|
|
assert listing["slots"][0]["empty"] is False
|
|
assert listing["slots"][0]["monitoring_enabled"] is False
|