82 lines
2.4 KiB
Python
82 lines
2.4 KiB
Python
|
"""Server batch zombie recovery and stuck detection helpers."""
|
||
|
|
|
||
|
|
import json
|
||
|
|
from datetime import datetime, timedelta, timezone
|
||
|
|
|
||
|
|
from server.domain.models import ServerBatchJob
|
||
|
|
from server.infrastructure.database.server_batch_job_repo import job_row_to_api
|
||
|
|
from server.infrastructure.redis import server_batch_store as sbs
|
||
|
|
|
||
|
|
|
||
|
|
def test_pending_server_ids():
|
||
|
|
live = {
|
||
|
|
"server_ids": [1, 2, 3],
|
||
|
|
"results": {"1": {"server_id": 1, "success": True}},
|
||
|
|
}
|
||
|
|
assert sbs.pending_server_ids(live) == [2, 3]
|
||
|
|
|
||
|
|
|
||
|
|
def test_apply_pending_failures_marks_stuck():
|
||
|
|
live = {"server_ids": [10], "results": {}}
|
||
|
|
added = sbs.apply_pending_failures(
|
||
|
|
live,
|
||
|
|
server_ids=[10],
|
||
|
|
error=sbs.STUCK_MESSAGE,
|
||
|
|
auto_stuck=True,
|
||
|
|
server_name_fallback="S10",
|
||
|
|
)
|
||
|
|
assert added == 1
|
||
|
|
entry = live["results"]["10"]
|
||
|
|
assert entry["success"] is False
|
||
|
|
assert entry["auto_stuck"] is True
|
||
|
|
assert entry["error"] == sbs.STUCK_MESSAGE
|
||
|
|
|
||
|
|
|
||
|
|
def test_finalize_status_after_pending_failures():
|
||
|
|
live = {
|
||
|
|
"server_ids": [1, 2],
|
||
|
|
"results": {},
|
||
|
|
"status": "running",
|
||
|
|
}
|
||
|
|
sbs.apply_pending_failures(
|
||
|
|
live,
|
||
|
|
server_ids=[1, 2],
|
||
|
|
error=sbs.INTERRUPTED_MESSAGE,
|
||
|
|
)
|
||
|
|
status = sbs.finalize_status(live)
|
||
|
|
assert status == "failed"
|
||
|
|
assert live["status"] == "failed"
|
||
|
|
|
||
|
|
|
||
|
|
def test_is_stale_running_true_when_old_update():
|
||
|
|
old = (datetime.now(timezone.utc) - timedelta(hours=2)).isoformat()
|
||
|
|
live = {"status": "running", "updated_at": old}
|
||
|
|
assert sbs.is_stale_running(live) is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_is_stale_running_false_when_recent():
|
||
|
|
recent = datetime.now(timezone.utc).isoformat()
|
||
|
|
live = {"status": "running", "updated_at": recent}
|
||
|
|
assert sbs.is_stale_running(live) is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_live_from_mysql_row_rebuilds_results_map():
|
||
|
|
row = ServerBatchJob(
|
||
|
|
id=20,
|
||
|
|
op="install-agent",
|
||
|
|
label="批量安装 Agent",
|
||
|
|
server_ids=json.dumps([290, 346]),
|
||
|
|
status="running",
|
||
|
|
results=json.dumps([
|
||
|
|
{"server_id": 290, "server_name": "A", "success": True, "stdout": "ok", "error": ""},
|
||
|
|
]),
|
||
|
|
operator="admin",
|
||
|
|
started_at=datetime(2026, 6, 8, 10, 0, tzinfo=timezone.utc),
|
||
|
|
)
|
||
|
|
live = sbs.live_from_mysql_row(row)
|
||
|
|
assert live["id"] == 20
|
||
|
|
assert live["status"] == "running"
|
||
|
|
assert sbs.pending_server_ids(live) == [346]
|
||
|
|
api = job_row_to_api(row)
|
||
|
|
assert api["remaining"] == 1
|