28 lines
988 B
Python
28 lines
988 B
Python
|
"""Background reconciliation for server batch jobs — stuck detection (60s interval)."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import asyncio
|
||
|
|
import logging
|
||
|
|
|
||
|
|
from server.application.services.server_batch_service import detect_stuck_batch_jobs
|
||
|
|
from server.infrastructure.redis.server_batch_store import RECONCILE_INTERVAL_SECONDS
|
||
|
|
|
||
|
|
logger = logging.getLogger("nexus.server_batch_reconcile")
|
||
|
|
|
||
|
|
|
||
|
|
async def server_batch_reconcile_loop() -> None:
|
||
|
|
"""Every minute: auto-finalize batch jobs with no Redis progress for 1h."""
|
||
|
|
logger.info(
|
||
|
|
"Server batch reconcile loop started (interval=%ss)",
|
||
|
|
RECONCILE_INTERVAL_SECONDS,
|
||
|
|
)
|
||
|
|
while True:
|
||
|
|
await asyncio.sleep(RECONCILE_INTERVAL_SECONDS)
|
||
|
|
try:
|
||
|
|
stuck = await detect_stuck_batch_jobs()
|
||
|
|
if stuck:
|
||
|
|
logger.info("Server batch reconcile: %s stuck job(s) finalized", stuck)
|
||
|
|
except Exception as e:
|
||
|
|
logger.error("Server batch reconcile loop error: %s", e)
|