"""Nexus — Retry Runner Background Task Every 5 minutes: check pending PushRetryJobs, retry failed pushes with exponential backoff. """ import asyncio import logging from datetime import datetime, timezone, timedelta from server.infrastructure.database.session import AsyncSessionLocal from server.domain.models import PushRetryJob logger = logging.getLogger("nexus.retry_runner") RETRY_CHECK_INTERVAL = 300 # 5 minutes RETRY_BACKOFF_BASE = 60 # seconds — doubles each retry async def retry_runner_loop(): """Background task: every 5min, retry pending retry jobs that are due.""" logger.info("Retry runner loop started (interval: 5min)") while True: await asyncio.sleep(RETRY_CHECK_INTERVAL) try: async with AsyncSessionLocal() as session: from sqlalchemy import select now = datetime.now(timezone.utc) result = await session.execute( select(PushRetryJob).where( PushRetryJob.status == "pending", PushRetryJob.retry_count < PushRetryJob.max_retries, PushRetryJob.next_retry_at <= now, ) ) jobs = result.scalars().all() if not jobs: continue retried = 0 for job in jobs: try: from server.application.services.sync_engine_v2 import SyncEngineV2 from server.infrastructure.database.server_repo import ServerRepositoryImpl from server.infrastructure.database.sync_log_repo import SyncLogRepositoryImpl from server.infrastructure.database.audit_log_repo import AuditLogRepositoryImpl from server.infrastructure.database.push_schedule_repo import PushRetryJobRepositoryImpl engine = SyncEngineV2( server_repo=ServerRepositoryImpl(session), sync_log_repo=SyncLogRepositoryImpl(session), audit_repo=AuditLogRepositoryImpl(session), retry_repo=PushRetryJobRepositoryImpl(session), ) sync_result = await engine.sync_files( server_ids=[job.server_id], source_path=job.source_path, target_path=job.target_path, trigger_type="retry", operator=job.operator or "retry_runner", ) job.retry_count += 1 if sync_result["failed"] == 0: job.status = "completed" logger.info(f"Retry job {job.id}: success after {job.retry_count} attempts") else: # Still failing — schedule next retry with backoff backoff = RETRY_BACKOFF_BASE * (2 ** min(job.retry_count - 1, 6)) job.next_retry_at = now + timedelta(seconds=backoff) job.last_error = f"Retry {job.retry_count} failed" logger.warning( f"Retry job {job.id}: attempt {job.retry_count} failed, " f"next retry in {backoff}s" ) await session.commit() retried += 1 except Exception as e: logger.error(f"Retry job {job.id} execution error: {e}") job.retry_count += 1 backoff = RETRY_BACKOFF_BASE * (2 ** min(job.retry_count - 1, 6)) job.next_retry_at = now + timedelta(seconds=backoff) job.last_error = str(e)[:500] await session.commit() if retried: logger.info(f"Retry runner: {retried} jobs processed") except Exception as e: logger.error(f"Retry runner check failed: {e}")