Your Name
104 lines
4.7 KiB
Python
104 lines
4.7 KiB
Python
"""Nexus — Retry Runner Background Task
|
|
Every 5 minutes: check pending PushRetryJobs, retry failed pushes with exponential backoff.
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
from datetime import datetime, timezone, timedelta
|
|
|
|
from server.infrastructure.database.session import AsyncSessionLocal
|
|
from server.domain.models import PushRetryJob
|
|
|
|
logger = logging.getLogger("nexus.retry_runner")
|
|
|
|
RETRY_CHECK_INTERVAL = 300 # 5 minutes
|
|
RETRY_BACKOFF_BASE = 60 # seconds — doubles each retry
|
|
|
|
|
|
async def retry_runner_loop():
|
|
"""Background task: every 5min, retry pending retry jobs that are due."""
|
|
logger.info("Retry runner loop started (interval: 5min)")
|
|
while True:
|
|
await asyncio.sleep(RETRY_CHECK_INTERVAL)
|
|
|
|
try:
|
|
async with AsyncSessionLocal() as session:
|
|
from sqlalchemy import select
|
|
now = datetime.now(timezone.utc)
|
|
|
|
result = await session.execute(
|
|
select(PushRetryJob).where(
|
|
PushRetryJob.status == "pending",
|
|
PushRetryJob.retry_count < PushRetryJob.max_retries,
|
|
PushRetryJob.next_retry_at <= now,
|
|
)
|
|
)
|
|
jobs = result.scalars().all()
|
|
|
|
if not jobs:
|
|
continue
|
|
|
|
retried = 0
|
|
for job in jobs:
|
|
try:
|
|
from server.application.services.sync_engine_v2 import SyncEngineV2
|
|
from server.infrastructure.database.server_repo import ServerRepositoryImpl
|
|
from server.infrastructure.database.sync_log_repo import SyncLogRepositoryImpl
|
|
from server.infrastructure.database.audit_log_repo import AuditLogRepositoryImpl
|
|
from server.infrastructure.database.push_schedule_repo import PushRetryJobRepositoryImpl
|
|
|
|
engine = SyncEngineV2(
|
|
server_repo=ServerRepositoryImpl(session),
|
|
sync_log_repo=SyncLogRepositoryImpl(session),
|
|
audit_repo=AuditLogRepositoryImpl(session),
|
|
retry_repo=PushRetryJobRepositoryImpl(session),
|
|
)
|
|
|
|
sync_result = await engine.sync_files(
|
|
server_ids=[job.server_id],
|
|
source_path=job.source_path,
|
|
target_path=job.target_path,
|
|
trigger_type="retry",
|
|
operator=job.operator or "retry_runner",
|
|
)
|
|
|
|
job.retry_count += 1
|
|
|
|
if sync_result["failed"] == 0:
|
|
job.status = "completed"
|
|
logger.info(f"Retry job {job.id}: success after {job.retry_count} attempts")
|
|
elif job.retry_count >= job.max_retries:
|
|
job.status = "failed"
|
|
job.last_error = f"Retry exhausted after {job.retry_count} attempts"
|
|
logger.warning(f"Retry job {job.id}: max retries reached, marking failed")
|
|
else:
|
|
# Still failing — schedule next retry with backoff
|
|
backoff = RETRY_BACKOFF_BASE * (2 ** min(job.retry_count - 1, 6))
|
|
job.next_retry_at = now + timedelta(seconds=backoff)
|
|
job.last_error = f"Retry {job.retry_count} failed"
|
|
logger.warning(
|
|
f"Retry job {job.id}: attempt {job.retry_count} failed, "
|
|
f"next retry in {backoff}s"
|
|
)
|
|
|
|
await session.commit()
|
|
retried += 1
|
|
|
|
except Exception as e:
|
|
logger.error(f"Retry job {job.id} execution error: {e}")
|
|
job.retry_count += 1
|
|
job.last_error = str(e)[:500]
|
|
if job.retry_count >= job.max_retries:
|
|
job.status = "failed"
|
|
logger.warning(f"Retry job {job.id}: max retries reached (exception), marking failed")
|
|
else:
|
|
backoff = RETRY_BACKOFF_BASE * (2 ** min(job.retry_count - 1, 6))
|
|
job.next_retry_at = now + timedelta(seconds=backoff)
|
|
await session.commit()
|
|
|
|
if retried:
|
|
logger.info(f"Retry runner: {retried} jobs processed")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Retry runner check failed: {e}")
|