Your Name
814e11588e
1. schedule_runner.py: cron-based schedule execution every 60s
- Parses 5-field cron expressions (*, */N, ranges, lists)
- Triggers SyncEngineV2.sync_files for due schedules
- Prevents re-trigger within same minute
2. retry_runner.py: automatic retry of failed pushes every 5min
- Exponential backoff (60s base, doubles per retry, max 64x)
- Respects max_retries from PushRetryJob model
3. GET /api/servers/stats: dashboard aggregation endpoint
- Returns total/online/offline counts + category breakdown
- Reads real-time status from Redis, falls back to MySQL
- Registered before /{id} to avoid path collision
4. main.py: register schedule_runner + retry_runner as background tasks
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
96 lines
4.1 KiB
Python
96 lines
4.1 KiB
Python
"""Nexus — Retry Runner Background Task
|
|
Every 5 minutes: check pending PushRetryJobs, retry failed pushes with exponential backoff.
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
from datetime import datetime, timezone, timedelta
|
|
|
|
from server.infrastructure.database.session import AsyncSessionLocal
|
|
from server.domain.models import PushRetryJob
|
|
|
|
logger = logging.getLogger("nexus.retry_runner")
|
|
|
|
RETRY_CHECK_INTERVAL = 300 # 5 minutes
|
|
RETRY_BACKOFF_BASE = 60 # seconds — doubles each retry
|
|
|
|
|
|
async def retry_runner_loop():
|
|
"""Background task: every 5min, retry pending retry jobs that are due."""
|
|
logger.info("Retry runner loop started (interval: 5min)")
|
|
while True:
|
|
await asyncio.sleep(RETRY_CHECK_INTERVAL)
|
|
|
|
try:
|
|
async with AsyncSessionLocal() as session:
|
|
from sqlalchemy import select
|
|
now = datetime.now(timezone.utc)
|
|
|
|
result = await session.execute(
|
|
select(PushRetryJob).where(
|
|
PushRetryJob.status == "pending",
|
|
PushRetryJob.retry_count < PushRetryJob.max_retries,
|
|
PushRetryJob.next_retry_at <= now,
|
|
)
|
|
)
|
|
jobs = result.scalars().all()
|
|
|
|
if not jobs:
|
|
continue
|
|
|
|
retried = 0
|
|
for job in jobs:
|
|
try:
|
|
from server.application.services.sync_engine_v2 import SyncEngineV2
|
|
from server.infrastructure.database.server_repo import ServerRepositoryImpl
|
|
from server.infrastructure.database.sync_log_repo import SyncLogRepositoryImpl
|
|
from server.infrastructure.database.audit_log_repo import AuditLogRepositoryImpl
|
|
from server.infrastructure.database.push_schedule_repo import PushRetryJobRepositoryImpl
|
|
|
|
engine = SyncEngineV2(
|
|
server_repo=ServerRepositoryImpl(session),
|
|
sync_log_repo=SyncLogRepositoryImpl(session),
|
|
audit_repo=AuditLogRepositoryImpl(session),
|
|
retry_repo=PushRetryJobRepositoryImpl(session),
|
|
)
|
|
|
|
sync_result = await engine.sync_files(
|
|
server_ids=[job.server_id],
|
|
source_path=job.source_path,
|
|
target_path=job.target_path,
|
|
trigger_type="retry",
|
|
operator=job.operator or "retry_runner",
|
|
)
|
|
|
|
job.retry_count += 1
|
|
|
|
if sync_result["failed"] == 0:
|
|
job.status = "completed"
|
|
logger.info(f"Retry job {job.id}: success after {job.retry_count} attempts")
|
|
else:
|
|
# Still failing — schedule next retry with backoff
|
|
backoff = RETRY_BACKOFF_BASE * (2 ** min(job.retry_count - 1, 6))
|
|
job.next_retry_at = now + timedelta(seconds=backoff)
|
|
job.last_error = f"Retry {job.retry_count} failed"
|
|
logger.warning(
|
|
f"Retry job {job.id}: attempt {job.retry_count} failed, "
|
|
f"next retry in {backoff}s"
|
|
)
|
|
|
|
await session.commit()
|
|
retried += 1
|
|
|
|
except Exception as e:
|
|
logger.error(f"Retry job {job.id} execution error: {e}")
|
|
job.retry_count += 1
|
|
backoff = RETRY_BACKOFF_BASE * (2 ** min(job.retry_count - 1, 6))
|
|
job.next_retry_at = now + timedelta(seconds=backoff)
|
|
job.last_error = str(e)[:500]
|
|
await session.commit()
|
|
|
|
if retried:
|
|
logger.info(f"Retry runner: {retried} jobs processed")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Retry runner check failed: {e}")
|