#!/usr/bin/env bash # Nexus — External Health Monitor (Layer 3 of 3-tier guardian) # Runs via crontab every minute. Works on Docker (1Panel) and Supervisor bare-metal. # # Layer 1: Docker restart policy / Supervisor autorestart # Layer 2: Python self_monitor (30s) # Layer 3: This script — HTTP /health, 3 failures → restart + optional Telegram set -euo pipefail DEPLOY_DIR="${NEXUS_DEPLOY_DIR:-/opt/nexus}" NEXUS_SERVICE="${NEXUS_SERVICE:-nexus}" FAIL_FILE="/tmp/nexus_health_fail_count" LAST_RESTART_FILE="/tmp/nexus_health_last_restart" MAX_FAIL=3 RESTART_COOLDOWN_SEC="${NEXUS_HEALTH_RESTART_COOLDOWN:-300}" ENV_PROD="${DEPLOY_DIR}/docker/.env.prod" resolve_publish_port() { local port port="$(grep -E '^NEXUS_PUBLISH_PORT=' "$ENV_PROD" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '\r" ' || true)" echo "${port:-8600}" } PUBLISH_PORT="$(resolve_publish_port)" HEALTH_URL="http://127.0.0.1:${PUBLISH_PORT}/health" resolve_nexus_container() { docker ps --format '{{.Names}}' 2>/dev/null | grep -E 'nexus-prod-nexus|nexus-nexus-1' | head -1 || true } read_env_var() { local key="$1" file="$2" grep -E "^${key}=" "$file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '\r"' || true } load_telegram_config() { local cname="$1" dotenv BOT_TOKEN="" CHAT_ID="" for dotenv in \ "${DEPLOY_DIR}/.env" \ "$ENV_PROD" \ ; do [[ -f "$dotenv" ]] || continue BOT_TOKEN="$(read_env_var NEXUS_TELEGRAM_BOT_TOKEN "$dotenv")" CHAT_ID="$(read_env_var NEXUS_TELEGRAM_CHAT_ID "$dotenv")" [[ -n "$BOT_TOKEN" && -n "$CHAT_ID" ]] && return 0 done if [[ -n "$cname" ]]; then local line while IFS= read -r line; do case "$line" in NEXUS_TELEGRAM_BOT_TOKEN=*) BOT_TOKEN="${line#NEXUS_TELEGRAM_BOT_TOKEN=}" ;; NEXUS_TELEGRAM_CHAT_ID=*) CHAT_ID="${line#NEXUS_TELEGRAM_CHAT_ID=}" ;; esac done < <(docker exec "$cname" grep -E '^NEXUS_TELEGRAM_' /app/.env 2>/dev/null || true) fi } send_telegram() { [[ -n "${BOT_TOKEN:-}" && -n "${CHAT_ID:-}" ]] || return 1 curl -sf --max-time 10 "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \ -d "chat_id=${CHAT_ID}" \ -d "text=$1" \ -d "parse_mode=HTML" \ > /dev/null 2>&1 } bump_fail_count() { local fail=0 if command -v flock >/dev/null 2>&1; then ( flock -x 9 fail="$(cat "$FAIL_FILE" 2>/dev/null || echo 0)" fail=$((fail + 1)) echo "$fail" > "$FAIL_FILE" ) 9>"${FAIL_FILE}.lock" else fail="$(cat "$FAIL_FILE" 2>/dev/null || echo 0)" fail=$((fail + 1)) echo "$fail" > "$FAIL_FILE" fi echo "$fail" } reset_fail_count() { echo "0" > "$FAIL_FILE" } restart_cooldown_active() { local now last elapsed [[ -f "$LAST_RESTART_FILE" ]] || return 1 now="$(date +%s)" last="$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0)" elapsed=$((now - last)) [[ "$elapsed" -lt "$RESTART_COOLDOWN_SEC" ]] } mark_restart_attempt() { date +%s > "$LAST_RESTART_FILE" } restart_nexus() { local cname="$1" via="" if [[ -n "$cname" ]]; then docker restart "$cname" >/dev/null 2>&1 && via="Docker 容器 ${cname}" elif command -v supervisorctl >/dev/null 2>&1 && supervisorctl status "$NEXUS_SERVICE" >/dev/null 2>&1; then supervisorctl restart "$NEXUS_SERVICE" >/dev/null 2>&1 && via="Supervisor ${NEXUS_SERVICE}" else return 1 fi echo "$via" } NEXUS_CONTAINER="$(resolve_nexus_container)" load_telegram_config "$NEXUS_CONTAINER" if curl -sf --max-time 5 "$HEALTH_URL" > /dev/null 2>&1; then reset_fail_count exit 0 fi FAIL="$(bump_fail_count)" if [[ "$FAIL" -lt "$MAX_FAIL" ]]; then exit 0 fi if restart_cooldown_active; then send_telegram "🔴 Nexus /health 连续失败\n冷却期内跳过自动重启(${RESTART_COOLDOWN_SEC}s)\n请检查: docker logs ${NEXUS_CONTAINER:-nexus}" || true exit 0 fi send_telegram "🔴 Nexus 连续 ${MAX_FAIL} 次无响应\n正在自动重启…" || true mark_restart_attempt if VIA="$(restart_nexus "$NEXUS_CONTAINER")"; then sleep 5 if curl -sf --max-time 5 "$HEALTH_URL" > /dev/null 2>&1; then send_telegram "🟢 Nexus 已恢复\n${VIA} 重启成功" || true reset_fail_count else send_telegram "🔴 Nexus 重启后仍未恢复\n${VIA} — 需人工介入" || true fi else send_telegram "🔴 Nexus 无法自动重启\n未找到 Docker 容器或 Supervisor 服务" || true fi