Nexus/deploy/health_monitor.sh

#!/usr/bin/env bash
# Nexus — External Health Monitor (Layer 3 of 3-tier guardian)
# Runs via crontab every minute. Works on Docker (1Panel) and Supervisor bare-metal.
#
# Layer 1: Docker restart policy / Supervisor autorestart
# Layer 2: Python self_monitor (30s)
# Layer 3: This script — HTTP /health, 3 failures → restart + optional Telegram
set -euo pipefail

DEPLOY_DIR="${NEXUS_DEPLOY_DIR:-/opt/nexus}"
NEXUS_SERVICE="${NEXUS_SERVICE:-nexus}"
FAIL_FILE="/tmp/nexus_health_fail_count"
LAST_RESTART_FILE="/tmp/nexus_health_last_restart"
MAX_FAIL=3
RESTART_COOLDOWN_SEC="${NEXUS_HEALTH_RESTART_COOLDOWN:-300}"
ENV_PROD="${DEPLOY_DIR}/docker/.env.prod"

resolve_publish_port() {
    local port
    port="$(grep -E '^NEXUS_PUBLISH_PORT=' "$ENV_PROD" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '\r" ' || true)"
    echo "${port:-8600}"
}

PUBLISH_PORT="$(resolve_publish_port)"
HEALTH_URL="http://127.0.0.1:${PUBLISH_PORT}/health"

resolve_nexus_container() {
    docker ps --format '{{.Names}}' 2>/dev/null | grep -E 'nexus-prod-nexus|nexus-nexus-1' | head -1 || true
}

read_env_var() {
    local key="$1" file="$2"
    grep -E "^${key}=" "$file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '\r"' || true
}

load_telegram_from_db() {
    local cname="$1"
    [[ -n "$cname" ]] || return 1
    [[ -n "${BOT_TOKEN:-}" && -n "${CHAT_ID:-}" ]] && return 0
    local out token chat
    out="$(docker exec "$cname" python3 -c "
import asyncio
from server.config import settings as s
from server.infrastructure.database.session import AsyncSessionLocal

async def main():
    async with AsyncSessionLocal() as db:
        await s.load_settings_from_db(db)
    t = (s.TELEGRAM_BOT_TOKEN or '').strip()
    c = (s.TELEGRAM_CHAT_ID or '').strip()
    if t and c:
        print(t)
        print(c)

asyncio.run(main())
" 2>/dev/null)" || true
    [[ -n "$out" ]] || return 1
    token="$(printf '%s\n' "$out" | sed -n '1p')"
    chat="$(printf '%s\n' "$out" | sed -n '2p')"
    [[ -n "$token" && -n "$chat" ]] || return 1
    BOT_TOKEN="$token"
    CHAT_ID="$chat"
}

load_telegram_config() {
    local cname="$1" dotenv
    BOT_TOKEN=""
    CHAT_ID=""
    for dotenv in \
        "${DEPLOY_DIR}/.env" \
        "$ENV_PROD" \
        ; do
        [[ -f "$dotenv" ]] || continue
        BOT_TOKEN="$(read_env_var NEXUS_TELEGRAM_BOT_TOKEN "$dotenv")"
        CHAT_ID="$(read_env_var NEXUS_TELEGRAM_CHAT_ID "$dotenv")"
        [[ -n "$BOT_TOKEN" && -n "$CHAT_ID" ]] && return 0
    done
    if [[ -n "$cname" ]]; then
        local line
        while IFS= read -r line; do
            case "$line" in
                NEXUS_TELEGRAM_BOT_TOKEN=*) BOT_TOKEN="${line#NEXUS_TELEGRAM_BOT_TOKEN=}" ;;
                NEXUS_TELEGRAM_CHAT_ID=*) CHAT_ID="${line#NEXUS_TELEGRAM_CHAT_ID=}" ;;
            esac
        done < <(docker exec "$cname" grep -E '^NEXUS_TELEGRAM_' /app/.env 2>/dev/null || true)
        [[ -n "$BOT_TOKEN" && -n "$CHAT_ID" ]] && return 0
        load_telegram_from_db "$cname" || true
    fi
}

send_telegram() {
    [[ -n "${BOT_TOKEN:-}" && -n "${CHAT_ID:-}" ]] || return 1
    curl -sf --max-time 10 "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
        -d "chat_id=${CHAT_ID}" \
        -d "text=$1" \
        -d "parse_mode=HTML" \
        > /dev/null 2>&1
}

bump_fail_count() {
    local fail=0
    if command -v flock >/dev/null 2>&1; then
        (
            flock -x 9
            fail="$(cat "$FAIL_FILE" 2>/dev/null || echo 0)"
            fail=$((fail + 1))
            echo "$fail" > "$FAIL_FILE"
        ) 9>"${FAIL_FILE}.lock"
    else
        fail="$(cat "$FAIL_FILE" 2>/dev/null || echo 0)"
        fail=$((fail + 1))
        echo "$fail" > "$FAIL_FILE"
    fi
    echo "$fail"
}

reset_fail_count() {
    echo "0" > "$FAIL_FILE"
}

restart_cooldown_active() {
    local now last elapsed
    [[ -f "$LAST_RESTART_FILE" ]] || return 1
    now="$(date +%s)"
    last="$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0)"
    elapsed=$((now - last))
    [[ "$elapsed" -lt "$RESTART_COOLDOWN_SEC" ]]
}

mark_restart_attempt() {
    date +%s > "$LAST_RESTART_FILE"
}

restart_nexus() {
    local cname="$1" via=""
    if [[ -n "$cname" ]]; then
        docker restart "$cname" >/dev/null 2>&1 && via="Docker 容器 ${cname}"
    elif command -v supervisorctl >/dev/null 2>&1 && supervisorctl status "$NEXUS_SERVICE" >/dev/null 2>&1; then
        supervisorctl restart "$NEXUS_SERVICE" >/dev/null 2>&1 && via="Supervisor ${NEXUS_SERVICE}"
    else
        return 1
    fi
    echo "$via"
}

NEXUS_CONTAINER="$(resolve_nexus_container)"
load_telegram_config "$NEXUS_CONTAINER"

if curl -sf --max-time 5 "$HEALTH_URL" > /dev/null 2>&1; then
    reset_fail_count
    exit 0
fi

FAIL="$(bump_fail_count)"
if [[ "$FAIL" -lt "$MAX_FAIL" ]]; then
    exit 0
fi

if restart_cooldown_active; then
    send_telegram "🔴 <b>Nexus /health 连续失败</b>\n冷却期内跳过自动重启（${RESTART_COOLDOWN_SEC}s）\n请检查: docker logs ${NEXUS_CONTAINER:-nexus}" || true
    exit 0
fi

send_telegram "🔴 <b>Nexus 连续 ${MAX_FAIL} 次无响应</b>\n正在自动重启…" || true
mark_restart_attempt

if VIA="$(restart_nexus "$NEXUS_CONTAINER")"; then
    sleep 5
    if curl -sf --max-time 5 "$HEALTH_URL" > /dev/null 2>&1; then
        send_telegram "🟢 <b>Nexus 已恢复</b>\n${VIA} 重启成功" || true
        reset_fail_count
    else
        send_telegram "🔴 <b>Nexus 重启后仍未恢复</b>\n${VIA} — 需人工介入" || true
    fi
else
    send_telegram "🔴 <b>Nexus 无法自动重启</b>\n未找到 Docker 容器或 Supervisor 服务" || true
fi