Nexus/deploy/health_monitor.sh

#!/bin/bash
# Nexus — External Health Monitor (Layer 3 of 3-tier guardian)
# Runs via crontab every minute.
# Does NOT depend on Python — even if Python completely crashes, this script works.
#
# Guardian layers:
#   Layer 1: Supervisor — auto-restart crashed Python process
#   Layer 2: Python self_monitor — internal Redis/MySQL/WS checks (30s)
#   Layer 3: Shell health_monitor.sh — external HTTP health check (1min cron)
#
# Flow:
#   1. HTTP GET http://127.0.0.1:8600/health
#   2. If fails >= 3 consecutive times → restart via supervisorctl + Telegram alert
#   3. If restart succeeds → Telegram recovery notification
#   4. If restart fails → Telegram "needs human intervention" alert
#
# *** Configure DEPLOY_DIR and NEXUS_SERVICE before deploying ***

DEPLOY_DIR="${NEXUS_DEPLOY_DIR:-/opt/nexus}"
NEXUS_SERVICE="${NEXUS_SERVICE:-nexus}"
FAIL_FILE="/tmp/nexus_health_fail_count"
MAX_FAIL=3
HEALTH_URL="http://127.0.0.1:8600/health"

# ── Read Telegram config from .env file ──
DOTENV="$DEPLOY_DIR/.env"

BOT_TOKEN=""
CHAT_ID=""

if [ -f "$DOTENV" ]; then
    BOT_TOKEN=$(grep '^NEXUS_TELEGRAM_BOT_TOKEN=' "$DOTENV" 2>/dev/null | cut -d'=' -f2- || echo "")
    CHAT_ID=$(grep '^NEXUS_TELEGRAM_CHAT_ID=' "$DOTENV" 2>/dev/null | cut -d'=' -f2- || echo "")
fi

send_telegram() {
    [ -z "$BOT_TOKEN" ] || [ -z "$CHAT_ID" ] && return 1
    curl -sf --max-time 10 "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
        -d chat_id="$CHAT_ID" \
        -d text="$1" \
        -d parse_mode=HTML \
        > /dev/null 2>&1
}

# ── Health Check ──
if ! curl -sf --max-time 5 "$HEALTH_URL" > /dev/null 2>&1; then
    # Python is NOT responding
    FAIL=$(cat "$FAIL_FILE" 2>/dev/null || echo "0")
    FAIL=$((FAIL + 1))
    echo "$FAIL" > "$FAIL_FILE"

    if [ "$FAIL" -ge "$MAX_FAIL" ]; then
        # Consecutive failures >= threshold → attempt restart
        send_telegram "🔴 <b>Nexus后端连续${MAX_FAIL}次无响应</b>\n正在通过Supervisor自动重启..."
        supervisorctl restart "$NEXUS_SERVICE" > /dev/null 2>&1
        sleep 5

        # Verify restart
        if curl -sf --max-time 5 "$HEALTH_URL" > /dev/null 2>&1; then
            send_telegram "🟢 <b>Nexus后端已恢复</b>\nSupervisor重启成功"
            echo "0" > "$FAIL_FILE"
        else
            send_telegram "🔴 <b>Nexus重启后仍未恢复！请人工介入</b>\nSupervisor重启失败，需要手动检查"
            # Don't reset fail count — keep retrying each cron cycle
        fi
    fi
else
    # Python is responding normally
    echo "0" > "$FAIL_FILE"
fi