Nexus/deploy/health_monitor.sh

#!/usr/bin/env bash
# Nexus — External Health Monitor (Layer 3 of 3-tier guardian)
# Runs via crontab every minute. Works on Docker (1Panel) and Supervisor bare-metal.
#
# Layer 1: Docker restart policy / Supervisor autorestart
# Layer 2: Python self_monitor (30s)
# Layer 3: This script — HTTP /health, 3 failures → restart + optional Telegram
set -euo pipefail

DEPLOY_DIR="${NEXUS_DEPLOY_DIR:-/opt/nexus}"
NEXUS_SERVICE="${NEXUS_SERVICE:-nexus}"
FAIL_FILE="/tmp/nexus_health_fail_count"
LAST_RESTART_FILE="/tmp/nexus_health_last_restart"
MAX_FAIL=3
RESTART_COOLDOWN_SEC="${NEXUS_HEALTH_RESTART_COOLDOWN:-300}"
ENV_PROD="${DEPLOY_DIR}/docker/.env.prod"

resolve_publish_port() {
    local port
    port="$(grep -E '^NEXUS_PUBLISH_PORT=' "$ENV_PROD" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '\r" ' || true)"
    echo "${port:-8600}"
}

PUBLISH_PORT="$(resolve_publish_port)"
HEALTH_URL="http://127.0.0.1:${PUBLISH_PORT}/health"

resolve_nexus_container() {
    docker ps --format '{{.Names}}' 2>/dev/null | grep -E 'nexus-prod-nexus|nexus-nexus-1' | head -1 || true
}

read_env_var() {
    local key="$1" file="$2"
    grep -E "^${key}=" "$file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '\r"' || true
}

normalize_env_value() {
    local v="${1:-}"
    v="${v//$'\r'/}"
    if [[ ${#v} -ge 2 && ${v:0:1} == '"' && ${v: -1} == '"' ]]; then
        v="${v:1:-1}"
        v="${v//\\n/$'\n'}"
        v="${v//\\\"/\"}"
        v="${v//\\\\/\\}"
    fi
    echo "$v"
}

load_telegram_from_db() {
    local cname="$1"
    [[ -n "$cname" ]] || return 1
    [[ -n "${BOT_TOKEN:-}" && -n "${CHAT_ID:-}" ]] && return 0
    local out token chat
    out="$(docker exec "$cname" python3 -c "
import asyncio
from server.config import settings as s
from server.infrastructure.database.session import AsyncSessionLocal

async def main():
    async with AsyncSessionLocal() as db:
        await s.load_settings_from_db(db)
    t = (s.TELEGRAM_BOT_TOKEN or '').strip()
    c = (s.TELEGRAM_CHAT_ID or '').strip()
    if t and c:
        print(t)
        print(c)

asyncio.run(main())
" 2>/dev/null)" || true
    [[ -n "$out" ]] || return 1
    token="$(printf '%s\n' "$out" | sed -n '1p')"
    chat="$(printf '%s\n' "$out" | sed -n '2p')"
    [[ -n "$token" && -n "$chat" ]] || return 1
    BOT_TOKEN="$token"
    CHAT_ID="$chat"
}

load_telegram_config() {
    local cname="$1" dotenv
    BOT_TOKEN=""
    CHAT_ID=""
    for dotenv in \
        "${DEPLOY_DIR}/.env" \
        "$ENV_PROD" \
        ; do
        [[ -f "$dotenv" ]] || continue
        BOT_TOKEN="$(read_env_var NEXUS_TELEGRAM_BOT_TOKEN "$dotenv")"
        CHAT_ID="$(read_env_var NEXUS_TELEGRAM_CHAT_ID "$dotenv")"
        [[ -n "$BOT_TOKEN" && -n "$CHAT_ID" ]] && return 0
    done
    if [[ -n "$cname" ]]; then
        local line
        while IFS= read -r line; do
            case "$line" in
                NEXUS_TELEGRAM_BOT_TOKEN=*) BOT_TOKEN="${line#NEXUS_TELEGRAM_BOT_TOKEN=}" ;;
                NEXUS_TELEGRAM_CHAT_ID=*) CHAT_ID="${line#NEXUS_TELEGRAM_CHAT_ID=}" ;;
            esac
        done < <(docker exec "$cname" grep -E '^NEXUS_TELEGRAM_' /app/.env 2>/dev/null || true)
        [[ -n "$BOT_TOKEN" && -n "$CHAT_ID" ]] && return 0
        load_telegram_from_db "$cname" || true
    fi
    BOT_TOKEN="$(normalize_env_value "$BOT_TOKEN")"
    CHAT_ID="$(normalize_env_value "$CHAT_ID")"
}

send_telegram() {
    local msg="${1//\\n/$'\n'}" resp http log="${NEXUS_HEALTH_LOG:-/var/log/nexus_health.log}"
    BOT_TOKEN="$(normalize_env_value "${BOT_TOKEN:-}")"
    CHAT_ID="$(normalize_env_value "${CHAT_ID:-}")"
    [[ -n "$BOT_TOKEN" && -n "$CHAT_ID" ]] || return 1
    resp="$(curl -s --max-time 15 -w '\n%{http_code}' \
        "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
        -d "chat_id=${CHAT_ID}" \
        --data-urlencode "text=${msg}" \
        -d "parse_mode=HTML" 2>/dev/null)" || return 1
    http="${resp##*$'\n'}"
    if [[ "$http" != "200" ]]; then
        echo "$(date -Is 2>/dev/null || date) telegram send failed http=${http}" >> "$log" 2>/dev/null || true
        return 1
    fi
}

bump_fail_count() {
    local fail=0
    if command -v flock >/dev/null 2>&1; then
        (
            flock -x 9
            fail="$(cat "$FAIL_FILE" 2>/dev/null || echo 0)"
            fail=$((fail + 1))
            echo "$fail" > "$FAIL_FILE"
        ) 9>"${FAIL_FILE}.lock"
        fail="$(cat "$FAIL_FILE" 2>/dev/null || echo 0)"
    else
        fail="$(cat "$FAIL_FILE" 2>/dev/null || echo 0)"
        fail=$((fail + 1))
        echo "$fail" > "$FAIL_FILE"
    fi
    echo "$fail"
}

reset_fail_count() {
    echo "0" > "$FAIL_FILE"
}

restart_cooldown_active() {
    local now last elapsed
    [[ -f "$LAST_RESTART_FILE" ]] || return 1
    now="$(date +%s)"
    last="$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0)"
    elapsed=$((now - last))
    [[ "$elapsed" -lt "$RESTART_COOLDOWN_SEC" ]]
}

mark_restart_attempt() {
    date +%s > "$LAST_RESTART_FILE"
}

restart_nexus() {
    local cname="$1" via=""
    if [[ -n "$cname" ]]; then
        docker restart "$cname" >/dev/null 2>&1 && via="Docker 容器 ${cname}"
    elif command -v supervisorctl >/dev/null 2>&1 && supervisorctl status "$NEXUS_SERVICE" >/dev/null 2>&1; then
        supervisorctl restart "$NEXUS_SERVICE" >/dev/null 2>&1 && via="Supervisor ${NEXUS_SERVICE}"
    else
        return 1
    fi
    echo "$via"
}

NEXUS_CONTAINER="$(resolve_nexus_container)"
load_telegram_config "$NEXUS_CONTAINER"
HOST_LABEL="$(hostname -f 2>/dev/null || hostname 2>/dev/null || echo unknown)"
TS_LABEL="$(date '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null || date)"

if curl -sf --max-time 5 "$HEALTH_URL" > /dev/null 2>&1; then
    reset_fail_count
    exit 0
fi

FAIL="$(bump_fail_count)"
if [[ "$FAIL" -lt "$MAX_FAIL" ]]; then
    exit 0
fi

if restart_cooldown_active; then
    send_telegram "🔴 <b>【Layer 3 宿主机巡检】健康检查连续失败（冷却期内）</b>\n宿主机：${HOST_LABEL}\n检测地址：${HEALTH_URL}\n连续失败：已达 ${MAX_FAIL} 次\nDocker 容器：${NEXUS_CONTAINER:-未识别}\n具体情况：/health 仍不可达，但距上次自动重启不足 ${RESTART_COOLDOWN_SEC} 秒，本次跳过重启以免频繁抖动\n建议操作：查看 docker logs ${NEXUS_CONTAINER:-nexus} 与 /var/log/nexus_health.log\n时间：${TS_LABEL}" || true
    exit 0
fi

send_telegram "🔴 <b>【Layer 3 宿主机巡检】Nexus 连续 ${MAX_FAIL} 次无响应</b>\n宿主机：${HOST_LABEL}\n检测地址：${HEALTH_URL}\n连续失败：${FAIL} 次（阈值 ${MAX_FAIL} 次）\nDocker 容器：${NEXUS_CONTAINER:-未识别}\n具体情况：宿主机 cron 无法访问 API 健康端点，平台可能已宕机或端口异常\n正在执行：自动重启 Nexus 服务…\n时间：${TS_LABEL}" || true
mark_restart_attempt

if VIA="$(restart_nexus "$NEXUS_CONTAINER")"; then
    sleep 5
    if curl -sf --max-time 5 "$HEALTH_URL" > /dev/null 2>&1; then
        send_telegram "🟢 <b>【Layer 3 宿主机巡检】Nexus 已自动恢复</b>\n宿主机：${HOST_LABEL}\n检测地址：${HEALTH_URL}\n恢复方式：${VIA}\n具体情况：自动重启后 /health 检测通过，服务已恢复在线\n时间：${TS_LABEL}" || true
        reset_fail_count
    else
        send_telegram "🔴 <b>【Layer 3 宿主机巡检】自动重启后仍未恢复</b>\n宿主机：${HOST_LABEL}\n检测地址：${HEALTH_URL}\n已尝试：${VIA}\n具体情况：重启完成后 /health 仍失败，可能存在数据库/Redis 故障或配置错误\n建议操作：立即 SSH 登录排查 docker logs、MySQL、Redis，需人工介入\n时间：${TS_LABEL}" || true
    fi
else
    send_telegram "🔴 <b>【Layer 3 宿主机巡检】无法执行自动重启</b>\n宿主机：${HOST_LABEL}\n检测地址：${HEALTH_URL}\n具体情况：未找到可重启的 Docker 容器或 Supervisor 服务（${NEXUS_SERVICE}）\n建议操作：确认 nexus 容器名称与 docker ps 输出，手动执行 docker restart 或 nx update\n时间：${TS_LABEL}" || true
fi