Nexus/deploy/health_monitor.sh

#!/usr/bin/env bash
# Nexus — External Health Monitor (Layer 3 of 3-tier guardian)
# Runs via crontab every minute. Works on Docker (1Panel) and Supervisor bare-metal.
#
# Layer 1: Docker restart policy / Supervisor autorestart
# Layer 2: Python self_monitor (30s)
# Layer 3: This script — HTTP /health, 3 failures → restart + optional Telegram
set -euo pipefail

DEPLOY_DIR="${NEXUS_DEPLOY_DIR:-/opt/nexus}"
NEXUS_SERVICE="${NEXUS_SERVICE:-nexus}"
STATE_DIR=""
FAIL_FILE=""
LAST_RESTART_FILE=""
# Legacy paths (cron user may have created root-owned files under /tmp)
LEGACY_FAIL_FILE="/tmp/nexus_health_fail_count"
LEGACY_LAST_RESTART_FILE="/tmp/nexus_health_last_restart"
MAX_FAIL=3
RESTART_COOLDOWN_SEC="${NEXUS_HEALTH_RESTART_COOLDOWN:-300}"
ENV_PROD="${DEPLOY_DIR}/docker/.env.prod"

ensure_state_paths() {
    if [[ -n "$STATE_DIR" ]]; then
        return 0
    fi
    local preferred="${DEPLOY_DIR}/var/layer3-health"
    local fallback="${HOME:-/tmp}/.nexus/layer3-health"
    if mkdir -p "$preferred" 2>/dev/null && [[ -w "$preferred" ]]; then
        STATE_DIR="$preferred"
    elif command -v sudo >/dev/null 2>&1 && sudo mkdir -p "$preferred" 2>/dev/null \
        && sudo chown -R "$(id -un):$(id -gn)" "$preferred" 2>/dev/null \
        && [[ -w "$preferred" ]]; then
        STATE_DIR="$preferred"
    else
        mkdir -p "$fallback"
        STATE_DIR="$fallback"
    fi
    FAIL_FILE="${STATE_DIR}/fail_count"
    LAST_RESTART_FILE="${STATE_DIR}/last_restart"
}

docker_cmd() {
    if docker "$@" 2>/dev/null; then
        return 0
    fi
    if command -v sudo >/dev/null 2>&1; then
        sudo docker "$@" 2>/dev/null
    else
        return 1
    fi
}

init_layer3_state() {
    ensure_state_paths
    chmod 755 "$STATE_DIR" 2>/dev/null || true
    if [[ ! -f "$FAIL_FILE" && -f "$LEGACY_FAIL_FILE" ]]; then
        cp "$LEGACY_FAIL_FILE" "$FAIL_FILE" 2>/dev/null || true
    fi
    if [[ ! -f "$LAST_RESTART_FILE" && -f "$LEGACY_LAST_RESTART_FILE" ]]; then
        cp "$LEGACY_LAST_RESTART_FILE" "$LAST_RESTART_FILE" 2>/dev/null || true
    fi
}

write_state_file() {
    local path="$1" value="$2"
    init_layer3_state
    printf '%s\n' "$value" >"$path" 2>/dev/null || return 1
}

resolve_publish_port() {
    local port
    port="$(grep -E '^NEXUS_PUBLISH_PORT=' "$ENV_PROD" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '\r" ' || true)"
    echo "${port:-8600}"
}

PUBLISH_PORT="$(resolve_publish_port)"
HEALTH_URL="http://127.0.0.1:${PUBLISH_PORT}/health"

resolve_nexus_container() {
    docker_cmd ps --format '{{.Names}}' 2>/dev/null | grep -E 'nexus-prod-nexus|nexus-nexus-1' | head -1 || true
}

read_env_var() {
    local key="$1" file="$2"
    grep -E "^${key}=" "$file" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '\r"' || true
}

normalize_env_value() {
    local v="${1:-}"
    v="${v//$'\r'/}"
    if [[ ${#v} -ge 2 && ${v:0:1} == '"' && ${v: -1} == '"' ]]; then
        v="${v:1:-1}"
        v="${v//\\n/$'\n'}"
        v="${v//\\\"/\"}"
        v="${v//\\\\/\\}"
    fi
    echo "$v"
}

load_telegram_from_db() {
    local cname="$1"
    [[ -n "$cname" ]] || return 1
    [[ -n "${BOT_TOKEN:-}" && -n "${CHAT_ID:-}" ]] && return 0
    local out token chat
    out="$(docker_cmd exec "$cname" python3 -c "
import asyncio
from server.config import settings as s
from server.infrastructure.database.session import AsyncSessionLocal

async def main():
    async with AsyncSessionLocal() as db:
        await s.load_settings_from_db(db)
    t = (s.TELEGRAM_BOT_TOKEN or '').strip()
    c = (s.TELEGRAM_CHAT_ID or '').strip()
    if t and c:
        print(t)
        print(c)

asyncio.run(main())
" 2>/dev/null)" || true
    [[ -n "$out" ]] || return 1
    token="$(printf '%s\n' "$out" | sed -n '1p')"
    chat="$(printf '%s\n' "$out" | sed -n '2p')"
    [[ -n "$token" && -n "$chat" ]] || return 1
    BOT_TOKEN="$token"
    CHAT_ID="$chat"
}

load_telegram_config() {
    local cname="$1" dotenv
    BOT_TOKEN=""
    CHAT_ID=""
    for dotenv in \
        "${DEPLOY_DIR}/.env" \
        "$ENV_PROD" \
        ; do
        [[ -f "$dotenv" ]] || continue
        BOT_TOKEN="$(read_env_var NEXUS_TELEGRAM_BOT_TOKEN "$dotenv")"
        CHAT_ID="$(read_env_var NEXUS_TELEGRAM_CHAT_ID "$dotenv")"
        [[ -n "$BOT_TOKEN" && -n "$CHAT_ID" ]] && return 0
    done
    if [[ -n "$cname" ]]; then
        local line
        while IFS= read -r line; do
            case "$line" in
                NEXUS_TELEGRAM_BOT_TOKEN=*) BOT_TOKEN="${line#NEXUS_TELEGRAM_BOT_TOKEN=}" ;;
                NEXUS_TELEGRAM_CHAT_ID=*) CHAT_ID="${line#NEXUS_TELEGRAM_CHAT_ID=}" ;;
            esac
        done < <(docker_cmd exec "$cname" grep -E '^NEXUS_TELEGRAM_' /app/.env 2>/dev/null || true)
        [[ -n "$BOT_TOKEN" && -n "$CHAT_ID" ]] && return 0
        load_telegram_from_db "$cname" || true
    fi
    BOT_TOKEN="$(normalize_env_value "$BOT_TOKEN")"
    CHAT_ID="$(normalize_env_value "$CHAT_ID")"
}

send_telegram() {
    local msg="${1}" resp http log="${NEXUS_HEALTH_LOG:-/var/log/nexus_health.log}"
    BOT_TOKEN="$(normalize_env_value "${BOT_TOKEN:-}")"
    CHAT_ID="$(normalize_env_value "${CHAT_ID:-}")"
    [[ -n "$BOT_TOKEN" && -n "$CHAT_ID" ]] || return 1
    resp="$(curl -s --max-time 15 -w '\n%{http_code}' \
        "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
        -d "chat_id=${CHAT_ID}" \
        --data-urlencode "text=${msg}" \
        -d "parse_mode=HTML" 2>/dev/null)" || return 1
    http="${resp##*$'\n'}"
    if [[ "$http" != "200" ]]; then
        echo "$(date -Is 2>/dev/null || date) telegram send failed http=${http}" >> "$log" 2>/dev/null || true
        return 1
    fi
}

bump_fail_count() {
    local fail=0
    init_layer3_state
    if command -v flock >/dev/null 2>&1; then
        (
            flock -x 9
            fail="$(cat "$FAIL_FILE" 2>/dev/null || echo 0)"
            fail=$((fail + 1))
            write_state_file "$FAIL_FILE" "$fail" || true
        ) 9>"${FAIL_FILE}.lock"
        fail="$(cat "$FAIL_FILE" 2>/dev/null || echo 0)"
    else
        fail="$(cat "$FAIL_FILE" 2>/dev/null || echo 0)"
        fail=$((fail + 1))
        write_state_file "$FAIL_FILE" "$fail" || true
    fi
    echo "$fail"
}

reset_fail_count() {
    write_state_file "$FAIL_FILE" "0" || true
}

restart_cooldown_active() {
    local now last elapsed
    [[ -f "$LAST_RESTART_FILE" ]] || return 1
    now="$(date +%s)"
    last="$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0)"
    elapsed=$((now - last))
    [[ "$elapsed" -lt "$RESTART_COOLDOWN_SEC" ]]
}

mark_restart_attempt() {
    write_state_file "$LAST_RESTART_FILE" "$(date +%s)" || true
}

restart_nexus() {
    local cname="$1" via=""
    if [[ -n "$cname" ]]; then
        docker_cmd restart "$cname" >/dev/null 2>&1 && via="Docker 容器 ${cname}"
    elif command -v supervisorctl >/dev/null 2>&1 && supervisorctl status "$NEXUS_SERVICE" >/dev/null 2>&1; then
        supervisorctl restart "$NEXUS_SERVICE" >/dev/null 2>&1 && via="Supervisor ${NEXUS_SERVICE}"
    else
        return 1
    fi
    echo "$via"
}

telegram_health_fail_cooldown() {
    local body
    body="$(cat <<EOF
🔴 <b>【Layer 3 宿主机巡检】健康检查连续失败（冷却期内）</b>
宿主机：${HOST_LABEL}
检测地址：${HEALTH_URL}
连续失败：已达 ${MAX_FAIL} 次
Docker 容器：${NEXUS_CONTAINER:-未识别}
具体情况：/health 仍不可达，但距上次自动重启不足 ${RESTART_COOLDOWN_SEC} 秒，本次跳过重启以免频繁抖动
建议操作：查看 docker logs ${NEXUS_CONTAINER:-nexus} 与 ${NEXUS_HEALTH_LOG:-/var/log/nexus_health.log}
时间：${TS_LABEL}
EOF
)"
    send_telegram "$body" || true
}

telegram_health_fail_restart() {
    local body
    body="$(cat <<EOF
🔴 <b>【Layer 3 宿主机巡检】Nexus 连续 ${MAX_FAIL} 次无响应</b>
宿主机：${HOST_LABEL}
检测地址：${HEALTH_URL}
连续失败：${FAIL} 次（阈值 ${MAX_FAIL} 次）
Docker 容器：${NEXUS_CONTAINER:-未识别}
具体情况：宿主机 cron 无法访问 API 健康端点，平台可能已宕机或端口异常
正在执行：自动重启 Nexus 服务…
时间：${TS_LABEL}
EOF
)"
    send_telegram "$body" || true
}

telegram_health_recovered() {
    local via="$1" body
    body="$(cat <<EOF
🟢 <b>【Layer 3 宿主机巡检】Nexus 已自动恢复</b>
宿主机：${HOST_LABEL}
检测地址：${HEALTH_URL}
恢复方式：${via}
具体情况：自动重启后 /health 检测通过，服务已恢复在线
时间：${TS_LABEL}
EOF
)"
    send_telegram "$body" || true
}

telegram_health_still_down() {
    local via="$1" body
    body="$(cat <<EOF
🔴 <b>【Layer 3 宿主机巡检】自动重启后仍未恢复</b>
宿主机：${HOST_LABEL}
检测地址：${HEALTH_URL}
已尝试：${via}
具体情况：重启完成后 /health 仍失败，可能存在数据库/Redis 故障或配置错误
建议操作：立即 SSH 登录排查 docker logs、MySQL、Redis，需人工介入
时间：${TS_LABEL}
EOF
)"
    send_telegram "$body" || true
}

telegram_health_restart_failed() {
    local body
    body="$(cat <<EOF
🔴 <b>【Layer 3 宿主机巡检】无法执行自动重启</b>
宿主机：${HOST_LABEL}
检测地址：${HEALTH_URL}
具体情况：未找到可重启的 Docker 容器或 Supervisor 服务（${NEXUS_SERVICE}）
建议操作：确认 nexus 容器名称与 docker ps 输出，手动执行 docker restart 或 nx update
时间：${TS_LABEL}
EOF
)"
    send_telegram "$body" || true
}

NEXUS_CONTAINER="$(resolve_nexus_container)"
load_telegram_config "$NEXUS_CONTAINER"
init_layer3_state
HOST_LABEL="$(hostname -f 2>/dev/null || hostname 2>/dev/null || echo unknown)"
TS_LABEL="$(date '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null || date)"

if curl -sf --max-time 5 "$HEALTH_URL" > /dev/null 2>&1; then
    reset_fail_count
    exit 0
fi

FAIL="$(bump_fail_count)"
if [[ "$FAIL" -lt "$MAX_FAIL" ]]; then
    exit 0
fi

if restart_cooldown_active; then
    telegram_health_fail_cooldown
    exit 0
fi

telegram_health_fail_restart
mark_restart_attempt

if VIA="$(restart_nexus "$NEXUS_CONTAINER")"; then
    sleep 5
    if curl -sf --max-time 5 "$HEALTH_URL" > /dev/null 2>&1; then
        telegram_health_recovered "$VIA"
        reset_fail_count
    else
        telegram_health_still_down "$VIA"
    fi
else
    telegram_health_restart_failed
fi