#!/usr/bin/env bash # monitor-ai 健康检查兜底脚本 # # 用途:当 monitor-ai 自身的健康检查失败时,通过 webhook 直接发送告警 # 部署:添加到 crontab,每 5 分钟执行一次 # # crontab 配置: # */5 * * * * /root/docker/monitor-ai/scripts/monitor-watchdog.sh >> /var/log/monitor-watchdog.log 2>&1 # # 日志文件:/var/log/monitor-watchdog.log(建议配合 logrotate) set -euo pipefail # ============================================================ # 配置 # ============================================================ HEALTH_URL="${MONITOR_HEALTH_URL:-https://monitor.tlyq.ai/api/health}" TIMEOUT=10 MAX_RETRIES=2 RETRY_DELAY=5 # 企业微信 Webhook(从 .env 读取或使用环境变量) MONITOR_DIR="/root/docker/monitor-ai" if [[ -f "$MONITOR_DIR/.env" ]]; then # 从 .env 读取第一个启用的 webhook URL WEBHOOK_URL="${WEBHOOK_URL:-$(grep -E '^WEBHOOK_URL=' "$MONITOR_DIR/.env" 2>/dev/null | head -1 | cut -d= -f2-)}" fi # 如果没有配置 webhook,尝试从数据库读取 if [[ -z "${WEBHOOK_URL:-}" ]]; then DB_PATH="${MONITOR_DIR}/data/monitor.db" if [[ -f "$DB_PATH" ]]; then WEBHOOK_URL=$(sqlite3 "$DB_PATH" "SELECT webhook_url FROM alert_channels WHERE enabled=1 LIMIT 1;" 2>/dev/null || echo "") fi fi # 日志函数 log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" } # ============================================================ # 健康检查 # ============================================================ check_health() { local attempt=0 local http_code="" while [[ $attempt -lt $MAX_RETRIES ]]; do attempt=$((attempt + 1)) http_code=$(curl -s -o /dev/null -w '%{http_code}' \ --connect-timeout "$TIMEOUT" \ --max-time "$TIMEOUT" \ "$HEALTH_URL" 2>/dev/null) || http_code="000" if [[ "$http_code" == "200" ]]; then return 0 fi if [[ $attempt -lt $MAX_RETRIES ]]; then log "健康检查失败 (HTTP $http_code),${RETRY_DELAY}s 后重试..." sleep "$RETRY_DELAY" fi done return 1 } # ============================================================ # 发送告警 # ============================================================ send_alert() { local message="$1" if [[ -z "${WEBHOOK_URL:-}" ]]; then log "WARNING: 未配置企业微信 Webhook URL,无法发送告警" return 1 fi local payload payload=$(cat < 告警时间: $(date '+%Y-%m-%d %H:%M:%S')\n\n**健康检查地址**: ${HEALTH_URL}\n\n**失败原因**: ${message}\n\n**处理建议**: 请检查 monitor-ai 容器状态\n\n此告警由 monitor-watchdog 发送" } } EOF ) local response response=$(curl -s -X POST \ --connect-timeout 10 \ --max-time 10 \ -H "Content-Type: application/json" \ -d "$payload" \ "$WEBHOOK_URL" 2>&1) if [[ $? -eq 0 ]]; then log "告警已发送: $response" else log "WARNING: 告警发送失败: $response" return 1 fi } # ============================================================ # 主逻辑 # ============================================================ main() { log "开始健康检查: $HEALTH_URL" if check_health; then log "健康检查通过 (HTTP 200)" exit 0 fi log "ERROR: 健康检查失败,尝试发送告警..." send_alert "HTTP 健康检查失败,连续 ${MAX_RETRIES} 次请求均未返回 200" # 检查容器状态 local container_status container_status=$(docker inspect -f '{{.State.Status}}' monitor-ai 2>/dev/null || echo "not_found") log "容器状态: $container_status" if [[ "$container_status" != "running" ]]; then log "ERROR: 容器未运行,尝试重启..." cd "$MONITOR_DIR" && docker compose up -d 2>&1 | while read -r line; do log " $line"; done # 等待容器启动后再次检查 sleep 10 if check_health; then log "容器已重启并通过健康检查" else log "ERROR: 重启后健康检查仍然失败,请人工介入" send_alert "容器已重启,但健康检查仍然失败,请人工介入" fi fi } main "$@"