diff --git a/monitor-watchdog.sh b/monitor-watchdog.sh new file mode 100644 index 0000000..b470b67 --- /dev/null +++ b/monitor-watchdog.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# monitor-ai 健康检查兜底脚本 +# +# 用途:当 monitor-ai 自身的健康检查失败时,通过 webhook 直接发送告警 +# 部署:添加到 crontab,每 5 分钟执行一次 +# +# crontab 配置: +# */5 * * * * /root/docker/monitor-ai/scripts/monitor-watchdog.sh >> /var/log/monitor-watchdog.log 2>&1 +# +# 日志文件:/var/log/monitor-watchdog.log(建议配合 logrotate) + +set -euo pipefail + +# ============================================================ +# 配置 +# ============================================================ +HEALTH_URL="${MONITOR_HEALTH_URL:-https://monitor.tlyq.ai/api/health}" +TIMEOUT=10 +MAX_RETRIES=2 +RETRY_DELAY=5 + +# 企业微信 Webhook(从 .env 读取或使用环境变量) +MONITOR_DIR="/root/docker/monitor-ai" +if [[ -f "$MONITOR_DIR/.env" ]]; then + # 从 .env 读取第一个启用的 webhook URL + WEBHOOK_URL="${WEBHOOK_URL:-$(grep -E '^WEBHOOK_URL=' "$MONITOR_DIR/.env" 2>/dev/null | head -1 | cut -d= -f2-)}" +fi + +# 如果没有配置 webhook,尝试从数据库读取 +if [[ -z "${WEBHOOK_URL:-}" ]]; then + DB_PATH="${MONITOR_DIR}/data/monitor.db" + if [[ -f "$DB_PATH" ]]; then + WEBHOOK_URL=$(sqlite3 "$DB_PATH" "SELECT webhook_url FROM alert_channels WHERE enabled=1 LIMIT 1;" 2>/dev/null || echo "") + fi +fi + +# 日志函数 +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" +} + +# ============================================================ +# 健康检查 +# ============================================================ +check_health() { + local attempt=0 + local http_code="" + + while [[ $attempt -lt $MAX_RETRIES ]]; do + attempt=$((attempt + 1)) + + http_code=$(curl -s -o /dev/null -w '%{http_code}' \ + --connect-timeout "$TIMEOUT" \ + --max-time "$TIMEOUT" \ + "$HEALTH_URL" 2>/dev/null) || http_code="000" + + if [[ "$http_code" == "200" ]]; then + return 0 + fi + + if [[ $attempt -lt $MAX_RETRIES ]]; then + log "健康检查失败 (HTTP $http_code),${RETRY_DELAY}s 后重试..." + sleep "$RETRY_DELAY" + fi + done + + return 1 +} + +# ============================================================ +# 发送告警 +# ============================================================ +send_alert() { + local message="$1" + + if [[ -z "${WEBHOOK_URL:-}" ]]; then + log "WARNING: 未配置企业微信 Webhook URL,无法发送告警" + return 1 + fi + + local payload + payload=$(cat < 告警时间: $(date '+%Y-%m-%d %H:%M:%S')\n\n**健康检查地址**: ${HEALTH_URL}\n\n**失败原因**: ${message}\n\n**处理建议**: 请检查 monitor-ai 容器状态\n\n此告警由 monitor-watchdog 发送" + } +} +EOF +) + + local response + response=$(curl -s -X POST \ + --connect-timeout 10 \ + --max-time 10 \ + -H "Content-Type: application/json" \ + -d "$payload" \ + "$WEBHOOK_URL" 2>&1) + + if [[ $? -eq 0 ]]; then + log "告警已发送: $response" + else + log "WARNING: 告警发送失败: $response" + return 1 + fi +} + +# ============================================================ +# 主逻辑 +# ============================================================ +main() { + log "开始健康检查: $HEALTH_URL" + + if check_health; then + log "健康检查通过 (HTTP 200)" + exit 0 + fi + + log "ERROR: 健康检查失败,尝试发送告警..." + send_alert "HTTP 健康检查失败,连续 ${MAX_RETRIES} 次请求均未返回 200" + + # 检查容器状态 + local container_status + container_status=$(docker inspect -f '{{.State.Status}}' monitor-ai 2>/dev/null || echo "not_found") + log "容器状态: $container_status" + + if [[ "$container_status" != "running" ]]; then + log "ERROR: 容器未运行,尝试重启..." + cd "$MONITOR_DIR" && docker compose up -d 2>&1 | while read -r line; do log " $line"; done + + # 等待容器启动后再次检查 + sleep 10 + if check_health; then + log "容器已重启并通过健康检查" + else + log "ERROR: 重启后健康检查仍然失败,请人工介入" + send_alert "容器已重启,但健康检查仍然失败,请人工介入" + fi + fi +} + +main "$@"