143 lines
4.3 KiB
Bash
143 lines
4.3 KiB
Bash
#!/usr/bin/env bash
|
||
# monitor-ai 健康检查兜底脚本
|
||
#
|
||
# 用途：当 monitor-ai 自身的健康检查失败时，通过 webhook 直接发送告警
|
||
# 部署：添加到 crontab，每 5 分钟执行一次
|
||
#
|
||
# crontab 配置：
|
||
# */5 * * * * /root/docker/monitor-ai/scripts/monitor-watchdog.sh >> /var/log/monitor-watchdog.log 2>&1
|
||
#
|
||
# 日志文件：/var/log/monitor-watchdog.log（建议配合 logrotate）
|
||
|
||
set -euo pipefail
|
||
|
||
# ============================================================
|
||
# 配置
|
||
# ============================================================
|
||
HEALTH_URL="${MONITOR_HEALTH_URL:-https://monitor.tlyq.ai/api/health}"
|
||
TIMEOUT=10
|
||
MAX_RETRIES=2
|
||
RETRY_DELAY=5
|
||
|
||
# 企业微信 Webhook（从 .env 读取或使用环境变量）
|
||
MONITOR_DIR="/root/docker/monitor-ai"
|
||
if [[ -f "$MONITOR_DIR/.env" ]]; then
|
||
# 从 .env 读取第一个启用的 webhook URL
|
||
WEBHOOK_URL="${WEBHOOK_URL:-$(grep -E '^WEBHOOK_URL=' "$MONITOR_DIR/.env" 2>/dev/null | head -1 | cut -d= -f2-)}"
|
||
fi
|
||
|
||
# 如果没有配置 webhook，尝试从数据库读取
|
||
if [[ -z "${WEBHOOK_URL:-}" ]]; then
|
||
DB_PATH="${MONITOR_DIR}/data/monitor.db"
|
||
if [[ -f "$DB_PATH" ]]; then
|
||
WEBHOOK_URL=$(sqlite3 "$DB_PATH" "SELECT webhook_url FROM alert_channels WHERE enabled=1 LIMIT 1;" 2>/dev/null || echo "")
|
||
fi
|
||
fi
|
||
|
||
# 日志函数
|
||
log() {
|
||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
|
||
}
|
||
|
||
# ============================================================
|
||
# 健康检查
|
||
# ============================================================
|
||
check_health() {
|
||
local attempt=0
|
||
local http_code=""
|
||
|
||
while [[ $attempt -lt $MAX_RETRIES ]]; do
|
||
attempt=$((attempt + 1))
|
||
|
||
http_code=$(curl -s -o /dev/null -w '%{http_code}' \
|
||
--connect-timeout "$TIMEOUT" \
|
||
--max-time "$TIMEOUT" \
|
||
"$HEALTH_URL" 2>/dev/null) || http_code="000"
|
||
|
||
if [[ "$http_code" == "200" ]]; then
|
||
return 0
|
||
fi
|
||
|
||
if [[ $attempt -lt $MAX_RETRIES ]]; then
|
||
log "健康检查失败 (HTTP $http_code)，${RETRY_DELAY}s 后重试..."
|
||
sleep "$RETRY_DELAY"
|
||
fi
|
||
done
|
||
|
||
return 1
|
||
}
|
||
|
||
# ============================================================
|
||
# 发送告警
|
||
# ============================================================
|
||
send_alert() {
|
||
local message="$1"
|
||
|
||
if [[ -z "${WEBHOOK_URL:-}" ]]; then
|
||
log "WARNING: 未配置企业微信 Webhook URL，无法发送告警"
|
||
return 1
|
||
fi
|
||
|
||
local payload
|
||
payload=$(cat <<EOF
|
||
{
|
||
"msgtype": "markdown",
|
||
"markdown": {
|
||
"content": "## ⚠️ monitor-ai 自身异常\n\n> 告警时间: $(date '+%Y-%m-%d %H:%M:%S')\n\n**健康检查地址**: ${HEALTH_URL}\n\n**失败原因**: ${message}\n\n**处理建议**: 请检查 monitor-ai 容器状态\n\n<font color=\"warning\">此告警由 monitor-watchdog 发送</font>"
|
||
}
|
||
}
|
||
EOF
|
||
)
|
||
|
||
local response
|
||
response=$(curl -s -X POST \
|
||
--connect-timeout 10 \
|
||
--max-time 10 \
|
||
-H "Content-Type: application/json" \
|
||
-d "$payload" \
|
||
"$WEBHOOK_URL" 2>&1)
|
||
|
||
if [[ $? -eq 0 ]]; then
|
||
log "告警已发送: $response"
|
||
else
|
||
log "WARNING: 告警发送失败: $response"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# ============================================================
|
||
# 主逻辑
|
||
# ============================================================
|
||
main() {
|
||
log "开始健康检查: $HEALTH_URL"
|
||
|
||
if check_health; then
|
||
log "健康检查通过 (HTTP 200)"
|
||
exit 0
|
||
fi
|
||
|
||
log "ERROR: 健康检查失败，尝试发送告警..."
|
||
send_alert "HTTP 健康检查失败，连续 ${MAX_RETRIES} 次请求均未返回 200"
|
||
|
||
# 检查容器状态
|
||
local container_status
|
||
container_status=$(docker inspect -f '{{.State.Status}}' monitor-ai 2>/dev/null || echo "not_found")
|
||
log "容器状态: $container_status"
|
||
|
||
if [[ "$container_status" != "running" ]]; then
|
||
log "ERROR: 容器未运行，尝试重启..."
|
||
cd "$MONITOR_DIR" && docker compose up -d 2>&1 | while read -r line; do log " $line"; done
|
||
|
||
# 等待容器启动后再次检查
|
||
sleep 10
|
||
if check_health; then
|
||
log "容器已重启并通过健康检查"
|
||
else
|
||
log "ERROR: 重启后健康检查仍然失败，请人工介入"
|
||
send_alert "容器已重启，但健康检查仍然失败，请人工介入"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
main "$@"
|