// src/app/api/monitor/scan-emails/route.ts import { NextRequest, NextResponse } from 'next/server' import { initDatabase } from '@/lib/db-schema' import { getDb } from '@/lib/db' import { getCurrentUser } from '@/lib/auth' import { hasPermission } from '@/lib/permissions' import { getMonitorConfig } from '@/lib/monitor/settings-manager' import Imap from 'imap' import { simpleParser } from 'mailparser' import * as cheerio from 'cheerio' import { formatBeijingTime } from '@/lib/monitor/types' import { writeAuditLog, getClientIP } from '@/lib/audit' import { getScanState, setScanState, isCancelRequested, resetCancelFlag } from '@/lib/monitor/scan-state' import { fetchPOP3Emails, normalizeMessageId } from '@/lib/monitor/pop3-fetcher' import { notifyTicketCreated } from '@/lib/monitor/ticket-notifier' interface ScanResult { status: 'running' | 'completed' | 'error' | 'cancelling' startedAt: string completedAt?: string timeRange: { value: number | null; unit: string } stats: { total: number matched: number imported: number skipped: number errors: number } details: { msg_id: string subject: string date: string order_number: string | null status: 'imported' | 'skipped' | 'error' ticket_no?: string error?: string }[] detailsTruncated: boolean error?: string } const VALID_UNITS = ['minute', 'hour', 'day', 'week', 'month', 'all'] const MAX_VALUE = 365 const MAX_DETAILS = 500 const SCAN_TIMEOUT_MS = 5 * 60 * 1000 // 5 分钟超时 // 辅助函数:添加 detail 并追踪是否被截断 function addDetail( scanState: ScanResult, detail: ScanResult['details'][0], counter: { total: number } ): void { counter.total++ if (scanState.details.length < MAX_DETAILS) { scanState.details.push(detail) } else { scanState.detailsTruncated = true } } function extractOrderNumber(subject: string): string | null { const match = subject.match(/【服务器故障单】(\d+),/) return match?.[1] ?? null } function getDateRange(value: number | null, unit: string): Date { const now = new Date() if (!value || unit === 'all') { // 全部:搜索最近 30 天 now.setDate(now.getDate() - 30) return now } switch (unit) { case 'minute': now.setMinutes(now.getMinutes() - value) break case 'hour': now.setHours(now.getHours() - value) break case 'day': now.setDate(now.getDate() - value) break case 'week': now.setDate(now.getDate() - value * 7) break case 'month': now.setMonth(now.getMonth() - value) break default: now.setDate(now.getDate() - 7) } return now } export async function POST(request: NextRequest) { initDatabase() const user = await getCurrentUser() if (!user || !hasPermission(user, 'monitor:write')) { return NextResponse.json({ error: '权限不足' }, { status: 403 }) } const clientIP = getClientIP(request) // 检查是否有正在运行的扫描(包括取消中的扫描) const currentScan = getScanState() if (currentScan?.status === 'running' || currentScan?.status === 'cancelling') { return NextResponse.json({ error: '扫描正在进行中,请稍后再试' }, { status: 409 }) } const body = await request.json() const { value, unit } = body.timeRange || { value: 7, unit: 'day' } // 输入验证 if (unit !== 'all') { if (typeof value !== 'number' || value < 1 || value > MAX_VALUE) { return NextResponse.json({ error: `时间范围值必须在 1-${MAX_VALUE} 之间` }, { status: 400 }) } } if (!VALID_UNITS.includes(unit)) { return NextResponse.json({ error: `无效的时间单位: ${unit}` }, { status: 400 }) } const config = getMonitorConfig() const db = getDb() const since = getDateRange(value, unit) // 初始化扫描结果 const scanState: ScanResult = { status: 'running', startedAt: formatBeijingTime(), timeRange: { value, unit }, stats: { total: 0, matched: 0, imported: 0, skipped: 0, errors: 0 }, details: [], detailsTruncated: false, } setScanState(scanState) // 重置取消标志 resetCancelFlag() // 异步执行扫描 scanEmails(config, db, since, scanState, clientIP, user.id).catch(err => { // 内层已设置错误状态,此处不再重复设置 console.error('[Scan] Unexpected error:', err) }) return NextResponse.json({ success: true, message: '扫描已开始' }) } async function scanEmails( config: ReturnType, db: ReturnType, since: Date, scanState: ScanResult, clientIP: string | null, userId: number ): Promise { // 使用 imap 库(与 Python 版本兼容)替代 imapflow // imapflow 对单部分 text/html 邮件的 source 获取有 bug const imap = new Imap({ user: config.mail.address, password: config.mail.password, host: config.mail.imap_server, port: config.mail.imap_port, tls: true, connTimeout: 15000, authTimeout: 15000, }) // 整体超时控制 const startTime = Date.now() const checkTimeout = (): boolean => { if (Date.now() - startTime > SCAN_TIMEOUT_MS) { scanState.status = 'error' scanState.completedAt = formatBeijingTime() scanState.error = `扫描超时(超过 ${SCAN_TIMEOUT_MS / 60000} 分钟)` return true } return false } // 将 IMAP 回调风格封装为 Promise function imapConnect(): Promise { return new Promise((resolve, reject) => { imap.once('ready', () => resolve()) imap.once('error', (err: Error) => reject(err)) imap.connect() }) } function imapOpenBox(boxName: string, readOnly: boolean): Promise { return new Promise((resolve, reject) => { imap.openBox(boxName, readOnly, (err: Error | null, box: Imap.Box) => { if (err) reject(err) else resolve(box) }) }) } function imapSearch(criteria: any[]): Promise { return new Promise((resolve, reject) => { // imap.search 返回 UID(imap.fetch 默认也使用 UID) imap.search(criteria, (err: Error | null, results: number[]) => { if (err) reject(err) else resolve(results) }) }) } // 获取单封邮件的原始内容(使用 UID 获取) function fetchEmailRaw(uid: number): Promise { return new Promise((resolve, reject) => { console.log(`[Scan] fetchEmailRaw(uid=${uid}) 开始获取`) // imap.fetch 默认使用 UID,无需 uid: true const fetch = imap.fetch(uid, { bodies: '', markSeen: false }) let emailBuffer = Buffer.alloc(0) let messageCount = 0 let bodyCount = 0 fetch.on('message', (msg: any, seqno: number) => { messageCount++ console.log(`[Scan] fetchEmailRaw(uid=${uid}) message 事件触发, seqno=${seqno}`) msg.on('body', (stream: any, info: any) => { bodyCount++ console.log(`[Scan] fetchEmailRaw(uid=${uid}) body 事件触发, info=${JSON.stringify(info)}`) const chunks: Buffer[] = [] stream.on('data', (chunk: Buffer) => { chunks.push(chunk) console.log(`[Scan] fetchEmailRaw(uid=${uid}) 收到数据块, 大小=${chunk.length}`) }) stream.on('end', () => { emailBuffer = Buffer.concat(chunks) console.log(`[Scan] fetchEmailRaw(uid=${uid}) body 结束, 总大小=${emailBuffer.length}`) }) }) }) fetch.once('end', () => { console.log(`[Scan] fetchEmailRaw(uid=${uid}) fetch 结束, messageCount=${messageCount}, bodyCount=${bodyCount}, 最终大小=${emailBuffer.length}`) resolve(emailBuffer) }) fetch.once('error', (err: Error) => { console.error(`[Scan] fetchEmailRaw(uid=${uid}) fetch 错误:`, err.message) reject(err) }) }) } // 通过 IMAP ENVELOPE 获取邮件的 Message-ID(BODY 为空时 ENVELOPE 仍可用) // 用于与 POP3 邮件精确匹配,避免脆弱的位置对齐 // 注意:只请求 envelope,不能带 bodies(BODY 为空时 attributes 事件不会触发) function fetchEmailMessageId(uid: number): Promise { return new Promise((resolve) => { let messageId: string | null = null try { const fetch = imap.fetch(uid, { envelope: true } as any) fetch.on('message', (msg: any) => { msg.on('attributes', (attrs: any) => { messageId = attrs?.envelope?.messageId || null }) }) fetch.once('end', () => resolve(normalizeMessageId(messageId))) fetch.once('error', () => resolve(null)) } catch { resolve(null) } }) } const detailsCounter = { total: 0 } try { await imapConnect() console.log('[Scan] IMAP 连接成功') await imapOpenBox('INBOX', true) // 搜索邮件:将 since 日期转换为 IMAP SINCE 格式 const sinceStr = since.toLocaleDateString('en-US', { day: '2-digit', month: 'short', year: 'numeric' }) const uids = await imapSearch([['SINCE', sinceStr]]) console.log(`[Scan] 搜索条件 SINCE "${sinceStr}",找到 ${uids.length} 封邮件`) if (!uids || uids.length === 0) { scanState.status = 'completed' scanState.completedAt = formatBeijingTime() imap.end() saveScanHistory(scanState, userId) return } scanState.stats.total = uids.length // POP3 fallback 状态:IMAP 近期邮件 BODY 为空时自动切换 // pop3Map: Message-ID → 原始邮件 Buffer;pop3Attempted: 是否已尝试(无论成败,避免重连风暴) let pop3Map: Map = new Map() let pop3Attempted = false // 逐封处理邮件 for (let idx = 0; idx < uids.length; idx++) { const uid = uids[idx] if (isCancelRequested()) { scanState.status = 'completed' scanState.completedAt = formatBeijingTime() scanState.error = '用户取消' break } if (checkTimeout()) break try { // 获取完整邮件原始内容(RFC822 格式),优先 IMAP,失败时 POP3 fallback let rawEmail: Buffer | null = await fetchEmailRaw(uid) // IMAP BODY 为空时,首次触发 POP3 fallback(一次性拉取所有邮件建立 Message-ID 索引) if ((!rawEmail || rawEmail.length === 0) && !pop3Attempted) { console.log(`[Scan] IMAP 返回空 (uid=${uid}),启用 POP3 fallback`) pop3Attempted = true // 无论成败都置位,避免每封空邮件都重连 try { pop3Map = await fetchPOP3Emails(config.mail.address, config.mail.password, uids.length, config.mail.pop3_server, config.mail.pop3_port) } catch (pop3Err) { console.error('[Scan] POP3 fallback 失败:', pop3Err instanceof Error ? pop3Err.message : pop3Err) } } // 从 POP3 缓存中按 Message-ID 精确匹配(通过 IMAP ENVELOPE 获取当前 UID 的 Message-ID) if ((!rawEmail || rawEmail.length === 0) && pop3Map.size > 0) { const targetMsgId = await fetchEmailMessageId(uid) if (targetMsgId && pop3Map.has(targetMsgId)) { rawEmail = pop3Map.get(targetMsgId)! console.log(`[Scan] POP3 fallback 命中: uid=${uid}, msgId=${targetMsgId}, size=${rawEmail.length}`) } else { console.log(`[Scan] POP3 fallback 未命中: uid=${uid}, msgId=${targetMsgId || '(未获取到)'}`) } } if (!rawEmail || rawEmail.length === 0) { console.log(`[Scan] 邮件 ${uid} 内容为空(IMAP + POP3 均失败),跳过`) continue } // 使用 mailparser 解析邮件 const parsed = await simpleParser(rawEmail) const subject = parsed.subject || '' const from = parsed.from?.text || '' const date = parsed.date ? formatBeijingTime(parsed.date) : formatBeijingTime() const msgId = parsed.messageId || String(uid) console.log(`[Scan] 邮件 ${uid}: subject="${subject}", from="${from}", 大小=${rawEmail.length}`) // 邮件过滤:检查发件人是否匹配 if (config.filter.sender_email) { const senderMatch = from.toLowerCase().includes(config.filter.sender_email.toLowerCase()) if (!senderMatch) { console.log(`[Scan] 邮件 ${uid}: 发件人不匹配,跳过`) continue } } // 邮件过滤:检查主题关键词是否匹配 if (config.filter.subject_keywords && config.filter.subject_keywords.length > 0) { const keywordMatch = config.filter.subject_keywords.some(kw => subject.toLowerCase().includes(kw.toLowerCase()) ) if (!keywordMatch) { console.log(`[Scan] 邮件 ${uid}: 主题关键词不匹配,跳过`) continue } } // 提取工单号 const orderNumber = extractOrderNumber(subject) if (!orderNumber) { scanState.stats.matched++ addDetail(scanState, { msg_id: msgId, subject, date, order_number: null, status: 'skipped', error: '无法提取工单号', }, detailsCounter) continue } // 检查工单是否已存在 const existing = db.prepare('SELECT id FROM tickets WHERE id = ?').get(parseInt(orderNumber)) if (existing) { scanState.stats.skipped++ scanState.stats.matched++ addDetail(scanState, { msg_id: msgId, subject, date, order_number: orderNumber, status: 'skipped', ticket_no: orderNumber, error: '工单已存在', }, detailsCounter) continue } // 检查是否已处理过 const processed = db.prepare('SELECT msg_id FROM processed_emails WHERE msg_id = ?').get(msgId) if (processed) { scanState.stats.skipped++ scanState.stats.matched++ addDetail(scanState, { msg_id: msgId, subject, date, order_number: orderNumber, status: 'skipped', error: '邮件已处理过', }, detailsCounter) continue } // 从邮件 HTML 正文中提取故障信息(参考 Python 脚本的 extract_fault_info) const htmlContent = parsed.html || '' const ticketInfo = buildTicketContent(subject, orderNumber, htmlContent) // 根据设备 IP 查询 assets 站点获取节点名称 if (ticketInfo.device_ip) { try { const assetsUrl = process.env.ASSETS_URL || 'https://assets.tlyq.ai' const assetsApiKey = process.env.ASSETS_API_KEY || '' if (!assetsApiKey) { console.log(`[Scan] 跳过设备名称查询: ASSETS_API_KEY 未配置`) } else { const headers: Record = { 'Authorization': `Bearer ${assetsApiKey}` } const controller = new AbortController() const timeout = setTimeout(() => controller.abort(), 5000) try { const assetsResp = await fetch(`${assetsUrl}/api/assets?filter_business_ip=${encodeURIComponent(ticketInfo.device_ip)}`, { headers, signal: controller.signal }) clearTimeout(timeout) if (assetsResp.ok) { const assetsData = await assetsResp.json() if (assetsData.data && assetsData.data.length > 0) { ticketInfo.device_name = assetsData.data[0].node_name || null console.log(`[Scan] 设备 IP ${ticketInfo.device_ip} → 节点名称: ${ticketInfo.device_name}`) } } else { console.error(`[Scan] assets API 返回 ${assetsResp.status} (IP: ${ticketInfo.device_ip})`) } } catch (fetchErr) { clearTimeout(timeout) throw fetchErr } } } catch (e) { console.error(`[Scan] 查询设备名称失败 (IP: ${ticketInfo.device_ip}): ${e instanceof Error ? e.message : e}`) } } // 创建工单,填入提取的字段 const ticketId = parseInt(orderNumber) db.prepare(` INSERT INTO tickets (id, content, device_ip, device_sn, device_name, ticket_type, assign_time, current_status, created_by, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, 'open', ?, datetime('now', '+8 hours'), datetime('now', '+8 hours')) `).run(ticketId, ticketInfo.content, ticketInfo.device_ip, ticketInfo.device_sn, ticketInfo.device_name, ticketInfo.ticket_type, ticketInfo.assign_time, userId) db.prepare("INSERT OR IGNORE INTO processed_emails (msg_id, subject) VALUES (?, ?)").run(msgId, subject) writeAuditLog({ userId, apiKeyId: null, action: 'import', entityType: 'ticket', entityId: ticketId, details: { created: { ticket_no: orderNumber, source: 'email_scan' } }, ipAddress: clientIP, }) void notifyTicketCreated({ ticket_no: orderNumber, device_ip: ticketInfo.device_ip, device_sn: ticketInfo.device_sn, device_name: ticketInfo.device_name, ticket_type: ticketInfo.ticket_type, content: ticketInfo.content, assign_time: ticketInfo.assign_time, }).catch(e => console.error('[scan-emails] notify 失败:', e)) scanState.stats.imported++ scanState.stats.matched++ addDetail(scanState, { msg_id: msgId, subject, date, order_number: orderNumber, status: 'imported', ticket_no: orderNumber, }, detailsCounter) } catch (err) { scanState.stats.errors++ addDetail(scanState, { msg_id: String(uid), subject: '', date: '', order_number: null, status: 'error', error: err instanceof Error ? err.message : '处理失败', }, detailsCounter) } } if (scanState.detailsTruncated) { console.log(`[Scan] 详情列表已截断: 共 ${detailsCounter.total} 条,仅保存前 ${MAX_DETAILS} 条`) } if (scanState.status === 'running') { scanState.status = 'completed' scanState.completedAt = formatBeijingTime() } } catch (err) { scanState.status = 'error' scanState.completedAt = formatBeijingTime() scanState.error = err instanceof Error ? err.message : '连接失败' } finally { try { imap.end() } catch {} } saveScanHistory(scanState, userId) } // 提取的邮件信息结构 interface ExtractedTicketInfo { content: string device_ip: string | null device_sn: string | null device_name: string | null ticket_type: string | null fault_time: string | null fault_info: string | null auth_note: string | null assign_time: string | null } // 从邮件 HTML 正文中提取故障信息,参考 Python 脚本的 extract_fault_info function buildTicketContent(subject: string, orderNumber: string, htmlContent: string): ExtractedTicketInfo { const result: ExtractedTicketInfo = { content: '', device_ip: null, device_sn: null, device_name: null, ticket_type: null, fault_time: null, fault_info: null, auth_note: null, assign_time: null, } // 提取工单类型 if (subject.includes('OEM诊断')) { result.ticket_type = 'OEM诊断' } else if (subject.includes('OEM维修')) { result.ticket_type = 'OEM维修' } // 使用 cheerio 解析 HTML 表格 if (htmlContent) { const $ = cheerio.load(htmlContent) // 先定位包含"故障信息"的目标表格,避免混入无关表格 let targetTable = $('table').filter((_, table) => { return $(table).text().includes('故障信息') }).first() // 如果没找到包含"故障信息"的表格,使用第一个表格 if (targetTable.length === 0) { targetTable = $('table').first() } // 遍历目标表格的行,提取键值对 targetTable.find('tr').each((_, row) => { const cells = $(row).find('td') if (cells.length >= 2) { const key = $(cells[0]).text().trim() const value = $(cells[1]).text().trim() switch (key) { case '故障发生时间:': case '故障发生时间': result.fault_time = value result.assign_time = value break case '服务器SN:': case '服务器SN': result.device_sn = value break case '服务器IP地址:': case '服务器IP地址': case '服务器IP:': case '服务器IP': result.device_ip = value break case '故障信息:': case '故障信息': result.fault_info = value break case '授权说明:': case '授权说明': result.auth_note = value break } } }) // Fallback:如果表格提取失败,使用正则从纯文本中提取 if (!result.fault_info) { const textContent = $.text().replace(/\s+/g, ' ') const faultMatch = textContent.match(/故障信息[::\s]*(.+?)(?=授权说明|$)/i) if (faultMatch) { result.fault_info = faultMatch[1].trim() } } if (!result.device_ip) { const textContent = $.text().replace(/\s+/g, ' ') const ipMatch = textContent.match(/(?:服务器IP地址|服务器IP|IP)[::\s]*(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/) if (ipMatch) { result.device_ip = ipMatch[1] } } if (!result.device_sn) { const textContent = $.text().replace(/\s+/g, ' ') const snMatch = textContent.match(/服务器SN[::\s]*([A-Za-z0-9]+)/) if (snMatch) { result.device_sn = snMatch[1] } } } // content 只包含故障信息 result.content = result.fault_info || null return result } // 保存扫描历史到数据库 function saveScanHistory(scanState: ScanResult, userId: number): void { try { const db = getDb() db.prepare(` INSERT INTO scan_history ( status, started_at, completed_at, time_range_value, time_range_unit, total_count, matched_count, imported_count, skipped_count, error_count, details_truncated, error_message, details_json, created_by ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `).run( scanState.status, scanState.startedAt, scanState.completedAt || null, scanState.timeRange.value, scanState.timeRange.unit, scanState.stats.total, scanState.stats.matched, scanState.stats.imported, scanState.stats.skipped, scanState.stats.errors, scanState.detailsTruncated ? 1 : 0, scanState.error || null, JSON.stringify(scanState.details), userId ) // 保留最近 50 条扫描历史,删除更早的(用 id 单调排序,避免 created_at 同秒重复) db.prepare(`DELETE FROM scan_history WHERE id NOT IN ( SELECT id FROM scan_history ORDER BY id DESC LIMIT 50 )`).run() console.log(`[Scan] 扫描历史已保存: 状态=${scanState.status}, 导入=${scanState.stats.imported}`) } catch (err) { console.error('[Scan] 保存扫描历史失败:', err) } }