issue-ai/src/app/api/monitor/scan-emails/route.ts

649 lines
23 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// src/app/api/monitor/scan-emails/route.ts
import { NextRequest, NextResponse } from 'next/server'
import { initDatabase } from '@/lib/db-schema'
import { getDb } from '@/lib/db'
import { getCurrentUser } from '@/lib/auth'
import { hasPermission } from '@/lib/permissions'
import { getMonitorConfig } from '@/lib/monitor/settings-manager'
import Imap from 'imap'
import { simpleParser } from 'mailparser'
import * as cheerio from 'cheerio'
import { formatBeijingTime } from '@/lib/monitor/types'
import { writeAuditLog, getClientIP } from '@/lib/audit'
import { getScanState, setScanState, isCancelRequested, resetCancelFlag } from '@/lib/monitor/scan-state'
import { fetchPOP3Emails, normalizeMessageId } from '@/lib/monitor/pop3-fetcher'
import { notifyTicketCreated } from '@/lib/monitor/ticket-notifier'
interface ScanResult {
status: 'running' | 'completed' | 'error' | 'cancelling'
startedAt: string
completedAt?: string
timeRange: { value: number | null; unit: string }
stats: {
total: number
matched: number
imported: number
skipped: number
errors: number
}
details: {
msg_id: string
subject: string
date: string
order_number: string | null
status: 'imported' | 'skipped' | 'error'
ticket_no?: string
error?: string
}[]
detailsTruncated: boolean
error?: string
}
const VALID_UNITS = ['minute', 'hour', 'day', 'week', 'month', 'all']
const MAX_VALUE = 365
const MAX_DETAILS = 500
const SCAN_TIMEOUT_MS = 5 * 60 * 1000 // 5 分钟超时
// 辅助函数:添加 detail 并追踪是否被截断
function addDetail(
scanState: ScanResult,
detail: ScanResult['details'][0],
counter: { total: number }
): void {
counter.total++
if (scanState.details.length < MAX_DETAILS) {
scanState.details.push(detail)
} else {
scanState.detailsTruncated = true
}
}
function extractOrderNumber(subject: string): string | null {
const match = subject.match(/【服务器故障单】(\d+),/)
return match?.[1] ?? null
}
function getDateRange(value: number | null, unit: string): Date {
const now = new Date()
if (!value || unit === 'all') {
// 全部:搜索最近 30 天
now.setDate(now.getDate() - 30)
return now
}
switch (unit) {
case 'minute':
now.setMinutes(now.getMinutes() - value)
break
case 'hour':
now.setHours(now.getHours() - value)
break
case 'day':
now.setDate(now.getDate() - value)
break
case 'week':
now.setDate(now.getDate() - value * 7)
break
case 'month':
now.setMonth(now.getMonth() - value)
break
default:
now.setDate(now.getDate() - 7)
}
return now
}
export async function POST(request: NextRequest) {
initDatabase()
const user = await getCurrentUser()
if (!user || !hasPermission(user, 'monitor:write')) {
return NextResponse.json({ error: '权限不足' }, { status: 403 })
}
const clientIP = getClientIP(request)
// 检查是否有正在运行的扫描(包括取消中的扫描)
const currentScan = getScanState()
if (currentScan?.status === 'running' || currentScan?.status === 'cancelling') {
return NextResponse.json({ error: '扫描正在进行中,请稍后再试' }, { status: 409 })
}
const body = await request.json()
const { value, unit } = body.timeRange || { value: 7, unit: 'day' }
// 输入验证
if (unit !== 'all') {
if (typeof value !== 'number' || value < 1 || value > MAX_VALUE) {
return NextResponse.json({ error: `时间范围值必须在 1-${MAX_VALUE} 之间` }, { status: 400 })
}
}
if (!VALID_UNITS.includes(unit)) {
return NextResponse.json({ error: `无效的时间单位: ${unit}` }, { status: 400 })
}
const config = getMonitorConfig()
const db = getDb()
const since = getDateRange(value, unit)
// 初始化扫描结果
const scanState: ScanResult = {
status: 'running',
startedAt: formatBeijingTime(),
timeRange: { value, unit },
stats: { total: 0, matched: 0, imported: 0, skipped: 0, errors: 0 },
details: [],
detailsTruncated: false,
}
setScanState(scanState)
// 重置取消标志
resetCancelFlag()
// 异步执行扫描
scanEmails(config, db, since, scanState, clientIP, user.id).catch(err => {
// 内层已设置错误状态,此处不再重复设置
console.error('[Scan] Unexpected error:', err)
})
return NextResponse.json({ success: true, message: '扫描已开始' })
}
async function scanEmails(
config: ReturnType<typeof getMonitorConfig>,
db: ReturnType<typeof getDb>,
since: Date,
scanState: ScanResult,
clientIP: string | null,
userId: number
): Promise<void> {
// 使用 imap 库(与 Python 版本兼容)替代 imapflow
// imapflow 对单部分 text/html 邮件的 source 获取有 bug
const imap = new Imap({
user: config.mail.address,
password: config.mail.password,
host: config.mail.imap_server,
port: config.mail.imap_port,
tls: true,
connTimeout: 15000,
authTimeout: 15000,
})
// 整体超时控制
const startTime = Date.now()
const checkTimeout = (): boolean => {
if (Date.now() - startTime > SCAN_TIMEOUT_MS) {
scanState.status = 'error'
scanState.completedAt = formatBeijingTime()
scanState.error = `扫描超时(超过 ${SCAN_TIMEOUT_MS / 60000} 分钟)`
return true
}
return false
}
// 将 IMAP 回调风格封装为 Promise
function imapConnect(): Promise<void> {
return new Promise((resolve, reject) => {
imap.once('ready', () => resolve())
imap.once('error', (err: Error) => reject(err))
imap.connect()
})
}
function imapOpenBox(boxName: string, readOnly: boolean): Promise<Imap.Box> {
return new Promise((resolve, reject) => {
imap.openBox(boxName, readOnly, (err: Error | null, box: Imap.Box) => {
if (err) reject(err)
else resolve(box)
})
})
}
function imapSearch(criteria: any[]): Promise<number[]> {
return new Promise((resolve, reject) => {
// imap.search 返回 UIDimap.fetch 默认也使用 UID
imap.search(criteria, (err: Error | null, results: number[]) => {
if (err) reject(err)
else resolve(results)
})
})
}
// 获取单封邮件的原始内容(使用 UID 获取)
function fetchEmailRaw(uid: number): Promise<Buffer> {
return new Promise((resolve, reject) => {
console.log(`[Scan] fetchEmailRaw(uid=${uid}) 开始获取`)
// imap.fetch 默认使用 UID无需 uid: true
const fetch = imap.fetch(uid, { bodies: '', markSeen: false })
let emailBuffer = Buffer.alloc(0)
let messageCount = 0
let bodyCount = 0
fetch.on('message', (msg: any, seqno: number) => {
messageCount++
console.log(`[Scan] fetchEmailRaw(uid=${uid}) message 事件触发, seqno=${seqno}`)
msg.on('body', (stream: any, info: any) => {
bodyCount++
console.log(`[Scan] fetchEmailRaw(uid=${uid}) body 事件触发, info=${JSON.stringify(info)}`)
const chunks: Buffer[] = []
stream.on('data', (chunk: Buffer) => {
chunks.push(chunk)
console.log(`[Scan] fetchEmailRaw(uid=${uid}) 收到数据块, 大小=${chunk.length}`)
})
stream.on('end', () => {
emailBuffer = Buffer.concat(chunks)
console.log(`[Scan] fetchEmailRaw(uid=${uid}) body 结束, 总大小=${emailBuffer.length}`)
})
})
})
fetch.once('end', () => {
console.log(`[Scan] fetchEmailRaw(uid=${uid}) fetch 结束, messageCount=${messageCount}, bodyCount=${bodyCount}, 最终大小=${emailBuffer.length}`)
resolve(emailBuffer)
})
fetch.once('error', (err: Error) => {
console.error(`[Scan] fetchEmailRaw(uid=${uid}) fetch 错误:`, err.message)
reject(err)
})
})
}
// 通过 IMAP ENVELOPE 获取邮件的 Message-IDBODY 为空时 ENVELOPE 仍可用)
// 用于与 POP3 邮件精确匹配,避免脆弱的位置对齐
// 注意:只请求 envelope不能带 bodiesBODY 为空时 attributes 事件不会触发)
function fetchEmailMessageId(uid: number): Promise<string | null> {
return new Promise((resolve) => {
let messageId: string | null = null
try {
const fetch = imap.fetch(uid, { envelope: true } as any)
fetch.on('message', (msg: any) => {
msg.on('attributes', (attrs: any) => {
messageId = attrs?.envelope?.messageId || null
})
})
fetch.once('end', () => resolve(normalizeMessageId(messageId)))
fetch.once('error', () => resolve(null))
} catch {
resolve(null)
}
})
}
const detailsCounter = { total: 0 }
try {
await imapConnect()
console.log('[Scan] IMAP 连接成功')
await imapOpenBox('INBOX', true)
// 搜索邮件:将 since 日期转换为 IMAP SINCE 格式
const sinceStr = since.toLocaleDateString('en-US', { day: '2-digit', month: 'short', year: 'numeric' })
const uids = await imapSearch([['SINCE', sinceStr]])
console.log(`[Scan] 搜索条件 SINCE "${sinceStr}",找到 ${uids.length} 封邮件`)
if (!uids || uids.length === 0) {
scanState.status = 'completed'
scanState.completedAt = formatBeijingTime()
imap.end()
saveScanHistory(scanState, userId)
return
}
scanState.stats.total = uids.length
// POP3 fallback 状态IMAP 近期邮件 BODY 为空时自动切换
// pop3Map: Message-ID → 原始邮件 Bufferpop3Attempted: 是否已尝试(无论成败,避免重连风暴)
let pop3Map: Map<string, Buffer> = new Map()
let pop3Attempted = false
// 逐封处理邮件
for (let idx = 0; idx < uids.length; idx++) {
const uid = uids[idx]
if (isCancelRequested()) {
scanState.status = 'completed'
scanState.completedAt = formatBeijingTime()
scanState.error = '用户取消'
break
}
if (checkTimeout()) break
try {
// 获取完整邮件原始内容RFC822 格式),优先 IMAP失败时 POP3 fallback
let rawEmail: Buffer | null = await fetchEmailRaw(uid)
// IMAP BODY 为空时,首次触发 POP3 fallback一次性拉取所有邮件建立 Message-ID 索引)
if ((!rawEmail || rawEmail.length === 0) && !pop3Attempted) {
console.log(`[Scan] IMAP 返回空 (uid=${uid}),启用 POP3 fallback`)
pop3Attempted = true // 无论成败都置位,避免每封空邮件都重连
try {
pop3Map = await fetchPOP3Emails(config.mail.address, config.mail.password, uids.length, config.mail.pop3_server, config.mail.pop3_port)
} catch (pop3Err) {
console.error('[Scan] POP3 fallback 失败:', pop3Err instanceof Error ? pop3Err.message : pop3Err)
}
}
// 从 POP3 缓存中按 Message-ID 精确匹配(通过 IMAP ENVELOPE 获取当前 UID 的 Message-ID
if ((!rawEmail || rawEmail.length === 0) && pop3Map.size > 0) {
const targetMsgId = await fetchEmailMessageId(uid)
if (targetMsgId && pop3Map.has(targetMsgId)) {
rawEmail = pop3Map.get(targetMsgId)!
console.log(`[Scan] POP3 fallback 命中: uid=${uid}, msgId=${targetMsgId}, size=${rawEmail.length}`)
} else {
console.log(`[Scan] POP3 fallback 未命中: uid=${uid}, msgId=${targetMsgId || '(未获取到)'}`)
}
}
if (!rawEmail || rawEmail.length === 0) {
console.log(`[Scan] 邮件 ${uid} 内容为空IMAP + POP3 均失败),跳过`)
continue
}
// 使用 mailparser 解析邮件
const parsed = await simpleParser(rawEmail)
const subject = parsed.subject || ''
const from = parsed.from?.text || ''
const date = parsed.date ? formatBeijingTime(parsed.date) : formatBeijingTime()
const msgId = parsed.messageId || String(uid)
console.log(`[Scan] 邮件 ${uid}: subject="${subject}", from="${from}", 大小=${rawEmail.length}`)
// 邮件过滤:检查发件人是否匹配
if (config.filter.sender_email) {
const senderMatch = from.toLowerCase().includes(config.filter.sender_email.toLowerCase())
if (!senderMatch) {
console.log(`[Scan] 邮件 ${uid}: 发件人不匹配,跳过`)
continue
}
}
// 邮件过滤:检查主题关键词是否匹配
if (config.filter.subject_keywords && config.filter.subject_keywords.length > 0) {
const keywordMatch = config.filter.subject_keywords.some(kw =>
subject.toLowerCase().includes(kw.toLowerCase())
)
if (!keywordMatch) {
console.log(`[Scan] 邮件 ${uid}: 主题关键词不匹配,跳过`)
continue
}
}
// 提取工单号
const orderNumber = extractOrderNumber(subject)
if (!orderNumber) {
scanState.stats.matched++
addDetail(scanState, {
msg_id: msgId, subject, date,
order_number: null, status: 'skipped', error: '无法提取工单号',
}, detailsCounter)
continue
}
// 检查工单是否已存在
const existing = db.prepare('SELECT id FROM tickets WHERE id = ?').get(parseInt(orderNumber))
if (existing) {
scanState.stats.skipped++
scanState.stats.matched++
addDetail(scanState, {
msg_id: msgId, subject, date,
order_number: orderNumber, status: 'skipped', ticket_no: orderNumber, error: '工单已存在',
}, detailsCounter)
continue
}
// 检查是否已处理过
const processed = db.prepare('SELECT msg_id FROM processed_emails WHERE msg_id = ?').get(msgId)
if (processed) {
scanState.stats.skipped++
scanState.stats.matched++
addDetail(scanState, {
msg_id: msgId, subject, date,
order_number: orderNumber, status: 'skipped', error: '邮件已处理过',
}, detailsCounter)
continue
}
// 从邮件 HTML 正文中提取故障信息(参考 Python 脚本的 extract_fault_info
const htmlContent = parsed.html || ''
const ticketInfo = buildTicketContent(subject, orderNumber, htmlContent)
// 根据设备 IP 查询 assets 站点获取节点名称
if (ticketInfo.device_ip) {
try {
const assetsUrl = process.env.ASSETS_URL || 'https://assets.tlyq.ai'
const assetsApiKey = process.env.ASSETS_API_KEY || ''
if (!assetsApiKey) {
console.log(`[Scan] 跳过设备名称查询: ASSETS_API_KEY 未配置`)
} else {
const headers: Record<string, string> = { 'Authorization': `Bearer ${assetsApiKey}` }
const controller = new AbortController()
const timeout = setTimeout(() => controller.abort(), 5000)
try {
const assetsResp = await fetch(`${assetsUrl}/api/assets?filter_business_ip=${encodeURIComponent(ticketInfo.device_ip)}`, { headers, signal: controller.signal })
clearTimeout(timeout)
if (assetsResp.ok) {
const assetsData = await assetsResp.json()
if (assetsData.data && assetsData.data.length > 0) {
ticketInfo.device_name = assetsData.data[0].node_name || null
console.log(`[Scan] 设备 IP ${ticketInfo.device_ip} → 节点名称: ${ticketInfo.device_name}`)
}
} else {
console.error(`[Scan] assets API 返回 ${assetsResp.status} (IP: ${ticketInfo.device_ip})`)
}
} catch (fetchErr) {
clearTimeout(timeout)
throw fetchErr
}
}
} catch (e) {
console.error(`[Scan] 查询设备名称失败 (IP: ${ticketInfo.device_ip}): ${e instanceof Error ? e.message : e}`)
}
}
// 创建工单,填入提取的字段
const ticketId = parseInt(orderNumber)
db.prepare(`
INSERT INTO tickets (id, content, device_ip, device_sn, device_name, ticket_type, assign_time, current_status, created_by, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, 'open', ?, datetime('now', '+8 hours'), datetime('now', '+8 hours'))
`).run(ticketId, ticketInfo.content, ticketInfo.device_ip, ticketInfo.device_sn, ticketInfo.device_name, ticketInfo.ticket_type, ticketInfo.assign_time, userId)
db.prepare("INSERT OR IGNORE INTO processed_emails (msg_id, subject) VALUES (?, ?)").run(msgId, subject)
writeAuditLog({
userId, apiKeyId: null, action: 'import', entityType: 'ticket', entityId: ticketId,
details: { created: { ticket_no: orderNumber, source: 'email_scan' } },
ipAddress: clientIP,
})
void notifyTicketCreated({
ticket_no: orderNumber,
device_ip: ticketInfo.device_ip,
device_sn: ticketInfo.device_sn,
device_name: ticketInfo.device_name,
ticket_type: ticketInfo.ticket_type,
content: ticketInfo.content,
assign_time: ticketInfo.assign_time,
}).catch(e => console.error('[scan-emails] notify 失败:', e))
scanState.stats.imported++
scanState.stats.matched++
addDetail(scanState, {
msg_id: msgId, subject, date,
order_number: orderNumber, status: 'imported', ticket_no: orderNumber,
}, detailsCounter)
} catch (err) {
scanState.stats.errors++
addDetail(scanState, {
msg_id: String(uid), subject: '', date: '',
order_number: null, status: 'error',
error: err instanceof Error ? err.message : '处理失败',
}, detailsCounter)
}
}
if (scanState.detailsTruncated) {
console.log(`[Scan] 详情列表已截断: 共 ${detailsCounter.total} 条,仅保存前 ${MAX_DETAILS}`)
}
if (scanState.status === 'running') {
scanState.status = 'completed'
scanState.completedAt = formatBeijingTime()
}
} catch (err) {
scanState.status = 'error'
scanState.completedAt = formatBeijingTime()
scanState.error = err instanceof Error ? err.message : '连接失败'
} finally {
try { imap.end() } catch {}
}
saveScanHistory(scanState, userId)
}
// 提取的邮件信息结构
interface ExtractedTicketInfo {
content: string
device_ip: string | null
device_sn: string | null
device_name: string | null
ticket_type: string | null
fault_time: string | null
fault_info: string | null
auth_note: string | null
assign_time: string | null
}
// 从邮件 HTML 正文中提取故障信息,参考 Python 脚本的 extract_fault_info
function buildTicketContent(subject: string, orderNumber: string, htmlContent: string): ExtractedTicketInfo {
const result: ExtractedTicketInfo = {
content: '',
device_ip: null,
device_sn: null,
device_name: null,
ticket_type: null,
fault_time: null,
fault_info: null,
auth_note: null,
assign_time: null,
}
// 提取工单类型
if (subject.includes('OEM诊断')) {
result.ticket_type = 'OEM诊断'
} else if (subject.includes('OEM维修')) {
result.ticket_type = 'OEM维修'
}
// 使用 cheerio 解析 HTML 表格
if (htmlContent) {
const $ = cheerio.load(htmlContent)
// 先定位包含"故障信息"的目标表格,避免混入无关表格
let targetTable = $('table').filter((_, table) => {
return $(table).text().includes('故障信息')
}).first()
// 如果没找到包含"故障信息"的表格,使用第一个表格
if (targetTable.length === 0) {
targetTable = $('table').first()
}
// 遍历目标表格的行,提取键值对
targetTable.find('tr').each((_, row) => {
const cells = $(row).find('td')
if (cells.length >= 2) {
const key = $(cells[0]).text().trim()
const value = $(cells[1]).text().trim()
switch (key) {
case '故障发生时间:':
case '故障发生时间':
result.fault_time = value
result.assign_time = value
break
case '服务器SN':
case '服务器SN':
result.device_sn = value
break
case '服务器IP地址':
case '服务器IP地址':
case '服务器IP':
case '服务器IP':
result.device_ip = value
break
case '故障信息:':
case '故障信息':
result.fault_info = value
break
case '授权说明:':
case '授权说明':
result.auth_note = value
break
}
}
})
// Fallback如果表格提取失败使用正则从纯文本中提取
if (!result.fault_info) {
const textContent = $.text().replace(/\s+/g, ' ')
const faultMatch = textContent.match(/故障信息[:\s]*(.+?)(?=授权说明|$)/i)
if (faultMatch) {
result.fault_info = faultMatch[1].trim()
}
}
if (!result.device_ip) {
const textContent = $.text().replace(/\s+/g, ' ')
const ipMatch = textContent.match(/(?:服务器IP地址|服务器IP|IP)[:\s]*(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/)
if (ipMatch) {
result.device_ip = ipMatch[1]
}
}
if (!result.device_sn) {
const textContent = $.text().replace(/\s+/g, ' ')
const snMatch = textContent.match(/服务器SN[:\s]*([A-Za-z0-9]+)/)
if (snMatch) {
result.device_sn = snMatch[1]
}
}
}
// content 只包含故障信息
result.content = result.fault_info || null
return result
}
// 保存扫描历史到数据库
function saveScanHistory(scanState: ScanResult, userId: number): void {
try {
const db = getDb()
db.prepare(`
INSERT INTO scan_history (
status, started_at, completed_at,
time_range_value, time_range_unit,
total_count, matched_count, imported_count, skipped_count, error_count,
details_truncated, error_message, details_json, created_by
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
scanState.status,
scanState.startedAt,
scanState.completedAt || null,
scanState.timeRange.value,
scanState.timeRange.unit,
scanState.stats.total,
scanState.stats.matched,
scanState.stats.imported,
scanState.stats.skipped,
scanState.stats.errors,
scanState.detailsTruncated ? 1 : 0,
scanState.error || null,
JSON.stringify(scanState.details),
userId
)
// 保留最近 50 条扫描历史,删除更早的(用 id 单调排序,避免 created_at 同秒重复)
db.prepare(`DELETE FROM scan_history WHERE id NOT IN (
SELECT id FROM scan_history ORDER BY id DESC LIMIT 50
)`).run()
console.log(`[Scan] 扫描历史已保存: 状态=${scanState.status}, 导入=${scanState.stats.imported}`)
} catch (err) {
console.error('[Scan] 保存扫描历史失败:', err)
}
}