优化底层游标索引性能;优化HTTPAPI索引逻辑;优化导出图片的索引写入逻辑

This commit is contained in:
cc committed 2026-04-22 23:02:17 +08:00
1 parent 5802cf36c6
commit 777f5b82db
12 files changed
+1127 -314

No files matched your search

+2 -1
View File
@@ -2390,6 +2390,8 @@ function registerIpcHandlers() {
allowStaleCache?: boolean
preferAccurateSpecialTypes?: boolean
cacheOnly?: boolean
beginTimestamp?: number
endTimestamp?: number
}) => {
return chatService.getExportSessionStats(sessionIds, options)
})
@@ -3935,4 +3937,3 @@ app.on('window-all-closed', () => {
+2 -1
View File
@@ -219,6 +219,8 @@ contextBridge.exposeInMainWorld('electronAPI', {
allowStaleCache?: boolean
preferAccurateSpecialTypes?: boolean
cacheOnly?: boolean
beginTimestamp?: number
endTimestamp?: number
}
) => ipcRenderer.invoke('chat:getExportSessionStats', sessionIds, options),
getGroupMyMessageCountHint: (chatroomId: string) =>
@@ -565,4 +567,3 @@ contextBridge.exposeInMainWorld('electronAPI', {
validateWeiboUid: (uid: string) => ipcRenderer.invoke('social:validateWeiboUid', uid)
}
})
+351 -90
View File
@@ -198,6 +198,8 @@ interface ExportSessionStatsOptions {
allowStaleCache?: boolean
preferAccurateSpecialTypes?: boolean
cacheOnly?: boolean
beginTimestamp?: number
endTimestamp?: number
}
interface ExportSessionStatsCacheMeta {
@@ -2178,28 +2180,31 @@ class ChatService {
return { success: false, error: connectResult.error || '数据库未连接' }
}
const batchSize = Math.max(1, limit)
const cursorResult = await wcdbService.openMessageCursor(sessionId, batchSize, false, 0, 0)
if (!cursorResult.success || !cursorResult.cursor) {
return { success: false, error: cursorResult.error || '打开消息游标失败' }
// 聊天页首屏优先走稳定路径:直接拉取固定窗口并做本地确定性排序,
// 避免游标首批在极端数据分布下出现不稳定边界。
const pageLimit = Math.max(1, Math.floor(limit || this.messageBatchDefault))
const probeLimit = Math.min(500, pageLimit + 1)
const result = await wcdbService.getMessages(sessionId, probeLimit, 0)
if (!result.success || !Array.isArray(result.messages)) {
return { success: false, error: result.error || '获取最新消息失败' }
}
try {
const collected = await this.collectVisibleMessagesFromCursor(sessionId, cursorResult.cursor, limit)
if (!collected.success) {
return { success: false, error: collected.error || '获取消息失败' }
}
console.log(
`[ChatService] getLatestMessages session=${sessionId} rawRowsConsumed=${collected.rawRowsConsumed || 0} visibleMessagesReturned=${collected.messages?.length || 0} filteredOut=${collected.filteredOut || 0} nextOffset=${collected.rawRowsConsumed || 0} hasMore=${collected.hasMore === true}`
)
return {
success: true,
messages: collected.messages,
hasMore: collected.hasMore,
nextOffset: collected.rawRowsConsumed || 0
}
} finally {
await wcdbService.closeMessageCursor(cursorResult.cursor)
const rawRows = result.messages as Record<string, any>[]
const hasMore = rawRows.length > pageLimit
const selectedRows = hasMore ? rawRows.slice(0, pageLimit) : rawRows
const mapped = this.mapRowsToMessages(selectedRows)
const visible = mapped.filter((msg) => this.isMessageVisibleForSession(sessionId, msg))
const normalized = this.normalizeMessageOrder(visible)
await this.repairEmojiMessages(normalized)
console.log(
`[ChatService] getLatestMessages(stable) session=${sessionId} rawRows=${rawRows.length} visibleMessagesReturned=${normalized.length} nextOffset=${selectedRows.length} hasMore=${hasMore}`
)
return {
success: true,
messages: normalized,
hasMore,
nextOffset: selectedRows.length
}
} catch (e) {
console.error('ChatService: 获取最新消息失败:', e)
@@ -2241,16 +2246,59 @@ class ChatService {
}
}
private compareMessagesByTimeline(a: Message, b: Message): number {
const aSortSeq = Math.max(0, Number(a.sortSeq || 0))
const bSortSeq = Math.max(0, Number(b.sortSeq || 0))
const aCreateTime = Math.max(0, Number(a.createTime || 0))
const bCreateTime = Math.max(0, Number(b.createTime || 0))
const aLocalId = Math.max(0, Number(a.localId || 0))
const bLocalId = Math.max(0, Number(b.localId || 0))
const aServerId = Math.max(0, Number(a.serverId || 0))
const bServerId = Math.max(0, Number(b.serverId || 0))
// 与 C++ 侧归并规则一致:当两侧都有 sortSeq 时优先 sortSeq,否则先看 createTime。
if (aSortSeq > 0 && bSortSeq > 0 && aSortSeq !== bSortSeq) {
return aSortSeq - bSortSeq
}
if (aCreateTime !== bCreateTime) {
return aCreateTime - bCreateTime
}
if (aSortSeq !== bSortSeq) {
return aSortSeq - bSortSeq
}
if (aLocalId !== bLocalId) {
return aLocalId - bLocalId
}
if (aServerId !== bServerId) {
return aServerId - bServerId
}
const aKey = String(a.messageKey || '')
const bKey = String(b.messageKey || '')
if (aKey < bKey) return -1
if (aKey > bKey) return 1
return 0
}
private normalizeMessageOrder(messages: Message[]): Message[] {
if (messages.length < 2) return messages
const first = messages[0]
const last = messages[messages.length - 1]
const firstKey = first.sortSeq || first.createTime || first.localId || 0
const lastKey = last.sortSeq || last.createTime || last.localId || 0
if (firstKey > lastKey) {
return [...messages].reverse()
const withIndex = messages.map((msg, index) => ({ msg, index }))
withIndex.sort((left, right) => {
const diff = this.compareMessagesByTimeline(left.msg, right.msg)
if (diff !== 0) return diff
return left.index - right.index
})
let changed = false
for (let index = 0; index < withIndex.length; index += 1) {
if (withIndex[index].msg !== messages[index]) {
changed = true
break
}
}
return messages
if (!changed) return messages
return withIndex.map((entry) => entry.msg)
}
private encodeMessageKeySegment(value: unknown): string {
@@ -2436,6 +2484,95 @@ class ChatService {
return Number.isFinite(parsed) ? parsed : fallback
}
private parseCompactDateTimeDigitsToSeconds(raw: string): number {
const text = String(raw || '').trim()
if (!/^\d{8}(?:\d{4}(?:\d{2})?)?$/.test(text)) return 0
const year = Number.parseInt(text.slice(0, 4), 10)
const month = Number.parseInt(text.slice(4, 6), 10)
const day = Number.parseInt(text.slice(6, 8), 10)
const hour = text.length >= 12 ? Number.parseInt(text.slice(8, 10), 10) : 0
const minute = text.length >= 12 ? Number.parseInt(text.slice(10, 12), 10) : 0
const second = text.length >= 14 ? Number.parseInt(text.slice(12, 14), 10) : 0
if (!Number.isFinite(year) || year < 1990 || year > 2200) return 0
if (!Number.isFinite(month) || month < 1 || month > 12) return 0
if (!Number.isFinite(day) || day < 1 || day > 31) return 0
if (!Number.isFinite(hour) || hour < 0 || hour > 23) return 0
if (!Number.isFinite(minute) || minute < 0 || minute > 59) return 0
if (!Number.isFinite(second) || second < 0 || second > 59) return 0
const dt = new Date(year, month - 1, day, hour, minute, second)
if (
dt.getFullYear() !== year ||
dt.getMonth() !== month - 1 ||
dt.getDate() !== day ||
dt.getHours() !== hour ||
dt.getMinutes() !== minute ||
dt.getSeconds() !== second
) {
return 0
}
const ts = Math.floor(dt.getTime() / 1000)
return Number.isFinite(ts) && ts > 0 ? ts : 0
}
private parseDateTimeTextToSeconds(raw: unknown): number {
const text = String(raw ?? '').trim()
if (!text) return 0
const compactDigits = this.parseCompactDateTimeDigitsToSeconds(text)
if (compactDigits > 0) return compactDigits
if (/[zZ]|[+-]\d{2}:?\d{2}$/.test(text)) {
const parsed = Date.parse(text)
const seconds = Math.floor(parsed / 1000)
if (Number.isFinite(seconds) && seconds > 0) return seconds
}
const normalized = text.replace('T', ' ').replace(/\.\d+$/, '').replace(/\//g, '-')
const match = normalized.match(/^(\d{4})-(\d{1,2})-(\d{1,2})(?:\s+(\d{1,2}):(\d{1,2})(?::(\d{1,2}))?)?$/)
if (!match) return 0
const year = Number.parseInt(match[1], 10)
const month = Number.parseInt(match[2], 10)
const day = Number.parseInt(match[3], 10)
const hour = Number.parseInt(match[4] || '0', 10)
const minute = Number.parseInt(match[5] || '0', 10)
const second = Number.parseInt(match[6] || '0', 10)
if (!Number.isFinite(year) || !Number.isFinite(month) || !Number.isFinite(day)) return 0
const dt = new Date(year, month - 1, day, hour, minute, second)
const ts = Math.floor(dt.getTime() / 1000)
return Number.isFinite(ts) && ts > 0 ? ts : 0
}
private normalizeTimestampLikeToSeconds(raw: unknown): number {
if (raw === undefined || raw === null || raw === '') return 0
const text = String(raw ?? '').trim()
if (!text) return 0
const compactDigits = this.parseCompactDateTimeDigitsToSeconds(text)
if (compactDigits > 0) return compactDigits
const parsed = this.coerceRowNumber(raw)
if (Number.isFinite(parsed) && parsed > 0) {
let normalized = Math.floor(parsed)
while (normalized > 10000000000) {
normalized = Math.floor(normalized / 1000)
}
return normalized
}
return this.parseDateTimeTextToSeconds(text)
}
private getRowTimestampSeconds(row: Record<string, any>, keys: string[], fallback = 0): number {
const raw = this.getRowField(row, keys)
if (raw === undefined || raw === null || raw === '') return fallback
const parsed = this.normalizeTimestampLikeToSeconds(raw)
return parsed > 0 ? parsed : fallback
}
private hasAnyContactExtendedFieldKey(row: Record<string, any>): boolean {
for (const key of Object.keys(row || {})) {
if (this.contactExtendedFieldCandidateSet.has(String(key || '').toLowerCase())) {
@@ -3066,13 +3203,13 @@ class ChatService {
if (typeof raw === 'number') return raw
if (typeof raw === 'bigint') return Number(raw)
if (Buffer.isBuffer(raw)) {
return parseInt(raw.toString('utf-8'), 10)
return this.coerceRowNumber(raw.toString('utf-8'))
}
if (raw instanceof Uint8Array) {
return parseInt(Buffer.from(raw).toString('utf-8'), 10)
return this.coerceRowNumber(Buffer.from(raw).toString('utf-8'))
}
if (Array.isArray(raw)) {
return parseInt(Buffer.from(raw).toString('utf-8'), 10)
return this.coerceRowNumber(Buffer.from(raw).toString('utf-8'))
}
if (typeof raw === 'object') {
if ('value' in raw) return this.coerceRowNumber(raw.value)
@@ -3088,13 +3225,21 @@ class ChatService {
}
const text = raw.toString ? String(raw) : ''
if (text && text !== '[object Object]') {
const parsed = parseInt(text, 10)
return Number.isFinite(parsed) ? parsed : NaN
return this.coerceRowNumber(text)
}
return NaN
}
const parsed = parseInt(String(raw), 10)
return Number.isFinite(parsed) ? parsed : NaN
const text = String(raw).trim()
if (!text) return NaN
if (/^[+-]?\d+$/.test(text)) {
const parsed = Number(text)
return Number.isFinite(parsed) ? parsed : NaN
}
if (/^[+-]?\d+\.\d+$/.test(text)) {
const parsed = Number(text)
return Number.isFinite(parsed) ? parsed : NaN
}
return NaN
}
private buildIdentityKeys(raw: string): string[] {
@@ -3656,7 +3801,11 @@ class ChatService {
return this.extractXmlValue(content, 'type')
}
private async collectSpecialMessageCountsByCursorScan(sessionId: string): Promise<{
private async collectSpecialMessageCountsByCursorScan(
sessionId: string,
beginTimestamp: number = 0,
endTimestamp: number = 0
): Promise<{
transferMessages: number
redPacketMessages: number
callMessages: number
@@ -3667,7 +3816,7 @@ class ChatService {
callMessages: 0
}
const cursorResult = await wcdbService.openMessageCursorLite(sessionId, 500, false, 0, 0)
const cursorResult = await wcdbService.openMessageCursorLite(sessionId, 500, false, beginTimestamp, endTimestamp)
if (!cursorResult.success || !cursorResult.cursor) {
return counters
}
@@ -3713,7 +3862,9 @@ class ChatService {
private async collectSessionExportStatsByCursorScan(
sessionId: string,
selfIdentitySet: Set<string>
selfIdentitySet: Set<string>,
beginTimestamp: number = 0,
endTimestamp: number = 0
): Promise<ExportSessionStats> {
const stats: ExportSessionStats = {
totalMessages: 0,
@@ -3731,7 +3882,7 @@ class ChatService {
}
const senderIdentities = new Set<string>()
const cursorResult = await wcdbService.openMessageCursorLite(sessionId, 500, false, 0, 0)
const cursorResult = await wcdbService.openMessageCursorLite(sessionId, 500, false, beginTimestamp, endTimestamp)
if (!cursorResult.success || !cursorResult.cursor) {
return stats
}
@@ -3806,7 +3957,7 @@ class ChatService {
if (sessionId.endsWith('@chatroom')) {
stats.groupActiveSpeakers = senderIdentities.size
if (Number.isFinite(stats.groupMyMessages)) {
if ((beginTimestamp <= 0 && endTimestamp <= 0) && Number.isFinite(stats.groupMyMessages)) {
this.setGroupMyMessageCountHintEntry(sessionId, stats.groupMyMessages as number)
}
}
@@ -3816,7 +3967,9 @@ class ChatService {
private async collectSessionExportStats(
sessionId: string,
selfIdentitySet: Set<string>,
preferAccurateSpecialTypes: boolean = false
preferAccurateSpecialTypes: boolean = false,
beginTimestamp: number = 0,
endTimestamp: number = 0
): Promise<ExportSessionStats> {
const stats: ExportSessionStats = {
totalMessages: 0,
@@ -3834,9 +3987,9 @@ class ChatService {
stats.groupActiveSpeakers = 0
}
const nativeResult = await wcdbService.getSessionMessageTypeStats(sessionId, 0, 0)
const nativeResult = await wcdbService.getSessionMessageTypeStats(sessionId, beginTimestamp, endTimestamp)
if (!nativeResult.success || !nativeResult.data) {
return this.collectSessionExportStatsByCursorScan(sessionId, selfIdentitySet)
return this.collectSessionExportStatsByCursorScan(sessionId, selfIdentitySet, beginTimestamp, endTimestamp)
}
const data = nativeResult.data as Record<string, any>
@@ -3856,7 +4009,7 @@ class ChatService {
if (preferAccurateSpecialTypes) {
try {
const preciseCounters = await this.collectSpecialMessageCountsByCursorScan(sessionId)
const preciseCounters = await this.collectSpecialMessageCountsByCursorScan(sessionId, beginTimestamp, endTimestamp)
stats.transferMessages = preciseCounters.transferMessages
stats.redPacketMessages = preciseCounters.redPacketMessages
stats.callMessages = preciseCounters.callMessages
@@ -3868,14 +4021,19 @@ class ChatService {
if (isGroup) {
stats.groupMyMessages = Math.max(0, Math.floor(Number(data.group_my_messages || 0)))
stats.groupActiveSpeakers = Math.max(0, Math.floor(Number(data.group_sender_count || 0)))
if (Number.isFinite(stats.groupMyMessages)) {
if ((beginTimestamp <= 0 && endTimestamp <= 0) && Number.isFinite(stats.groupMyMessages)) {
this.setGroupMyMessageCountHintEntry(sessionId, stats.groupMyMessages as number)
}
}
return stats
}
private toExportSessionStatsFromNativeTypeRow(sessionId: string, row: Record<string, any>): ExportSessionStats {
private toExportSessionStatsFromNativeTypeRow(
sessionId: string,
row: Record<string, any>,
options?: { updateGroupHint?: boolean }
): ExportSessionStats {
const updateGroupHint = options?.updateGroupHint !== false
const stats: ExportSessionStats = {
totalMessages: Math.max(0, Math.floor(Number(row?.total_messages || 0))),
voiceMessages: Math.max(0, Math.floor(Number(row?.voice_messages || 0))),
@@ -3895,7 +4053,7 @@ class ChatService {
if (sessionId.endsWith('@chatroom')) {
stats.groupMyMessages = Math.max(0, Math.floor(Number(row?.group_my_messages || 0)))
stats.groupActiveSpeakers = Math.max(0, Math.floor(Number(row?.group_sender_count || 0)))
if (Number.isFinite(stats.groupMyMessages)) {
if (updateGroupHint && Number.isFinite(stats.groupMyMessages)) {
this.setGroupMyMessageCountHintEntry(sessionId, stats.groupMyMessages as number)
}
}
@@ -4025,9 +4183,17 @@ class ChatService {
sessionId: string,
selfIdentitySet: Set<string>,
includeRelations: boolean,
preferAccurateSpecialTypes: boolean = false
preferAccurateSpecialTypes: boolean = false,
beginTimestamp: number = 0,
endTimestamp: number = 0
): Promise<ExportSessionStats> {
const stats = await this.collectSessionExportStats(sessionId, selfIdentitySet, preferAccurateSpecialTypes)
const stats = await this.collectSessionExportStats(
sessionId,
selfIdentitySet,
preferAccurateSpecialTypes,
beginTimestamp,
endTimestamp
)
const isGroup = sessionId.endsWith('@chatroom')
if (isGroup) {
@@ -4066,7 +4232,9 @@ class ChatService {
sessionIds: string[],
includeRelations: boolean,
selfIdentitySet: Set<string>,
preferAccurateSpecialTypes: boolean = false
preferAccurateSpecialTypes: boolean = false,
beginTimestamp: number = 0,
endTimestamp: number = 0
): Promise<Record<string, ExportSessionStats>> {
const normalizedSessionIds = Array.from(
new Set(
@@ -4127,8 +4295,8 @@ class ChatService {
try {
const quickMode = !includeRelations && normalizedSessionIds.length > 1
const nativeBatch = await wcdbService.getSessionMessageTypeStatsBatch(normalizedSessionIds, {
beginTimestamp: 0,
endTimestamp: 0,
beginTimestamp,
endTimestamp,
quickMode,
includeGroupSenderCount: true
})
@@ -4136,7 +4304,9 @@ class ChatService {
for (const sessionId of normalizedSessionIds) {
const row = nativeBatch.data?.[sessionId] as Record<string, any> | undefined
if (!row || typeof row !== 'object') continue
nativeBatchStats[sessionId] = this.toExportSessionStatsFromNativeTypeRow(sessionId, row)
nativeBatchStats[sessionId] = this.toExportSessionStatsFromNativeTypeRow(sessionId, row, {
updateGroupHint: beginTimestamp <= 0 && endTimestamp <= 0
})
}
hasNativeBatchStats = Object.keys(nativeBatchStats).length > 0
} else {
@@ -4151,7 +4321,13 @@ class ChatService {
try {
const stats = hasNativeBatchStats && nativeBatchStats[sessionId]
? { ...nativeBatchStats[sessionId] }
: await this.collectSessionExportStats(sessionId, selfIdentitySet, preferAccurateSpecialTypes)
: await this.collectSessionExportStats(
sessionId,
selfIdentitySet,
preferAccurateSpecialTypes,
beginTimestamp,
endTimestamp
)
if (sessionId.endsWith('@chatroom')) {
if (shouldLoadGroupMemberCount) {
stats.groupMemberCount = typeof memberCountMap[sessionId] === 'number'
@@ -4181,10 +4357,12 @@ class ChatService {
sessionId: string,
includeRelations: boolean,
selfIdentitySet: Set<string>,
preferAccurateSpecialTypes: boolean = false
preferAccurateSpecialTypes: boolean = false,
beginTimestamp: number = 0,
endTimestamp: number = 0
): Promise<ExportSessionStats> {
if (preferAccurateSpecialTypes) {
return this.computeSessionExportStats(sessionId, selfIdentitySet, includeRelations, true)
return this.computeSessionExportStats(sessionId, selfIdentitySet, includeRelations, true, beginTimestamp, endTimestamp)
}
const scopedKey = this.buildScopedSessionStatsKey(sessionId)
@@ -4199,8 +4377,13 @@ class ChatService {
if (pendingFull) return pendingFull
}
const shouldUsePendingPool = beginTimestamp <= 0 && endTimestamp <= 0
if (!shouldUsePendingPool) {
return this.computeSessionExportStats(sessionId, selfIdentitySet, includeRelations, false, beginTimestamp, endTimestamp)
}
const targetMap = includeRelations ? this.sessionStatsPendingFull : this.sessionStatsPendingBasic
const pending = this.computeSessionExportStats(sessionId, selfIdentitySet, includeRelations, false)
const pending = this.computeSessionExportStats(sessionId, selfIdentitySet, includeRelations, false, beginTimestamp, endTimestamp)
targetMap.set(scopedKey, pending)
try {
return await pending
@@ -4216,6 +4399,55 @@ class ChatService {
return this.mapRowsToMessages(rows)
}
mapRowsToMessagesLiteForApi(rows: Record<string, any>[]): Message[] {
const myWxid = String(this.configService.get('myWxid') || '').trim()
const messages: Message[] = []
for (const row of rows) {
const sourceInfo = this.getMessageSourceInfo(row)
const localType = this.getRowInt(row, ['local_type'], 1)
const createTime = this.getRowTimestampSeconds(row, ['create_time', 'createTime', 'msg_time', 'msgTime', 'time'], 0)
const sortSeq = this.getRowInt(row, ['sort_seq'], createTime > 0 ? createTime * 1000 : 0)
const localId = this.getRowInt(row, ['local_id'], 0)
const serverId = this.getRowInt(row, ['server_id'], 0)
const content = this.decodeMessageContent(row.message_content, row.compress_content)
const isSendRaw = row.computed_is_send ?? row.is_send
const parsedRawIsSend = isSendRaw === null || isSendRaw === undefined
? null
: parseInt(String(isSendRaw), 10)
const normalizedIsSend = typeof parsedRawIsSend === 'number' && Number.isFinite(parsedRawIsSend)
? parsedRawIsSend
: null
const senderFromRow = String(row.sender_username || '').trim() || this.extractSenderUsernameFromContent(content) || null
const { isSend } = this.resolveMessageIsSend(normalizedIsSend, senderFromRow)
const senderUsername = senderFromRow || (isSend === 1 && myWxid ? myWxid : null)
messages.push({
messageKey: this.buildMessageKey({
localId,
serverId,
createTime,
sortSeq,
senderUsername,
localType,
...sourceInfo
}),
localId,
serverId,
localType,
createTime,
sortSeq,
isSend,
senderUsername,
parsedContent: '',
rawContent: content,
content,
_db_path: sourceInfo.dbPath
})
}
return messages
}
private mapRowsToMessages(rows: Record<string, any>[]): Message[] {
const myWxid = this.configService.get('myWxid')
@@ -4233,7 +4465,7 @@ class ChatService {
|| this.extractSenderUsernameFromContent(content)
|| null
const { isSend } = this.resolveMessageIsSend(parsedRawIsSend, senderUsername)
const createTime = this.getRowInt(row, ['create_time'], 0)
const createTime = this.getRowTimestampSeconds(row, ['create_time', 'createTime', 'msg_time', 'msgTime', 'time'], 0)
if (senderUsername && !myWxid) {
// [DEBUG] Issue #34: 未配置 myWxid,无法判断是否发送
@@ -7042,6 +7274,9 @@ class ChatService {
const allowStaleCache = options.allowStaleCache === true
const preferAccurateSpecialTypes = options.preferAccurateSpecialTypes === true
const cacheOnly = options.cacheOnly === true
const beginTimestamp = this.normalizeTimestampSeconds(Number(options.beginTimestamp || 0))
const endTimestamp = this.normalizeTimestampSeconds(Number(options.endTimestamp || 0))
const useRangeFilter = beginTimestamp > 0 || endTimestamp > 0
const normalizedSessionIds = Array.from(
new Set(
@@ -7065,7 +7300,7 @@ class ChatService {
? this.getGroupMyMessageCountHintEntry(sessionId)
: null
const cachedResult = this.getSessionStatsCacheEntry(sessionId)
const canUseCache = cacheOnly || (!forceRefresh && !preferAccurateSpecialTypes)
const canUseCache = !useRangeFilter && (cacheOnly || (!forceRefresh && !preferAccurateSpecialTypes))
if (canUseCache && cachedResult && this.supportsRequestedRelation(cachedResult.entry, includeRelations)) {
const stale = now - cachedResult.entry.updatedAt > this.sessionStatsCacheTtlMs
if (!stale || allowStaleCache || cacheOnly) {
@@ -7103,31 +7338,16 @@ class ChatService {
if (pendingSessionIds.length === 1) {
const sessionId = pendingSessionIds[0]
try {
const stats = await this.getOrComputeSessionExportStats(sessionId, includeRelations, selfIdentitySet, preferAccurateSpecialTypes)
resultMap[sessionId] = stats
const updatedAt = this.setSessionStatsCacheEntry(sessionId, stats, includeRelations)
cacheMeta[sessionId] = {
updatedAt,
stale: false,
includeRelations,
source: 'fresh'
}
usedBatchedCompute = true
} catch {
usedBatchedCompute = false
}
} else {
try {
const batchedStatsMap = await this.computeSessionExportStatsBatch(
pendingSessionIds,
const stats = await this.getOrComputeSessionExportStats(
sessionId,
includeRelations,
selfIdentitySet,
preferAccurateSpecialTypes
preferAccurateSpecialTypes,
beginTimestamp,
endTimestamp
)
for (const sessionId of pendingSessionIds) {
const stats = batchedStatsMap[sessionId]
if (!stats) continue
resultMap[sessionId] = stats
resultMap[sessionId] = stats
if (!useRangeFilter) {
const updatedAt = this.setSessionStatsCacheEntry(sessionId, stats, includeRelations)
cacheMeta[sessionId] = {
updatedAt,
@@ -7140,19 +7360,56 @@ class ChatService {
} catch {
usedBatchedCompute = false
}
} else {
try {
const batchedStatsMap = await this.computeSessionExportStatsBatch(
pendingSessionIds,
includeRelations,
selfIdentitySet,
preferAccurateSpecialTypes,
beginTimestamp,
endTimestamp
)
for (const sessionId of pendingSessionIds) {
const stats = batchedStatsMap[sessionId]
if (!stats) continue
resultMap[sessionId] = stats
if (!useRangeFilter) {
const updatedAt = this.setSessionStatsCacheEntry(sessionId, stats, includeRelations)
cacheMeta[sessionId] = {
updatedAt,
stale: false,
includeRelations,
source: 'fresh'
}
}
}
usedBatchedCompute = true
} catch {
usedBatchedCompute = false
}
}
if (!usedBatchedCompute) {
await this.forEachWithConcurrency(pendingSessionIds, 3, async (sessionId) => {
try {
const stats = await this.getOrComputeSessionExportStats(sessionId, includeRelations, selfIdentitySet, preferAccurateSpecialTypes)
resultMap[sessionId] = stats
const updatedAt = this.setSessionStatsCacheEntry(sessionId, stats, includeRelations)
cacheMeta[sessionId] = {
updatedAt,
stale: false,
const stats = await this.getOrComputeSessionExportStats(
sessionId,
includeRelations,
source: 'fresh'
selfIdentitySet,
preferAccurateSpecialTypes,
beginTimestamp,
endTimestamp
)
resultMap[sessionId] = stats
if (!useRangeFilter) {
const updatedAt = this.setSessionStatsCacheEntry(sessionId, stats, includeRelations)
cacheMeta[sessionId] = {
updatedAt,
stale: false,
includeRelations,
source: 'fresh'
}
}
} catch {
resultMap[sessionId] = this.buildEmptyExportSessionStats(sessionId, includeRelations)
@@ -8892,7 +9149,11 @@ class ChatService {
private normalizeTimestampSeconds(value: number): number {
const numeric = Number(value || 0)
if (!Number.isFinite(numeric) || numeric <= 0) return 0
return numeric > 1e12 ? Math.floor(numeric / 1000) : Math.floor(numeric)
let normalized = Math.floor(numeric)
while (normalized > 10000000000) {
normalized = Math.floor(normalized / 1000)
}
return normalized
}
private toSafeInt(value: unknown, fallback = 0): number {
@@ -10532,8 +10793,8 @@ class ChatService {
const serverIdRaw = this.normalizeUnsignedIntegerToken(row.server_id)
const serverId = this.getRowInt(row, ['server_id'], 0)
const localType = this.getRowInt(row, ['local_type'], 0)
const createTime = this.getRowInt(row, ['create_time'], 0)
const sortSeq = this.getRowInt(row, ['sort_seq'], createTime)
const createTime = this.getRowTimestampSeconds(row, ['create_time', 'createTime', 'msg_time', 'msgTime', 'time'], 0)
const sortSeq = this.getRowInt(row, ['sort_seq'], createTime > 0 ? createTime * 1000 : 0)
const rawIsSend = row.computed_is_send ?? row.is_send
const senderUsername = await this.resolveSenderUsernameForMessageRow(row, rawContent)
const sendState = this.resolveMessageIsSend(rawIsSend === null ? null : parseInt(rawIsSend, 10), senderUsername)
+617 -156
View File
@@ -321,10 +321,35 @@ class ExportService {
)
}
private normalizeTimestampSeconds(value: unknown): number {
const raw = Number(value)
if (!Number.isFinite(raw) || raw <= 0) return 0
let normalized = Math.floor(raw)
// 兼容毫秒/微秒/纳秒时间戳输入,统一降到秒级。
while (normalized > 10000000000) {
normalized = Math.floor(normalized / 1000)
}
return normalized
}
private normalizeExportDateRange(dateRange?: { start: number; end: number } | null): { start: number; end: number } | null {
if (!dateRange) return null
let start = this.normalizeTimestampSeconds(dateRange.start)
let end = this.normalizeTimestampSeconds(dateRange.end)
if (start > 0 && end > 0 && start > end) {
const tmp = start
start = end
end = tmp
}
if (start <= 0 && end <= 0) return null
return { start, end }
}
private getExportStatsDateRangeToken(dateRange?: { start: number; end: number } | null): string {
if (!dateRange) return 'all'
const start = Number.isFinite(dateRange.start) ? Math.max(0, Math.floor(dateRange.start)) : 0
const end = Number.isFinite(dateRange.end) ? Math.max(0, Math.floor(dateRange.end)) : 0
const normalized = this.normalizeExportDateRange(dateRange)
if (!normalized) return 'all'
const start = normalized.start
const end = normalized.end
return `${start}-${end}`
}
@@ -528,8 +553,9 @@ class ExportService {
}
private formatDateTokenBySeconds(seconds?: number): string | null {
if (!Number.isFinite(seconds) || (seconds || 0) <= 0) return null
const date = new Date(Math.floor(Number(seconds)) * 1000)
const normalizedSeconds = this.normalizeTimestampSeconds(seconds)
if (normalizedSeconds <= 0) return null
const date = new Date(normalizedSeconds * 1000)
if (Number.isNaN(date.getTime())) return null
const y = date.getFullYear()
const m = `${date.getMonth() + 1}`.padStart(2, '0')
@@ -956,10 +982,7 @@ class ExportService {
}
private isUnboundedDateRange(dateRange?: { start: number; end: number } | null): boolean {
if (!dateRange) return true
const start = Number.isFinite(dateRange.start) ? dateRange.start : 0
const end = Number.isFinite(dateRange.end) ? dateRange.end : 0
return start <= 0 && end <= 0
return this.normalizeExportDateRange(dateRange) === null
}
private shouldUseFastTextCollection(options: ExportOptions): boolean {
@@ -1114,6 +1137,135 @@ class ExportService {
return { mode }
}
private resolveFastMediaStreamType(
collectMode: MessageCollectMode,
targetMediaTypes: Set<number> | null
): 'image' | 'video' | null {
if (collectMode !== 'media-fast' || !targetMediaTypes || targetMediaTypes.size !== 1) return null
if (targetMediaTypes.has(3)) return 'image'
if (targetMediaTypes.has(43)) return 'video'
return null
}
private async collectMessagesByFastMediaStream(
sessionId: string,
cleanedMyWxid: string,
normalizedDateRange: { start: number; end: number } | null,
useCursorTimeRange: boolean,
normalizedSenderUsernameFilter: string,
mediaType: 'image' | 'video',
onCollectProgress?: (payload: { fetched: number }) => void,
control?: ExportTaskControl
): Promise<{
success: boolean
rows: any[]
senderUsernames: string[]
firstTime: number | null
lastTime: number | null
error?: string
}> {
const rows: any[] = []
const senderSet = new Set<string>()
let firstTime: number | null = null
let lastTime: number | null = null
let offset = 0
let hasMore = true
const PAGE_LIMIT = 480
while (hasMore) {
this.throwIfStopRequested(control)
const streamResult = await wcdbService.getMediaStream({
sessionId,
mediaType,
beginTimestamp: useCursorTimeRange ? (normalizedDateRange?.start || 0) : 0,
endTimestamp: useCursorTimeRange ? (normalizedDateRange?.end || 0) : 0,
limit: PAGE_LIMIT,
offset
})
if (!streamResult.success) {
return {
success: false,
rows,
senderUsernames: Array.from(senderSet),
firstTime,
lastTime,
error: streamResult.error || '媒体快速流读取失败'
}
}
const items = Array.isArray(streamResult.items) ? streamResult.items : []
if (items.length === 0) {
hasMore = false
break
}
for (const item of items) {
const createTime = this.normalizeRowTimestampSeconds(item?.createTime)
if (normalizedDateRange) {
if (createTime > 0 && normalizedDateRange.start > 0 && createTime < normalizedDateRange.start) continue
if (createTime > 0 && normalizedDateRange.end > 0 && createTime > normalizedDateRange.end) continue
}
const localTypeRaw = Number(item?.localType || 0)
let localType = Number.isFinite(localTypeRaw) ? Math.floor(localTypeRaw) : 0
if (localType <= 0) {
localType = mediaType === 'video' ? 43 : 3
}
const isSend = Number(item?.isSend) === 1
const senderUsernameRaw = String(item?.senderUsername || '').trim()
const actualSender = isSend ? cleanedMyWxid : (senderUsernameRaw || sessionId)
if (normalizedSenderUsernameFilter && !this.isSameWxid(actualSender, normalizedSenderUsernameFilter)) {
continue
}
senderSet.add(actualSender)
const localIdRaw = Number(item?.localId || 0)
const localId = Number.isFinite(localIdRaw) ? Math.floor(localIdRaw) : 0
const serverIdRawToken = this.normalizeUnsignedIntToken(item?.serverId)
const serverIdValue = Number.parseInt(serverIdRawToken, 10)
const imageMd5 = String(item?.imageMd5 || '').trim().toLowerCase()
const imageDatName = String(item?.imageDatName || '').trim().toLowerCase()
const videoMd5 = String(item?.videoMd5 || '').trim().toLowerCase()
rows.push({
localId,
serverId: Number.isFinite(serverIdValue) ? serverIdValue : 0,
serverIdRaw: serverIdRawToken !== '0' ? serverIdRawToken : undefined,
createTime,
localType,
content: String(item?.content || ''),
senderUsername: actualSender,
isSend,
imageMd5: imageMd5 || undefined,
imageDatName: imageDatName || undefined,
videoMd5: videoMd5 || undefined
})
if (createTime > 0) {
if (firstTime === null || createTime < firstTime) firstTime = createTime
if (lastTime === null || createTime > lastTime) lastTime = createTime
}
}
onCollectProgress?.({ fetched: rows.length })
const nextOffset = Number(streamResult.nextOffset)
const safeNextOffset = Number.isFinite(nextOffset) && nextOffset > offset
? Math.floor(nextOffset)
: offset + items.length
offset = safeNextOffset
hasMore = Boolean(streamResult.hasMore) && items.length > 0
}
return {
success: true,
rows,
senderUsernames: Array.from(senderSet),
firstTime,
lastTime
}
}
private createCollectProgressReporter(
sessionName: string,
onProgress?: (progress: ExportProgress) => void,
@@ -1182,6 +1334,106 @@ class ExportService {
return fallback
}
private parseDateTimeTextToSeconds(value: string): number {
const raw = String(value || '').trim()
if (!raw) return 0
const compactDigits = this.parseCompactDateTimeDigitsToSeconds(raw)
if (compactDigits > 0) return compactDigits
// 优先处理带时区信息的格式(例如 2026-04-22T21:33:12Z / +08:00)
if (/[zZ]|[+-]\d{2}:?\d{2}$/.test(raw)) {
const parsed = Date.parse(raw)
const seconds = Math.floor(parsed / 1000)
if (Number.isFinite(seconds) && seconds > 0) return seconds
}
const normalized = raw.replace('T', ' ').replace(/\.\d+$/, '').replace(/\//g, '-')
const match = normalized.match(/^(\d{4})-(\d{1,2})-(\d{1,2})(?:[ ](\d{1,2}):(\d{1,2})(?::(\d{1,2}))?)?$/)
if (!match) return 0
const year = Number.parseInt(match[1], 10)
const month = Number.parseInt(match[2], 10)
const day = Number.parseInt(match[3], 10)
const hour = Number.parseInt(match[4] || '0', 10)
const minute = Number.parseInt(match[5] || '0', 10)
const second = Number.parseInt(match[6] || '0', 10)
if (!Number.isFinite(year) || !Number.isFinite(month) || !Number.isFinite(day)) return 0
const dt = new Date(year, month - 1, day, hour, minute, second)
const ts = Math.floor(dt.getTime() / 1000)
return Number.isFinite(ts) && ts > 0 ? ts : 0
}
private parseCompactDateTimeDigitsToSeconds(value: string): number {
const raw = String(value || '').trim()
if (!/^\d{8}(?:\d{4}(?:\d{2})?)?$/.test(raw)) return 0
const year = Number.parseInt(raw.slice(0, 4), 10)
const month = Number.parseInt(raw.slice(4, 6), 10)
const day = Number.parseInt(raw.slice(6, 8), 10)
const hour = raw.length >= 12 ? Number.parseInt(raw.slice(8, 10), 10) : 0
const minute = raw.length >= 12 ? Number.parseInt(raw.slice(10, 12), 10) : 0
const second = raw.length >= 14 ? Number.parseInt(raw.slice(12, 14), 10) : 0
if (!Number.isFinite(year) || year < 1990 || year > 2200) return 0
if (!Number.isFinite(month) || month < 1 || month > 12) return 0
if (!Number.isFinite(day) || day < 1 || day > 31) return 0
if (!Number.isFinite(hour) || hour < 0 || hour > 23) return 0
if (!Number.isFinite(minute) || minute < 0 || minute > 59) return 0
if (!Number.isFinite(second) || second < 0 || second > 59) return 0
const dt = new Date(year, month - 1, day, hour, minute, second)
if (
dt.getFullYear() !== year ||
dt.getMonth() !== month - 1 ||
dt.getDate() !== day ||
dt.getHours() !== hour ||
dt.getMinutes() !== minute ||
dt.getSeconds() !== second
) {
return 0
}
const ts = Math.floor(dt.getTime() / 1000)
return Number.isFinite(ts) && ts > 0 ? ts : 0
}
private normalizeRowTimestampSeconds(value: unknown): number {
if (value === undefined || value === null || value === '') return 0
const rawText = String(value || '').trim()
if (!rawText) return 0
// 纯数字且看起来是年月日时间串时,优先按日期解析,避免误当作毫秒。
const compactDigits = this.parseCompactDateTimeDigitsToSeconds(rawText)
if (compactDigits > 0) return compactDigits
const numeric = Number(rawText)
if (Number.isFinite(numeric) && numeric > 0) {
return this.normalizeTimestampSeconds(numeric)
}
return this.parseDateTimeTextToSeconds(rawText)
}
private getTimestampSecondsFromRow(row: Record<string, any>): number {
const rawPrimary = this.getRowField(row, [
'create_time', 'createTime', 'createtime',
'msg_create_time', 'msgCreateTime',
'msg_time', 'msgTime', 'time',
'WCDB_CT_create_time'
])
let primary = this.normalizeRowTimestampSeconds(rawPrimary)
const rawSortSeq = this.getRowField(row, ['sort_seq', 'sortSeq', 'server_seq', 'serverSeq'])
const sortSeqSeconds = this.normalizeRowTimestampSeconds(rawSortSeq)
// 对异常小时间戳兜底(例如 parseInt("2026-...") => 2026),优先回退 sort_seq。
if (primary > 0 && primary < 946684800 && sortSeqSeconds > 946684800) {
return sortSeqSeconds
}
if (primary > 0) return primary
if (sortSeqSeconds > 0) return sortSeqSeconds
return 0
}
private getRowField(row: Record<string, any>, keys: string[]): any {
for (const key of keys) {
if (row && Object.prototype.hasOwnProperty.call(row, key)) {
@@ -3787,43 +4039,31 @@ class ExportService {
dirCache?.add(imagesDir)
}
// 使用消息对象中已提取的字段
const imageMd5 = msg.imageMd5
const imageDatName = msg.imageDatName
const tryResolveImagePath = async (imageMd5?: string, imageDatName?: string): Promise<string | null> => {
if (!imageMd5 && !imageDatName) return null
if (!imageMd5 && !imageDatName) {
return null
}
const missingRunCacheKey = this.getImageMissingRunCacheKey(
sessionId,
imageMd5,
imageDatName
)
if (missingRunCacheKey && this.mediaRunMissingImageKeys.has(missingRunCacheKey)) {
return null
}
const result = await imageDecryptService.decryptImage({
sessionId,
imageMd5,
imageDatName,
createTime: msg.createTime,
force: true, // 导出优先高清,失败再回退缩略图
preferFilePath: true,
hardlinkOnly: true,
disableUpdateCheck: true,
allowCacheIndex: !imageMd5,
suppressEvents: true
})
if (!result.success || !result.localPath) {
if (result.failureKind === 'decrypt_failed') {
console.log(`[Export] 图片解密失败 (localId=${msg.localId}): imageMd5=${imageMd5}, imageDatName=${imageDatName}, error=${result.error || '未知'}`)
} else {
console.log(`[Export] 图片本地无数据 (localId=${msg.localId}): imageMd5=${imageMd5}, imageDatName=${imageDatName}, error=${result.error || '未知'}`)
const decryptResult = await imageDecryptService.decryptImage({
sessionId,
imageMd5,
imageDatName,
createTime: msg.createTime,
force: true, // 导出优先高清,失败再回退缩略图
preferFilePath: true,
hardlinkOnly: true,
disableUpdateCheck: true,
allowCacheIndex: !imageMd5,
suppressEvents: true
})
if (decryptResult.success && decryptResult.localPath) {
return decryptResult.localPath
}
// 尝试获取缩略图
if (decryptResult.failureKind === 'decrypt_failed') {
console.log(`[Export] 图片解密失败 (localId=${msg.localId}): imageMd5=${imageMd5 || ''}, imageDatName=${imageDatName || ''}, error=${decryptResult.error || '未知'}`)
} else {
console.log(`[Export] 图片本地无数据 (localId=${msg.localId}): imageMd5=${imageMd5 || ''}, imageDatName=${imageDatName || ''}, error=${decryptResult.error || '未知'}`)
}
const thumbResult = await imageDecryptService.resolveCachedImage({
sessionId,
imageMd5,
@@ -3836,14 +4076,38 @@ class ExportService {
})
if (thumbResult.success && thumbResult.localPath) {
console.log(`[Export] 使用缩略图替代 (localId=${msg.localId}): ${thumbResult.localPath}`)
result.localPath = thumbResult.localPath
} else {
console.log(`[Export] 缩略图也获取失败,所有方式均失败 → 将显示 [图片] 占位符`)
if (missingRunCacheKey) {
this.mediaRunMissingImageKeys.add(missingRunCacheKey)
}
return null
return thumbResult.localPath
}
return null
}
// 使用消息对象中已提取的字段,先尝试快速导出。
let imageMd5 = String(msg.imageMd5 || '').trim().toLowerCase() || undefined
let imageDatName = String(msg.imageDatName || '').trim().toLowerCase() || undefined
const initialMissingRunCacheKey = this.getImageMissingRunCacheKey(sessionId, imageMd5, imageDatName)
if (initialMissingRunCacheKey && this.mediaRunMissingImageKeys.has(initialMissingRunCacheKey)) {
return null
}
let sourcePath = await tryResolveImagePath(imageMd5, imageDatName)
// 快速流字段存在偏差时,按 localId 强制回填再重试一次,避免“导出进度前进但写入 0”。
if (!sourcePath) {
const localId = Number(msg?.localId || 0)
if (Number.isFinite(localId) && localId > 0) {
await this.backfillMediaFieldsFromMessageDetail(sessionId, [msg], new Set([3]), undefined, { force: true })
imageMd5 = String(msg.imageMd5 || '').trim().toLowerCase() || undefined
imageDatName = String(msg.imageDatName || '').trim().toLowerCase() || undefined
sourcePath = await tryResolveImagePath(imageMd5, imageDatName)
}
}
if (!sourcePath) {
const missingRunCacheKey = this.getImageMissingRunCacheKey(sessionId, imageMd5, imageDatName)
console.log(`[Export] 缩略图也获取失败,所有方式均失败 → 将显示 [图片] 占位符`)
if (missingRunCacheKey) {
this.mediaRunMissingImageKeys.add(missingRunCacheKey)
}
return null
}
// 为每条消息生成稳定且唯一的文件名前缀,避免跨日期/消息发生同名覆盖
@@ -3851,7 +4115,6 @@ class ExportService {
const imageKey = (imageMd5 || imageDatName || 'image').replace(/[^a-zA-Z0-9_-]/g, '')
// 从 data URL 或 file URL 获取实际路径
let sourcePath: string = result.localPath!
if (sourcePath.startsWith('data:')) {
// 是 data URL,需要保存为文件
const base64Data = sourcePath.split(',')[1]
@@ -4121,8 +4384,24 @@ class ExportService {
includePoster = false
): Promise<MediaExportItem | null> {
try {
const videoMd5 = msg.videoMd5
if (!videoMd5) return null
let videoMd5 = String(msg.videoMd5 || '').trim().toLowerCase()
const resolveVideoInfo = async (token: string) => {
if (!token) return null
const videoInfo = await videoService.getVideoInfo(token, { includePoster })
if (!videoInfo.exists || !videoInfo.videoUrl) return null
return videoInfo
}
let videoInfo = await resolveVideoInfo(videoMd5)
if (!videoInfo) {
const localId = Number(msg?.localId || 0)
if (Number.isFinite(localId) && localId > 0) {
await this.backfillMediaFieldsFromMessageDetail(sessionId, [msg], new Set([43]), undefined, { force: true })
videoMd5 = String(msg.videoMd5 || '').trim().toLowerCase()
videoInfo = await resolveVideoInfo(videoMd5)
}
}
if (!videoInfo) return null
const videosDir = path.join(mediaRootDir, mediaRelativePrefix, 'videos')
if (!dirCache?.has(videosDir)) {
@@ -4130,11 +4409,6 @@ class ExportService {
dirCache?.add(videosDir)
}
const videoInfo = await videoService.getVideoInfo(videoMd5, { includePoster })
if (!videoInfo.exists || !videoInfo.videoUrl) {
return null
}
const sourcePath = videoInfo.videoUrl
const fileName = path.basename(sourcePath)
const destPath = path.join(videosDir, fileName)
@@ -4764,6 +5038,13 @@ class ExportService {
return this.mediaExportTelemetry?.doneFiles ?? 0
}
private formatMediaPhaseLabel(processed: number, total: number, beforeDoneFiles: number): string {
const safeProcessed = Math.max(0, Math.floor(processed || 0))
const safeTotal = Math.max(0, Math.floor(total || 0))
const writtenNow = Math.max(0, this.getMediaDoneFilesCount() - Math.max(0, Math.floor(beforeDoneFiles || 0)))
return `导出媒体 ${Math.min(safeProcessed, safeTotal)}/${safeTotal}(已写入 ${writtenNow})`
}
private buildFileOnlyExportFailure(
options: ExportOptions,
mediaMessages: any[],
@@ -4834,79 +5115,136 @@ class ExportService {
collectMode: MessageCollectMode = 'full',
targetMediaTypes?: Set<number>,
control?: ExportTaskControl,
onCollectProgress?: (payload: { fetched: number }) => void
): Promise<{ rows: any[]; memberSet: Map<string, { member: ChatLabMember; avatarUrl?: string }>; firstTime: number | null; lastTime: number | null }> {
onCollectProgress?: (payload: { fetched: number }) => void,
allowLiteFallback = true,
allowRangeFallback = true,
useCursorTimeRange = true,
allowModeFallback = true
): Promise<{ rows: any[]; memberSet: Map<string, { member: ChatLabMember; avatarUrl?: string }>; firstTime: number | null; lastTime: number | null; error?: string }> {
const rows: any[] = []
const memberSet = new Map<string, { member: ChatLabMember; avatarUrl?: string }>()
const senderSet = new Set<string>()
let firstTime: number | null = null
let lastTime: number | null = null
const mediaTypeFilter = collectMode === 'media-fast' && targetMediaTypes && targetMediaTypes.size > 0
const mediaTypeFilter = targetMediaTypes && targetMediaTypes.size > 0
? targetMediaTypes
: null
const fileOnlyMediaFilter = this.isFileOnlyMediaFilter(mediaTypeFilter)
// 修复时间范围:0 表示不限制,而不是时间戳 0
const beginTime = dateRange?.start || 0
const endTime = dateRange?.end && dateRange.end > 0 ? dateRange.end : 0
const normalizedDateRange = this.normalizeExportDateRange(dateRange)
const normalizedSenderUsernameFilter = String(senderUsernameFilter || '').trim()
const beginTime = useCursorTimeRange ? (normalizedDateRange?.start || 0) : 0
const endTime = useCursorTimeRange ? (normalizedDateRange?.end || 0) : 0
const batchSize = (collectMode === 'text-fast' || collectMode === 'media-fast') ? 2000 : 500
this.throwIfStopRequested(control)
const cursor = collectMode === 'media-fast'
? await wcdbService.openMessageCursorLite(
const fastMediaType = this.resolveFastMediaStreamType(collectMode, mediaTypeFilter)
let usedFastMediaStream = false
let usedLiteCursor = false
if (fastMediaType) {
const streamCollected = await this.collectMessagesByFastMediaStream(
sessionId,
batchSize,
true,
beginTime,
endTime
cleanedMyWxid,
normalizedDateRange,
useCursorTimeRange,
normalizedSenderUsernameFilter,
fastMediaType,
onCollectProgress,
control
)
: await wcdbService.openMessageCursor(
sessionId,
batchSize,
true,
beginTime,
endTime
)
if (!cursor.success || !cursor.cursor) {
console.error(`[Export] 打开游标失败: ${cursor.error || '未知错误'}`)
return { rows, memberSet, firstTime, lastTime }
if (streamCollected.success) {
usedFastMediaStream = true
rows.push(...streamCollected.rows)
for (const username of streamCollected.senderUsernames) {
senderSet.add(username)
}
firstTime = streamCollected.firstTime
lastTime = streamCollected.lastTime
} else {
console.warn(`[Export] 媒体快速流读取失败,回退游标链路: session=${sessionId}, type=${fastMediaType}, error=${streamCollected.error || 'unknown'}`)
}
}
try {
let hasMore = true
let batchCount = 0
while (hasMore) {
this.throwIfStopRequested(control)
const batch = await wcdbService.fetchMessageBatch(cursor.cursor)
batchCount++
if (!batch.success) {
console.error(`[Export] 获取批次 ${batchCount} 失败: ${batch.error}`)
break
if (!usedFastMediaStream) {
// 媒体导出链路必须优先保证字段完整性,否则会出现“进度前进但无文件写出”。
// 轻量游标仅用于文本快路径;媒体快路径保留标准游标。
usedLiteCursor = allowLiteFallback && collectMode === 'text-fast'
let cursor = usedLiteCursor
? await wcdbService.openMessageCursorLite(
sessionId,
batchSize,
true,
beginTime,
endTime
)
: await wcdbService.openMessageCursor(
sessionId,
batchSize,
true,
beginTime,
endTime
)
if (!cursor.success || !cursor.cursor) {
if (usedLiteCursor && allowLiteFallback) {
console.warn(`[Export] 轻量游标打开失败,回退标准游标重试: ${cursor.error || '未知错误'}`)
return this.collectMessages(
sessionId,
cleanedMyWxid,
normalizedDateRange,
senderUsernameFilter,
collectMode,
targetMediaTypes,
control,
onCollectProgress,
false,
allowRangeFallback,
useCursorTimeRange,
allowModeFallback
)
}
if (!batch.rows) break
let rowIndex = 0
for (const row of batch.rows) {
console.error(`[Export] 打开游标失败: ${cursor.error || '未知错误'}`)
return {
rows,
memberSet,
firstTime,
lastTime,
error: cursor.error || '打开消息游标失败'
}
}
try {
let hasMore = true
let batchCount = 0
while (hasMore) {
this.throwIfStopRequested(control)
const batch = await wcdbService.fetchMessageBatch(cursor.cursor)
batchCount++
if (!batch.success) {
console.error(`[Export] 获取批次 ${batchCount} 失败: ${batch.error}`)
break
}
if (!batch.rows) break
let rowIndex = 0
for (const row of batch.rows) {
if ((rowIndex++ & 0x7f) === 0) {
this.throwIfStopRequested(control)
}
const createTime = this.getIntFromRow(row, [
'create_time', 'createTime', 'createtime',
'msg_create_time', 'msgCreateTime',
'msg_time', 'msgTime', 'time',
'WCDB_CT_create_time'
], 0)
if (dateRange) {
if (createTime < dateRange.start || createTime > dateRange.end) continue
const createTime = this.getTimestampSecondsFromRow(row)
if (normalizedDateRange) {
if (createTime > 0 && normalizedDateRange.start > 0 && createTime < normalizedDateRange.start) continue
if (createTime > 0 && normalizedDateRange.end > 0 && createTime > normalizedDateRange.end) continue
}
const localType = this.getIntFromRow(row, [
'local_type', 'localType', 'type', 'msg_type', 'msgType', 'WCDB_CT_local_type'
], 1)
const rowFileHints = this.getFileAppMessageHints(row)
const allowFileProbe = fileOnlyMediaFilter && this.hasFileAppMessageHints(row)
const rowFileHints = collectMode === 'text-fast'
? {}
: this.getFileAppMessageHints(row)
const allowFileProbe = collectMode !== 'text-fast' && fileOnlyMediaFilter && this.hasFileAppMessageHints(row)
if (mediaTypeFilter && !mediaTypeFilter.has(localType) && !allowFileProbe) {
continue
}
@@ -4964,11 +5302,27 @@ class ExportService {
actualSender = isSend ? cleanedMyWxid : (senderUsername || sessionId)
}
if (senderUsernameFilter && !this.isSameWxid(actualSender, senderUsernameFilter)) {
if (normalizedSenderUsernameFilter && !this.isSameWxid(actualSender, normalizedSenderUsernameFilter)) {
continue
}
senderSet.add(actualSender)
if (collectMode === 'text-fast') {
rows.push({
localId,
serverId,
serverIdRaw: serverIdRaw !== '0' ? serverIdRaw : undefined,
createTime,
localType,
content,
senderUsername: actualSender,
isSend
})
if (firstTime === null || createTime < firstTime) firstTime = createTime
if (lastTime === null || createTime > lastTime) lastTime = createTime
continue
}
// 提取媒体相关字段(轻量模式下跳过)
let imageMd5: string | undefined
let imageDatName: string | undefined
@@ -5083,22 +5437,77 @@ class ExportService {
if (firstTime === null || createTime < firstTime) firstTime = createTime
if (lastTime === null || createTime > lastTime) lastTime = createTime
}
onCollectProgress?.({ fetched: rows.length })
hasMore = batch.hasMore === true
}
onCollectProgress?.({ fetched: rows.length })
hasMore = batch.hasMore === true
}
} catch (err) {
if (this.isStopError(err)) throw err
console.error(`[Export] 收集消息异常:`, err)
} finally {
try {
await wcdbService.closeMessageCursor(cursor.cursor)
} catch (err) {
console.error(`[Export] 关闭游标失败:`, err)
if (this.isStopError(err)) throw err
console.error(`[Export] 收集消息异常:`, err)
} finally {
try {
await wcdbService.closeMessageCursor(cursor.cursor)
} catch (err) {
console.error(`[Export] 关闭游标失败:`, err)
}
}
}
if (rows.length === 0 && usedLiteCursor && allowLiteFallback) {
console.warn(`[Export] 轻量游标返回 0 条,回退标准游标重试: session=${sessionId}, range=${beginTime}-${endTime}`)
return this.collectMessages(
sessionId,
cleanedMyWxid,
normalizedDateRange,
senderUsernameFilter,
collectMode,
targetMediaTypes,
control,
onCollectProgress,
false,
allowRangeFallback,
useCursorTimeRange,
allowModeFallback
)
}
if (rows.length === 0 && collectMode === 'media-fast' && allowModeFallback) {
console.warn(`[Export] media-fast 返回 0 条,回退 full 模式重试: session=${sessionId}`)
return this.collectMessages(
sessionId,
cleanedMyWxid,
normalizedDateRange,
senderUsernameFilter,
'full',
mediaTypeFilter || undefined,
control,
onCollectProgress,
false,
allowRangeFallback,
useCursorTimeRange,
false
)
}
if (rows.length === 0 && allowRangeFallback && normalizedDateRange && useCursorTimeRange) {
console.warn(`[Export] 时间范围游标返回 0 条,回退为全量游标+本地过滤重试: session=${sessionId}, range=${normalizedDateRange.start}-${normalizedDateRange.end}`)
return this.collectMessages(
sessionId,
cleanedMyWxid,
normalizedDateRange,
senderUsernameFilter,
collectMode,
targetMediaTypes,
control,
onCollectProgress,
allowLiteFallback,
false,
false,
allowModeFallback
)
}
this.throwIfStopRequested(control)
if (collectMode === 'media-fast' && mediaTypeFilter && rows.length > 0) {
await this.backfillMediaFieldsFromMessageDetail(sessionId, rows, mediaTypeFilter, control)
@@ -5136,10 +5545,15 @@ class ExportService {
sessionId: string,
rows: any[],
targetMediaTypes: Set<number>,
control?: ExportTaskControl
control?: ExportTaskControl,
options?: { force?: boolean }
): Promise<void> {
const force = options?.force === true
const fileOnlyMediaFilter = this.isFileOnlyMediaFilter(targetMediaTypes)
const needsBackfill = rows.filter((msg) => {
if (force) {
return Number(msg?.localId || 0) > 0
}
const isFileCandidate = this.isFileAppLocalType(Number(msg.localType || 0)) || (fileOnlyMediaFilter && this.hasFileAppMessageHints(msg))
if (isFileCandidate) {
return !msg.xmlType || !msg.fileName || !msg.fileMd5 || !msg.fileSize || !msg.fileExt
@@ -5175,8 +5589,8 @@ class ExportService {
const supplementalPayload = `${this.decodeMaybeCompressed(String(packedInfoRaw || ''))}\n${this.decodeMaybeCompressed(String(reserved0Raw || ''))}`
if (msg.localType === 3) {
const imageMd5 = String(row.image_md5 || row.imageMd5 || '').trim() || this.extractImageMd5(content)
const imageDatName = String(row.image_dat_name || row.imageDatName || '').trim() || this.extractImageDatName(content)
const imageMd5 = (String(row.image_md5 || row.imageMd5 || '').trim() || this.extractImageMd5(content) || '').toLowerCase()
const imageDatName = (String(row.image_dat_name || row.imageDatName || '').trim() || this.extractImageDatName(content) || '').toLowerCase()
if (imageMd5) msg.imageMd5 = imageMd5
if (imageDatName) msg.imageDatName = imageDatName
return
@@ -5198,7 +5612,7 @@ class ExportService {
}
if (msg.localType === 43) {
const videoMd5 = this.extractVideoFileNameFromRow(row, content)
const videoMd5 = String(this.extractVideoFileNameFromRow(row, content) || '').trim().toLowerCase()
if (videoMd5) msg.videoMd5 = videoMd5
return
}
@@ -5686,7 +6100,7 @@ class ExportService {
// 如果没有消息,不创建文件
if (totalMessages === 0) {
return { success: false, error: '该会话在指定时间范围内没有消息' }
return { success: false, error: collected.error || '该会话在指定时间范围内没有消息' }
}
await this.hydrateEmojiCaptionsForMessages(sessionId, allMessages, control)
@@ -5739,12 +6153,12 @@ class ExportService {
}
}
allMessages.sort((a, b) => a.createTime - b.createTime)
const allMessagesInCursorOrder = allMessages
const { exportMediaEnabled, mediaRootDir, mediaRelativePrefix } = this.getMediaLayout(outputPath, options)
// ========== 阶段1:并行导出媒体文件 ==========
const mediaMessages = this.collectMediaMessagesForExport(allMessages, options)
const mediaMessages = this.collectMediaMessagesForExport(allMessagesInCursorOrder, options)
const mediaCache = new Map<string, MediaExportItem | null>()
const mediaDirCache = new Set<string>()
@@ -5767,7 +6181,7 @@ class ExportService {
phase: 'exporting-media',
phaseProgress: 0,
phaseTotal: mediaMessages.length,
phaseLabel: `导出媒体 0/${mediaMessages.length}`,
phaseLabel: this.formatMediaPhaseLabel(0, mediaMessages.length, beforeMediaDoneFiles),
...this.getMediaTelemetrySnapshot(),
estimatedTotalMessages: totalMessages
})
@@ -5801,7 +6215,7 @@ class ExportService {
phase: 'exporting-media',
phaseProgress: mediaExported,
phaseTotal: mediaMessages.length,
phaseLabel: `导出媒体 ${mediaExported}/${mediaMessages.length}`,
phaseLabel: this.formatMediaPhaseLabel(mediaExported, mediaMessages.length, beforeMediaDoneFiles),
...this.getMediaTelemetrySnapshot()
})
}
@@ -6218,7 +6632,7 @@ class ExportService {
// 如果没有消息,不创建文件
if (totalMessages === 0) {
return { success: false, error: '该会话在指定时间范围内没有消息' }
return { success: false, error: collected.error || '该会话在指定时间范围内没有消息' }
}
await this.hydrateEmojiCaptionsForMessages(sessionId, collected.rows, control)
@@ -6272,7 +6686,7 @@ class ExportService {
phase: 'exporting-media',
phaseProgress: 0,
phaseTotal: mediaMessages.length,
phaseLabel: `导出媒体 0/${mediaMessages.length}`,
phaseLabel: this.formatMediaPhaseLabel(0, mediaMessages.length, beforeMediaDoneFiles),
...this.getMediaTelemetrySnapshot(),
estimatedTotalMessages: totalMessages
})
@@ -6305,7 +6719,7 @@ class ExportService {
phase: 'exporting-media',
phaseProgress: mediaExported,
phaseTotal: mediaMessages.length,
phaseLabel: `导出媒体 ${mediaExported}/${mediaMessages.length}`,
phaseLabel: this.formatMediaPhaseLabel(mediaExported, mediaMessages.length, beforeMediaDoneFiles),
...this.getMediaTelemetrySnapshot()
})
}
@@ -6948,7 +7362,7 @@ class ExportService {
// 如果没有消息,不创建文件
if (totalMessages === 0) {
return { success: false, error: '该会话在指定时间范围内没有消息' }
return { success: false, error: collected.error || '该会话在指定时间范围内没有消息' }
}
await this.hydrateEmojiCaptionsForMessages(sessionId, collected.rows, control)
@@ -7108,7 +7522,7 @@ class ExportService {
// 填充数据
const sortedMessages = collected.rows.sort((a, b) => a.createTime - b.createTime)
const sortedMessages = collected.rows
// 媒体导出设置
const { exportMediaEnabled, mediaRootDir, mediaRelativePrefix } = this.getMediaLayout(outputPath, options)
@@ -7137,7 +7551,7 @@ class ExportService {
phase: 'exporting-media',
phaseProgress: 0,
phaseTotal: mediaMessages.length,
phaseLabel: `导出媒体 0/${mediaMessages.length}`,
phaseLabel: this.formatMediaPhaseLabel(0, mediaMessages.length, beforeMediaDoneFiles),
...this.getMediaTelemetrySnapshot(),
estimatedTotalMessages: totalMessages
})
@@ -7170,7 +7584,7 @@ class ExportService {
phase: 'exporting-media',
phaseProgress: mediaExported,
phaseTotal: mediaMessages.length,
phaseLabel: `导出媒体 ${mediaExported}/${mediaMessages.length}`,
phaseLabel: this.formatMediaPhaseLabel(mediaExported, mediaMessages.length, beforeMediaDoneFiles),
...this.getMediaTelemetrySnapshot()
})
}
@@ -7818,7 +8232,7 @@ class ExportService {
// 如果没有消息,不创建文件
if (totalMessages === 0) {
return { success: false, error: '该会话在指定时间范围内没有消息' }
return { success: false, error: collected.error || '该会话在指定时间范围内没有消息' }
}
await this.hydrateEmojiCaptionsForMessages(sessionId, collected.rows, control)
@@ -7855,7 +8269,7 @@ class ExportService {
? await this.getGroupNicknamesForRoom(sessionId, groupNicknameCandidates)
: new Map<string, string>()
const sortedMessages = collected.rows.sort((a, b) => a.createTime - b.createTime)
const sortedMessages = collected.rows
const { exportMediaEnabled, mediaRootDir, mediaRelativePrefix } = this.getMediaLayout(outputPath, options)
const mediaMessages = this.collectMediaMessagesForExport(sortedMessages, options)
@@ -7881,7 +8295,7 @@ class ExportService {
phase: 'exporting-media',
phaseProgress: 0,
phaseTotal: mediaMessages.length,
phaseLabel: `导出媒体 0/${mediaMessages.length}`,
phaseLabel: this.formatMediaPhaseLabel(0, mediaMessages.length, beforeMediaDoneFiles),
...this.getMediaTelemetrySnapshot(),
estimatedTotalMessages: totalMessages
})
@@ -7914,7 +8328,7 @@ class ExportService {
phase: 'exporting-media',
phaseProgress: mediaExported,
phaseTotal: mediaMessages.length,
phaseLabel: `导出媒体 ${mediaExported}/${mediaMessages.length}`,
phaseLabel: this.formatMediaPhaseLabel(mediaExported, mediaMessages.length, beforeMediaDoneFiles),
...this.getMediaTelemetrySnapshot()
})
}
@@ -7968,7 +8382,24 @@ class ExportService {
exportedMessages: 0
})
const lines: string[] = []
const stream = fs.createWriteStream(outputPath, { encoding: 'utf-8' })
const writeChunk = async (chunk: string): Promise<void> => {
await new Promise<void>((resolve, _reject) => {
this.throwIfStopRequested(control)
if (!stream.write(chunk)) {
stream.once('drain', resolve)
} else {
resolve()
}
})
}
const WRITE_BATCH = 120
let writeBuffer: string[] = []
const flushWriteBuffer = async (): Promise<void> => {
if (writeBuffer.length === 0) return
await writeChunk(writeBuffer.join(''))
writeBuffer = []
}
const senderProfileCache = new Map<string, ExportDisplayProfile>()
for (let i = 0; i < totalMessages; i++) {
@@ -8083,9 +8514,10 @@ class ExportService {
}
}
lines.push(`${this.formatTimestamp(msg.createTime)} '${senderRole}'`)
lines.push(enrichedContentValue)
lines.push('')
writeBuffer.push(`${this.formatTimestamp(msg.createTime)} '${senderRole}'\n${enrichedContentValue}\n\n`)
if (writeBuffer.length >= WRITE_BATCH) {
await flushWriteBuffer()
}
if ((i + 1) % 200 === 0) {
const progress = 60 + Math.floor((i + 1) / sortedMessages.length * 30)
@@ -8101,6 +8533,8 @@ class ExportService {
}
}
await flushWriteBuffer()
onProgress?.({
current: 92,
total: 100,
@@ -8112,7 +8546,10 @@ class ExportService {
})
this.throwIfStopRequested(control)
await fs.promises.writeFile(outputPath, lines.join('\n'), 'utf-8')
await new Promise<void>((resolve, reject) => {
stream.on('error', reject)
stream.end(() => resolve())
})
onProgress?.({
current: 100,
@@ -8186,7 +8623,7 @@ class ExportService {
)
let totalMessages = collected.rows.length
if (totalMessages === 0) {
return { success: false, error: '该会话在指定时间范围内没有消息' }
return { success: false, error: collected.error || '该会话在指定时间范围内没有消息' }
}
await this.hydrateEmojiCaptionsForMessages(sessionId, collected.rows, control)
@@ -8215,7 +8652,6 @@ class ExportService {
: new Map<string, string>()
const sortedMessages = collected.rows
.sort((a, b) => a.createTime - b.createTime)
.filter((msg) => !this.isQuotedReplyMessage(msg.localType, msg.content || ''))
totalMessages = sortedMessages.length
if (totalMessages === 0) {
@@ -8254,7 +8690,7 @@ class ExportService {
phase: 'exporting-media',
phaseProgress: 0,
phaseTotal: mediaMessages.length,
phaseLabel: `导出媒体 0/${mediaMessages.length}`,
phaseLabel: this.formatMediaPhaseLabel(0, mediaMessages.length, beforeMediaDoneFiles),
...this.getMediaTelemetrySnapshot(),
estimatedTotalMessages: totalMessages
})
@@ -8287,7 +8723,7 @@ class ExportService {
phase: 'exporting-media',
phaseProgress: mediaExported,
phaseTotal: mediaMessages.length,
phaseLabel: `导出媒体 ${mediaExported}/${mediaMessages.length}`,
phaseLabel: this.formatMediaPhaseLabel(mediaExported, mediaMessages.length, beforeMediaDoneFiles),
...this.getMediaTelemetrySnapshot()
})
}
@@ -8341,8 +8777,25 @@ class ExportService {
exportedMessages: 0
})
const lines: string[] = []
lines.push('id,MsgSvrID,type_name,is_sender,talker,msg,src,CreateTime')
const stream = fs.createWriteStream(outputPath, { encoding: 'utf-8' })
const writeChunk = async (chunk: string): Promise<void> => {
await new Promise<void>((resolve, _reject) => {
this.throwIfStopRequested(control)
if (!stream.write(chunk)) {
stream.once('drain', resolve)
} else {
resolve()
}
})
}
const WRITE_BATCH = 160
let writeBuffer: string[] = []
const flushWriteBuffer = async (): Promise<void> => {
if (writeBuffer.length === 0) return
await writeChunk(writeBuffer.join(''))
writeBuffer = []
}
await writeChunk('\uFEFFid,MsgSvrID,type_name,is_sender,talker,msg,src,CreateTime\r\n')
const senderProfileCache = new Map<string, ExportDisplayProfile>()
for (let i = 0; i < totalMessages; i++) {
@@ -8423,7 +8876,10 @@ class ExportService {
this.formatIsoTimestamp(msg.createTime)
]
lines.push(row.map((value) => this.escapeCsvCell(value)).join(','))
writeBuffer.push(`${row.map((value) => this.escapeCsvCell(value)).join(',')}\r\n`)
if (writeBuffer.length >= WRITE_BATCH) {
await flushWriteBuffer()
}
if ((i + 1) % 200 === 0) {
const progress = 60 + Math.floor((i + 1) / sortedMessages.length * 30)
@@ -8439,6 +8895,8 @@ class ExportService {
}
}
await flushWriteBuffer()
onProgress?.({
current: 92,
total: 100,
@@ -8450,7 +8908,10 @@ class ExportService {
})
this.throwIfStopRequested(control)
await fs.promises.writeFile(outputPath, `\uFEFF${lines.join('\r\n')}`, 'utf-8')
await new Promise<void>((resolve, reject) => {
stream.on('error', reject)
stream.end(() => resolve())
})
onProgress?.({
current: 100,
@@ -8612,7 +9073,7 @@ class ExportService {
// 如果没有消息,不创建文件
if (collected.rows.length === 0) {
return { success: false, error: '该会话在指定时间范围内没有消息' }
return { success: false, error: collected.error || '该会话在指定时间范围内没有消息' }
}
const totalMessages = collected.rows.length
@@ -8645,7 +9106,7 @@ class ExportService {
this.throwIfStopRequested(control)
await this.mergeGroupMembers(sessionId, collected.memberSet, options.exportAvatars === true)
}
const sortedMessages = collected.rows.sort((a, b) => a.createTime - b.createTime)
const sortedMessages = collected.rows
const { exportMediaEnabled, mediaRootDir, mediaRelativePrefix } = this.getMediaLayout(outputPath, options)
const mediaMessages = this.collectMediaMessagesForExport(sortedMessages, options)
@@ -8671,7 +9132,7 @@ class ExportService {
phase: 'exporting-media',
phaseProgress: 0,
phaseTotal: mediaMessages.length,
phaseLabel: `导出媒体 0/${mediaMessages.length}`,
phaseLabel: this.formatMediaPhaseLabel(0, mediaMessages.length, beforeMediaDoneFiles),
...this.getMediaTelemetrySnapshot(),
estimatedTotalMessages: totalMessages
})
@@ -8705,7 +9166,7 @@ class ExportService {
phase: 'exporting-media',
phaseProgress: mediaExported,
phaseTotal: mediaMessages.length,
phaseLabel: `导出媒体 ${mediaExported}/${mediaMessages.length}`,
phaseLabel: this.formatMediaPhaseLabel(mediaExported, mediaMessages.length, beforeMediaDoneFiles),
...this.getMediaTelemetrySnapshot()
})
}
+114 -51
View File
@@ -516,27 +516,29 @@ class HttpService {
limit: number,
startTime: number,
endTime: number,
ascending: boolean
ascending: boolean,
useLiteMapping: boolean = true
): Promise<{ success: boolean; messages?: Message[]; hasMore?: boolean; error?: string }> {
try {
// 使用固定 batch 大小(与 limit 相同或最多 500)来减少循环次数
const batchSize = Math.min(limit, 500)
// 深分页时放大 batch,避免 offset 很大时出现大量小批次循环。
const batchSize = Math.min(2000, Math.max(500, limit))
const beginTimestamp = startTime > 10000000000 ? Math.floor(startTime / 1000) : startTime
const endTimestamp = endTime > 10000000000 ? Math.floor(endTime / 1000) : endTime
const cursorResult = await wcdbService.openMessageCursor(talker, batchSize, ascending, beginTimestamp, endTimestamp)
const cursorResult = await wcdbService.openMessageCursorLite(talker, batchSize, ascending, beginTimestamp, endTimestamp)
if (!cursorResult.success || !cursorResult.cursor) {
return { success: false, error: cursorResult.error || '打开消息游标失败' }
}
const cursor = cursorResult.cursor
try {
const allRows: Record<string, any>[] = []
const collectedRows: Record<string, any>[] = []
let hasMore = true
let skipped = 0
let reachedLimit = false
// 循环获取消息,处理 offset 跳过 + limit 累积
while (allRows.length < limit && hasMore) {
while (collectedRows.length < limit && hasMore) {
const batch = await wcdbService.fetchMessageBatch(cursor)
if (!batch.success || !batch.rows || batch.rows.length === 0) {
hasMore = false
@@ -557,12 +559,20 @@ class HttpService {
skipped = offset
}
allRows.push(...rows)
const remainingCapacity = limit - collectedRows.length
if (rows.length > remainingCapacity) {
collectedRows.push(...rows.slice(0, remainingCapacity))
reachedLimit = true
break
}
collectedRows.push(...rows)
}
const trimmedRows = allRows.slice(0, limit)
const finalHasMore = hasMore || allRows.length > limit
const messages = chatService.mapRowsToMessagesForApi(trimmedRows)
const finalHasMore = hasMore || reachedLimit
const messages = useLiteMapping
? chatService.mapRowsToMessagesLiteForApi(collectedRows)
: chatService.mapRowsToMessagesForApi(collectedRows)
await this.backfillMissingSenderUsernames(talker, messages)
return { success: true, messages, hasMore: finalHasMore }
} finally {
@@ -590,32 +600,70 @@ class HttpService {
if (targets.length === 0) return
const myWxid = (this.configService.get('myWxid') || '').trim()
for (const msg of targets) {
const localId = Number(msg.localId || 0)
if (Number.isFinite(localId) && localId > 0) {
try {
const detail = await wcdbService.getMessageById(talker, localId)
if (detail.success && detail.message) {
const hydrated = chatService.mapRowsToMessagesForApi([detail.message])[0]
if (hydrated?.senderUsername) {
msg.senderUsername = hydrated.senderUsername
}
if ((msg.isSend === null || msg.isSend === undefined) && hydrated?.isSend !== undefined) {
msg.isSend = hydrated.isSend
}
if (!msg.rawContent && hydrated?.rawContent) {
msg.rawContent = hydrated.rawContent
}
}
} catch (error) {
console.warn('[HttpService] backfill sender failed:', error)
const MAX_DETAIL_BACKFILL = 120
if (targets.length > MAX_DETAIL_BACKFILL) {
for (const msg of targets) {
if (!msg.senderUsername && msg.isSend === 1 && myWxid) {
msg.senderUsername = myWxid
}
}
return
}
if (!msg.senderUsername && msg.isSend === 1 && myWxid) {
msg.senderUsername = myWxid
const queue = [...targets]
const workerCount = Math.max(1, Math.min(6, queue.length))
const state = {
attempted: 0,
hydrated: 0,
consecutiveMiss: 0
}
const MAX_DETAIL_LOOKUPS = 80
const MAX_CONSECUTIVE_MISS = 36
const runWorker = async (): Promise<void> => {
while (queue.length > 0) {
if (state.attempted >= MAX_DETAIL_LOOKUPS) break
if (state.consecutiveMiss >= MAX_CONSECUTIVE_MISS && state.hydrated <= 0) break
const msg = queue.shift()
if (!msg) break
const localId = Number(msg.localId || 0)
if (Number.isFinite(localId) && localId > 0) {
state.attempted += 1
try {
const detail = await wcdbService.getMessageById(talker, localId)
if (detail.success && detail.message) {
const hydrated = chatService.mapRowsToMessagesForApi([detail.message])[0]
if (hydrated?.senderUsername) {
msg.senderUsername = hydrated.senderUsername
}
if ((msg.isSend === null || msg.isSend === undefined) && hydrated?.isSend !== undefined) {
msg.isSend = hydrated.isSend
}
if (!msg.rawContent && hydrated?.rawContent) {
msg.rawContent = hydrated.rawContent
}
if (msg.senderUsername) {
state.hydrated += 1
state.consecutiveMiss = 0
} else {
state.consecutiveMiss += 1
}
} else {
state.consecutiveMiss += 1
}
} catch (error) {
console.warn('[HttpService] backfill sender failed:', error)
state.consecutiveMiss += 1
}
}
if (!msg.senderUsername && msg.isSend === 1 && myWxid) {
msg.senderUsername = myWxid
}
}
}
await Promise.all(Array.from({ length: workerCount }, () => runWorker()))
}
private parseBooleanParam(url: URL, keys: string[], defaultValue: boolean = false): boolean {
@@ -663,7 +711,7 @@ class HttpService {
const talker = (url.searchParams.get('talker') || '').trim()
const limit = this.parseIntParam(url.searchParams.get('limit'), 100, 1, 10000)
const offset = this.parseIntParam(url.searchParams.get('offset'), 0, 0, Number.MAX_SAFE_INTEGER)
const keyword = (url.searchParams.get('keyword') || '').trim().toLowerCase()
const keyword = (url.searchParams.get('keyword') || '').trim()
const startParam = url.searchParams.get('start')
const endParam = url.searchParams.get('end')
const chatlab = this.parseBooleanParam(url, ['chatlab'], false)
@@ -683,26 +731,41 @@ class HttpService {
const startTime = this.parseTimeParam(startParam)
const endTime = this.parseTimeParam(endParam, true)
const queryOffset = keyword ? 0 : offset
const queryLimit = keyword ? 10000 : limit
const result = await this.fetchMessagesBatch(talker, queryOffset, queryLimit, startTime, endTime, false)
if (!result.success || !result.messages) {
this.sendError(res, 500, result.error || 'Failed to get messages')
return
}
let messages = result.messages
let hasMore = result.hasMore === true
let messages: Message[] = []
let hasMore = false
if (keyword) {
const filtered = messages.filter((msg) => {
const content = (msg.parsedContent || msg.rawContent || '').toLowerCase()
return content.includes(keyword)
})
const endIndex = offset + limit
hasMore = filtered.length > endIndex
messages = filtered.slice(offset, endIndex)
const searchLimit = Math.max(1, limit) + 1
const searchResult = await chatService.searchMessages(
keyword,
talker,
searchLimit,
offset,
startTime,
endTime
)
if (!searchResult.success || !searchResult.messages) {
this.sendError(res, 500, searchResult.error || 'Failed to search messages')
return
}
hasMore = searchResult.messages.length > limit
messages = hasMore ? searchResult.messages.slice(0, limit) : searchResult.messages
} else {
const result = await this.fetchMessagesBatch(
talker,
offset,
limit,
startTime,
endTime,
false,
!mediaOptions.enabled
)
if (!result.success || !result.messages) {
this.sendError(res, 500, result.error || 'Failed to get messages')
return
}
messages = result.messages
hasMore = result.hasMore === true
}
const mediaMap = mediaOptions.enabled
@@ -812,7 +875,7 @@ class HttpService {
const endTime = endParam ? this.parseTimeParam(endParam, true) : 0
try {
const result = await this.fetchMessagesBatch(sessionId, offset, limit, startTime, endTime, true)
const result = await this.fetchMessagesBatch(sessionId, offset, limit, startTime, endTime, true, true)
if (!result.success || !result.messages) {
this.sendError(res, 500, result.error || 'Failed to get messages')
return
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+6 -2
View File
@@ -3445,10 +3445,10 @@ function ChatPage(props: ChatPageProps) {
if (result.success && result.messages) {
const resultMessages = result.messages
if (offset === 0) {
setNoMessageTable(false)
setMessages(resultMessages)
persistSessionPreviewCache(sessionId, resultMessages)
if (resultMessages.length === 0) {
setNoMessageTable(true)
setHasMoreMessages(false)
}
@@ -3549,7 +3549,10 @@ function ChatPage(props: ChatPageProps) {
: offset + resultMessages.length
setCurrentOffset(nextOffset)
} else if (!result.success) {
setNoMessageTable(true)
const errorText = String(result.error || '')
const shouldMarkNoTable =
/schema mismatch|no message db|no table|消息数据库未找到|消息表|message schema/i.test(errorText)
setNoMessageTable(shouldMarkNoTable)
setHasMoreMessages(false)
}
} catch (e) {
@@ -3557,6 +3560,7 @@ function ChatPage(props: ChatPageProps) {
setConnectionError('加载消息失败')
setHasMoreMessages(false)
if (offset === 0 && currentSessionRef.current === sessionId) {
setNoMessageTable(false)
setMessages([])
}
} finally {
+33 -12
View File
@@ -1899,7 +1899,7 @@ const TaskCenterModal = memo(function TaskCenterModal({
? `缓存命中 ${mediaCacheHitFiles}/${mediaCacheTotal}`
: ''
const mediaMissMetricLabel = mediaCacheMissFiles > 0
? `未导出 ${mediaCacheMissFiles} 个文件/媒体`
? `缓存未命中 ${mediaCacheMissFiles}`
: ''
const mediaDedupMetricLabel = mediaDedupReuseFiles > 0
? `复用 ${mediaDedupReuseFiles}`
@@ -1914,7 +1914,7 @@ const TaskCenterModal = memo(function TaskCenterModal({
)
: ''
const mediaLiveMetricLabel = task.progress.phase === 'exporting-media'
? (mediaDoneFiles > 0 ? `已处理 ${mediaDoneFiles}` : '')
? (mediaDoneFiles > 0 ? `已写入 ${mediaDoneFiles}` : '')
: ''
const sessionProgressLabel = completedSessionTotal > 0
? `会话 ${completedSessionCount}/${completedSessionTotal}`
@@ -2238,6 +2238,27 @@ function ExportPage() {
exportConcurrency: 2
})
const exportStatsRangeOptions = useMemo(() => {
if (options.useAllTime || !options.dateRange) return null
const beginTimestamp = Math.floor(options.dateRange.start.getTime() / 1000)
const endTimestamp = Math.floor(options.dateRange.end.getTime() / 1000)
if (!Number.isFinite(beginTimestamp) || !Number.isFinite(endTimestamp)) return null
if (beginTimestamp <= 0 && endTimestamp <= 0) return null
return {
beginTimestamp: Math.max(0, beginTimestamp),
endTimestamp: Math.max(0, endTimestamp)
}
}, [options.useAllTime, options.dateRange])
const withExportStatsRange = useCallback((statsOptions: Record<string, any>): Record<string, any> => {
if (!exportStatsRangeOptions) return statsOptions
return {
...statsOptions,
beginTimestamp: exportStatsRangeOptions.beginTimestamp,
endTimestamp: exportStatsRangeOptions.endTimestamp
}
}, [exportStatsRangeOptions])
const [exportDialog, setExportDialog] = useState<ExportDialogState>({
open: false,
intent: 'manual',
@@ -4003,7 +4024,7 @@ function ExportPage() {
const cacheResult = await withTimeout(
window.electronAPI.chat.getExportSessionStats(
batchSessionIds,
{ includeRelations: false, allowStaleCache: true, cacheOnly: true }
withExportStatsRange({ includeRelations: false, allowStaleCache: true, cacheOnly: true })
),
12000,
'cacheOnly'
@@ -4018,7 +4039,7 @@ function ExportPage() {
const freshResult = await withTimeout(
window.electronAPI.chat.getExportSessionStats(
missingSessionIds,
{ includeRelations: false, allowStaleCache: true }
withExportStatsRange({ includeRelations: false, allowStaleCache: true })
),
45000,
'fresh'
@@ -4062,7 +4083,7 @@ function ExportPage() {
void runSessionMediaMetricWorker(runId)
}
}
}, [applySessionMediaMetricsFromStats, isSessionMediaMetricReady, patchSessionLoadTraceStage])
}, [applySessionMediaMetricsFromStats, isSessionMediaMetricReady, patchSessionLoadTraceStage, withExportStatsRange])
const scheduleSessionMediaMetricWorker = useCallback(() => {
if (activeTaskCountRef.current > 0) return
@@ -7243,7 +7264,7 @@ function ExportPage() {
try {
const quickStatsResult = await window.electronAPI.chat.getExportSessionStats(
[normalizedSessionId],
{ includeRelations: false, allowStaleCache: true, cacheOnly: true }
withExportStatsRange({ includeRelations: false, allowStaleCache: true, cacheOnly: true })
)
if (requestSeq !== detailRequestSeqRef.current) return
if (quickStatsResult.success) {
@@ -7270,7 +7291,7 @@ function ExportPage() {
try {
const relationCacheResult = await window.electronAPI.chat.getExportSessionStats(
[normalizedSessionId],
{ includeRelations: true, allowStaleCache: true, cacheOnly: true }
withExportStatsRange({ includeRelations: true, allowStaleCache: true, cacheOnly: true })
)
if (requestSeq !== detailRequestSeqRef.current) return
if (relationCacheResult.success && relationCacheResult.data) {
@@ -7295,7 +7316,7 @@ function ExportPage() {
// 后台补齐非关系统计,不走精确特型扫描,避免阻塞列表统计队列。
const freshResult = await window.electronAPI.chat.getExportSessionStats(
[normalizedSessionId],
{ includeRelations: false, forceRefresh: true }
withExportStatsRange({ includeRelations: false, forceRefresh: true })
)
if (requestSeq !== detailRequestSeqRef.current) return
if (freshResult.success && freshResult.data) {
@@ -7330,7 +7351,7 @@ function ExportPage() {
setIsLoadingSessionDetailExtra(false)
}
}
}, [applySessionDetailStats, contactByUsername, mergeSessionContentMetrics, sessionContentMetrics, sessionMessageCounts, sessionRowByUsername])
}, [applySessionDetailStats, contactByUsername, mergeSessionContentMetrics, sessionContentMetrics, sessionMessageCounts, sessionRowByUsername, withExportStatsRange])
const loadSessionRelationStats = useCallback(async (options?: { forceRefresh?: boolean }) => {
const normalizedSessionId = String(sessionDetail?.wxid || '').trim()
@@ -7343,7 +7364,7 @@ function ExportPage() {
if (!forceRefresh) {
const relationCacheResult = await window.electronAPI.chat.getExportSessionStats(
[normalizedSessionId],
{ includeRelations: true, allowStaleCache: true, cacheOnly: true }
withExportStatsRange({ includeRelations: true, allowStaleCache: true, cacheOnly: true })
)
if (requestSeq !== detailRequestSeqRef.current) return
@@ -7361,7 +7382,7 @@ function ExportPage() {
const relationResult = await window.electronAPI.chat.getExportSessionStats(
[normalizedSessionId],
{ includeRelations: true, forceRefresh, preferAccurateSpecialTypes: true }
withExportStatsRange({ includeRelations: true, forceRefresh, preferAccurateSpecialTypes: true })
)
if (requestSeq !== detailRequestSeqRef.current) return
@@ -7381,7 +7402,7 @@ function ExportPage() {
setIsLoadingSessionRelationStats(false)
}
}
}, [applySessionDetailStats, isLoadingSessionRelationStats, sessionDetail?.wxid])
}, [applySessionDetailStats, isLoadingSessionRelationStats, sessionDetail?.wxid, withExportStatsRange])
const handleRefreshTableData = useCallback(async () => {
const scopeKey = await ensureExportCacheScope()
+2 -1
View File
@@ -311,6 +311,8 @@ export interface ElectronAPI {
allowStaleCache?: boolean
preferAccurateSpecialTypes?: boolean
cacheOnly?: boolean
beginTimestamp?: number
endTimestamp?: number
}
) => Promise<{
success: boolean
@@ -1220,4 +1222,3 @@ declare global {
export { }