This commit is contained in:
cc committed 2026-07-01 15:34:37 +08:00
1 parent 99dcb6e43b
commit 301cb5abce
48 files changed
+5444 -1015

No files matched your search

+209 -143
View File
@@ -20,7 +20,7 @@ import { EXPORT_HTML_STYLES } from '../../exportHtmlStyles'
import { LRUCache } from '../../../utils/LRUCache.js'
import { normalizeTimestampSeconds, formatTimestamp, formatIsoTimestamp, parseCompactDateTimeDigitsToSeconds, parseDateTimeTextToSeconds, normalizeExportDateRange, normalizeRowTimestampSeconds, getTimestampSecondsFromRow } from '../../export/utils/timestamp';
import { escapeHtml, escapeAttribute, renderMultilineText, decodeHtmlEntities } from '../../export/utils/htmlEscape';
import { sanitizeExportFileNamePart, resolveFileAttachmentExtensionDir, normalizeFileNamingMode, formatDateTokenBySeconds, buildDateRangeFileNamePart, buildSessionExportBaseName, reserveUniqueOutputPath } from '../../export/utils/fileNaming';
import { sanitizeExportFileNamePart, resolveFileAttachmentExtensionDir, normalizeFileNamingMode, normalizeExportConflictStrategy, formatDateTokenBySeconds, buildDateRangeFileNamePart, buildSessionExportBaseName, reserveUniqueOutputPath } from '../../export/utils/fileNaming';
import { extractXmlValue, extractXmlAttribute, extractAppMessageType, normalizeAppMessageContent } from '../../export/parsers/xmlExtractor';
import { decodeMessageContent, decodeMaybeCompressed, decodeBinaryContent, looksLikeHex, looksLikeBase64 } from '../../export/parsers/contentDecoder';
import { parseVoipMessage } from '../../export/parsers/voipParser';
@@ -177,17 +177,24 @@ export class ExportContext {
const normalizedDateRange = normalizeExportDateRange(options.dateRange);
const normalizedMaxFileSizeMb = this.normalizeMaxFileSizeMb(options.maxFileSizeMb);
const normalizedWriteLayout = this.resolveExportWriteLayout(options);
const exportMedia = options.exportMedia === true;
const mediaContentType = this.getMediaContentType(options);
const exportMedia = options.exportMedia === true || this.isMediaOptionSelected(options) || mediaContentType !== null;
const resolveMediaOption = (key: keyof Pick<ExportOptions, 'exportImages' | 'exportVoices' | 'exportVideos' | 'exportEmojis' | 'exportFiles'>, type: MediaContentType): boolean | undefined => {
if (mediaContentType) return mediaContentType === type
return exportMedia ? options[key] !== false : options[key]
}
return {
...options,
dateRange: normalizedDateRange,
maxFileSizeMb: normalizedMaxFileSizeMb,
exportConflictStrategy: this.resolveExportConflictStrategy(options),
exportWriteLayout: normalizedWriteLayout,
exportImages: exportMedia ? options.exportImages !== false : options.exportImages,
exportVoices: exportMedia ? options.exportVoices !== false : options.exportVoices,
exportVideos: exportMedia ? options.exportVideos !== false : options.exportVideos,
exportEmojis: exportMedia ? options.exportEmojis !== false : options.exportEmojis,
exportFiles: exportMedia ? options.exportFiles !== false : options.exportFiles
exportMedia,
exportImages: resolveMediaOption('exportImages', 'image'),
exportVoices: resolveMediaOption('exportVoices', 'voice'),
exportVideos: resolveMediaOption('exportVideos', 'video'),
exportEmojis: resolveMediaOption('exportEmojis', 'emoji'),
exportFiles: resolveMediaOption('exportFiles', 'file')
}
}
@@ -200,6 +207,14 @@ export class ExportContext {
: 'B'
}
public resolveExportConflictStrategy(options?: Pick<ExportOptions, 'exportConflictStrategy'> | null): 'incremental' | 'overwrite' | 'rename' {
return normalizeExportConflictStrategy(options?.exportConflictStrategy)
}
public shouldReuseExistingExportFile(options?: Pick<ExportOptions, 'exportConflictStrategy'> | null): boolean {
return this.resolveExportConflictStrategy(options) === 'incremental'
}
public resolveExportPathStyle(options?: Pick<ExportOptions, 'exportPathStyle'> | null): 'posix' | 'windows' {
const style = options?.exportPathStyle
if (style === 'posix') return 'posix'
@@ -521,8 +536,15 @@ export class ExportContext {
return code === 'EXDEV' || code === 'EPERM' || code === 'EACCES' || code === 'EINVAL' || code === 'ENOSYS' || code === 'ENOTSUP'
}
private async copyMediaWithCacheAndDedup(kind: 'image' | 'video' | 'emoji', sourcePath: string, destPath: string, control?: ExportTaskControl): Promise<{ success: boolean; code?: string }> {
private async copyMediaWithCacheAndDedup(kind: 'image' | 'video' | 'emoji', sourcePath: string, destPath: string, control?: ExportTaskControl, options?: Pick<ExportOptions, 'exportConflictStrategy'>): Promise<{ success: boolean; code?: string }> {
const existedBeforeCopy = await pathExists(destPath);
if (existedBeforeCopy && this.shouldReuseExistingExportFile(options)) {
this.noteMediaTelemetry({
doneFiles: 1,
dedupReuseFiles: 1
})
return { success: true }
}
const resolved = await this.resolvePreferredMediaSource(kind, sourcePath);
if (resolved.cacheHit) {
this.noteMediaTelemetry({ cacheHitFiles: 1 })
@@ -649,9 +671,12 @@ export class ExportContext {
}
}
private isMediaExportEnabled(options: ExportOptions): boolean {
return options.exportMedia === true &&
Boolean(options.exportImages || options.exportVoices || options.exportVideos || options.exportEmojis || options.exportFiles)
private isMediaOptionSelected(options: Pick<ExportOptions, 'exportImages' | 'exportVoices' | 'exportVideos' | 'exportEmojis' | 'exportFiles'>): boolean {
return Boolean(options.exportImages || options.exportVoices || options.exportVideos || options.exportEmojis || options.exportFiles)
}
public isMediaExportEnabled(options: ExportOptions): boolean {
return this.getMediaContentType(options) !== null || this.isMediaOptionSelected(options)
}
public isUnboundedDateRange(dateRange?: { start: number; end: number } | null): boolean {
@@ -2706,6 +2731,46 @@ export class ExportContext {
/**
* 导出媒体文件到指定目录
*/
private resolveWeliveMediaPath(msg: any): string {
const nestedMedia = msg?.media && typeof msg.media === 'object' ? msg.media : null
const candidates = [
msg?.mediaPath,
msg?.media_path,
msg?.videoPath,
msg?.video_path,
msg?.imagePath,
msg?.image_path,
msg?.voicePath,
msg?.voice_path,
msg?.filePath,
msg?.file_path,
msg?.localPath,
msg?.local_path,
msg?.sourcePath,
msg?.source_path,
msg?.fullPath,
msg?.full_path,
msg?.absolutePath,
msg?.absolute_path,
nestedMedia?.path,
nestedMedia?.mediaPath,
nestedMedia?.media_path,
nestedMedia?.videoPath,
nestedMedia?.video_path,
nestedMedia?.localPath,
nestedMedia?.local_path,
nestedMedia?.filePath,
nestedMedia?.file_path
]
for (const candidate of candidates) {
const value = String(candidate || '').trim()
if (value) return value
}
return ''
}
public async exportMediaForMessage(msg: any, sessionId: string, mediaRootDir: string, mediaRelativePrefix: string, options: {
exportImages?: boolean
exportVoices?: boolean
@@ -2714,13 +2779,14 @@ export class ExportContext {
exportFiles?: boolean
maxFileSizeMb?: number
exportVoiceAsText?: boolean
exportConflictStrategy?: ExportOptions['exportConflictStrategy']
includeVideoPoster?: boolean
includeVoiceWithTranscript?: boolean
dirCache?: Set<string>
control?: ExportTaskControl
}): Promise<MediaExportItem | null> {
const localType = msg.localType;
const weliveMediaPath = String(msg?.mediaPath || msg?.media_path || '').trim()
const weliveMediaPath = this.resolveWeliveMediaPath(msg)
if (weliveMediaPath) {
const adopted = await this.adoptWeliveExportedMedia(msg, weliveMediaPath, mediaRootDir, mediaRelativePrefix, {
exportImages: options.exportImages,
@@ -2729,6 +2795,7 @@ export class ExportContext {
exportEmojis: options.exportEmojis,
exportFiles: options.exportFiles,
maxFileSizeMb: options.maxFileSizeMb,
exportConflictStrategy: options.exportConflictStrategy,
dirCache: options.dirCache,
control: options.control
})
@@ -2742,7 +2809,8 @@ export class ExportContext {
mediaRootDir,
mediaRelativePrefix,
options.dirCache,
options.control
options.control,
options
)
if (result) {
}
@@ -2751,7 +2819,7 @@ export class ExportContext {
if (localType === 34) {
if (options.exportVoices) {
return this.exportVoice(msg, sessionId, mediaRootDir, mediaRelativePrefix, options.dirCache, options.control)
return this.exportVoice(msg, sessionId, mediaRootDir, mediaRelativePrefix, options.dirCache, options.control, options)
}
if (options.exportVoiceAsText) {
return null
@@ -2759,7 +2827,7 @@ export class ExportContext {
}
if (localType === 47 && options.exportEmojis) {
const result = await this.exportEmoji(msg, sessionId, mediaRootDir, mediaRelativePrefix, options.dirCache, options.control)
const result = await this.exportEmoji(msg, sessionId, mediaRootDir, mediaRelativePrefix, options.dirCache, options.control, options)
if (result) {
}
return result
@@ -2773,7 +2841,8 @@ export class ExportContext {
mediaRelativePrefix,
options.dirCache,
options.includeVideoPoster === true,
options.control
options.control,
options
)
}
@@ -2784,7 +2853,8 @@ export class ExportContext {
mediaRelativePrefix,
options.maxFileSizeMb,
options.dirCache,
options.control
options.control,
options
)
}
@@ -2798,6 +2868,7 @@ export class ExportContext {
exportEmojis?: boolean
exportFiles?: boolean
maxFileSizeMb?: number
exportConflictStrategy?: ExportOptions['exportConflictStrategy']
dirCache?: Set<string>
control?: ExportTaskControl
}): Promise<MediaExportItem | null> {
@@ -2846,14 +2917,18 @@ export class ExportContext {
const destPath = path.join(targetDir, destFileName)
if (path.resolve(sourcePath) !== path.resolve(destPath)) {
if (kind === 'image' || kind === 'emoji' || kind === 'video') {
const copied = await this.copyMediaWithCacheAndDedup(kind, sourcePath, destPath, options.control)
const copied = await this.copyMediaWithCacheAndDedup(kind, sourcePath, destPath, options.control, options)
if (!copied.success) return null
} else {
const existedBeforeCopy = await pathExists(destPath)
const copied = await copyFileOptimized(sourcePath, destPath)
if (!copied.success) return null
if (!existedBeforeCopy) options.control?.recordCreatedFile?.(destPath)
this.noteMediaTelemetry({ doneFiles: 1, bytesWritten: stat.size })
if (existedBeforeCopy && this.shouldReuseExistingExportFile(options)) {
this.noteMediaTelemetry({ doneFiles: 1, dedupReuseFiles: 1 })
} else {
const copied = await copyFileOptimized(sourcePath, destPath)
if (!copied.success) return null
if (!existedBeforeCopy) options.control?.recordCreatedFile?.(destPath)
this.noteMediaTelemetry({ doneFiles: 1, bytesWritten: stat.size })
}
}
}
@@ -2869,7 +2944,7 @@ export class ExportContext {
/**
* 导出图片文件
*/
private async exportImage(msg: any, sessionId: string, mediaRootDir: string, mediaRelativePrefix: string, dirCache?: Set<string>, control?: ExportTaskControl): Promise<MediaExportItem | null> {
private async exportImage(msg: any, sessionId: string, mediaRootDir: string, mediaRelativePrefix: string, dirCache?: Set<string>, control?: ExportTaskControl, options?: Pick<ExportOptions, 'exportConflictStrategy'>): Promise<MediaExportItem | null> {
try {
const imagesDir = path.join(mediaRootDir, mediaRelativePrefix, 'images')
await ensureExportDir(imagesDir, control, dirCache)
@@ -2992,6 +3067,14 @@ export class ExportContext {
const fileName = `${messageId}_${imageKey}${ext}`
const destPath = path.join(imagesDir, fileName)
if (await pathExists(destPath) && this.shouldReuseExistingExportFile(options)) {
this.noteMediaTelemetry({ doneFiles: 1, dedupReuseFiles: 1 })
return {
relativePath: path.posix.join(mediaRelativePrefix, 'images', fileName),
kind: 'image'
}
}
const buffer = Buffer.from(base64Data, 'base64')
await this.recordCreatedFileBeforeWrite(destPath, control)
await fs.promises.writeFile(destPath, buffer)
@@ -3013,7 +3096,7 @@ export class ExportContext {
const ext = path.extname(sourcePath) || '.jpg'
const fileName = `${messageId}_${imageKey}${ext}`
const destPath = path.join(imagesDir, fileName)
const copied = await this.copyMediaWithCacheAndDedup('image', sourcePath, destPath, control)
const copied = await this.copyMediaWithCacheAndDedup('image', sourcePath, destPath, control, options)
if (!copied.success) {
if (copied.code === 'ENOENT') {
console.log(`[Export] 源图片文件不存在 (localId=${msg.localId}): ${sourcePath} → 将显示 [图片] 占位符`)
@@ -3033,10 +3116,19 @@ export class ExportContext {
}
}
public async preloadMediaLookupCaches(_sessionId: string, messages: any[], options: { exportImages?: boolean; exportVideos?: boolean }, control?: ExportTaskControl): Promise<void> {
public async preloadMediaLookupCaches(sessionId: string, messages: any[], options: { exportImages?: boolean; exportVideos?: boolean }, control?: ExportTaskControl): Promise<void> {
if (!Array.isArray(messages) || messages.length === 0) return
const md5Pattern = /^[a-f0-9]{32}$/i;
const imageMd5Set = new Set<string>();
const videoTokenSet = new Set<string>();
if (options.exportVideos) {
const videoMessages = messages.filter((msg) => Number(msg?.localType || 0) === 43)
if (videoMessages.length > 0) {
await this.backfillMediaFieldsFromMessageDetail(sessionId, videoMessages, new Set([43]), control)
}
}
let scanIndex = 0;
for (const msg of messages) {
if ((scanIndex++ & 0x7f) === 0) {
@@ -3054,12 +3146,27 @@ export class ExportContext {
}
}
if (options.exportVideos && msg?.localType === 43) {
const videoMd5 = String(msg?.videoMd5 || '').trim().toLowerCase()
if (videoMd5) {
videoTokenSet.add(videoMd5)
continue
}
const parsedVideoMd5 = String(videoService.parseVideoMd5(String(msg?.content || '')) || '').trim().toLowerCase()
if (parsedVideoMd5) {
videoTokenSet.add(parsedVideoMd5)
}
}
}
const preloadTasks: Array<Promise<void>> = [];
if (imageMd5Set.size > 0) {
preloadTasks.push(imageDecryptService.preloadImageHardlinkMd5s(Array.from(imageMd5Set)))
}
if (videoTokenSet.size > 0) {
preloadTasks.push(videoService.getVideoInfoBatch(Array.from(videoTokenSet), { includePoster: false }).then(() => undefined))
}
if (preloadTasks.length === 0) return
await Promise.all(preloadTasks.map((task) => task.catch(() => { })))
@@ -3112,7 +3219,7 @@ export class ExportContext {
/**
* 导出语音文件
*/
private async exportVoice(msg: any, sessionId: string, mediaRootDir: string, mediaRelativePrefix: string, dirCache?: Set<string>, control?: ExportTaskControl): Promise<MediaExportItem | null> {
private async exportVoice(msg: any, sessionId: string, mediaRootDir: string, mediaRelativePrefix: string, dirCache?: Set<string>, control?: ExportTaskControl, options?: Pick<ExportOptions, 'exportConflictStrategy'>): Promise<MediaExportItem | null> {
try {
const voicesDir = path.join(mediaRootDir, mediaRelativePrefix, 'voices')
await ensureExportDir(voicesDir, control, dirCache)
@@ -3125,8 +3232,8 @@ export class ExportContext {
const fileName = `voice_${safeSession}_${stableKey || msgId}.wav`
const destPath = path.join(voicesDir, fileName)
// 如果已存在则跳过
if (await pathExists(destPath)) {
if (await pathExists(destPath) && this.shouldReuseExistingExportFile(options)) {
this.noteMediaTelemetry({ doneFiles: 1, dedupReuseFiles: 1 })
return {
relativePath: path.posix.join(mediaRelativePrefix, 'voices', fileName),
kind: 'voice'
@@ -3181,7 +3288,7 @@ export class ExportContext {
/**
* 导出表情文件
*/
private async exportEmoji(msg: any, sessionId: string, mediaRootDir: string, mediaRelativePrefix: string, dirCache?: Set<string>, control?: ExportTaskControl): Promise<MediaExportItem | null> {
private async exportEmoji(msg: any, sessionId: string, mediaRootDir: string, mediaRelativePrefix: string, dirCache?: Set<string>, control?: ExportTaskControl, options?: Pick<ExportOptions, 'exportConflictStrategy'>): Promise<MediaExportItem | null> {
try {
const emojisDir = path.join(mediaRootDir, mediaRelativePrefix, 'emojis')
await ensureExportDir(emojisDir, control, dirCache)
@@ -3211,7 +3318,7 @@ export class ExportContext {
const key = emojiMd5 || crypto.createHash('md5').update(emojiUrl || localPath || String(msg.localId || '')).digest('hex')
const fileName = `${key}${ext}`
const destPath = path.join(emojisDir, fileName)
const copied = await this.copyMediaWithCacheAndDedup('emoji', localPath, destPath, control)
const copied = await this.copyMediaWithCacheAndDedup('emoji', localPath, destPath, control, options)
if (!copied.success) return null
return {
@@ -3227,9 +3334,34 @@ export class ExportContext {
/**
* 导出视频文件
*/
private async exportVideo(msg: any, sessionId: string, mediaRootDir: string, mediaRelativePrefix: string, dirCache?: Set<string>, includePoster = false, control?: ExportTaskControl): Promise<MediaExportItem | null> {
private async exportVideo(msg: any, sessionId: string, mediaRootDir: string, mediaRelativePrefix: string, dirCache?: Set<string>, includePoster = false, control?: ExportTaskControl, options?: Pick<ExportOptions, 'exportConflictStrategy'>): Promise<MediaExportItem | null> {
try {
let videoMd5 = String(msg.videoMd5 || '').trim().toLowerCase()
const collectLookupTokens = () => {
const contents = [
msg?.content,
msg?.messageContent,
msg?.message_content,
msg?.rawContent,
msg?.raw_content,
msg?.parsedContent,
msg?.parsed_content
].map((item) => String(item || '')).filter(Boolean)
const tokens = [
msg?.videoMd5,
msg?.video_md5,
msg?.rawMd5,
msg?.raw_md5,
this.extractVideoFileNameFromRow(msg, contents[0] || ''),
this.normalizeVideoFileToken(this.resolveWeliveMediaPath(msg))
]
for (const content of contents) {
tokens.push(videoService.parseVideoMd5(content))
tokens.push(this.extractVideoMd5(content))
}
return Array.from(new Set(tokens
.map((token) => this.normalizeVideoFileToken(token))
.filter((token): token is string => Boolean(token))))
}
const resolveVideoInfo = async (token: string) => {
if (!token) return null
const videoInfo = await videoService.getVideoInfo(token, { includePoster })
@@ -3237,13 +3369,19 @@ export class ExportContext {
return videoInfo
}
let videoInfo = await resolveVideoInfo(videoMd5)
let videoInfo = null
for (const token of collectLookupTokens()) {
videoInfo = await resolveVideoInfo(token)
if (videoInfo) break
}
if (!videoInfo) {
const localId = Number(msg?.localId || 0)
if (Number.isFinite(localId) && localId > 0) {
await this.backfillMediaFieldsFromMessageDetail(sessionId, [msg], new Set([43]), undefined, { force: true })
videoMd5 = String(msg.videoMd5 || '').trim().toLowerCase()
videoInfo = await resolveVideoInfo(videoMd5)
for (const token of collectLookupTokens()) {
videoInfo = await resolveVideoInfo(token)
if (videoInfo) break
}
}
}
if (!videoInfo) return null
@@ -3256,7 +3394,7 @@ export class ExportContext {
const fileName = path.basename(sourcePath)
const destPath = path.join(videosDir, fileName)
const copied = await this.copyMediaWithCacheAndDedup('video', sourcePath, destPath, control)
const copied = await this.copyMediaWithCacheAndDedup('video', sourcePath, destPath, control, options)
if (!copied.success) return null
return {
@@ -3398,12 +3536,19 @@ export class ExportContext {
private extractVideoMd5(content: string): string | undefined {
if (!content) return undefined
const attrMatch = /<videomsg[^>]*\smd5\s*=\s*['"]([a-fA-F0-9]+)['"]/i.exec(content);
if (attrMatch) {
return attrMatch[1].toLowerCase()
const attrPatterns = [
/<videomsg[^>]*\smd5\s*=\s*['"]([a-fA-F0-9]+)['"]/i,
/<videomsg[^>]*\srawmd5\s*=\s*['"]([a-fA-F0-9]+)['"]/i,
/<videomsg[^>]*\snewmd5\s*=\s*['"]([a-fA-F0-9]+)['"]/i,
/<videomsg[^>]*\soriginsourcemd5\s*=\s*['"]([a-fA-F0-9]+)['"]/i,
/(?:^|\s)(?:md5|rawmd5|newmd5|originsourcemd5)\s*=\s*['"]([a-fA-F0-9]+)['"]/i
]
for (const pattern of attrPatterns) {
const match = pattern.exec(content)
if (match?.[1]) return match[1].toLowerCase()
}
const tagMatch = /<md5>([^<]+)<\/md5>/i.exec(content);
const tagMatch = /<(?:md5|rawmd5|newmd5|originsourcemd5)>([a-fA-F0-9]+)<\/(?:md5|rawmd5|newmd5|originsourcemd5)>/i.exec(content);
return tagMatch?.[1]?.toLowerCase()
}
@@ -3752,7 +3897,7 @@ export class ExportContext {
return candidates
}
private async exportFileAttachment(msg: any, mediaRootDir: string, mediaRelativePrefix: string, maxFileSizeMb?: number, dirCache?: Set<string>, control?: ExportTaskControl): Promise<MediaExportItem | null> {
private async exportFileAttachment(msg: any, mediaRootDir: string, mediaRelativePrefix: string, maxFileSizeMb?: number, dirCache?: Set<string>, control?: ExportTaskControl, options?: Pick<ExportOptions, 'exportConflictStrategy'>): Promise<MediaExportItem | null> {
try {
const fileNameRaw = String(msg?.fileName || '').trim()
if (!fileNameRaw) return null
@@ -3804,6 +3949,14 @@ export class ExportContext {
const destFileName = `${messageId}_${safeBaseName}`
const destPath = path.join(fileDir, destFileName)
const existedBeforeCopy = await pathExists(destPath)
if (existedBeforeCopy && this.shouldReuseExistingExportFile(options)) {
this.noteMediaTelemetry({ doneFiles: 1, dedupReuseFiles: 1 })
return {
relativePath: path.posix.join(mediaRelativePrefix, 'file', fileExtDir, destFileName),
kind: 'file'
}
}
const copied = await copyFileOptimized(selected.sourcePath, destPath)
if (!copied.success) {
this.recordFileAttachmentMiss(msg, '附件复制失败', {
@@ -3882,8 +4035,7 @@ export class ExportContext {
mediaRootDir: string
mediaRelativePrefix: string
} {
const exportMediaEnabled = options.exportMedia === true &&
Boolean(options.exportImages || options.exportVoices || options.exportVideos || options.exportEmojis || options.exportFiles);
const exportMediaEnabled = this.isMediaExportEnabled(options);
const outputDir = path.dirname(outputPath);
const writeLayout = this.resolveExportWriteLayout(options);
if (writeLayout === 'A' && path.basename(outputDir) === 'texts') {
@@ -3904,7 +4056,7 @@ export class ExportContext {
public async resolveWeliveRawMediaItem(msg: any, mediaRootDir: string, mediaRelativePrefix: string, options: ExportOptions, control?: ExportTaskControl): Promise<MediaExportItem | null> {
if (!this.isWeliveRawExportMode()) return null
const sourcePathRaw = String(msg?.mediaPath || msg?.media_path || '').trim()
const sourcePathRaw = this.resolveWeliveMediaPath(msg)
const localType = Number(msg?.localType || msg?.local_type || 0)
const mediaType = String(msg?.mediaType || msg?.media_type || '').trim().toLowerCase()
@@ -3922,26 +4074,9 @@ export class ExportContext {
}
if (!kind) return null
if (!sourcePathRaw) {
const emojiUrl = kind === 'emoji'
? String(msg?.emojiCdnUrl || msg?.emoji_cdn_url || '').trim()
: ''
if (emojiUrl) {
return await this.exportEmoji({
...msg,
emojiCdnUrl: emojiUrl,
emojiMd5: this.normalizeEmojiMd5(msg?.emojiMd5 || msg?.emoji_md5)
}, String(msg?.sessionId || msg?.session_id || ''), mediaRootDir, mediaRelativePrefix, undefined, control)
}
return null
}
if (/^https?:\/\//i.test(sourcePathRaw)) {
if (kind === 'emoji') {
return await this.exportEmoji({
...msg,
emojiCdnUrl: sourcePathRaw,
emojiMd5: this.normalizeEmojiMd5(msg?.emojiMd5 || msg?.emoji_md5)
}, String(msg?.sessionId || msg?.session_id || ''), mediaRootDir, mediaRelativePrefix, undefined, control)
}
return { relativePath: sourcePathRaw, kind }
}
@@ -3954,88 +4089,17 @@ export class ExportContext {
}
public async preloadWeliveRawEmojiMedia(
messages: any[],
mediaCache: Map<string, MediaExportItem | null>,
mediaRootDir: string,
mediaRelativePrefix: string,
options: ExportOptions,
control?: ExportTaskControl,
onProgress?: (progress: ExportProgress) => void,
sessionName = '',
progressCurrent = 25
_messages: any[],
_mediaCache: Map<string, MediaExportItem | null>,
_mediaRootDir: string,
_mediaRelativePrefix: string,
_options: ExportOptions,
_control?: ExportTaskControl,
_onProgress?: (progress: ExportProgress) => void,
_sessionName = '',
_progressCurrent = 25
): Promise<void> {
if (!this.isWeliveRawExportMode() || options.exportEmojis !== true || !Array.isArray(messages) || messages.length === 0) return
const groups = new Map<string, { msg: any; mediaKeys: string[] }>()
for (const msg of messages) {
const localType = Number(msg?.localType || msg?.local_type || 0)
const mediaType = String(msg?.mediaType || msg?.media_type || '').trim().toLowerCase()
if (localType !== 47 && mediaType !== 'emoji') continue
const mediaKey = this.getMediaCacheKey(msg)
if (mediaCache.has(mediaKey)) continue
const sourcePathRaw = String(msg?.mediaPath || msg?.media_path || '').trim()
const emojiUrl = /^https?:\/\//i.test(sourcePathRaw)
? sourcePathRaw
: this.normalizeEmojiCdnUrl(msg?.emojiCdnUrl || msg?.emoji_cdn_url || this.extractEmojiUrl(String(msg?.content || '')))
if (!emojiUrl) continue
const emojiMd5 = this.normalizeEmojiMd5(msg?.emojiMd5 || msg?.emoji_md5)
const groupKey = emojiMd5 || emojiUrl
const existing = groups.get(groupKey)
if (existing) {
existing.mediaKeys.push(mediaKey)
} else {
groups.set(groupKey, {
msg: {
...msg,
mediaPath: /^https?:\/\//i.test(sourcePathRaw) ? sourcePathRaw : msg?.mediaPath,
media_path: /^https?:\/\//i.test(sourcePathRaw) ? sourcePathRaw : msg?.media_path,
emojiCdnUrl: emojiUrl,
emojiMd5: emojiMd5 || msg?.emojiMd5
},
mediaKeys: [mediaKey]
})
}
}
const workItems = Array.from(groups.values())
if (workItems.length === 0) return
const concurrencySeed = typeof options.exportConcurrency === 'number'
? options.exportConcurrency * 4
: undefined
const concurrency = this.getClampedConcurrency(concurrencySeed, 12, 24)
let completed = 0
let lastReportAt = 0
const report = (force = false) => {
if (!onProgress) return
const now = Date.now()
if (!force && now - lastReportAt < 350) return
lastReportAt = now
onProgress({
current: progressCurrent,
total: 100,
currentSession: sessionName,
phase: 'exporting-media',
phaseProgress: completed,
phaseTotal: workItems.length,
phaseLabel: `下载表情包 ${completed.toLocaleString()}/${workItems.length.toLocaleString()}`,
...this.getMediaTelemetrySnapshot()
})
}
report(true)
await parallelLimit(workItems, concurrency, async (item) => {
this.throwIfStopRequested(control)
const mediaItem = await this.resolveWeliveRawMediaItem(item.msg, mediaRootDir, mediaRelativePrefix, options, control)
for (const mediaKey of item.mediaKeys) {
mediaCache.set(mediaKey, mediaItem)
}
completed++
report(completed === workItems.length || completed % 10 === 0)
})
// WeLive raw export owns media resolution. Weflow must not scan/copy original media here.
}
public collectMediaMessagesForExport(messages: any[], options: ExportOptions): any[] {
@@ -4262,6 +4326,8 @@ export class ExportContext {
senderSet.add(actualSender)
rows.push({
sessionId,
session_id: sessionId,
localId: this.getIntFromRow(row, ['local_id', 'localId', 'LocalId', 'msg_local_id', 'msgLocalId', 'MsgLocalId', 'msg_id', 'msgId', 'MsgId', 'id', 'WCDB_CT_local_id'], 0),
serverId: this.getIntFromRow(row, ['server_id', 'serverId', 'ServerId', 'msg_server_id', 'msgServerId', 'MsgServerId', 'svr_id', 'svrId', 'msg_svr_id', 'msgSvrId', 'MsgSvrId', 'WCDB_CT_server_id'], 0),
serverIdRaw: this.normalizeUnsignedIntToken(this.getRowField(row, ['server_id', 'serverId', 'ServerId', 'msg_server_id', 'msgServerId', 'MsgServerId', 'svr_id', 'svrId', 'msg_svr_id', 'msgSvrId', 'MsgSvrId', 'WCDB_CT_server_id'])) || undefined,
@@ -4280,7 +4346,7 @@ export class ExportContext {
fileSize,
fileExt,
fileMd5,
mediaPath: String(row.media_path || row.mediaPath || '').trim() || undefined,
mediaPath: this.resolveWeliveMediaPath(row) || undefined,
mediaType: String(row.media_type || row.mediaType || '').trim() || undefined,
mediaError: String(row.media_error || row.mediaError || '').trim() || undefined,
locationLat,
@@ -18,7 +18,7 @@ import { EXPORT_HTML_STYLES } from '../../exportHtmlStyles'
import { LRUCache } from '../../../utils/LRUCache.js'
import { normalizeTimestampSeconds, formatTimestamp, formatIsoTimestamp, parseCompactDateTimeDigitsToSeconds, parseDateTimeTextToSeconds, normalizeExportDateRange, normalizeRowTimestampSeconds, getTimestampSecondsFromRow } from '../../export/utils/timestamp';
import { escapeHtml, escapeAttribute, renderMultilineText, decodeHtmlEntities } from '../../export/utils/htmlEscape';
import { sanitizeExportFileNamePart, resolveFileAttachmentExtensionDir, normalizeFileNamingMode, formatDateTokenBySeconds, buildDateRangeFileNamePart, buildSessionExportBaseName, reserveUniqueOutputPath } from '../../export/utils/fileNaming';
import { sanitizeExportFileNamePart, resolveFileAttachmentExtensionDir, normalizeExportConflictStrategy, formatDateTokenBySeconds, buildDateRangeFileNamePart, buildSessionExportBaseName, reserveUniqueOutputPath } from '../../export/utils/fileNaming';
import { extractXmlValue, extractXmlAttribute, extractAppMessageType, normalizeAppMessageContent } from '../../export/parsers/xmlExtractor';
import { decodeMessageContent, decodeMaybeCompressed, decodeBinaryContent, looksLikeHex, looksLikeBase64 } from '../../export/parsers/contentDecoder';
import { parseVoipMessage } from '../../export/parsers/voipParser';
@@ -37,6 +37,7 @@ import { ChatLabFormatter } from '../formatters/ChatLabFormatter';
import { ExcelFormatter } from '../formatters/ExcelFormatter';
import { HtmlFormatter } from '../formatters/HtmlFormatter';
import { JsonFormatter } from '../formatters/JsonFormatter';
import { MarkdownFormatter } from '../formatters/MarkdownFormatter';
import { TxtFormatter } from '../formatters/TxtFormatter';
import { WeCloneFormatter } from '../formatters/WeCloneFormatter';
@@ -92,6 +93,14 @@ export class ExportOrchestrator {
return formatter.export(sessionId, outputPath, options, onProgress, control);
}
/**
* 导出单个会话为 Markdown 格式
*/
async exportSessionToMarkdown(sessionId: string, outputPath: string, options: ExportOptions, onProgress?: (progress: ExportProgress) => void, control?: ExportTaskControl): Promise<{ success: boolean; error?: string }> {
const formatter = new MarkdownFormatter(this.context);
return formatter.export(sessionId, outputPath, options, onProgress, control);
}
/**
* 批量导出多个会话
*/
@@ -133,9 +142,9 @@ export class ExportOrchestrator {
const effectiveOptions: ExportOptions = this.context.isMediaContentBatchExport(normalizedOptions)
? { ...normalizedOptions, exportVoiceAsText: false }
: normalizedOptions
const conflictStrategy = normalizeExportConflictStrategy(effectiveOptions.exportConflictStrategy)
const exportMediaEnabled = effectiveOptions.exportMedia === true &&
Boolean(effectiveOptions.exportImages || effectiveOptions.exportVoices || effectiveOptions.exportVideos || effectiveOptions.exportEmojis || effectiveOptions.exportFiles)
const exportMediaEnabled = this.context.isMediaExportEnabled(effectiveOptions)
attachMediaTelemetry = exportMediaEnabled
if (exportMediaEnabled) {
this.context.triggerMediaFileCacheCleanup()
@@ -193,7 +202,7 @@ export class ExportOrchestrator {
this.context.isUnboundedDateRange(effectiveOptions.dateRange) &&
!String(effectiveOptions.senderUsername || '').trim()
const canFastSkipEmptySessions = false
const canTrySkipUnchangedTextSessions = canUseSessionSnapshotHints
const canTrySkipUnchangedTextSessions = canUseSessionSnapshotHints && conflictStrategy === 'incremental'
const precheckSessionIds = canFastSkipEmptySessions
? sessionIds.filter((sessionId) => !sessionMessageCountHints.has(sessionId))
: []
@@ -349,7 +358,6 @@ export class ExportOrchestrator {
phaseLabel: '准备导出'
})
const fileNamingMode = normalizeFileNamingMode(effectiveOptions.fileNamingMode)
const safeName = buildSessionExportBaseName(sessionId, sessionInfo.displayName, effectiveOptions)
const sessionNameWithTypePrefix = effectiveOptions.sessionNameWithTypePrefix !== false
const sessionTypePrefix = sessionNameWithTypePrefix ? await this.context.getSessionFilePrefix(sessionId) : ''
@@ -366,6 +374,7 @@ export class ExportOrchestrator {
if (effectiveOptions.format === 'chatlab-jsonl') ext = '.jsonl'
else if (effectiveOptions.format === 'excel') ext = '.xlsx'
else if (effectiveOptions.format === 'txt') ext = '.txt'
else if (effectiveOptions.format === 'markdown') ext = '.md'
else if (effectiveOptions.format === 'weclone') ext = '.csv'
else if (effectiveOptions.format === 'html') ext = '.html'
const preferredOutputPath = path.join(sessionDir, `${fileNameWithPrefix}${ext}`)
@@ -402,7 +411,7 @@ export class ExportOrchestrator {
}
}
const outputPath = fileNamingMode === 'date-range'
const outputPath = conflictStrategy === 'rename'
? await reserveUniqueOutputPath(preferredOutputPath, reservedOutputPaths)
: preferredOutputPath
@@ -415,6 +424,8 @@ export class ExportOrchestrator {
result = await this.exportSessionToExcel(sessionId, outputPath, effectiveOptions, sessionProgress, control)
} else if (effectiveOptions.format === 'txt') {
result = await this.exportSessionToTxt(sessionId, outputPath, effectiveOptions, sessionProgress, control)
} else if (effectiveOptions.format === 'markdown') {
result = await this.exportSessionToMarkdown(sessionId, outputPath, effectiveOptions, sessionProgress, control)
} else if (effectiveOptions.format === 'weclone') {
result = await this.exportSessionToWeCloneCsv(sessionId, outputPath, effectiveOptions, sessionProgress, control)
} else if (effectiveOptions.format === 'html') {
@@ -161,6 +161,7 @@ export class ChatLabFormatter {
exportFiles: options.exportFiles,
maxFileSizeMb: options.maxFileSizeMb,
exportVoiceAsText: options.exportVoiceAsText,
exportConflictStrategy: options.exportConflictStrategy,
includeVideoPoster: options.format === 'html',
dirCache: mediaDirCache,
control
@@ -276,6 +276,7 @@ export class ExcelFormatter {
exportFiles: options.exportFiles,
maxFileSizeMb: options.maxFileSizeMb,
exportVoiceAsText: options.exportVoiceAsText,
exportConflictStrategy: options.exportConflictStrategy,
includeVideoPoster: options.format === 'html',
dirCache: mediaDirCache,
control
@@ -141,6 +141,7 @@ export class HtmlFormatter {
exportFiles: options.exportFiles,
maxFileSizeMb: options.maxFileSizeMb,
exportVoiceAsText: options.exportVoiceAsText,
exportConflictStrategy: options.exportConflictStrategy,
includeVideoPoster: options.format === 'html',
includeVoiceWithTranscript: true,
exportVideos: options.exportVideos,
@@ -137,6 +137,7 @@ export class JsonFormatter {
exportFiles: options.exportFiles,
maxFileSizeMb: options.maxFileSizeMb,
exportVoiceAsText: options.exportVoiceAsText,
exportConflictStrategy: options.exportConflictStrategy,
includeVideoPoster: options.format === 'html',
dirCache: mediaDirCache,
control
@@ -0,0 +1,463 @@
import * as fs from 'fs';
import * as path from 'path';
import { resolveExportDisplayProfile } from '../../export/contacts/contactResolver';
import { buildGroupNicknameIdCandidates } from '../../export/contacts/groupNickname';
import { appendTransferDesc, isTransferExportContent, resolveTransferDesc } from '../../export/parsers/transferParser';
import { formatTimestamp } from '../../export/utils/timestamp';
import { parallelLimit } from '../../export/utils/parallelLimit';
import { ExportDisplayProfile, MediaExportItem } from '../../export/types';
import { wcdbService } from '../../wcdbService';
const escapeMarkdownText = (value: unknown): string => {
return String(value ?? '')
.replace(/\\/g, '\\\\')
.replace(/([`*_{}\[\]<>#+\-.!|])/g, '\\$1')
}
const escapeMarkdownLinkText = (value: unknown): string => {
return escapeMarkdownText(value).replace(/\r?\n/g, ' ')
}
const toMarkdownUrl = (value: string): string => {
return encodeURI(value)
.replace(/\(/g, '%28')
.replace(/\)/g, '%29')
.replace(/</g, '%3C')
.replace(/>/g, '%3E')
}
const buildMarkdownBlockquote = (value: string): string => {
return value
.split(/\r?\n/)
.map(line => `> ${line ? escapeMarkdownText(line) : ''}`)
.join('\n')
}
export class MarkdownFormatter {
constructor(private exportService: any) {}
public async export(sessionId: any, outputPath: any, options: any, onProgress: any, control: any): Promise<{ success: boolean; error?: string }> {
try {
this.exportService.throwIfStopRequested(control)
const conn = await this.exportService.ensureConnected()
if (!conn.success || !conn.cleanedWxid) return { success: false, error: conn.error }
const cleanedMyWxid = conn.cleanedWxid
const isGroup = sessionId.includes('@chatroom')
const rawMyWxid = this.exportService.getConfiguredMyWxid()
const sessionInfo = await this.exportService.getContactInfo(sessionId)
const myInfo = await this.exportService.getContactInfo(cleanedMyWxid)
const contactCache = new Map<string, { success: boolean; contact?: any; error?: string }>()
const getContactCached = async (username: string) => {
if (contactCache.has(username)) return contactCache.get(username)!
const result = await wcdbService.getContact(username)
contactCache.set(username, result)
return result
}
onProgress?.({
current: 0,
total: 100,
currentSession: sessionInfo.displayName,
phase: 'preparing'
})
const collectParams = this.exportService.resolveCollectParams(options)
const collectProgressReporter = this.exportService.createCollectProgressReporter(sessionInfo.displayName, onProgress, 5)
const collected = await this.exportService.collectMessages(
sessionId,
cleanedMyWxid,
options.dateRange,
options.senderUsername,
collectParams.mode,
collectParams.targetMediaTypes,
control,
collectProgressReporter
)
const totalMessages = collected.rows.length
if (totalMessages === 0) {
return { success: false, error: await this.exportService.buildNoMessagesError(sessionId, collected) }
}
await this.exportService.createWeliveRawOutputPlaceholder(outputPath, control)
await this.exportService.hydrateEmojiCaptionsForMessages(sessionId, collected.rows, control)
await this.exportService.resolveQuotedMessagesForExport(collected.rows, sessionId)
const voiceMessages = options.exportVoiceAsText
? collected.rows.filter((msg: any) => msg.localType === 34)
: []
if (options.exportVoiceAsText && voiceMessages.length > 0) {
await this.exportService.ensureVoiceModel(onProgress)
}
const senderUsernames = new Set<string>()
let senderScanIndex = 0
for (const msg of collected.rows) {
if ((senderScanIndex++ & 0x7f) === 0) {
this.exportService.throwIfStopRequested(control)
}
if (msg.senderUsername) senderUsernames.add(msg.senderUsername)
}
senderUsernames.add(sessionId)
await this.exportService.preloadContacts(senderUsernames, contactCache)
const groupNicknameCandidates = isGroup
? buildGroupNicknameIdCandidates([
...Array.from(senderUsernames.values()),
...collected.rows.map((msg: any) => msg.senderUsername),
cleanedMyWxid,
rawMyWxid
])
: []
const groupNicknamesMap = isGroup
? await this.exportService.getGroupNicknamesForRoom(sessionId, groupNicknameCandidates)
: new Map<string, string>()
const sortedMessages = collected.rows
const { mediaRootDir, mediaRelativePrefix } = this.exportService.getMediaLayout(outputPath, options)
const mediaMessages = this.exportService.collectMediaMessagesForExport(sortedMessages, options)
const mediaCache = new Map<string, MediaExportItem | null>()
const mediaDirCache = new Set<string>()
const beforeMediaDoneFiles = this.exportService.getMediaDoneFilesCount()
if (mediaMessages.length > 0) {
await this.exportService.preloadMediaLookupCaches(sessionId, mediaMessages, {
exportImages: options.exportImages,
exportVideos: options.exportVideos
}, control)
const voiceMediaMessages = mediaMessages.filter((msg: any) => msg.localType === 34)
if (voiceMediaMessages.length > 0) {
await this.exportService.preloadVoiceWavCache(sessionId, voiceMediaMessages, control)
}
onProgress?.({
current: 25,
total: 100,
currentSession: sessionInfo.displayName,
phase: 'exporting-media',
phaseProgress: 0,
phaseTotal: mediaMessages.length,
phaseLabel: this.exportService.formatMediaPhaseLabel(0, mediaMessages.length, beforeMediaDoneFiles),
...this.exportService.getMediaTelemetrySnapshot(),
estimatedTotalMessages: totalMessages
})
const mediaConcurrency = this.exportService.getClampedConcurrency(options.exportConcurrency)
let mediaExported = 0
await parallelLimit(mediaMessages, mediaConcurrency, async (msg: any) => {
this.exportService.throwIfStopRequested(control)
const mediaKey = this.exportService.getMediaCacheKey(msg)
if (!mediaCache.has(mediaKey)) {
const mediaItem = await this.exportService.exportMediaForMessage(msg, sessionId, mediaRootDir, mediaRelativePrefix, {
exportImages: options.exportImages,
exportVoices: options.exportVoices,
exportVideos: options.exportVideos,
exportEmojis: options.exportEmojis,
exportFiles: options.exportFiles,
maxFileSizeMb: options.maxFileSizeMb,
exportVoiceAsText: options.exportVoiceAsText,
exportConflictStrategy: options.exportConflictStrategy,
includeVoiceWithTranscript: true,
dirCache: mediaDirCache,
control
})
mediaCache.set(mediaKey, mediaItem)
}
mediaExported++
if (mediaExported % 5 === 0 || mediaExported === mediaMessages.length) {
onProgress?.({
current: 25,
total: 100,
currentSession: sessionInfo.displayName,
phase: 'exporting-media',
phaseProgress: mediaExported,
phaseTotal: mediaMessages.length,
phaseLabel: this.exportService.formatMediaPhaseLabel(mediaExported, mediaMessages.length, beforeMediaDoneFiles),
...this.exportService.getMediaTelemetrySnapshot()
})
}
})
}
await this.exportService.preloadWeliveRawEmojiMedia(sortedMessages, mediaCache, mediaRootDir, mediaRelativePrefix, options, control, onProgress, sessionInfo.displayName, 20)
const fileOnlyExportFailure = this.exportService.buildFileOnlyExportFailure(options, mediaMessages, beforeMediaDoneFiles)
if (fileOnlyExportFailure) return fileOnlyExportFailure
const voiceTranscriptMap = new Map<string, string>()
if (voiceMessages.length > 0) {
await this.exportService.preloadVoiceWavCache(sessionId, voiceMessages, control)
onProgress?.({
current: 45,
total: 100,
currentSession: sessionInfo.displayName,
phase: 'exporting-voice',
phaseProgress: 0,
phaseTotal: voiceMessages.length,
phaseLabel: `语音转文字 0/${voiceMessages.length}`,
estimatedTotalMessages: totalMessages
})
const VOICE_CONCURRENCY = 4
let voiceTranscribed = 0
await parallelLimit(voiceMessages, VOICE_CONCURRENCY, async (msg: any) => {
this.exportService.throwIfStopRequested(control)
const transcript = await this.exportService.transcribeVoice(sessionId, String(msg.localId), msg.createTime, msg.senderUsername)
voiceTranscriptMap.set(this.exportService.getStableMessageKey(msg), transcript)
voiceTranscribed++
onProgress?.({
current: 45,
total: 100,
currentSession: sessionInfo.displayName,
phase: 'exporting-voice',
phaseProgress: voiceTranscribed,
phaseTotal: voiceMessages.length,
phaseLabel: `语音转文字 ${voiceTranscribed}/${voiceMessages.length}`
})
})
}
onProgress?.({
current: 60,
total: 100,
currentSession: sessionInfo.displayName,
phase: 'writing',
estimatedTotalMessages: totalMessages,
collectedMessages: totalMessages,
exportedMessages: 0
})
await this.exportService.recordCreatedFileBeforeWrite(outputPath, control)
const stream = fs.createWriteStream(outputPath, { encoding: 'utf-8' })
const writeChunk = async (chunk: string): Promise<void> => {
await new Promise<void>((resolve, _reject) => {
this.exportService.throwIfStopRequested(control)
if (!stream.write(chunk)) stream.once('drain', resolve)
else resolve()
})
}
const exportMeta = this.exportService.getExportMeta(sessionId, sessionInfo, isGroup)
await writeChunk(`# ${escapeMarkdownText(sessionInfo.displayName || sessionId)}\n\n`)
await writeChunk(`- 会话ID: \`${String(sessionId).replace(/`/g, '\\`')}\`\n`)
await writeChunk(`- 会话类型: ${isGroup ? '群聊' : '私聊'}\n`)
await writeChunk(`- 消息数量: ${totalMessages}\n`)
await writeChunk(`- 导出时间: ${escapeMarkdownText(formatTimestamp(exportMeta.chatlab.exportedAt))}\n`)
await writeChunk(`- 导出工具: WeFlow\n\n---\n\n`)
const WRITE_BATCH = 80
let writeBuffer: string[] = []
const flushWriteBuffer = async (): Promise<void> => {
if (writeBuffer.length === 0) return
await writeChunk(writeBuffer.join(''))
writeBuffer = []
}
const senderProfileCache = new Map<string, ExportDisplayProfile>()
const getSenderRole = async (msg: any): Promise<string> => {
if (isGroup) {
const senderProfileKey = `${msg.isSend ? cleanedMyWxid : (msg.senderUsername || cleanedMyWxid)}::${msg.isSend ? '1' : '0'}`
let senderProfile = senderProfileCache.get(senderProfileKey)
if (!senderProfile) {
senderProfile = await resolveExportDisplayProfile(
msg.isSend ? cleanedMyWxid : (msg.senderUsername || cleanedMyWxid),
options.displayNamePreference,
getContactCached,
groupNicknamesMap,
msg.isSend ? (myInfo.displayName || cleanedMyWxid) : (msg.senderUsername || ''),
msg.isSend ? [rawMyWxid, cleanedMyWxid] : []
)
senderProfileCache.set(senderProfileKey, senderProfile)
}
return senderProfile.displayName
}
if (msg.isSend) return '我'
const contactDetail = await getContactCached(sessionId)
if (contactDetail.success && contactDetail.contact) {
return contactDetail.contact.remark || contactDetail.contact.nickName || sessionId
}
return sessionInfo.displayName || sessionId
}
const getRawMediaRelativePath = (msg: any): string => {
const rawPath = String(msg?.mediaPath || msg?.media_path || '').trim()
if (!rawPath || /^(?:https?:|data:|blob:)/i.test(rawPath)) return rawPath
const normalized = rawPath.replace(/\\/g, '/')
if (!path.isAbsolute(rawPath)) return normalized.replace(/^\.\//, '')
const relativeFromMediaRoot = path.relative(mediaRootDir, rawPath)
if (relativeFromMediaRoot && !relativeFromMediaRoot.startsWith('..') && !path.isAbsolute(relativeFromMediaRoot)) {
return relativeFromMediaRoot.split(path.sep).join('/')
}
return ''
}
const getFallbackMediaKind = (msg: any): MediaExportItem['kind'] | null => {
const localType = Number(msg?.localType || msg?.local_type || 0)
const mediaType = String(msg?.mediaType || msg?.media_type || '').trim().toLowerCase()
if (mediaType === 'image' || localType === 3) return 'image'
if (mediaType === 'emoji' || localType === 47) return 'emoji'
if (mediaType === 'voice' || localType === 34) return 'voice'
if (mediaType === 'video' || localType === 43) return 'video'
if (mediaType === 'file') return 'file'
return null
}
const buildMediaMarkdown = (mediaItem: MediaExportItem | null | undefined, typeName: string, fallbackRelativePath = '', fallbackKind: MediaExportItem['kind'] | null = null): string => {
const relativePath = mediaItem?.relativePath || fallbackRelativePath
if (!relativePath) return ''
const mediaUrl = toMarkdownUrl(this.exportService.formatExportMediaPath(relativePath, { exportPathStyle: 'posix' }, 'url'))
const fileName = path.basename(relativePath.replace(/\\/g, '/')) || typeName
const kind = mediaItem?.kind || fallbackKind
if (kind === 'image' || kind === 'emoji') {
return `![${escapeMarkdownLinkText(typeName)}](${mediaUrl})`
}
if (kind === 'voice') {
return `[${escapeMarkdownLinkText('语音文件')}](${mediaUrl})`
}
if (kind === 'video') {
return `[${escapeMarkdownLinkText('视频文件')}](${mediaUrl})`
}
return `[${escapeMarkdownLinkText(fileName)}](${mediaUrl})`
}
for (let i = 0; i < totalMessages; i++) {
if ((i & 0x7f) === 0) this.exportService.throwIfStopRequested(control)
const msg = sortedMessages[i]
const mediaKey = this.exportService.getMediaCacheKey(msg)
const mediaItem = mediaCache.has(mediaKey)
? mediaCache.get(mediaKey)
: await this.exportService.resolveWeliveRawMediaItem(msg, mediaRootDir, mediaRelativePrefix, options, control)
const senderRole = await getSenderRole(msg)
const typeName = this.exportService.getMessageTypeName(msg.localType, msg.content)
const shouldUseTranscript = msg.localType === 34 && options.exportVoiceAsText
const mediaMarkdown = buildMediaMarkdown(mediaItem, typeName, getRawMediaRelativePath(msg), getFallbackMediaKind(msg))
let contentValue = shouldUseTranscript
? this.exportService.formatPlainExportContent(
msg.content,
msg.localType,
options,
voiceTranscriptMap.get(this.exportService.getStableMessageKey(msg)),
cleanedMyWxid,
msg.senderUsername,
msg.isSend,
msg.emojiCaption
)
: this.exportService.formatPlainExportContent(
msg.content,
msg.localType,
options,
voiceTranscriptMap.get(this.exportService.getStableMessageKey(msg)),
cleanedMyWxid,
msg.senderUsername,
msg.isSend,
msg.emojiCaption
)
if (mediaMarkdown && !shouldUseTranscript && (msg.localType === 3 || msg.localType === 34 || msg.localType === 43 || msg.localType === 47)) {
contentValue = ''
}
if (isTransferExportContent(contentValue) && msg.content) {
const transferDesc = await resolveTransferDesc(
msg.content,
cleanedMyWxid,
groupNicknamesMap,
async (username: string) => {
const c = await getContactCached(username)
if (c.success && c.contact) {
return c.contact.remark || c.contact.nickName || c.contact.alias || username
}
return username
}
)
if (transferDesc) {
contentValue = appendTransferDesc(contentValue, transferDesc)
}
}
const quotedReplyDisplay = await this.exportService.resolveQuotedReplyDisplayWithNames({
content: msg.content,
isGroup,
displayNamePreference: options.displayNamePreference,
getContact: getContactCached,
groupNicknamesMap,
cleanedMyWxid,
rawMyWxid,
myDisplayName: myInfo.displayName || cleanedMyWxid
})
const linkCard = quotedReplyDisplay ? null : this.exportService.extractHtmlLinkCard(msg.content, msg.localType)
const parts: string[] = []
let contentIsMarkdown = false
if (quotedReplyDisplay) {
const quotedLabel = quotedReplyDisplay.quotedSender
? `${quotedReplyDisplay.quotedSender}: ${quotedReplyDisplay.quotedPreview}`
: quotedReplyDisplay.quotedPreview
parts.push(buildMarkdownBlockquote(quotedLabel))
contentValue = quotedReplyDisplay.replyText || contentValue
} else if (linkCard?.url) {
contentValue = `[${escapeMarkdownLinkText(linkCard.title || linkCard.url)}](${toMarkdownUrl(linkCard.url)})`
contentIsMarkdown = true
}
if (mediaMarkdown) parts.push(mediaMarkdown)
if (contentValue) parts.push(contentIsMarkdown ? contentValue : escapeMarkdownText(contentValue))
const body = parts.length > 0 ? parts.join('\n\n') : escapeMarkdownText(`[${typeName}]`)
writeBuffer.push(`## ${escapeMarkdownText(formatTimestamp(msg.createTime))} ${escapeMarkdownText(senderRole)}\n\n${body}\n\n`)
if (writeBuffer.length >= WRITE_BATCH) {
await flushWriteBuffer()
}
if ((i + 1) % 200 === 0) {
onProgress?.({
current: 60 + Math.floor((i + 1) / sortedMessages.length * 30),
total: 100,
currentSession: sessionInfo.displayName,
phase: 'writing',
estimatedTotalMessages: totalMessages,
collectedMessages: totalMessages,
exportedMessages: i + 1
})
}
}
await flushWriteBuffer()
this.exportService.throwIfStopRequested(control)
await new Promise<void>((resolve, reject) => {
stream.on('error', reject)
stream.end(() => resolve())
})
onProgress?.({
current: 100,
total: 100,
currentSession: sessionInfo.displayName,
phase: 'complete',
estimatedTotalMessages: totalMessages,
collectedMessages: totalMessages,
exportedMessages: totalMessages,
writtenFiles: 1
})
return { success: true }
} catch (e) {
if (this.exportService.isStopError(e)) {
return { success: false, error: '导出任务已停止' }
}
if (this.exportService.isPauseError(e)) {
return { success: false, error: '导出任务已暂停' }
}
return { success: false, error: String(e) }
}
}
}
@@ -144,6 +144,7 @@ export class TxtFormatter {
exportFiles: options.exportFiles,
maxFileSizeMb: options.maxFileSizeMb,
exportVoiceAsText: options.exportVoiceAsText,
exportConflictStrategy: options.exportConflictStrategy,
includeVideoPoster: options.format === 'html',
dirCache: mediaDirCache,
control
@@ -142,6 +142,7 @@ export class WeCloneFormatter {
exportFiles: options.exportFiles,
maxFileSizeMb: options.maxFileSizeMb,
exportVoiceAsText: options.exportVoiceAsText,
exportConflictStrategy: options.exportConflictStrategy,
includeVideoPoster: options.format === 'html',
dirCache: mediaDirCache,
control
+2 -1
View File
@@ -63,12 +63,13 @@ export interface ChatLabExport {
}
export interface ExportOptions {
format: 'chatlab' | 'chatlab-jsonl' | 'json' | 'arkme-json' | 'html' | 'txt' | 'excel' | 'weclone' | 'sql'
format: 'chatlab' | 'chatlab-jsonl' | 'json' | 'arkme-json' | 'html' | 'markdown' | 'txt' | 'excel' | 'weclone' | 'sql'
contentType?: 'text' | 'voice' | 'image' | 'video' | 'emoji' | 'file'
dateRange?: { start: number; end: number } | null
senderUsername?: string
fileNameSuffix?: string
fileNamingMode?: 'classic' | 'date-range'
exportConflictStrategy?: 'incremental' | 'overwrite' | 'rename'
exportMedia?: boolean
exportAvatars?: boolean
exportImages?: boolean
@@ -21,6 +21,13 @@ export function normalizeFileNamingMode(value: unknown): 'classic' | 'date-range
return String(value || '').trim().toLowerCase() === 'date-range' ? 'date-range' : 'classic'
}
export function normalizeExportConflictStrategy(value: unknown): 'incremental' | 'overwrite' | 'rename' {
const normalized = String(value || '').trim().toLowerCase()
if (normalized === 'overwrite') return 'overwrite'
if (normalized === 'rename') return 'rename'
return 'incremental'
}
export function formatDateTokenBySeconds(seconds?: number): string | null {
const normalizedSeconds = normalizeTimestampSeconds(seconds)
if (normalizedSeconds <= 0) return null
+588 -101
View File
@@ -1,8 +1,8 @@
import { app, BrowserWindow } from 'electron'
import { basename, dirname, extname, join } from 'path'
import { pathToFileURL } from 'url'
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, appendFileSync } from 'fs'
import { writeFile, rm, readdir } from 'fs/promises'
import { fileURLToPath, pathToFileURL } from 'url'
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync } from 'fs'
import { writeFile, rm, readdir, appendFile, readFile } from 'fs/promises'
import { homedir, tmpdir } from 'os'
import crypto from 'crypto'
import { ConfigService } from './config'
@@ -68,6 +68,8 @@ type CachedImagePayload = {
hardlinkOnly?: boolean
disableUpdateCheck?: boolean
allowCacheIndex?: boolean
allowCachePromotion?: boolean
allowFilesystemScan?: boolean
suppressEvents?: boolean
}
@@ -75,18 +77,39 @@ type DecryptImagePayload = CachedImagePayload & {
force?: boolean
}
type PendingImageTask<T extends DecryptResult> = {
promise: Promise<T>
emitsEvents: boolean
}
export class ImageDecryptService {
private configService = new ConfigService()
private resolvedCache = new Map<string, string>()
private pending = new Map<string, Promise<DecryptResult>>()
private pending = new Map<string, PendingImageTask<DecryptResult>>()
private resolvePending = new Map<string, PendingImageTask<DecryptResult & { hasUpdate?: boolean }>>()
private updateFlags = new Map<string, boolean>()
private nativeLogged = false
private runtimeConfig: { dbPath?: string; myWxid?: string; imageXorKey?: unknown; imageAesKey?: string } | null = null
private datNameScanMissAt = new Map<string, number>()
private readonly datNameScanMissTtlMs = 1200
private readonly maxDatNameScanMissEntries = 2400
private readonly accountDirCache = new Map<string, string>()
private cacheRootPath: string | null = null
private readonly ensuredDirs = new Set<string>()
private readonly maxResolvedCacheEntries = 12000
private pendingLogLines: string[] = []
private logFlushTimer: ReturnType<typeof setTimeout> | null = null
private corruptedCacheCheckPending = new Set<string>()
private corruptedCacheCheckQueue: string[] = []
private activeCorruptedCacheChecks = 0
private corruptedCacheCheckLastAt = new Map<string, number>()
private corruptedCachePathUntil = new Map<string, number>()
private readonly maxPendingLogLines = 1000
private readonly logFlushDelayMs = 200
private readonly corruptedCacheCheckCooldownMs = 5 * 60 * 1000
private readonly corruptedCacheQuarantineMs = 5 * 60 * 1000
private readonly maxConcurrentCorruptedCacheChecks = 1
private readonly maxCorruptedCacheChecks = 6000
private shouldEmitImageEvents(payload?: { suppressEvents?: boolean }): boolean {
if (payload?.suppressEvents === true) return false
@@ -145,14 +168,36 @@ export class ImageDecryptService {
}
private writeLog(line: string): void {
if (!line) return
this.pendingLogLines.push(line)
while (this.pendingLogLines.length > this.maxPendingLogLines) {
this.pendingLogLines.shift()
}
if (this.logFlushTimer !== null) return
this.logFlushTimer = setTimeout(() => {
this.logFlushTimer = null
void this.flushPendingLogs()
}, this.logFlushDelayMs)
}
private async flushPendingLogs(): Promise<void> {
if (this.pendingLogLines.length === 0) return
const lines = this.pendingLogLines.splice(0, this.pendingLogLines.length).join('')
try {
const logDir = join(this.getUserDataPath(), 'logs')
if (!existsSync(logDir)) {
mkdirSync(logDir, { recursive: true })
}
appendFileSync(join(logDir, 'wcdb.log'), line, { encoding: 'utf8' })
await appendFile(join(logDir, 'wcdb.log'), lines, { encoding: 'utf8' })
} catch (err) {
console.error('写入日志失败:', err)
} finally {
if (this.pendingLogLines.length > 0 && this.logFlushTimer === null) {
this.logFlushTimer = setTimeout(() => {
this.logFlushTimer = null
void this.flushPendingLogs()
}, this.logFlushDelayMs)
}
}
}
@@ -162,28 +207,51 @@ export class ImageDecryptService {
if (!cacheKey) {
return { success: false, error: '缺少图片标识', failureKind: 'not_found' }
}
const pendingKey = this.getResolvePendingKey(payload, cacheKey)
const pending = this.resolvePending.get(pendingKey)
if (pending) return this.resolvePendingResult(pending, payload, cacheKey)
const task = this.resolveCachedImageInternal({ ...payload, preferFilePath: true }, cacheKeys, cacheKey)
const pendingTask: PendingImageTask<DecryptResult & { hasUpdate?: boolean }> = {
promise: task,
emitsEvents: this.shouldEmitImageEvents(payload)
}
this.resolvePending.set(pendingKey, pendingTask)
try {
return await this.resolvePendingResult(pendingTask, payload, cacheKey)
} finally {
this.resolvePending.delete(pendingKey)
}
}
private async resolveCachedImageInternal(
payload: CachedImagePayload,
cacheKeys: string[],
cacheKey: string
): Promise<DecryptResult & { hasUpdate?: boolean }> {
for (const key of cacheKeys) {
const cached = this.resolvedCache.get(key)
const cached = this.getResolvedMediaCache(payload, key)
if (cached && existsSync(cached) && this.isUsableImageCacheFile(cached)) {
const upgraded = !this.isHdPath(cached)
const upgraded = !this.isHdPath(cached) && payload.allowCachePromotion !== false
? await this.tryPromoteThumbnailCache(payload, key, cached)
: null
const finalPath = upgraded || cached
const localPath = this.resolveLocalPathForPayload(finalPath, payload.preferFilePath)
const isNonHd = !this.isHdPath(finalPath)
const hasUpdate = isNonHd ? (this.updateFlags.get(key) ?? false) : false
const hasUpdate = isNonHd ? this.getUpdateFlag(payload, key) : false
if (isNonHd) {
if (this.shouldCheckImageUpdate(payload)) {
this.triggerUpdateCheck(payload, key, finalPath)
}
} else {
this.updateFlags.delete(key)
this.clearUpdateFlag(payload, key)
}
this.emitCacheResolved(payload, key, this.resolveEmitPath(finalPath, payload.preferFilePath))
return { success: true, localPath, hasUpdate }
}
if (cached && !this.isUsableImageCacheFile(cached)) {
this.resolvedCache.delete(key)
this.deleteResolvedMediaCache(payload, key)
}
}
@@ -199,26 +267,27 @@ export class ImageDecryptService {
allowThumbnail: true,
skipResolvedCache: false,
hardlinkOnly: true,
allowDatNameScanFallback: payload.allowCacheIndex !== false
allowDatNameScanFallback: payload.allowCacheIndex !== false,
allowFilesystemScan: payload.allowFilesystemScan !== false
}
)
if (datPath) {
const existing = this.findCachedOutputByDatPath(datPath, payload.sessionId, false)
if (existing) {
const upgraded = !this.isHdPath(existing)
const upgraded = !this.isHdPath(existing) && payload.allowCachePromotion !== false
? await this.tryPromoteThumbnailCache(payload, cacheKey, existing)
: null
const finalPath = upgraded || existing
this.cacheResolvedPaths(cacheKey, payload.imageMd5, payload.imageDatName, finalPath)
this.cacheResolvedPaths(payload, cacheKey, finalPath)
const localPath = this.resolveLocalPathForPayload(finalPath, payload.preferFilePath)
const isNonHd = !this.isHdPath(finalPath)
const hasUpdate = isNonHd ? (this.updateFlags.get(cacheKey) ?? false) : false
const hasUpdate = isNonHd ? this.getUpdateFlag(payload, cacheKey) : false
if (isNonHd) {
if (this.shouldCheckImageUpdate(payload)) {
this.triggerUpdateCheck(payload, cacheKey, finalPath)
}
} else {
this.updateFlags.delete(cacheKey)
this.clearUpdateFlag(payload, cacheKey)
}
this.emitCacheResolved(payload, cacheKey, this.resolveEmitPath(finalPath, payload.preferFilePath))
return { success: true, localPath, hasUpdate }
@@ -239,24 +308,24 @@ export class ImageDecryptService {
if (payload.force) {
for (const key of cacheKeys) {
const cached = this.resolvedCache.get(key)
const cached = this.getResolvedMediaCache(payload, key)
if (cached && existsSync(cached) && this.isUsableImageCacheFile(cached) && this.isHdPath(cached)) {
this.cacheResolvedPaths(cacheKey, payload.imageMd5, payload.imageDatName, cached)
this.clearUpdateFlags(cacheKey, payload.imageMd5, payload.imageDatName)
this.cacheResolvedPaths(payload, cacheKey, cached)
this.clearUpdateFlags(payload, cacheKey)
const localPath = this.resolveLocalPathForPayload(cached, payload.preferFilePath)
this.emitCacheResolved(payload, cacheKey, this.resolveEmitPath(cached, payload.preferFilePath))
this.emitDecryptProgress(payload, cacheKey, 'done', 100, 'done')
return { success: true, localPath }
}
if (cached && !this.isUsableImageCacheFile(cached)) {
this.resolvedCache.delete(key)
this.deleteResolvedMediaCache(payload, key)
}
}
}
if (!payload.force) {
const cached = this.resolvedCache.get(cacheKey)
const cached = this.getResolvedMediaCache(payload, cacheKey)
if (cached && existsSync(cached) && this.isUsableImageCacheFile(cached)) {
const upgraded = !this.isHdPath(cached)
? await this.tryPromoteThumbnailCache(payload, cacheKey, cached)
@@ -268,26 +337,34 @@ export class ImageDecryptService {
return { success: true, localPath }
}
if (cached && !this.isUsableImageCacheFile(cached)) {
this.resolvedCache.delete(cacheKey)
this.deleteResolvedMediaCache(payload, cacheKey)
}
}
const pending = this.pending.get(cacheKey)
const pendingKey = this.getDecryptPendingKey(payload, cacheKey)
const pending = this.pending.get(pendingKey)
if (pending) {
this.emitDecryptProgress(payload, cacheKey, 'queued', 8, 'running')
return pending
return this.resolvePendingResult(pending, payload, cacheKey)
}
const task = this.decryptImageInternal(payload, cacheKey)
this.pending.set(cacheKey, task)
const task = this.decryptImageInternal({ ...payload, preferFilePath: true }, cacheKey)
const pendingTask: PendingImageTask<DecryptResult> = {
promise: task,
emitsEvents: this.shouldEmitImageEvents(payload)
}
this.pending.set(pendingKey, pendingTask)
try {
return await task
return await this.resolvePendingResult(pendingTask, payload, cacheKey)
} finally {
this.pending.delete(cacheKey)
this.pending.delete(pendingKey)
}
}
async preloadImageHardlinkMd5s(md5List: string[]): Promise<void> {
async preloadImageHardlinkMd5s(
md5List: string[],
options?: { chunkSize?: number; yieldMs?: number; filesystemFallback?: boolean }
): Promise<void> {
const normalizedList = Array.from(
new Set((md5List || []).map((item) => String(item || '').trim().toLowerCase()).filter(Boolean))
)
@@ -301,19 +378,76 @@ export class ImageDecryptService {
if (!accountDir) return
try {
for (const md5 of normalizedList) {
if (!this.looksLikeMd5(md5)) continue
const selectedPath = this.selectBestDatPathByBase(accountDir, md5, undefined, undefined, true)
if (!selectedPath) continue
this.cacheDatPath(accountDir, md5, selectedPath)
const fileName = basename(selectedPath).toLowerCase()
if (fileName) this.cacheDatPath(accountDir, fileName, selectedPath)
const chunkSize = Math.max(1, Math.min(256, Math.floor(Number(options?.chunkSize || normalizedList.length))))
const yieldMs = Math.max(0, Math.min(100, Math.floor(Number(options?.yieldMs || 0))))
const allowFilesystemFallback = options?.filesystemFallback !== false
for (let offset = 0; offset < normalizedList.length; offset += chunkSize) {
const chunk = normalizedList.slice(offset, offset + chunkSize).filter((md5) => this.looksLikeMd5(md5))
if (chunk.length === 0) continue
const resolvedByBatch = new Set<string>()
try {
const result = await wcdbService.resolveImageHardlinkBatch(chunk.map((md5) => ({ md5, accountDir })))
const rows = Array.isArray(result?.rows) ? result.rows : []
for (const row of rows) {
const md5 = String(row?.md5 || '').trim().toLowerCase()
if (!md5 || !row?.success || !row.data) continue
const fileName = String(row.data.file_name || '').trim()
const fullPath = String(row.data.full_path || '').trim()
if (!fileName || !fullPath) continue
const selectedPath = this.normalizeHardlinkDatPathByFileName(fullPath, fileName)
if (!selectedPath || !existsSync(selectedPath)) continue
this.cacheDatPath(accountDir, md5, selectedPath)
const selectedFileName = basename(selectedPath).toLowerCase()
if (selectedFileName) this.cacheDatPath(accountDir, selectedFileName, selectedPath)
resolvedByBatch.add(md5)
}
} catch {
// fall through to optional filesystem fallback
}
if (!allowFilesystemFallback) {
if (yieldMs > 0 && offset + chunkSize < normalizedList.length) {
await new Promise((resolve) => setTimeout(resolve, yieldMs))
}
continue
}
for (const md5 of chunk) {
if (resolvedByBatch.has(md5)) continue
const selectedPath = this.selectBestDatPathByBase(accountDir, md5, undefined, undefined, true)
if (!selectedPath) continue
this.cacheDatPath(accountDir, md5, selectedPath)
const fileName = basename(selectedPath).toLowerCase()
if (fileName) this.cacheDatPath(accountDir, fileName, selectedPath)
}
if (yieldMs > 0 && offset + chunkSize < normalizedList.length) {
await new Promise((resolve) => setTimeout(resolve, yieldMs))
}
}
} catch {
// ignore preload failures
}
}
emitCacheResolvedPath(
payload: { sessionId?: string; imageMd5?: string; imageDatName?: string; createTime?: number; suppressEvents?: boolean },
cacheKey: string,
localPath: string
): void {
const normalizedCacheKey = String(cacheKey || '').trim()
const resolvedPath = String(localPath || '').trim()
if (!normalizedCacheKey || !resolvedPath) return
if (/^(data:image\/|blob:|https?:\/\/)/i.test(resolvedPath)) {
this.emitCacheResolved(payload, normalizedCacheKey, resolvedPath)
return
}
const filePath = /^file:\/\//i.test(resolvedPath)
? (this.fileUrlToPathSafely(resolvedPath) || resolvedPath)
: resolvedPath
this.emitCacheResolved(payload, normalizedCacheKey, this.resolveEmitPath(filePath, true))
}
private async decryptImageInternal(
payload: DecryptImagePayload,
cacheKey: string
@@ -353,7 +487,8 @@ export class ImageDecryptService {
allowThumbnail: false,
skipResolvedCache: false,
hardlinkOnly: payload.hardlinkOnly === true,
allowDatNameScanFallback: payload.allowCacheIndex !== false
allowDatNameScanFallback: payload.allowCacheIndex !== false,
allowFilesystemScan: payload.allowFilesystemScan !== false
}
)
if (!datPath) {
@@ -367,7 +502,8 @@ export class ImageDecryptService {
allowThumbnail: true,
skipResolvedCache: false,
hardlinkOnly: payload.hardlinkOnly === true,
allowDatNameScanFallback: payload.allowCacheIndex !== false
allowDatNameScanFallback: payload.allowCacheIndex !== false,
allowFilesystemScan: payload.allowFilesystemScan !== false
}
)
fallbackToThumbnail = Boolean(datPath)
@@ -389,7 +525,8 @@ export class ImageDecryptService {
allowThumbnail: true,
skipResolvedCache: false,
hardlinkOnly: payload.hardlinkOnly === true,
allowDatNameScanFallback: payload.allowCacheIndex !== false
allowDatNameScanFallback: payload.allowCacheIndex !== false,
allowFilesystemScan: payload.allowFilesystemScan !== false
}
)
}
@@ -407,7 +544,7 @@ export class ImageDecryptService {
this.emitDecryptProgress(payload, cacheKey, 'locating', 34, 'running')
if (!extname(datPath).toLowerCase().includes('dat')) {
this.cacheResolvedPaths(cacheKey, payload.imageMd5, payload.imageDatName, datPath)
this.cacheResolvedPaths(payload, cacheKey, datPath)
const localPath = this.resolveLocalPathForPayload(datPath, payload.preferFilePath)
const isThumb = this.isThumbnailPath(datPath)
this.emitCacheResolved(payload, cacheKey, this.resolveEmitPath(datPath, payload.preferFilePath))
@@ -421,7 +558,7 @@ export class ImageDecryptService {
this.logInfo('找到已解密文件(按DAT快速命中)', { existing: existingFast, isHd: this.isHdPath(existingFast) })
const isHd = this.isHdPath(existingFast)
if (!(payload.force && !isHd)) {
this.cacheResolvedPaths(cacheKey, payload.imageMd5, payload.imageDatName, existingFast)
this.cacheResolvedPaths(payload, cacheKey, existingFast)
const localPath = this.resolveLocalPathForPayload(existingFast, payload.preferFilePath)
const isThumb = this.isThumbnailPath(existingFast)
this.emitCacheResolved(payload, cacheKey, this.resolveEmitPath(existingFast, payload.preferFilePath))
@@ -456,7 +593,7 @@ export class ImageDecryptService {
this.logInfo('开始解密DAT文件', { datPath, xorKey, hasAesKey: Boolean(aesKeyForNative) })
this.emitDecryptProgress(payload, cacheKey, 'decrypting', 58, 'running')
const nativeResult = this.tryDecryptDatWithNative(datPath, xorKey, aesKeyForNative)
const nativeResult = await this.tryDecryptDatWithNative(datPath, xorKey, aesKeyForNative)
if (!nativeResult) {
this.emitDecryptProgress(payload, cacheKey, 'failed', 100, 'error', 'Rust原生解密不可用')
return { success: false, error: 'Rust原生解密不可用或解密失败,请检查 native 模块与密钥配置', failureKind: 'not_found' }
@@ -491,9 +628,9 @@ export class ImageDecryptService {
const isThumb = this.isThumbnailPath(datPath)
const isHdCache = this.isHdPath(outputPath)
this.removeDuplicateCacheCandidates(datPath, payload.sessionId, outputPath)
this.cacheResolvedPaths(cacheKey, payload.imageMd5, payload.imageDatName, outputPath)
this.cacheResolvedPaths(payload, cacheKey, outputPath)
if (isHdCache) {
this.clearUpdateFlags(cacheKey, payload.imageMd5, payload.imageDatName)
this.clearUpdateFlags(payload, cacheKey)
} else {
if (this.shouldCheckImageUpdate(payload)) {
this.triggerUpdateCheck(payload, cacheKey, outputPath)
@@ -584,12 +721,13 @@ export class ImageDecryptService {
imageDatName?: string,
sessionId?: string,
createTime?: number,
options?: { allowThumbnail?: boolean; skipResolvedCache?: boolean; hardlinkOnly?: boolean; allowDatNameScanFallback?: boolean }
options?: { allowThumbnail?: boolean; skipResolvedCache?: boolean; hardlinkOnly?: boolean; allowDatNameScanFallback?: boolean; allowFilesystemScan?: boolean }
): Promise<string | null> {
const allowThumbnail = options?.allowThumbnail ?? true
const skipResolvedCache = options?.skipResolvedCache ?? false
const hardlinkOnly = options?.hardlinkOnly ?? false
const allowDatNameScanFallback = options?.allowDatNameScanFallback ?? true
const allowFilesystemScan = options?.allowFilesystemScan !== false
this.logInfo('[ImageDecrypt] resolveDatPath', {
imageMd5,
imageDatName,
@@ -597,7 +735,8 @@ export class ImageDecryptService {
allowThumbnail,
skipResolvedCache,
hardlinkOnly,
allowDatNameScanFallback
allowDatNameScanFallback,
allowFilesystemScan
})
const lookupBases = this.collectLookupBasesForScan(imageMd5, imageDatName, allowDatNameScanFallback)
@@ -606,6 +745,23 @@ export class ImageDecryptService {
return null
}
const missKey = this.getResolveDatPathMissKey(
accountDir,
lookupBases,
imageMd5,
imageDatName,
sessionId,
createTime,
allowThumbnail,
allowDatNameScanFallback,
skipResolvedCache,
allowFilesystemScan
)
const lastMiss = this.getDatNameScanMissAt(missKey)
if (lastMiss && (Date.now() - lastMiss) < this.datNameScanMissTtlMs) {
return null
}
if (!skipResolvedCache) {
const cacheCandidates = Array.from(new Set([
...lookupBases,
@@ -614,13 +770,34 @@ export class ImageDecryptService {
].filter(Boolean)))
for (const cacheKey of cacheCandidates) {
const scopedKey = `${accountDir}|${cacheKey}`
const cached = this.resolvedCache.get(scopedKey)
const cached = this.getResolvedCache(scopedKey)
if (!cached || !existsSync(cached)) continue
if (!allowThumbnail && !this.isHdDatPath(cached)) continue
return cached
}
}
if (hardlinkOnly) {
for (const baseMd5 of lookupBases) {
if (!this.looksLikeMd5(baseMd5)) continue
const hardlinkPath = await this.resolveHardlinkPath(accountDir, baseMd5, sessionId)
if (!hardlinkPath) continue
if (!allowThumbnail && !this.isHdDatPath(hardlinkPath)) continue
this.cacheDatPath(accountDir, baseMd5, hardlinkPath)
if (imageMd5) this.cacheDatPath(accountDir, imageMd5, hardlinkPath)
if (imageDatName) this.cacheDatPath(accountDir, imageDatName, hardlinkPath)
const normalizedFileName = basename(hardlinkPath).toLowerCase()
if (normalizedFileName) this.cacheDatPath(accountDir, normalizedFileName, hardlinkPath)
this.datNameScanMissAt.delete(missKey)
return hardlinkPath
}
}
if (!allowFilesystemScan) {
this.setDatNameScanMissAt(missKey, Date.now())
return null
}
for (const baseMd5 of lookupBases) {
const selectedPath = this.selectBestDatPathByBase(accountDir, baseMd5, sessionId, createTime, allowThumbnail)
if (!selectedPath) continue
@@ -635,9 +812,11 @@ export class ImageDecryptService {
selectedPath,
allowThumbnail
})
this.datNameScanMissAt.delete(missKey)
return selectedPath
}
this.setDatNameScanMissAt(missKey, Date.now())
this.logInfo('[ImageDecrypt] resolveDatPath miss (dat scan)', {
imageMd5,
imageDatName,
@@ -696,7 +875,7 @@ export class ImageDecryptService {
}
private async tryPromoteThumbnailCache(
payload: { sessionId?: string; imageMd5?: string; imageDatName?: string; createTime?: number; preferFilePath?: boolean },
payload: { sessionId?: string; imageMd5?: string; imageDatName?: string; createTime?: number; preferFilePath?: boolean; suppressEvents?: boolean },
cacheKey: string,
cachedPath: string
): Promise<string | null> {
@@ -719,8 +898,8 @@ export class ImageDecryptService {
const existingHd = this.findCachedOutputByDatPath(hdDatPath, payload.sessionId, true)
if (existingHd && existsSync(existingHd) && this.isImageFile(existingHd) && this.isHdPath(existingHd)) {
this.cacheResolvedPaths(cacheKey, payload.imageMd5, payload.imageDatName, existingHd)
this.clearUpdateFlags(cacheKey, payload.imageMd5, payload.imageDatName)
this.cacheResolvedPaths(payload, cacheKey, existingHd)
this.clearUpdateFlags(payload, cacheKey)
this.removeThumbnailCacheFile(cachedPath, existingHd)
this.logInfo('[ImageDecrypt] thumbnail cache upgraded', {
cacheKey,
@@ -739,19 +918,20 @@ export class ImageDecryptService {
preferFilePath: true,
force: true,
hardlinkOnly: true,
disableUpdateCheck: true
disableUpdateCheck: true,
suppressEvents: payload.suppressEvents === true
})
if (!upgraded.success) return null
const cachedResult = this.resolvedCache.get(cacheKey)
const cachedResult = this.getResolvedMediaCache(payload, cacheKey)
const upgradedPath = (cachedResult && existsSync(cachedResult))
? cachedResult
: String(upgraded.localPath || '').trim()
if (!upgradedPath || !existsSync(upgradedPath)) return null
if (!this.isImageFile(upgradedPath) || !this.isHdPath(upgradedPath)) return null
this.cacheResolvedPaths(cacheKey, payload.imageMd5, payload.imageDatName, upgradedPath)
this.clearUpdateFlags(cacheKey, payload.imageMd5, payload.imageDatName)
this.cacheResolvedPaths(payload, cacheKey, upgradedPath)
this.clearUpdateFlags(payload, cacheKey)
this.removeThumbnailCacheFile(cachedPath, upgradedPath)
this.logInfo('[ImageDecrypt] thumbnail cache upgraded', {
cacheKey,
@@ -784,13 +964,13 @@ export class ImageDecryptService {
cachedPath: string
): void {
if (!this.shouldCheckImageUpdate(payload)) return
if (this.updateFlags.get(cacheKey)) return
if (this.getUpdateFlag(payload, cacheKey)) return
void this.checkHasUpdate(payload, cacheKey, cachedPath).then(async (hasUpdate) => {
if (!hasUpdate) return
this.updateFlags.set(cacheKey, true)
this.setUpdateFlag(payload, cacheKey, true)
const upgradedPath = await this.tryAutoRefreshBetterCache(payload, cacheKey, cachedPath)
if (upgradedPath) {
this.updateFlags.delete(cacheKey)
this.clearUpdateFlag(payload, cacheKey)
this.emitCacheResolved(payload, cacheKey, this.resolveEmitPath(upgradedPath, payload.preferFilePath))
return
}
@@ -828,7 +1008,7 @@ export class ImageDecryptService {
const refreshedPath = String(refreshed.localPath || '').trim()
if (!refreshedPath || !existsSync(refreshedPath)) return null
if (!this.isImageFile(refreshedPath)) return null
this.cacheResolvedPaths(cacheKey, payload.imageMd5, payload.imageDatName, refreshedPath)
this.cacheResolvedPaths(payload, cacheKey, refreshedPath)
this.removeThumbnailCacheFile(cachedPath, refreshedPath)
return refreshedPath
}
@@ -975,7 +1155,7 @@ export class ImageDecryptService {
const monthKey = this.resolveYearMonthFromCreateTime(createTime)
const missKey = `${accountDir}|scan|${String(sessionId || '').trim()}|${monthKey}|${baseMd5}|${allowThumbnail ? 'all' : 'hd'}`
const lastMiss = this.datNameScanMissAt.get(missKey) || 0
const lastMiss = this.getDatNameScanMissAt(missKey)
if (lastMiss && (Date.now() - lastMiss) < this.datNameScanMissTtlMs) {
return null
}
@@ -1001,7 +1181,7 @@ export class ImageDecryptService {
}
// 新策略:只扫描会话月目录,不做 account-wide 根目录回退。
this.datNameScanMissAt.set(missKey, Date.now())
this.setDatNameScanMissAt(missKey, Date.now())
this.logInfo('[ImageDecrypt] datName fallback precise scan miss', {
accountDir,
sessionId,
@@ -1025,6 +1205,71 @@ export class ImageDecryptService {
return `${y}-${m}`
}
private getResolveDatPathMissKey(
accountDir: string,
lookupBases: string[],
imageMd5?: string,
imageDatName?: string,
sessionId?: string,
createTime?: number,
allowThumbnail = true,
allowDatNameScanFallback = true,
skipResolvedCache = false,
allowFilesystemScan = true
): string {
const bases = Array.from(new Set(lookupBases.map((item) => String(item || '').trim().toLowerCase()).filter(Boolean)))
.sort()
.join(',')
const md5 = String(imageMd5 || '').trim().toLowerCase()
const datName = String(imageDatName || '').trim().toLowerCase()
const safeSessionId = String(sessionId || '').trim().toLowerCase()
const monthKey = this.resolveYearMonthFromCreateTime(createTime)
return [
accountDir,
'resolve-dat-miss',
safeSessionId,
monthKey,
allowThumbnail ? 'all' : 'hd',
allowDatNameScanFallback ? 'fallback1' : 'fallback0',
skipResolvedCache ? 'skip-cache1' : 'skip-cache0',
allowFilesystemScan ? 'fs-scan1' : 'fs-scan0',
md5,
datName,
bases
].join('|')
}
private getDatNameScanMissAt(key: string): number {
const missedAt = this.datNameScanMissAt.get(key) || 0
if (!missedAt) return 0
if ((Date.now() - missedAt) >= this.datNameScanMissTtlMs) {
this.datNameScanMissAt.delete(key)
return 0
}
this.datNameScanMissAt.delete(key)
this.datNameScanMissAt.set(key, missedAt)
return missedAt
}
private setDatNameScanMissAt(key: string, missedAt: number): void {
if (!key || !missedAt) return
if (this.datNameScanMissAt.has(key)) {
this.datNameScanMissAt.delete(key)
}
this.datNameScanMissAt.set(key, missedAt)
const expiresBefore = Date.now() - this.datNameScanMissTtlMs
for (const [missKey, value] of this.datNameScanMissAt.entries()) {
if (value >= expiresBefore) continue
this.datNameScanMissAt.delete(missKey)
}
while (this.datNameScanMissAt.size > this.maxDatNameScanMissEntries) {
const oldestKey = this.datNameScanMissAt.keys().next().value
if (!oldestKey) break
this.datNameScanMissAt.delete(oldestKey)
}
}
private collectDatCandidatesFromSessionMonth(
accountDir: string,
baseMd5: string,
@@ -1393,14 +1638,74 @@ export class ImageDecryptService {
return null
}
private cacheResolvedPaths(cacheKey: string, imageMd5: string | undefined, imageDatName: string | undefined, outputPath: string): void {
this.resolvedCache.set(cacheKey, outputPath)
if (imageMd5 && imageMd5 !== cacheKey) {
this.resolvedCache.set(imageMd5, outputPath)
private cacheResolvedPaths(
payload: { sessionId?: string; createTime?: number; imageMd5?: string; imageDatName?: string },
cacheKey: string,
outputPath: string
): void {
const normalizedCacheKey = String(cacheKey || '').trim().toLowerCase()
const imageMd5 = String(payload.imageMd5 || '').trim().toLowerCase()
const imageDatName = String(payload.imageDatName || '').trim().toLowerCase()
if (imageMd5) {
this.setResolvedCache(imageMd5, outputPath)
}
if (imageDatName && imageDatName !== cacheKey && imageDatName !== imageMd5) {
this.resolvedCache.set(imageDatName, outputPath)
if (normalizedCacheKey) {
this.setResolvedCache(this.getMediaResolvedCacheKey(payload, normalizedCacheKey), outputPath)
}
if (imageDatName) {
this.setResolvedCache(this.getMediaResolvedCacheKey(payload, imageDatName), outputPath)
}
}
private getResolvedMediaCache(payload: { sessionId?: string; createTime?: number; imageMd5?: string; imageDatName?: string }, key: string): string | undefined {
return this.getResolvedCache(this.getMediaResolvedCacheKey(payload, key))
}
private deleteResolvedMediaCache(payload: { sessionId?: string; createTime?: number; imageMd5?: string; imageDatName?: string }, key: string): void {
this.resolvedCache.delete(this.getMediaResolvedCacheKey(payload, key))
}
private deleteResolvedCachesByPath(filePath: string): void {
if (!filePath) return
for (const [key, value] of Array.from(this.resolvedCache.entries())) {
if (value !== filePath) continue
this.resolvedCache.delete(key)
this.updateFlags.delete(key)
}
}
private getResolvedCache(key: string): string | undefined {
const cached = this.resolvedCache.get(key)
if (cached === undefined) return undefined
this.resolvedCache.delete(key)
this.resolvedCache.set(key, cached)
return cached
}
private setResolvedCache(key: string, value: string): void {
if (!key || !value) return
if (this.resolvedCache.has(key)) {
this.resolvedCache.delete(key)
}
this.resolvedCache.set(key, value)
while (this.resolvedCache.size > this.maxResolvedCacheEntries) {
const oldestKey = this.resolvedCache.keys().next().value
if (!oldestKey) break
this.resolvedCache.delete(oldestKey)
this.updateFlags.delete(oldestKey)
}
}
private getMediaResolvedCacheKey(
payload: { sessionId?: string; createTime?: number; imageMd5?: string; imageDatName?: string },
key: string
): string {
const normalizedKey = String(key || '').trim().toLowerCase()
const imageMd5 = String(payload.imageMd5 || '').trim().toLowerCase()
if (imageMd5 && normalizedKey === imageMd5) return imageMd5
const sessionId = String(payload.sessionId || '').trim().toLowerCase()
const createTime = Number(payload.createTime || 0) || 0
return `dat:${sessionId}|${createTime}|${normalizedKey}`
}
private getCacheKeys(payload: { imageMd5?: string; imageDatName?: string }): string[] {
@@ -1420,19 +1725,125 @@ export class ImageDecryptService {
return keys
}
private cacheDatPath(accountDir: string, datName: string, datPath: string): void {
const key = `${accountDir}|${datName}`
this.resolvedCache.set(key, datPath)
const normalized = this.normalizeDatBase(datName)
if (normalized && normalized !== datName.toLowerCase()) {
this.resolvedCache.set(`${accountDir}|${normalized}`, datPath)
private getResolvePendingKey(payload: CachedImagePayload, cacheKey: string): string {
const sessionId = String(payload.sessionId || '').trim().toLowerCase()
const createTime = Number(payload.createTime || 0) || 0
return [
'resolve',
cacheKey.toLowerCase(),
sessionId,
String(createTime),
payload.hardlinkOnly === true ? 'hl1' : 'hl0',
payload.disableUpdateCheck === true ? 'du1' : 'du0',
payload.allowCacheIndex === false ? 'ci0' : 'ci1',
payload.allowCachePromotion === false ? 'cp0' : 'cp1'
].join('|')
}
private getDecryptPendingKey(payload: DecryptImagePayload, cacheKey: string): string {
const sessionId = String(payload.sessionId || '').trim().toLowerCase()
const createTime = Number(payload.createTime || 0) || 0
return [
'decrypt',
cacheKey.toLowerCase(),
sessionId,
String(createTime),
payload.force === true ? 'f1' : 'f0',
payload.hardlinkOnly === true ? 'hl1' : 'hl0',
payload.disableUpdateCheck === true ? 'du1' : 'du0',
payload.allowCacheIndex === false ? 'ci0' : 'ci1',
payload.allowCachePromotion === false ? 'cp0' : 'cp1'
].join('|')
}
private async resolvePendingResult<T extends DecryptResult>(
pending: PendingImageTask<T>,
payload: CachedImagePayload,
cacheKey: string
): Promise<T> {
const result = await pending.promise
if (!pending.emitsEvents && this.shouldEmitImageEvents(payload)) {
pending.emitsEvents = true
this.emitCacheResolvedFromResult(payload, cacheKey, result)
}
return this.formatPendingResultForPayload(result, payload)
}
private formatPendingResultForPayload<T extends DecryptResult>(result: T, payload: CachedImagePayload): T {
const localPath = String(result.localPath || '').trim()
if (!result.success || !localPath) return result
let formattedPath = localPath
if (payload.preferFilePath === true) {
if (/^file:\/\//i.test(localPath)) {
formattedPath = this.fileUrlToPathSafely(localPath) || localPath
}
} else if (!/^(data:image\/|blob:|https?:\/\/)/i.test(localPath)) {
const filePath = /^file:\/\//i.test(localPath)
? (this.fileUrlToPathSafely(localPath) || localPath)
: localPath
formattedPath = this.resolveLocalPathForPayload(filePath, false)
}
if (formattedPath === localPath) return result
return { ...result, localPath: formattedPath }
}
private fileUrlToPathSafely(value: string): string | null {
try {
const url = new URL(value)
url.search = ''
url.hash = ''
return fileURLToPath(url)
} catch {
return null
}
}
private clearUpdateFlags(cacheKey: string, imageMd5?: string, imageDatName?: string): void {
this.updateFlags.delete(cacheKey)
if (imageMd5) this.updateFlags.delete(imageMd5)
if (imageDatName) this.updateFlags.delete(imageDatName)
private emitCacheResolvedFromResult(payload: CachedImagePayload, cacheKey: string, result: DecryptResult): void {
if (!result.success || !result.localPath) return
const localPath = String(result.localPath || '').trim()
if (!localPath) return
const emitPath = payload.preferFilePath === true && !/^(data:image\/|blob:|https?:\/\/|file:\/\/)/i.test(localPath)
? this.filePathToUrl(localPath)
: localPath
this.emitCacheResolved(payload, cacheKey, emitPath)
}
private cacheDatPath(accountDir: string, datName: string, datPath: string): void {
const key = `${accountDir}|${datName}`
this.setResolvedCache(key, datPath)
const normalized = this.normalizeDatBase(datName)
if (normalized && normalized !== datName.toLowerCase()) {
this.setResolvedCache(`${accountDir}|${normalized}`, datPath)
}
}
private getUpdateFlag(payload: { sessionId?: string; createTime?: number; imageMd5?: string; imageDatName?: string }, cacheKey: string): boolean {
return this.updateFlags.get(this.getMediaResolvedCacheKey(payload, cacheKey)) ?? false
}
private setUpdateFlag(payload: { sessionId?: string; createTime?: number; imageMd5?: string; imageDatName?: string }, cacheKey: string, value: boolean): void {
const key = this.getMediaResolvedCacheKey(payload, cacheKey)
if (value) this.updateFlags.set(key, true)
else this.updateFlags.delete(key)
}
private clearUpdateFlag(payload: { sessionId?: string; createTime?: number; imageMd5?: string; imageDatName?: string }, cacheKey: string): void {
this.updateFlags.delete(this.getMediaResolvedCacheKey(payload, cacheKey))
}
private clearUpdateFlags(payload: { sessionId?: string; createTime?: number; imageMd5?: string; imageDatName?: string }, cacheKey: string): void {
const keys = new Set<string>()
const normalizedCacheKey = String(cacheKey || '').trim().toLowerCase()
const imageMd5 = String(payload.imageMd5 || '').trim().toLowerCase()
const imageDatName = String(payload.imageDatName || '').trim().toLowerCase()
if (normalizedCacheKey) keys.add(this.getMediaResolvedCacheKey(payload, normalizedCacheKey))
if (imageMd5) keys.add(this.getMediaResolvedCacheKey(payload, imageMd5))
if (imageDatName) keys.add(this.getMediaResolvedCacheKey(payload, imageDatName))
for (const key of keys) {
this.updateFlags.delete(key)
}
}
private getActiveWindowsSafely(): Array<{ isDestroyed: () => boolean; webContents: { send: (channel: string, payload: unknown) => void } }> {
@@ -1452,9 +1863,15 @@ export class ImageDecryptService {
}
}
private emitImageUpdate(payload: { sessionId?: string; imageMd5?: string; imageDatName?: string; suppressEvents?: boolean }, cacheKey: string): void {
private emitImageUpdate(payload: { sessionId?: string; imageMd5?: string; imageDatName?: string; createTime?: number; suppressEvents?: boolean }, cacheKey: string): void {
if (!this.shouldEmitImageEvents(payload)) return
const message = { cacheKey, imageMd5: payload.imageMd5, imageDatName: payload.imageDatName }
const message = {
cacheKey,
sessionId: payload.sessionId,
createTime: payload.createTime,
imageMd5: payload.imageMd5,
imageDatName: payload.imageDatName
}
for (const win of this.getActiveWindowsSafely()) {
if (!win.isDestroyed()) {
win.webContents.send('image:updateAvailable', message)
@@ -1462,9 +1879,16 @@ export class ImageDecryptService {
}
}
private emitCacheResolved(payload: { sessionId?: string; imageMd5?: string; imageDatName?: string; suppressEvents?: boolean }, cacheKey: string, localPath: string): void {
private emitCacheResolved(payload: { sessionId?: string; imageMd5?: string; imageDatName?: string; createTime?: number; suppressEvents?: boolean }, cacheKey: string, localPath: string): void {
if (!this.shouldEmitImageEvents(payload)) return
const message = { cacheKey, imageMd5: payload.imageMd5, imageDatName: payload.imageDatName, localPath }
const message = {
cacheKey,
sessionId: payload.sessionId,
createTime: payload.createTime,
imageMd5: payload.imageMd5,
imageDatName: payload.imageDatName,
localPath
}
for (const win of this.getActiveWindowsSafely()) {
if (!win.isDestroyed()) {
win.webContents.send('image:cacheResolved', message)
@@ -1520,11 +1944,11 @@ export class ImageDecryptService {
this.ensuredDirs.add(dirPath)
}
private tryDecryptDatWithNative(
private async tryDecryptDatWithNative(
datPath: string,
xorKey: number,
aesKey?: string
): { data: Buffer; ext: string; isWxgf: boolean } | null {
): Promise<{ data: Buffer; ext: string; isWxgf: boolean } | null> {
const result = decryptDatViaNative(datPath, xorKey, aesKey)
if (!this.nativeLogged) {
this.nativeLogged = true
@@ -1541,20 +1965,20 @@ export class ImageDecryptService {
}
}
if (result) return result
const fallback = this.tryDecryptDatWithJs(datPath, xorKey, aesKey)
const fallback = await this.tryDecryptDatWithJs(datPath, xorKey, aesKey)
if (fallback) {
this.logInfo('JS DAT 解密 fallback 已启用', { datPath, ext: fallback.ext })
}
return fallback
}
private tryDecryptDatWithJs(
private async tryDecryptDatWithJs(
datPath: string,
xorKey: number,
aesKey?: string
): { data: Buffer; ext: string; isWxgf: boolean } | null {
): Promise<{ data: Buffer; ext: string; isWxgf: boolean } | null> {
try {
const encrypted = readFileSync(datPath)
const encrypted = await readFile(datPath)
const directExt = this.detectImageExtension(encrypted)
if (directExt) return { data: encrypted, ext: directExt, isWxgf: false }
@@ -1727,22 +2151,79 @@ export class ImageDecryptService {
private isUsableImageCacheFile(filePath: string): boolean {
if (!this.isImageFile(filePath)) return false
if (!existsSync(filePath)) return false
if (this.isLikelyCorruptedDecodedImage(filePath)) {
this.logInfo('[ImageDecrypt] 跳过疑似损坏缓存文件', { filePath })
void rm(filePath, { force: true }).catch(() => { })
return false
}
const quarantinedUntil = this.corruptedCachePathUntil.get(filePath) || 0
if (quarantinedUntil > Date.now()) return false
this.scheduleCorruptedCacheValidation(filePath)
return true
}
private isLikelyCorruptedDecodedImage(filePath: string): boolean {
private scheduleCorruptedCacheValidation(filePath: string): void {
const ext = extname(filePath).toLowerCase()
if (ext !== '.jpg' && ext !== '.jpeg') return
const now = Date.now()
const lastCheckedAt = this.corruptedCacheCheckLastAt.get(filePath) || 0
if (lastCheckedAt && now - lastCheckedAt < this.corruptedCacheCheckCooldownMs) return
if (this.corruptedCacheCheckPending.has(filePath)) return
this.corruptedCacheCheckLastAt.set(filePath, now)
this.pruneCorruptedCacheCheckMaps(now)
this.corruptedCacheCheckPending.add(filePath)
this.corruptedCacheCheckQueue.push(filePath)
this.processCorruptedCacheValidationQueue()
}
private processCorruptedCacheValidationQueue(): void {
while (
this.activeCorruptedCacheChecks < this.maxConcurrentCorruptedCacheChecks &&
this.corruptedCacheCheckQueue.length > 0
) {
const filePath = this.corruptedCacheCheckQueue.shift()
if (!filePath) continue
this.activeCorruptedCacheChecks += 1
void this.validateCorruptedCacheFile(filePath).finally(() => {
this.activeCorruptedCacheChecks = Math.max(0, this.activeCorruptedCacheChecks - 1)
this.corruptedCacheCheckPending.delete(filePath)
this.processCorruptedCacheValidationQueue()
})
}
}
private async validateCorruptedCacheFile(filePath: string): Promise<void> {
try {
const ext = extname(filePath).toLowerCase()
if (ext !== '.jpg' && ext !== '.jpeg') return false
const data = readFileSync(filePath)
return this.isLikelyCorruptedJpegBuffer(data)
const data = await readFile(filePath)
if (!this.isLikelyCorruptedJpegBuffer(data)) return
this.logInfo('[ImageDecrypt] 后台移除疑似损坏缓存文件', { filePath })
this.corruptedCachePathUntil.set(filePath, Date.now() + this.corruptedCacheQuarantineMs)
this.deleteResolvedCachesByPath(filePath)
void rm(filePath, { force: true }).catch(() => { })
} catch {
return false
// Cache validation is best-effort and must not affect browsing.
}
}
private pruneCorruptedCacheCheckMaps(now = Date.now()): void {
const checkedExpiresBefore = now - this.corruptedCacheCheckCooldownMs
for (const [filePath, checkedAt] of this.corruptedCacheCheckLastAt.entries()) {
if (checkedAt >= checkedExpiresBefore) continue
this.corruptedCacheCheckLastAt.delete(filePath)
}
for (const [filePath, until] of this.corruptedCachePathUntil.entries()) {
if (until >= now) continue
this.corruptedCachePathUntil.delete(filePath)
}
while (this.corruptedCacheCheckLastAt.size > this.maxCorruptedCacheChecks) {
const oldest = this.corruptedCacheCheckLastAt.keys().next().value
if (!oldest) break
this.corruptedCacheCheckLastAt.delete(oldest)
}
while (this.corruptedCachePathUntil.size > this.maxCorruptedCacheChecks) {
const oldest = this.corruptedCachePathUntil.keys().next().value
if (!oldest) break
this.corruptedCachePathUntil.delete(oldest)
}
while (this.corruptedCacheCheckQueue.length > this.maxCorruptedCacheChecks) {
const dropped = this.corruptedCacheCheckQueue.shift()
if (dropped) this.corruptedCacheCheckPending.delete(dropped)
}
}
@@ -1995,7 +2476,7 @@ export class ImageDecryptService {
for (const attempt of attempts) {
// 清理上一轮的输出
try { if (existsSync(tmpOutput)) require('fs').unlinkSync(tmpOutput) } catch {}
await rm(tmpOutput, { force: true }).catch(() => {})
const result = await this.runFfmpegConvert(ffmpeg, attempt.inputArgs, tmpOutput, attempt.label, attempt.outputArgs)
if (!result) continue
@@ -2008,8 +2489,8 @@ export class ImageDecryptService {
this.logError('ffmpeg 转换异常', e)
return null
} finally {
try { if (existsSync(tmpInput)) require('fs').unlinkSync(tmpInput) } catch {}
try { if (existsSync(tmpOutput)) require('fs').unlinkSync(tmpOutput) } catch {}
await rm(tmpInput, { force: true }).catch(() => {})
await rm(tmpOutput, { force: true }).catch(() => {})
}
}
@@ -2046,11 +2527,11 @@ export class ImageDecryptService {
resolve(null)
}, 15000)
proc.on('close', (code: number) => {
proc.on('close', async (code: number) => {
clearTimeout(timer)
if (code === 0 && existsSync(tmpOutput)) {
try {
const jpgBuf = readFileSync(tmpOutput)
const jpgBuf = await readFile(tmpOutput)
if (jpgBuf.length > 0) {
this.logInfo(`ffmpeg [${label}] 成功`, { outputSize: jpgBuf.length })
resolve(jpgBuf)
@@ -2180,9 +2661,15 @@ export class ImageDecryptService {
async clearCache(): Promise<{ success: boolean; error?: string }> {
this.resolvedCache.clear()
this.pending.clear()
this.resolvePending.clear()
this.updateFlags.clear()
this.accountDirCache.clear()
this.ensuredDirs.clear()
this.corruptedCacheCheckPending.clear()
this.corruptedCacheCheckQueue = []
this.activeCorruptedCacheChecks = 0
this.corruptedCacheCheckLastAt.clear()
this.corruptedCachePathUntil.clear()
this.cacheRootPath = null
const configured = this.configService.get('cachePath')
+782 -22
View File
@@ -10,58 +10,761 @@ type PreloadImagePayload = {
type PreloadOptions = {
allowDecrypt?: boolean
allowCacheIndex?: boolean
allowFilesystemScan?: boolean
emitResolved?: boolean
scope?: string
priority?: 'high' | 'normal' | 'low'
}
type PreloadEnqueueResult = {
success: true
requested: number
accepted: number
mergedQueued: number
skippedActive: number
skippedPending: number
ignoredCanceled: number
rejectedCapacity: number
deferred: number
handledIdentities: string[]
acceptedIdentities: string[]
mergedQueuedIdentities: string[]
skippedActiveIdentities: string[]
skippedPendingIdentities: string[]
rejectedIdentities: string[]
deferredIdentities: string[]
}
type PreloadTask = PreloadImagePayload & {
key: string
identity: string
allowDecrypt: boolean
allowCacheIndex: boolean
allowFilesystemScan: boolean
scopes: Set<string>
emitResolvedScopes: Set<string>
priority: 'high' | 'normal' | 'low'
}
type PreloadHighWaterStats = {
queued: number
pending: number
queuedCache: number
queuedDecrypt: number
queuedHigh: number
queuedNormal: number
queuedLow: number
activeCache: number
activeDecrypt: number
activeIdentities: number
}
const isLikelyThumbnailCachePath = (localPath?: string): boolean => {
const normalized = String(localPath || '').trim().toLowerCase()
if (!normalized) return false
return (
normalized.includes('_thumb') ||
normalized.includes('.thumb') ||
normalized.includes('_t.') ||
normalized.includes('.t.')
)
}
export class ImagePreloadService {
private queue: PreloadTask[] = []
private pending = new Set<string>()
private queuedCacheByIdentity = new Map<string, PreloadTask>()
private queuedDecryptByIdentity = new Map<string, PreloadTask>()
private activeByIdentity = new Map<string, PreloadTask>()
private canceledScopes = new Map<string, number>()
private totals = {
accepted: 0,
mergedQueued: 0,
skippedActive: 0,
ignoredCanceled: 0,
droppedQueued: 0,
canceledQueued: 0,
canceledActive: 0,
promotedActive: 0,
rejectedCapacity: 0,
deferredLowPriority: 0,
lowPriorityIdleDeferrals: 0,
skippedPending: 0,
forcedThumbnailRefreshes: 0,
forcedThumbnailRefreshFailures: 0,
lowPriorityRejected: 0,
activeCacheSnapshots: 0,
activeCacheSnapshotSkipped: 0,
activeCacheSnapshotCanceled: 0,
started: 0,
completed: 0
}
private activeCache = 0
private activeDecrypt = 0
private readonly maxCacheConcurrent = 8
private queuedPriorityCounts: Record<PreloadTask['priority'], number> = {
high: 0,
normal: 0,
low: 0
}
private activeCacheSnapshotPending = new Set<string>()
private activeCacheSnapshotLastAt = new Map<string, number>()
private lastForegroundActivityAt = Date.now()
private lowPriorityWakeTimer: ReturnType<typeof setTimeout> | null = null
private highWater: PreloadHighWaterStats = {
queued: 0,
pending: 0,
queuedCache: 0,
queuedDecrypt: 0,
queuedHigh: 0,
queuedNormal: 0,
queuedLow: 0,
activeCache: 0,
activeDecrypt: 0,
activeIdentities: 0
}
private readonly maxCacheConcurrent = 4
private readonly maxDecryptConcurrent = 2
private readonly maxQueueSize = 320
private readonly maxLowPriorityQueueSize = 160
private readonly lowPrioritySoftQueueSize = 240
private readonly canceledScopeTtlMs = 2 * 60 * 1000
private readonly activeCacheSnapshotCooldownMs = 1800
private readonly lowPriorityIdleDelayMs = 1500
enqueue(payloads: PreloadImagePayload[], options?: PreloadOptions): void {
if (!Array.isArray(payloads) || payloads.length === 0) return
enqueue(payloads: PreloadImagePayload[], options?: PreloadOptions): PreloadEnqueueResult {
const result = this.createEnqueueResult(Array.isArray(payloads) ? payloads.length : 0)
if (!Array.isArray(payloads) || payloads.length === 0) return result
this.pruneCanceledScopes()
const allowDecrypt = options?.allowDecrypt !== false
const allowCacheIndex = options?.allowCacheIndex !== false
for (const payload of payloads) {
if (!allowDecrypt && this.queue.length >= this.maxQueueSize) break
const cacheKey = payload.imageMd5 || payload.imageDatName
if (!cacheKey) continue
const key = `${payload.sessionId || 'unknown'}|${cacheKey}`
if (this.pending.has(key)) continue
this.pending.add(key)
this.queue.push({ ...payload, key, allowDecrypt, allowCacheIndex })
const allowFilesystemScan = options?.allowFilesystemScan !== false
const emitResolved = options?.emitResolved === true
const scope = String(options?.scope || 'global').trim() || 'global'
const priority = options?.priority || (allowDecrypt ? 'low' : 'normal')
this.noteForegroundActivity(allowDecrypt, priority)
if (this.isScopeCanceled(scope)) {
this.totals.ignoredCanceled += payloads.length
result.ignoredCanceled += payloads.length
return result
}
for (const payload of payloads) {
const identity = this.getTaskIdentity(payload)
if (!identity) continue
const activeTask = this.activeByIdentity.get(identity)
if (activeTask && (!allowDecrypt || activeTask.allowDecrypt)) {
activeTask.scopes.add(scope)
activeTask.allowCacheIndex = activeTask.allowCacheIndex || allowCacheIndex
activeTask.allowFilesystemScan = activeTask.allowFilesystemScan || allowFilesystemScan
if (this.priorityScore(priority) < this.priorityScore(activeTask.priority)) {
activeTask.priority = priority
this.totals.promotedActive += 1
}
if (emitResolved) {
activeTask.emitResolvedScopes.add(scope)
}
if (!allowDecrypt && activeTask.allowDecrypt && emitResolved) {
this.emitActiveCacheSnapshot(payload, identity, allowCacheIndex, scope)
}
this.totals.skippedActive += 1
result.skippedActive += 1
result.handledIdentities.push(identity)
result.skippedActiveIdentities.push(identity)
continue
}
const mode = allowDecrypt ? 'decrypt' : 'cache'
const key = `${mode}|${identity}`
const queuedMap = this.getQueuedMap(allowDecrypt)
const queuedTask = queuedMap.get(identity)
if (queuedTask) {
this.totals.mergedQueued += 1
result.mergedQueued += 1
result.handledIdentities.push(identity)
result.mergedQueuedIdentities.push(identity)
this.mergeQueuedTaskPayload(queuedTask, payload)
queuedTask.allowCacheIndex = queuedTask.allowCacheIndex || allowCacheIndex
queuedTask.allowFilesystemScan = queuedTask.allowFilesystemScan || allowFilesystemScan
queuedTask.scopes.add(scope)
if (emitResolved) queuedTask.emitResolvedScopes.add(scope)
if (this.priorityScore(priority) < this.priorityScore(queuedTask.priority)) {
this.updateQueuedPriority(queuedTask, priority)
}
continue
}
if (this.shouldDeferLowPriorityDecrypt(allowDecrypt, priority)) {
this.deferLowPriority(result, identity)
continue
}
const queuedPeerTask = this.getQueuedMap(!allowDecrypt).get(identity)
const shouldKeepIncomingCacheSeparate = !allowDecrypt && queuedPeerTask?.allowDecrypt === true
if (queuedPeerTask && !shouldKeepIncomingCacheSeparate) {
if (this.shouldDeferPeerUpgrade(queuedPeerTask, allowDecrypt, priority)) {
this.deferLowPriority(result, identity)
continue
}
this.totals.mergedQueued += 1
result.mergedQueued += 1
result.handledIdentities.push(identity)
result.mergedQueuedIdentities.push(identity)
this.mergeQueuedTaskPayload(queuedPeerTask, payload)
queuedPeerTask.allowCacheIndex = queuedPeerTask.allowCacheIndex || allowCacheIndex
queuedPeerTask.allowFilesystemScan = queuedPeerTask.allowFilesystemScan || allowFilesystemScan
queuedPeerTask.scopes.add(scope)
if (emitResolved) queuedPeerTask.emitResolvedScopes.add(scope)
if (allowDecrypt && !queuedPeerTask.allowDecrypt) {
this.updateQueuedMode(queuedPeerTask, true)
}
if (this.priorityScore(priority) < this.priorityScore(queuedPeerTask.priority)) {
this.updateQueuedPriority(queuedPeerTask, priority)
}
continue
}
if (this.pending.has(key)) {
this.totals.skippedPending += 1
result.skippedPending += 1
result.handledIdentities.push(identity)
result.skippedPendingIdentities.push(identity)
continue
}
if (!this.ensureQueueCapacity(priority)) {
this.totals.rejectedCapacity += 1
if (this.isLowPrioritySoftLimitExceeded(priority)) {
this.totals.lowPriorityRejected += 1
}
result.rejectedCapacity += 1
result.rejectedIdentities.push(identity)
continue
}
this.pending.add(key)
const task = {
...payload,
key,
identity,
allowDecrypt,
allowCacheIndex,
allowFilesystemScan,
scopes: new Set([scope]),
emitResolvedScopes: emitResolved ? new Set([scope]) : new Set<string>(),
priority
}
this.addQueuedTask(task)
this.totals.accepted += 1
result.accepted += 1
result.handledIdentities.push(identity)
result.acceptedIdentities.push(identity)
this.recordHighWater()
}
this.sortQueue()
this.processQueue()
return result
}
cancelScope(scope: string): void {
const normalizedScope = String(scope || '').trim()
if (!normalizedScope) return
this.canceledScopes.set(normalizedScope, Date.now())
this.dropActiveCacheSnapshotsForScope(normalizedScope)
const kept: PreloadTask[] = []
for (const task of this.queue) {
task.scopes.delete(normalizedScope)
task.emitResolvedScopes.delete(normalizedScope)
if (this.isCanceled(task)) {
this.pending.delete(task.key)
this.removeQueuedTask(task)
this.totals.canceledQueued += 1
} else {
kept.push(task)
}
}
this.queue = kept
for (const task of this.activeByIdentity.values()) {
const hadScope = task.scopes.delete(normalizedScope)
task.emitResolvedScopes.delete(normalizedScope)
if (!hadScope) continue
if (this.isCanceled(task)) {
this.totals.canceledActive += 1
}
}
this.pruneCanceledScopes()
this.clearLowPriorityWakeIfUnneeded()
}
getStats(): {
queued: number
pending: number
activeCache: number
activeDecrypt: number
queuedIdentities: number
queuedCache: number
queuedDecrypt: number
queuedHigh: number
queuedNormal: number
queuedLow: number
activeIdentities: number
queuedScopes: number
activeScopes: number
canceledScopes: number
highWater: PreloadHighWaterStats
totals: {
accepted: number
mergedQueued: number
skippedActive: number
ignoredCanceled: number
droppedQueued: number
canceledQueued: number
canceledActive: number
promotedActive: number
rejectedCapacity: number
deferredLowPriority: number
lowPriorityIdleDeferrals: number
skippedPending: number
forcedThumbnailRefreshes: number
forcedThumbnailRefreshFailures: number
lowPriorityRejected: number
activeCacheSnapshots: number
activeCacheSnapshotSkipped: number
activeCacheSnapshotCanceled: number
started: number
completed: number
}
} {
this.recordHighWater()
const queuedIdentities = new Set([
...this.queuedCacheByIdentity.keys(),
...this.queuedDecryptByIdentity.keys()
]).size
const priorityCounts = this.countQueuedPriorities()
const highWater = { ...this.highWater }
this.resetHighWater()
return {
queued: this.queue.length,
pending: this.pending.size,
activeCache: this.activeCache,
activeDecrypt: this.activeDecrypt,
queuedIdentities,
queuedCache: this.queuedCacheByIdentity.size,
queuedDecrypt: this.queuedDecryptByIdentity.size,
queuedHigh: priorityCounts.high,
queuedNormal: priorityCounts.normal,
queuedLow: priorityCounts.low,
activeIdentities: this.activeByIdentity.size,
queuedScopes: this.countTaskScopes(this.queue),
activeScopes: this.countTaskScopes(this.activeByIdentity.values()),
canceledScopes: this.canceledScopes.size,
highWater,
totals: { ...this.totals }
}
}
private priorityScore(priority: PreloadTask['priority']): number {
if (priority === 'high') return 0
if (priority === 'normal') return 1
return 2
}
private noteForegroundActivity(allowDecrypt: boolean, priority: PreloadTask['priority']): void {
if (allowDecrypt && priority === 'low') return
this.lastForegroundActivityAt = Date.now()
}
private createEnqueueResult(requested: number): PreloadEnqueueResult {
return {
success: true,
requested,
accepted: 0,
mergedQueued: 0,
skippedActive: 0,
skippedPending: 0,
ignoredCanceled: 0,
rejectedCapacity: 0,
deferred: 0,
handledIdentities: [],
acceptedIdentities: [],
mergedQueuedIdentities: [],
skippedActiveIdentities: [],
skippedPendingIdentities: [],
rejectedIdentities: [],
deferredIdentities: []
}
}
private getTaskIdentity(payload: PreloadImagePayload): string {
const imageMd5 = String(payload.imageMd5 || '').trim().toLowerCase()
if (imageMd5) return `md5:${imageMd5}`
const imageDatName = String(payload.imageDatName || '').trim().toLowerCase()
if (imageDatName) {
const sessionId = String(payload.sessionId || '').trim().toLowerCase()
const createTime = Number(payload.createTime || 0) || 0
return `dat:${sessionId}|${createTime}|${imageDatName}`
}
return ''
}
private getQueuedMap(allowDecrypt: boolean): Map<string, PreloadTask> {
return allowDecrypt ? this.queuedDecryptByIdentity : this.queuedCacheByIdentity
}
private deferLowPriority(result: PreloadEnqueueResult, identity: string): void {
this.totals.deferredLowPriority += 1
result.deferred += 1
result.deferredIdentities.push(identity)
}
private shouldDeferLowPriorityDecrypt(allowDecrypt: boolean, priority: PreloadTask['priority']): boolean {
if (!allowDecrypt || priority !== 'low') return false
return (
this.queuedPriorityCounts.high > 0 ||
this.queuedPriorityCounts.normal > 0 ||
this.activeCache >= this.maxCacheConcurrent ||
(this.activeCache > 0 && this.queuedCacheByIdentity.size > 0)
)
}
private shouldDeferPeerUpgrade(
peerTask: PreloadTask,
allowDecrypt: boolean,
priority: PreloadTask['priority']
): boolean {
return allowDecrypt && !peerTask.allowDecrypt && priority === 'low' && (
peerTask.priority !== 'low' ||
peerTask.emitResolvedScopes.size > 0
)
}
private emitActiveCacheSnapshot(
payload: PreloadImagePayload,
identity: string,
allowCacheIndex: boolean,
scope: string
): void {
if (this.isScopeCanceled(scope)) return
const now = Date.now()
this.pruneActiveCacheSnapshotCooldowns(now)
const snapshotKey = `${scope}|${identity}`
if (this.activeCacheSnapshotPending.has(snapshotKey)) {
this.totals.activeCacheSnapshotSkipped += 1
return
}
const lastSnapshotAt = this.activeCacheSnapshotLastAt.get(snapshotKey) || 0
if (lastSnapshotAt && now - lastSnapshotAt < this.activeCacheSnapshotCooldownMs) {
this.totals.activeCacheSnapshotSkipped += 1
return
}
this.activeCacheSnapshotPending.add(snapshotKey)
this.activeCacheSnapshotLastAt.set(snapshotKey, now)
this.totals.activeCacheSnapshots += 1
const cacheKey = this.getSnapshotCacheKey(payload)
void imageDecryptService.resolveCachedImage({
sessionId: payload.sessionId,
imageMd5: payload.imageMd5,
imageDatName: payload.imageDatName,
createTime: payload.createTime,
preferFilePath: true,
hardlinkOnly: true,
disableUpdateCheck: true,
allowCacheIndex,
allowCachePromotion: false,
suppressEvents: true
}).then((result) => {
if (this.isScopeCanceled(scope)) {
this.totals.activeCacheSnapshotCanceled += 1
return
}
if (!result?.success || !result.localPath || !cacheKey) return
imageDecryptService.emitCacheResolvedPath({
sessionId: payload.sessionId,
imageMd5: payload.imageMd5,
imageDatName: payload.imageDatName,
createTime: payload.createTime
}, cacheKey, result.localPath)
}).catch(() => {
// Visible cache snapshots are best-effort; the active decrypt will still finish normally.
}).finally(() => {
this.activeCacheSnapshotPending.delete(snapshotKey)
})
}
private getSnapshotCacheKey(payload: PreloadImagePayload): string {
return String(payload.imageMd5 || payload.imageDatName || '').trim().toLowerCase()
}
private pruneActiveCacheSnapshotCooldowns(now = Date.now()): void {
const expiresBefore = now - this.activeCacheSnapshotCooldownMs
for (const [key, timestamp] of this.activeCacheSnapshotLastAt.entries()) {
if (timestamp >= expiresBefore) continue
this.activeCacheSnapshotLastAt.delete(key)
}
}
private dropActiveCacheSnapshotsForScope(scope: string): void {
const prefix = `${scope}|`
for (const key of Array.from(this.activeCacheSnapshotPending)) {
if (key.startsWith(prefix)) this.activeCacheSnapshotPending.delete(key)
}
for (const key of Array.from(this.activeCacheSnapshotLastAt.keys())) {
if (key.startsWith(prefix)) this.activeCacheSnapshotLastAt.delete(key)
}
}
private mergeQueuedTaskPayload(task: PreloadTask, payload: PreloadImagePayload): void {
const incomingSessionId = String(payload.sessionId || '').trim()
if (!task.sessionId && incomingSessionId) {
task.sessionId = incomingSessionId
}
const incomingCreateTime = Number(payload.createTime || 0) || 0
const currentCreateTime = Number(task.createTime || 0) || 0
if (!currentCreateTime && incomingCreateTime) {
task.createTime = incomingCreateTime
}
const incomingImageMd5 = String(payload.imageMd5 || '').trim().toLowerCase()
if (!task.imageMd5 && incomingImageMd5) {
task.imageMd5 = incomingImageMd5
}
const incomingDatName = String(payload.imageDatName || '').trim().toLowerCase()
if (!task.imageDatName && incomingDatName) {
task.imageDatName = incomingDatName
}
}
private addQueuedTask(task: PreloadTask): void {
this.queue.push(task)
this.getQueuedMap(task.allowDecrypt).set(task.identity, task)
this.queuedPriorityCounts[task.priority] += 1
}
private updateQueuedPriority(task: PreloadTask, priority: PreloadTask['priority']): void {
if (task.priority === priority) return
this.queuedPriorityCounts[task.priority] = Math.max(0, this.queuedPriorityCounts[task.priority] - 1)
task.priority = priority
this.queuedPriorityCounts[priority] += 1
}
private updateQueuedMode(task: PreloadTask, allowDecrypt: boolean): void {
if (task.allowDecrypt === allowDecrypt) return
this.getQueuedMap(task.allowDecrypt).delete(task.identity)
this.pending.delete(task.key)
task.allowDecrypt = allowDecrypt
task.key = `${allowDecrypt ? 'decrypt' : 'cache'}|${task.identity}`
this.getQueuedMap(task.allowDecrypt).set(task.identity, task)
this.pending.add(task.key)
}
private removeQueuedTask(task: PreloadTask): void {
this.getQueuedMap(task.allowDecrypt).delete(task.identity)
this.queuedPriorityCounts[task.priority] = Math.max(0, this.queuedPriorityCounts[task.priority] - 1)
}
private ensureQueueCapacity(priority: PreloadTask['priority']): boolean {
if (this.isLowPrioritySoftLimitExceeded(priority)) {
return false
}
if (this.queue.length < this.maxQueueSize) return true
const incomingScore = this.priorityScore(priority)
let dropIndex = -1
let dropScore = incomingScore
for (let index = this.queue.length - 1; index >= 0; index -= 1) {
const score = this.priorityScore(this.queue[index].priority)
if (score <= dropScore) continue
dropScore = score
dropIndex = index
}
if (dropIndex < 0) return false
const [dropped] = this.queue.splice(dropIndex, 1)
if (dropped) {
this.pending.delete(dropped.key)
this.removeQueuedTask(dropped)
this.totals.droppedQueued += 1
}
return true
}
private isLowPrioritySoftLimitExceeded(priority: PreloadTask['priority']): boolean {
return priority === 'low' && (
this.queue.length >= this.lowPrioritySoftQueueSize ||
this.queuedPriorityCounts.low >= this.maxLowPriorityQueueSize
)
}
private isScopeCanceled(scope: string): boolean {
return this.canceledScopes.has(scope)
}
private isCanceled(task: PreloadTask): boolean {
if (task.scopes.size === 0) return true
for (const scope of task.scopes) {
if (!this.isScopeCanceled(scope)) return false
}
return true
}
private hasRunnableEmitScope(task: PreloadTask): boolean {
for (const scope of Array.from(task.emitResolvedScopes)) {
if (task.scopes.has(scope) && !this.isScopeCanceled(scope)) return true
task.emitResolvedScopes.delete(scope)
}
return false
}
private emitTaskCacheResolved(task: PreloadTask, result?: { success?: boolean; localPath?: string }): void {
if (!result?.success || !result.localPath) return
if (!this.hasRunnableEmitScope(task)) return
const cacheKey = this.getSnapshotCacheKey(task)
if (!cacheKey) return
imageDecryptService.emitCacheResolvedPath({
sessionId: task.sessionId,
imageMd5: task.imageMd5,
imageDatName: task.imageDatName,
createTime: task.createTime
}, cacheKey, result.localPath)
}
private pruneCanceledScopes(): void {
if (this.canceledScopes.size === 0) return
const referencedScopes = new Set<string>()
for (const task of this.queue) {
for (const scope of task.scopes) referencedScopes.add(scope)
}
for (const task of this.activeByIdentity.values()) {
for (const scope of task.scopes) referencedScopes.add(scope)
}
const expiresBefore = Date.now() - this.canceledScopeTtlMs
for (const [scope, canceledAt] of this.canceledScopes.entries()) {
if (referencedScopes.has(scope)) continue
if (canceledAt >= expiresBefore) continue
this.canceledScopes.delete(scope)
}
}
private countTaskScopes(tasks: Iterable<PreloadTask>): number {
const scopes = new Set<string>()
for (const task of tasks) {
for (const scope of task.scopes) {
scopes.add(scope)
}
}
return scopes.size
}
private countQueuedPriorities(): Record<PreloadTask['priority'], number> {
return { ...this.queuedPriorityCounts }
}
private recordHighWater(): void {
const priorityCounts = this.countQueuedPriorities()
this.highWater.queued = Math.max(this.highWater.queued, this.queue.length)
this.highWater.pending = Math.max(this.highWater.pending, this.pending.size)
this.highWater.queuedCache = Math.max(this.highWater.queuedCache, this.queuedCacheByIdentity.size)
this.highWater.queuedDecrypt = Math.max(this.highWater.queuedDecrypt, this.queuedDecryptByIdentity.size)
this.highWater.queuedHigh = Math.max(this.highWater.queuedHigh, priorityCounts.high)
this.highWater.queuedNormal = Math.max(this.highWater.queuedNormal, priorityCounts.normal)
this.highWater.queuedLow = Math.max(this.highWater.queuedLow, priorityCounts.low)
this.highWater.activeCache = Math.max(this.highWater.activeCache, this.activeCache)
this.highWater.activeDecrypt = Math.max(this.highWater.activeDecrypt, this.activeDecrypt)
this.highWater.activeIdentities = Math.max(this.highWater.activeIdentities, this.activeByIdentity.size)
}
private resetHighWater(): void {
const priorityCounts = this.countQueuedPriorities()
this.highWater = {
queued: this.queue.length,
pending: this.pending.size,
queuedCache: this.queuedCacheByIdentity.size,
queuedDecrypt: this.queuedDecryptByIdentity.size,
queuedHigh: priorityCounts.high,
queuedNormal: priorityCounts.normal,
queuedLow: priorityCounts.low,
activeCache: this.activeCache,
activeDecrypt: this.activeDecrypt,
activeIdentities: this.activeByIdentity.size
}
}
private sortQueue(): void {
this.queue.sort((a, b) => this.priorityScore(a.priority) - this.priorityScore(b.priority))
}
private hasRunnableCapacity(): boolean {
return this.activeCache < this.maxCacheConcurrent || this.activeDecrypt < this.maxDecryptConcurrent
}
private canRunTask(task: PreloadTask): boolean {
if (this.activeByIdentity.has(task.identity)) return false
if (task.allowDecrypt && task.priority === 'low' && !this.canRunLowPriorityDecrypt()) return false
return task.allowDecrypt
? this.activeDecrypt < this.maxDecryptConcurrent
: this.activeCache < this.maxCacheConcurrent
}
private canRunLowPriorityDecrypt(): boolean {
if (this.activeDecrypt > 0) return false
if (this.activeCache > 0) return false
if (this.queuedCacheByIdentity.size > 0) return false
if (this.queuedPriorityCounts.high > 0 || this.queuedPriorityCounts.normal > 0) return false
return Date.now() - this.lastForegroundActivityAt >= this.lowPriorityIdleDelayMs
}
private hasQueuedLowPriorityDecrypt(): boolean {
return this.queue.some((task) => task.allowDecrypt && task.priority === 'low')
}
private scheduleLowPriorityWakeIfNeeded(): void {
if (this.lowPriorityWakeTimer !== null) return
if (!this.hasQueuedLowPriorityDecrypt()) return
if (this.activeDecrypt > 0 || this.activeCache > 0) return
if (this.queuedCacheByIdentity.size > 0) return
if (this.queuedPriorityCounts.high > 0 || this.queuedPriorityCounts.normal > 0) return
const elapsed = Date.now() - this.lastForegroundActivityAt
const delayMs = Math.max(20, this.lowPriorityIdleDelayMs - elapsed)
this.totals.lowPriorityIdleDeferrals += 1
this.lowPriorityWakeTimer = setTimeout(() => {
this.lowPriorityWakeTimer = null
this.processQueue()
}, delayMs)
}
private clearLowPriorityWakeIfUnneeded(): void {
if (this.lowPriorityWakeTimer === null) return
if (this.hasQueuedLowPriorityDecrypt()) return
clearTimeout(this.lowPriorityWakeTimer)
this.lowPriorityWakeTimer = null
}
private takeNextRunnableTask(): PreloadTask | null {
if (!this.hasRunnableCapacity()) return null
const taskIndex = this.queue.findIndex((task) => this.canRunTask(task))
if (taskIndex < 0) return null
const task = this.queue.splice(taskIndex, 1)[0]
if (!task) return null
this.removeQueuedTask(task)
return task
}
private processQueue(): void {
while (this.queue.length > 0) {
const taskIndex = this.queue.findIndex((task) => (
task.allowDecrypt
? this.activeDecrypt < this.maxDecryptConcurrent
: this.activeCache < this.maxCacheConcurrent
))
if (taskIndex < 0) return
const task = this.queue.splice(taskIndex, 1)[0]
if (!task) return
const task = this.takeNextRunnableTask()
if (!task) {
this.scheduleLowPriorityWakeIfNeeded()
return
}
this.activeByIdentity.set(task.identity, task)
this.totals.started += 1
if (task.allowDecrypt) this.activeDecrypt += 1
else this.activeCache += 1
this.recordHighWater()
void this.handleTask(task).finally(() => {
if (task.allowDecrypt) this.activeDecrypt = Math.max(0, this.activeDecrypt - 1)
else this.activeCache = Math.max(0, this.activeCache - 1)
this.activeByIdentity.delete(task.identity)
this.pending.delete(task.key)
this.totals.completed += 1
this.pruneCanceledScopes()
this.processQueue()
})
}
@@ -70,6 +773,7 @@ export class ImagePreloadService {
private async handleTask(task: PreloadTask): Promise<void> {
const cacheKey = task.imageMd5 || task.imageDatName
if (!cacheKey) return
if (this.isCanceled(task)) return
try {
const cached = await imageDecryptService.resolveCachedImage({
sessionId: task.sessionId,
@@ -78,12 +782,52 @@ export class ImagePreloadService {
createTime: task.createTime,
preferFilePath: true,
hardlinkOnly: true,
disableUpdateCheck: !task.allowDecrypt,
disableUpdateCheck: true,
allowCacheIndex: task.allowCacheIndex,
allowCachePromotion: false,
allowFilesystemScan: task.allowFilesystemScan,
suppressEvents: true
})
if (cached.success) return
if (cached.success) {
this.emitTaskCacheResolved(task, cached)
if (!task.allowDecrypt || this.isCanceled(task)) return
const shouldRefreshCachedPreview = cached.hasUpdate === true || isLikelyThumbnailCachePath(cached.localPath)
if (!shouldRefreshCachedPreview) return
this.totals.forcedThumbnailRefreshes += 1
const refreshed = await imageDecryptService.decryptImage({
sessionId: task.sessionId,
imageMd5: task.imageMd5,
imageDatName: task.imageDatName,
createTime: task.createTime,
preferFilePath: true,
force: true,
hardlinkOnly: true,
disableUpdateCheck: true,
allowFilesystemScan: task.allowFilesystemScan,
suppressEvents: true
})
if (!refreshed.success) {
this.totals.forcedThumbnailRefreshFailures += 1
}
if (this.isCanceled(task)) return
const resolved = await imageDecryptService.resolveCachedImage({
sessionId: task.sessionId,
imageMd5: task.imageMd5,
imageDatName: task.imageDatName,
createTime: task.createTime,
preferFilePath: true,
hardlinkOnly: true,
disableUpdateCheck: true,
allowCacheIndex: task.allowCacheIndex,
allowCachePromotion: false,
allowFilesystemScan: task.allowFilesystemScan,
suppressEvents: true
})
this.emitTaskCacheResolved(task, resolved)
return
}
if (!task.allowDecrypt) return
if (this.isCanceled(task)) return
await imageDecryptService.decryptImage({
sessionId: task.sessionId,
imageMd5: task.imageMd5,
@@ -92,8 +836,24 @@ export class ImagePreloadService {
preferFilePath: true,
hardlinkOnly: true,
disableUpdateCheck: true,
allowFilesystemScan: task.allowFilesystemScan,
suppressEvents: true
})
if (this.isCanceled(task)) return
const resolved = await imageDecryptService.resolveCachedImage({
sessionId: task.sessionId,
imageMd5: task.imageMd5,
imageDatName: task.imageDatName,
createTime: task.createTime,
preferFilePath: true,
hardlinkOnly: true,
disableUpdateCheck: true,
allowCacheIndex: task.allowCacheIndex,
allowCachePromotion: false,
allowFilesystemScan: task.allowFilesystemScan,
suppressEvents: true
})
this.emitTaskCacheResolved(task, resolved)
} catch {
// ignore preload failures
}
+115 -42
View File
@@ -1,5 +1,6 @@
import { join } from 'path'
import { existsSync, readdirSync, statSync, readFileSync, appendFileSync, mkdirSync } from 'fs'
import { constants, existsSync, mkdirSync } from 'fs'
import { access, appendFile, readFile, readdir } from 'fs/promises'
import { pathToFileURL } from 'url'
import { app } from 'electron'
import { ConfigService } from './config'
@@ -25,30 +26,68 @@ interface VideoIndexEntry {
type PosterFormat = 'dataUrl' | 'fileUrl'
const pathExists = async (filePath: string): Promise<boolean> => {
try {
await access(filePath, constants.F_OK)
return true
} catch {
return false
}
}
class VideoService {
private configService: ConfigService
private hardlinkResolveCache = new Map<string, TimedCacheEntry<string | null>>()
private videoInfoCache = new Map<string, TimedCacheEntry<VideoInfo>>()
private videoDirIndexCache = new Map<string, TimedCacheEntry<Map<string, VideoIndexEntry>>>()
private pendingVideoInfo = new Map<string, Promise<VideoInfo>>()
private pendingVideoDirIndex = new Map<string, Promise<Map<string, VideoIndexEntry>>>()
private readonly hardlinkCacheTtlMs = 10 * 60 * 1000
private readonly videoInfoCacheTtlMs = 2 * 60 * 1000
private readonly videoIndexCacheTtlMs = 90 * 1000
private readonly maxCacheEntries = 2000
private readonly maxIndexEntries = 6
private logFlushTimer: ReturnType<typeof setTimeout> | null = null
private pendingLogLines: string[] = []
private readonly logFlushDelayMs = 200
private readonly maxPendingLogLines = 600
constructor() {
this.configService = new ConfigService()
}
private log(message: string, meta?: Record<string, unknown>): void {
if (!this.configService.get('logEnabled')) return
const timestamp = new Date().toISOString()
const metaStr = meta ? ` ${JSON.stringify(meta)}` : ''
this.pendingLogLines.push(`[${timestamp}] [VideoService] ${message}${metaStr}\n`)
while (this.pendingLogLines.length > this.maxPendingLogLines) {
this.pendingLogLines.shift()
}
if (this.logFlushTimer) return
this.logFlushTimer = setTimeout(() => {
this.logFlushTimer = null
void this.flushPendingLogs()
}, this.logFlushDelayMs)
}
private async flushPendingLogs(): Promise<void> {
if (this.pendingLogLines.length === 0) return
const lines = this.pendingLogLines.splice(0, this.pendingLogLines.length).join('')
try {
const timestamp = new Date().toISOString()
const metaStr = meta ? ` ${JSON.stringify(meta)}` : ''
const logDir = join(app.getPath('userData'), 'logs')
if (!existsSync(logDir)) mkdirSync(logDir, { recursive: true })
appendFileSync(join(logDir, 'wcdb.log'), `[${timestamp}] [VideoService] ${message}${metaStr}\n`, 'utf8')
} catch { }
await appendFile(join(logDir, 'wcdb.log'), lines, 'utf8')
} catch {
// Logging is diagnostic only; video browsing must never wait on it.
} finally {
if (this.pendingLogLines.length > 0 && this.logFlushTimer === null) {
this.logFlushTimer = setTimeout(() => {
this.logFlushTimer = null
void this.flushPendingLogs()
}, this.logFlushDelayMs)
}
}
}
private readTimedCache<T>(cache: Map<string, TimedCacheEntry<T>>, key: string): T | undefined {
@@ -272,21 +311,33 @@ class VideoService {
void md5List
}
private fileToPosterUrl(filePath: string | undefined, mimeType: string, posterFormat: PosterFormat): string | undefined {
private async fileToPosterUrl(filePath: string | undefined, mimeType: string, posterFormat: PosterFormat): Promise<string | undefined> {
try {
if (!filePath || !existsSync(filePath)) return undefined
if (!filePath || !(await pathExists(filePath))) return undefined
if (posterFormat === 'fileUrl') return pathToFileURL(filePath).toString()
const buffer = readFileSync(filePath)
const buffer = await readFile(filePath)
return `data:${mimeType};base64,${buffer.toString('base64')}`
} catch {
return undefined
}
}
private getOrBuildVideoIndex(videoBaseDir: string): Map<string, VideoIndexEntry> {
private async getOrBuildVideoIndex(videoBaseDir: string): Promise<Map<string, VideoIndexEntry>> {
const cached = this.readTimedCache(this.videoDirIndexCache, videoBaseDir)
if (cached) return cached
const pending = this.pendingVideoDirIndex.get(videoBaseDir)
if (pending) return pending
const task = this.buildVideoIndex(videoBaseDir)
this.pendingVideoDirIndex.set(videoBaseDir, task)
try {
return await task
} finally {
this.pendingVideoDirIndex.delete(videoBaseDir)
}
}
private async buildVideoIndex(videoBaseDir: string): Promise<Map<string, VideoIndexEntry>> {
const index = new Map<string, VideoIndexEntry>()
const ensureEntry = (key: string): VideoIndexEntry => {
let entry = index.get(key)
@@ -298,22 +349,18 @@ class VideoService {
}
try {
const yearMonthDirs = readdirSync(videoBaseDir)
.filter((dir) => {
const dirPath = join(videoBaseDir, dir)
try {
return statSync(dirPath).isDirectory()
} catch {
return false
}
})
const yearMonthDirs = (await readdir(videoBaseDir, { withFileTypes: true }))
.filter((dir) => dir.isDirectory())
.map((dir) => dir.name)
.sort((a, b) => b.localeCompare(a))
for (const yearMonth of yearMonthDirs) {
const dirPath = join(videoBaseDir, yearMonth)
let files: string[] = []
try {
files = readdirSync(dirPath)
files = (await readdir(dirPath, { withFileTypes: true }))
.filter((entry) => entry.isFile())
.map((entry) => entry.name)
} catch {
continue
}
@@ -369,12 +416,12 @@ class VideoService {
return index
}
private getVideoInfoFromIndex(
private async getVideoInfoFromIndex(
index: Map<string, VideoIndexEntry>,
md5: string,
includePoster = true,
posterFormat: PosterFormat = 'dataUrl'
): VideoInfo | null {
): Promise<VideoInfo | null> {
const normalizedMd5 = String(md5 || '').trim().toLowerCase()
if (!normalizedMd5) return null
@@ -389,7 +436,7 @@ class VideoService {
for (const key of candidates) {
const entry = index.get(key)
if (!entry?.videoPath) continue
if (!existsSync(entry.videoPath)) continue
if (!(await pathExists(entry.videoPath))) continue
if (!includePoster) {
return {
videoUrl: entry.videoPath,
@@ -398,8 +445,8 @@ class VideoService {
}
return {
videoUrl: entry.videoPath,
coverUrl: this.fileToPosterUrl(entry.coverPath, 'image/jpeg', posterFormat),
thumbUrl: this.fileToPosterUrl(entry.thumbPath, 'image/jpeg', posterFormat),
coverUrl: await this.fileToPosterUrl(entry.coverPath, 'image/jpeg', posterFormat),
thumbUrl: await this.fileToPosterUrl(entry.thumbPath, 'image/jpeg', posterFormat),
exists: true
}
}
@@ -424,28 +471,22 @@ class VideoService {
return String(fallback?.[1] || '').toLowerCase()
}
private fallbackScanVideo(
private async fallbackScanVideo(
videoBaseDir: string,
realVideoMd5: string,
includePoster = true,
posterFormat: PosterFormat = 'dataUrl'
): VideoInfo | null {
): Promise<VideoInfo | null> {
try {
const yearMonthDirs = readdirSync(videoBaseDir)
.filter((dir) => {
const dirPath = join(videoBaseDir, dir)
try {
return statSync(dirPath).isDirectory()
} catch {
return false
}
})
const yearMonthDirs = (await readdir(videoBaseDir, { withFileTypes: true }))
.filter((dir) => dir.isDirectory())
.map((dir) => dir.name)
.sort((a, b) => b.localeCompare(a))
for (const yearMonth of yearMonthDirs) {
const dirPath = join(videoBaseDir, yearMonth)
const videoPath = join(dirPath, `${realVideoMd5}.mp4`)
if (!existsSync(videoPath)) continue
if (!(await pathExists(videoPath))) continue
if (!includePoster) {
return {
videoUrl: videoPath,
@@ -457,8 +498,8 @@ class VideoService {
const thumbPath = join(dirPath, `${baseMd5}_thumb.jpg`)
return {
videoUrl: videoPath,
coverUrl: this.fileToPosterUrl(coverPath, 'image/jpeg', posterFormat),
thumbUrl: this.fileToPosterUrl(thumbPath, 'image/jpeg', posterFormat),
coverUrl: await this.fileToPosterUrl(coverPath, 'image/jpeg', posterFormat),
thumbUrl: await this.fileToPosterUrl(thumbPath, 'image/jpeg', posterFormat),
exists: true
}
}
@@ -506,21 +547,21 @@ class VideoService {
const realVideoMd5 = this.normalizeVideoLookupKey(normalizedMd5) || normalizedMd5
const videoBaseDir = this.resolveVideoBaseDir(dbPath, wxid)
if (!existsSync(videoBaseDir)) {
if (!(await pathExists(videoBaseDir))) {
const miss = { exists: false }
this.writeTimedCache(this.videoInfoCache, cacheKey, miss, this.videoInfoCacheTtlMs, this.maxCacheEntries)
return miss
}
const index = this.getOrBuildVideoIndex(videoBaseDir)
const indexed = this.getVideoInfoFromIndex(index, realVideoMd5, includePoster, posterFormat)
const index = await this.getOrBuildVideoIndex(videoBaseDir)
const indexed = await this.getVideoInfoFromIndex(index, realVideoMd5, includePoster, posterFormat)
if (indexed) {
const withPoster = await this.ensurePoster(indexed, includePoster, posterFormat)
this.writeTimedCache(this.videoInfoCache, cacheKey, withPoster, this.videoInfoCacheTtlMs, this.maxCacheEntries)
return withPoster
}
const fallback = this.fallbackScanVideo(videoBaseDir, realVideoMd5, includePoster, posterFormat)
const fallback = await this.fallbackScanVideo(videoBaseDir, realVideoMd5, includePoster, posterFormat)
if (fallback) {
const withPoster = await this.ensurePoster(fallback, includePoster, posterFormat)
this.writeTimedCache(this.videoInfoCache, cacheKey, withPoster, this.videoInfoCacheTtlMs, this.maxCacheEntries)
@@ -541,6 +582,38 @@ class VideoService {
}
}
async getVideoInfoBatch(
videoMd5List: string[],
options?: { includePoster?: boolean; posterFormat?: PosterFormat }
): Promise<Array<{ index: number; md5: string; info: VideoInfo }>> {
const rows: Array<{ index: number; md5: string; info: VideoInfo }> = new Array(videoMd5List.length)
const seen = new Map<string, Promise<VideoInfo>>()
for (let index = 0; index < videoMd5List.length; index += 1) {
const md5 = String(videoMd5List[index] || '').trim().toLowerCase()
if (!md5) {
rows[index] = { index, md5, info: { exists: false } }
continue
}
let task = seen.get(md5)
if (!task) {
task = this.getVideoInfo(md5, options)
seen.set(md5, task)
}
rows[index] = { index, md5, info: { exists: false } }
}
await Promise.all(rows.map(async (row, index) => {
if (!row.md5) return
try {
rows[index] = { ...row, info: await seen.get(row.md5)! }
} catch {
rows[index] = { ...row, info: { exists: false } }
}
}))
return rows
}
/**
* 根据消息内容解析视频MD5
*/
+402 -48
View File
@@ -1,5 +1,6 @@
import { join, dirname, basename } from 'path'
import { appendFileSync, existsSync, mkdirSync, readdirSync, statSync, readFileSync } from 'fs'
import { existsSync, mkdirSync, readdirSync, statSync, readFileSync } from 'fs'
import { appendFile } from 'fs/promises'
import { tmpdir } from 'os'
import * as fzstd from 'fzstd'
import { expandHomePath } from '../utils/pathUtils'
@@ -24,6 +25,23 @@ function cleanAccountDirName(dirName: string): string {
return trimmed
}
type MediaStreamPageCacheEntry = {
key: string
startOffset: number
endOffset: number
items: any[]
hasMore: boolean
updatedAt: number
}
type MediaStreamInflightEntry = {
startOffset: number
endOffset: number
promise: Promise<void>
resolve: () => void
finished: boolean
}
export class WcdbCore {
private resourcesPath: string | null = null
private userDataPath: string | null = null
@@ -136,6 +154,9 @@ export class WcdbCore {
private monitorPipePath: string = ''
private displayNameCache: Map<string, { displayName: string; updatedAt: number }> = new Map()
private readonly displayNameCacheTtlMs = 10 * 60 * 1000
private readonly displayNameCacheMaxEntries = 20000
private avatarUrlCache: Map<string, { url?: string; updatedAt: number }> = new Map()
private readonly avatarCacheTtlMs = 10 * 60 * 1000
private imageHardlinkCache: Map<string, { result: { success: boolean; data?: any; error?: string }; updatedAt: number }> = new Map()
@@ -145,10 +166,18 @@ export class WcdbCore {
private mediaStreamSessionCache: Array<{ sessionId: string; displayName: string; sortTimestamp: number }> | null = null
private mediaStreamSessionCacheAt = 0
private readonly mediaStreamSessionCacheTtlMs = 12 * 1000
private mediaStreamPageCache: Map<string, MediaStreamPageCacheEntry[]> = new Map()
private mediaStreamInflight = new Map<string, MediaStreamInflightEntry[]>()
private readonly mediaStreamPageCacheTtlMs = 30 * 1000
private readonly mediaStreamPageCacheMaxEntries = 16
private logTimer: NodeJS.Timeout | null = null
private logFlushTimer: ReturnType<typeof setTimeout> | null = null
private pendingLogLines: string[] = []
private lastLogTail: string | null = null
private lastResolvedLogPath: string | null = null
private lastCursorForceReopenAt = 0
private readonly maxPendingLogLines = 1200
private readonly logFlushDelayMs = 200
private readonly cursorForceReopenCooldownMs = 15000
setPaths(resourcesPath: string, userDataPath: string): void {
@@ -371,28 +400,52 @@ export class WcdbCore {
private writeLog(message: string, force = false): void {
if (!force && !this.isLogEnabled()) return
const line = `[${new Date().toISOString()}] ${message}`
const line = `[${new Date().toISOString()}] ${message}\n`
this.pendingLogLines.push(line)
while (this.pendingLogLines.length > this.maxPendingLogLines) {
this.pendingLogLines.shift()
}
if (this.logFlushTimer) return
this.logFlushTimer = setTimeout(() => {
this.logFlushTimer = null
void this.flushPendingLogs()
}, this.logFlushDelayMs)
}
private getLogFileCandidates(): string[] {
const candidates: string[] = []
if (this.userDataPath) candidates.push(join(this.userDataPath, 'logs', 'wcdb.log'))
if (process.env.WCDB_LOG_DIR) candidates.push(join(process.env.WCDB_LOG_DIR, 'logs', 'wcdb.log'))
candidates.push(join(process.cwd(), 'logs', 'wcdb.log'))
candidates.push(join(tmpdir(), 'weflow-wcdb.log'))
return Array.from(new Set(candidates))
}
const uniq = Array.from(new Set(candidates))
for (const filePath of uniq) {
private async flushPendingLogs(): Promise<void> {
if (this.pendingLogLines.length === 0) return
const lines = this.pendingLogLines.splice(0, this.pendingLogLines.length).join('')
const candidates = this.getLogFileCandidates()
for (const filePath of candidates) {
try {
const dir = dirname(filePath)
if (!existsSync(dir)) mkdirSync(dir, { recursive: true })
appendFileSync(filePath, line + '\n', { encoding: 'utf8' })
await appendFile(filePath, lines, { encoding: 'utf8' })
this.lastResolvedLogPath = filePath
return
} catch (e) {
console.error(`[wcdbCore] writeLog failed path=${filePath}:`, e)
console.error(`[wcdbCore] flushPendingLogs failed path=${filePath}:`, e)
}
}
console.error('[wcdbCore] writeLog failed for all candidates:', uniq.join(' | '))
console.error('[wcdbCore] flushPendingLogs failed for all candidates:', candidates.join(' | '))
if (this.pendingLogLines.length > 0 && this.logFlushTimer === null) {
this.logFlushTimer = setTimeout(() => {
this.logFlushTimer = null
void this.flushPendingLogs()
}, this.logFlushDelayMs)
}
}
private formatSqlForLog(sql: string, maxLen = 240): string {
@@ -1545,6 +1598,172 @@ export class WcdbCore {
this.mediaStreamSessionCacheAt = 0
}
private clearDisplayNameCache(): void {
this.displayNameCache.clear()
}
private writeDisplayNameCache(username: string, displayName: string, now = Date.now()): void {
const normalizedUsername = String(username || '').trim()
const normalizedDisplayName = String(displayName || '').trim()
if (!normalizedUsername || !normalizedDisplayName) return
this.displayNameCache.set(normalizedUsername, {
displayName: normalizedDisplayName,
updatedAt: now
})
if (this.displayNameCache.size <= this.displayNameCacheMaxEntries) return
const expiresBefore = now - this.displayNameCacheTtlMs
for (const [key, entry] of this.displayNameCache) {
if (entry.updatedAt < expiresBefore) this.displayNameCache.delete(key)
}
while (this.displayNameCache.size > this.displayNameCacheMaxEntries) {
const oldestKey = this.displayNameCache.keys().next().value as string | undefined
if (!oldestKey) break
this.displayNameCache.delete(oldestKey)
}
}
private clearMediaStreamPageCache(): void {
this.mediaStreamPageCache.clear()
for (const entries of this.mediaStreamInflight.values()) {
for (const entry of entries) {
entry.finished = true
entry.resolve()
}
}
this.mediaStreamInflight.clear()
}
private pruneMediaStreamPageCache(now = Date.now()): void {
for (const [key, entries] of this.mediaStreamPageCache.entries()) {
const liveEntries = entries.filter((entry) => now - entry.updatedAt <= this.mediaStreamPageCacheTtlMs)
if (liveEntries.length > 0) {
this.mediaStreamPageCache.set(key, liveEntries)
} else {
this.mediaStreamPageCache.delete(key)
}
}
while (this.countMediaStreamPageCacheEntries() > this.mediaStreamPageCacheMaxEntries) {
let oldestKey = ''
let oldestIndex = -1
let oldestUpdatedAt = Number.POSITIVE_INFINITY
for (const [key, entries] of this.mediaStreamPageCache.entries()) {
for (let index = 0; index < entries.length; index += 1) {
const updatedAt = entries[index].updatedAt
if (updatedAt >= oldestUpdatedAt) continue
oldestUpdatedAt = updatedAt
oldestKey = key
oldestIndex = index
}
}
if (!oldestKey || oldestIndex < 0) break
const entries = this.mediaStreamPageCache.get(oldestKey) || []
entries.splice(oldestIndex, 1)
if (entries.length > 0) {
this.mediaStreamPageCache.set(oldestKey, entries)
} else {
this.mediaStreamPageCache.delete(oldestKey)
}
}
}
private countMediaStreamPageCacheEntries(): number {
let count = 0
for (const entries of this.mediaStreamPageCache.values()) {
count += entries.length
}
return count
}
private getMediaStreamPageCacheKey(options: {
sessionId?: string
mediaType?: string
beginTimestamp?: number
endTimestamp?: number
}): string {
return JSON.stringify([
this.currentPath || '',
this.currentWxid || '',
String(options.sessionId || '').trim(),
String(options.mediaType || 'all').trim(),
Number(options.beginTimestamp || 0) || 0,
Number(options.endTimestamp || 0) || 0
])
}
private readMediaStreamPageCache(key: string, offset: number, limit: number, startedAt = Date.now()): {
success: true
items: any[]
hasMore: boolean
nextOffset: number
streamSource: 'pageCache'
pageCacheHit: true
nativeRows: number
elapsedMs: number
} | null {
const now = Date.now()
this.pruneMediaStreamPageCache(now)
const entries = this.mediaStreamPageCache.get(key) || []
const entry = entries.find((candidate) => {
if (offset < candidate.startOffset) return false
if (offset > candidate.endOffset) return false
if (offset === candidate.endOffset && candidate.hasMore) return false
const start = offset - candidate.startOffset
const available = Math.max(0, candidate.items.length - start)
return available >= limit || !candidate.hasMore
})
if (!entry) return null
const start = offset - entry.startOffset
const available = Math.max(0, entry.items.length - start)
if (available <= 0) {
if (entry.hasMore) return null
return {
success: true,
items: [],
hasMore: false,
nextOffset: offset,
streamSource: 'pageCache',
pageCacheHit: true,
nativeRows: entry.items.length,
elapsedMs: Date.now() - startedAt
}
}
if (available < limit && entry.hasMore) return null
const items = entry.items.slice(start, start + limit)
entry.updatedAt = now
const nextOffset = offset + items.length
return {
success: true,
items,
hasMore: entry.hasMore || nextOffset < entry.endOffset,
nextOffset,
streamSource: 'pageCache',
pageCacheHit: true,
nativeRows: entry.items.length,
elapsedMs: Date.now() - startedAt
}
}
private writeMediaStreamPageCache(key: string, offset: number, items: any[], hasMore: boolean): void {
const now = Date.now()
this.pruneMediaStreamPageCache(now)
const nextEntry = {
key,
startOffset: offset,
endOffset: offset + items.length,
items,
hasMore,
updatedAt: now
}
const entries = (this.mediaStreamPageCache.get(key) || [])
.filter((entry) => entry.startOffset !== nextEntry.startOffset || entry.endOffset !== nextEntry.endOffset)
entries.push(nextEntry)
this.mediaStreamPageCache.set(key, entries)
this.pruneMediaStreamPageCache(now)
}
isReady(): boolean {
return this.ensureReady()
}
@@ -1621,6 +1840,8 @@ export class WcdbCore {
this.currentWxid = wxid
this.currentDbStoragePath = dbStoragePath
this.initialized = true
this.clearDisplayNameCache()
this.clearMediaStreamPageCache()
lastDllInitError = null
if (this.wcdbSetMyWxid && wxid) {
try {
@@ -1666,7 +1887,9 @@ export class WcdbCore {
this.currentDbStoragePath = null
this.initialized = false
this.clearHardlinkCaches()
this.clearDisplayNameCache()
this.clearMediaStreamSessionCache()
this.clearMediaStreamPageCache()
this.stopLogPolling()
}
}
@@ -1740,6 +1963,7 @@ export class WcdbCore {
return { success: false, error: message || `一键已读失败: ${result}` }
}
this.clearMediaStreamSessionCache()
this.clearMediaStreamPageCache()
this.writeLog('markAllSessionsRead ok')
return { success: true }
} catch (e) {
@@ -2098,10 +2322,17 @@ export class WcdbCore {
}>
hasMore?: boolean
nextOffset?: number
streamSource?: 'native' | 'pageCache' | 'inflight'
pageCacheHit?: boolean
inflightMerged?: boolean
nativeLimit?: number
nativeRows?: number
elapsedMs?: number
error?: string
}> {
if (!this.ensureReady()) return { success: false, error: 'WCDB 未连接' }
if (!this.wcdbScanMediaStream) return { success: false, error: '当前数据服务版本不支持资源扫描,请先更新 wcdb 数据服务' }
let finishMediaStreamInflight: (() => void) | null = null
try {
const toInt = (value: unknown): number => {
const n = Number(value || 0)
@@ -2295,7 +2526,12 @@ export class WcdbCore {
const fallback = candidates.find((item) => item.length >= 16 && item.length <= 64)
return String(fallback || '').toLowerCase()
}
const extractImageDatName = (row: Record<string, any>, content: string): string => {
const extractImageDatName = (
row: Record<string, any>,
content: string,
packedPayload?: string,
allowPackedFallback = true
): string => {
const direct = pickString(row, [
'image_path',
'imagePath',
@@ -2313,21 +2549,11 @@ export class WcdbCore {
const normalizedXml = normalizeDatBase(xmlCandidate)
if (normalizedXml) return normalizedXml
const packedRaw = pickRaw(row, [
'packed_info_data',
'packedInfoData',
'packed_info_blob',
'packedInfoBlob',
'packed_info',
'packedInfo',
'BytesExtra',
'bytes_extra',
'WCDB_CT_packed_info',
'reserved0',
'Reserved0',
'WCDB_CT_Reserved0'
])
const packedText = decodePackedToPrintable(packedRaw)
if (!allowPackedFallback) return ''
const packedText = packedPayload === undefined
? extractPackedPayload(row)
: packedPayload
if (packedText) {
const datLike = /([0-9a-fA-F]{8,})(?:\.t)?\.dat/i.exec(packedText)
if (datLike?.[1]) return String(datLike[1]).toLowerCase()
@@ -2371,6 +2597,62 @@ export class WcdbCore {
const endTimestamp = Math.max(0, toInt(options?.endTimestamp))
const offset = Math.max(0, toInt(options?.offset))
const limit = Math.min(1200, Math.max(40, toInt(options?.limit) || 240))
const nativeLimit = Math.min(1200, Math.max(limit, Math.min(Math.max(limit * 4, 320), 720)))
const pageCacheKey = this.getMediaStreamPageCacheKey({
sessionId: requestedSessionId,
mediaType,
beginTimestamp,
endTimestamp
})
const startedAt = Date.now()
const cachedPage = this.readMediaStreamPageCache(pageCacheKey, offset, limit, startedAt)
if (cachedPage) return cachedPage
const requestedEndOffset = offset + limit
const existingInflight = (this.mediaStreamInflight.get(pageCacheKey) || []).find((entry) => (
offset >= entry.startOffset &&
requestedEndOffset <= entry.endOffset
))
if (existingInflight) {
await existingInflight.promise.catch(() => { })
const inflightPage = this.readMediaStreamPageCache(pageCacheKey, offset, limit, startedAt)
if (inflightPage) {
return {
...inflightPage,
streamSource: 'inflight',
pageCacheHit: false,
inflightMerged: true,
elapsedMs: Date.now() - startedAt
}
}
}
let completeInflight: (() => void) | null = null
const inflightPromise = new Promise<void>((resolve) => {
completeInflight = resolve
})
const inflightEntry: MediaStreamInflightEntry = {
startOffset: offset,
endOffset: offset + nativeLimit,
promise: inflightPromise,
resolve: () => completeInflight?.(),
finished: false
}
const inflightEntries = this.mediaStreamInflight.get(pageCacheKey) || []
inflightEntries.push(inflightEntry)
this.mediaStreamInflight.set(pageCacheKey, inflightEntries)
const finishInflight = () => {
if (inflightEntry.finished) return
inflightEntry.finished = true
const entries = this.mediaStreamInflight.get(pageCacheKey) || []
const nextEntries = entries.filter((entry) => entry !== inflightEntry)
if (nextEntries.length > 0) {
this.mediaStreamInflight.set(pageCacheKey, nextEntries)
} else {
this.mediaStreamInflight.delete(pageCacheKey)
}
inflightEntry.resolve()
}
finishMediaStreamInflight = finishInflight
const getSessionRows = async (): Promise<{
success: boolean
@@ -2425,12 +2707,15 @@ export class WcdbCore {
} else {
const sessionsRowsRes = await getSessionRows()
if (!sessionsRowsRes.success || !Array.isArray(sessionsRowsRes.rows)) {
finishInflight()
return { success: false, error: sessionsRowsRes.error || '读取会话失败' }
}
sessionRows = sessionsRowsRes.rows
}
if (sessionRows.length === 0) {
this.writeMediaStreamPageCache(pageCacheKey, offset, [], false)
finishInflight()
return { success: true, items: [], hasMore: false, nextOffset: offset }
}
const sessionNameMap = new Map(sessionRows.map((row) => [row.sessionId, row.displayName || row.sessionId]))
@@ -2444,16 +2729,20 @@ export class WcdbCore {
mediaTypeCode,
beginTimestamp,
endTimestamp,
limit,
nativeLimit,
offset,
outPtr,
outHasMore
)
if (result !== 0 || !outPtr[0]) {
finishInflight()
return { success: false, error: `扫描资源失败: ${result}` }
}
const jsonStr = this.decodeJsonPtr(outPtr[0])
if (!jsonStr) return { success: false, error: '解析资源失败' }
if (!jsonStr) {
finishInflight()
return { success: false, error: '解析资源失败' }
}
const rows = JSON.parse(jsonStr)
const list = Array.isArray(rows) ? rows as Array<Record<string, any>> : []
@@ -2490,7 +2779,6 @@ export class WcdbCore {
if (!rawMessageContent && !rawCompressContent) return ''
return decodeMessageContent(rawMessageContent, rawCompressContent)
}
const packedPayload = extractPackedPayload(row)
const imageMd5ByColumn = pickString(row, ['image_md5', 'imageMd5'])
const videoMd5ByColumn = pickString(row, ['video_md5', 'videoMd5', 'raw_md5', 'rawMd5'])
const packedRaw = pickRaw(row, [
@@ -2507,6 +2795,11 @@ export class WcdbCore {
'Reserved0',
'WCDB_CT_Reserved0'
])
let packedPayload: string | undefined
const getPackedPayload = (): string => {
if (packedPayload === undefined) packedPayload = extractPackedPayload(row)
return packedPayload
}
let content = ''
let imageMd5: string | undefined
@@ -2514,24 +2807,25 @@ export class WcdbCore {
let videoMd5: string | undefined
if (localType === 3) {
imageMd5 = imageMd5ByColumn || extractHexMd5(packedPayload) || undefined
imageDatName = extractImageDatName(row, '') || undefined
imageMd5 = imageMd5ByColumn || undefined
imageDatName = extractImageDatName(row, '', undefined, false) || undefined
if (!imageMd5) imageMd5 = extractHexMd5(getPackedPayload()) || undefined
if (!imageMd5 || !imageDatName) {
content = decodeContentIfNeeded()
if (!imageMd5) imageMd5 = extractImageMd5(content) || extractHexMd5(packedPayload) || undefined
if (!imageDatName) imageDatName = extractImageDatName(row, content) || undefined
if (!imageMd5) imageMd5 = extractImageMd5(content) || extractHexMd5(getPackedPayload()) || undefined
if (!imageDatName) imageDatName = extractImageDatName(row, content, getPackedPayload()) || undefined
}
} else if (localType === 43) {
videoMd5 =
extractVideoFileNameFromPackedRaw(packedRaw) ||
normalizeVideoFileToken(videoMd5ByColumn) ||
extractHexMd5(packedPayload) ||
extractHexMd5(getPackedPayload()) ||
undefined
if (!videoMd5) {
content = decodeContentIfNeeded()
videoMd5 =
normalizeVideoFileToken(extractVideoMd5(content)) ||
extractHexMd5(packedPayload) ||
extractHexMd5(getPackedPayload()) ||
undefined
} else if (useRawMessageContent) {
// 占位态标题只依赖简单 XML,已带 md5 时不做额外解压
@@ -2547,7 +2841,7 @@ export class WcdbCore {
serverId: pickString(row, ['server_id', 'serverId']) || undefined,
createTime: toInt(row.create_time ?? row.createTime),
localType,
senderUsername: pickString(row, ['sender_username', 'senderUsername']) || undefined,
senderUsername: pickString(row, ['sender_username', 'senderUsername', 'real_sender_id', 'realSenderId']) || undefined,
isSend: row.is_send === null || row.is_send === undefined ? null : toInt(row.is_send),
imageMd5,
imageDatName,
@@ -2580,13 +2874,27 @@ export class WcdbCore {
}
}
const nativeHasMore = Number(outHasMore[0]) > 0
const canAdvance = items.length > 0
this.writeMediaStreamPageCache(pageCacheKey, offset, items, canAdvance && nativeHasMore)
const pageItems = items.slice(0, limit)
const nextOffset = offset + pageItems.length
const hasMore = nextOffset > offset && (nativeHasMore || pageItems.length < items.length)
finishInflight()
return {
success: true,
items,
hasMore: Number(outHasMore[0]) > 0,
nextOffset: offset + items.length
items: pageItems,
hasMore,
nextOffset,
streamSource: 'native',
pageCacheHit: false,
nativeLimit,
nativeRows: items.length,
elapsedMs: Date.now() - startedAt
}
} catch (e) {
finishMediaStreamInflight?.()
return { success: false, error: String(e) }
}
}
@@ -2597,14 +2905,35 @@ export class WcdbCore {
}
if (usernames.length === 0) return { success: true, map: {} }
try {
const now = Date.now()
const resultMap: Record<string, string> = {}
const toFetch: string[] = []
const seen = new Set<string>()
for (const username of usernames) {
const normalizedUsername = String(username || '').trim()
if (!normalizedUsername || seen.has(normalizedUsername)) continue
seen.add(normalizedUsername)
const cached = this.displayNameCache.get(normalizedUsername)
if (cached && now - cached.updatedAt < this.displayNameCacheTtlMs) {
resultMap[normalizedUsername] = cached.displayName
continue
}
toFetch.push(normalizedUsername)
}
if (toFetch.length === 0) return { success: true, map: resultMap }
if (process.platform === 'darwin') {
const uniq = Array.from(new Set(usernames.map((x) => String(x || '').trim()).filter(Boolean)))
if (uniq.length === 0) return { success: true, map: {} }
const inList = uniq.map((u) => `'${u.replace(/'/g, "''")}'`).join(',')
const inList = toFetch.map((u) => `'${u.replace(/'/g, "''")}'`).join(',')
const sql = `SELECT * FROM contact WHERE username IN (${inList})`
const q = await this.execQuery('contact', null, sql)
if (!q.success) return { success: false, error: q.error || '获取昵称失败' }
const map: Record<string, string> = {}
if (!q.success) {
if (Object.keys(resultMap).length > 0) {
return { success: true, map: resultMap, error: q.error || '获取昵称失败' }
}
return { success: false, error: q.error || '获取昵称失败' }
}
for (const row of (q.rows || []) as Array<Record<string, any>>) {
const username = this.pickFirstStringField(row, ['username', 'user_name', 'userName'])
if (!username) continue
@@ -2613,31 +2942,56 @@ export class WcdbCore {
'nick_name', 'nickName', 'nickname', 'NickName',
'alias', 'Alias'
]) || username
map[username] = display
resultMap[username] = display
this.writeDisplayNameCache(username, display, now)
}
// 保证每个请求用户名至少有回退值
for (const u of uniq) {
if (!map[u]) map[u] = u
for (const u of toFetch) {
if (!resultMap[u]) {
resultMap[u] = u
this.writeDisplayNameCache(u, u, now)
}
}
return { success: true, map }
return { success: true, map: resultMap }
}
// 让出控制权,避免阻塞事件循环
await new Promise(resolve => setImmediate(resolve))
const outPtr = [null as any]
const result = this.wcdbGetDisplayNames(this.handle, JSON.stringify(usernames), outPtr)
const result = this.wcdbGetDisplayNames(this.handle, JSON.stringify(toFetch), outPtr)
//数据服务调用后再次让出控制权
await new Promise(resolve => setImmediate(resolve))
if (result !== 0 || !outPtr[0]) {
if (Object.keys(resultMap).length > 0) {
return { success: true, map: resultMap, error: `获取昵称失败: ${result}` }
}
return { success: false, error: `获取昵称失败: ${result}` }
}
const jsonStr = this.decodeJsonPtr(outPtr[0])
if (!jsonStr) return { success: false, error: '解析昵称失败' }
if (!jsonStr) {
if (Object.keys(resultMap).length > 0) {
return { success: true, map: resultMap, error: '解析昵称失败' }
}
return { success: false, error: '解析昵称失败' }
}
const map = JSON.parse(jsonStr)
return { success: true, map }
if (map && typeof map === 'object') {
for (const username of toFetch) {
const display = String((map as Record<string, unknown>)[username] || '').trim()
if (!display) continue
resultMap[username] = display
this.writeDisplayNameCache(username, display, now)
}
}
for (const username of toFetch) {
if (resultMap[username]) continue
resultMap[username] = username
this.writeDisplayNameCache(username, username, now)
}
return { success: true, map: resultMap }
} catch (e) {
return { success: false, error: String(e) }
}
+6
View File
@@ -308,6 +308,12 @@ export class WcdbService {
}>
hasMore?: boolean
nextOffset?: number
streamSource?: 'native' | 'pageCache' | 'inflight'
pageCacheHit?: boolean
inflightMerged?: boolean
nativeLimit?: number
nativeRows?: number
elapsedMs?: number
error?: string
}> {
return this.callWorker('getMediaStream', { options })
+65 -3
View File
@@ -43,6 +43,7 @@ export interface WeliveExportRequest {
outputDir: string
exportsDir?: string
mediaDir?: string
mediaTypes?: Array<'image' | 'voice' | 'video' | 'emoji' | 'file'>
emojiCacheDir?: string
formattedDir?: string
format?: string
@@ -76,6 +77,7 @@ export interface WeliveExportResult {
rawResult?: WeliveExportEvent
stderr?: string
error?: string
diagnostics?: Record<string, unknown>
}
const platformDir = () => {
@@ -90,6 +92,36 @@ const archDir = () => {
const executableName = () => process.platform === 'win32' ? 'welive.exe' : 'welive'
const formatExitCode = (code: number | null, signal: NodeJS.Signals | null | undefined) => {
if (code === null) return signal ? `signal:${signal}` : 'unknown'
const unsigned = code >>> 0
const signed = unsigned > 0x7fffffff ? unsigned - 0x100000000 : unsigned
const hex = `0x${unsigned.toString(16).toUpperCase().padStart(8, '0')}`
return signed === code
? `${code} (${hex})`
: `${code} (${hex}, signed ${signed})`
}
const summarizeEvent = (event?: WeliveExportEvent) => {
if (!event) return undefined
const summary: Record<string, unknown> = { type: event.type }
for (const key of ['phase', 'label', 'session_id', 'current', 'total', 'exported_messages', 'estimated_total_messages']) {
if ((event as any)[key] !== undefined) summary[key] = (event as any)[key]
}
return summary
}
const formatDiagnostics = (diagnostics: Record<string, unknown>) => {
const parts = [
diagnostics.exit ? `exit=${diagnostics.exit}` : '',
diagnostics.exe ? `exe=${diagnostics.exe}` : '',
diagnostics.platform ? `platform=${diagnostics.platform}` : '',
diagnostics.arch ? `arch=${diagnostics.arch}` : '',
diagnostics.lastEvent ? `lastEvent=${JSON.stringify(diagnostics.lastEvent)}` : ''
].filter(Boolean)
return parts.length > 0 ? `[welive-diagnostics] ${parts.join(' ')}` : ''
}
export function resolveWeliveExecutable(resourcesPath: string, appPath?: string): string | null {
const platform = platformDir()
const arch = archDir()
@@ -137,6 +169,8 @@ export async function runWeliveExport(options: RunWeliveExportOptions): Promise<
let stderr = ''
let settled = false
let rawResult: WeliveExportEvent | undefined
let lastEvent: WeliveExportEvent | undefined
const eventTrail: Array<Record<string, unknown>> = []
const failedSessionErrors: Record<string, string> = {}
const sessionOutputPaths: Record<string, string> = {}
const rawSessionOutputPaths: Record<string, string> = {}
@@ -156,6 +190,12 @@ export async function runWeliveExport(options: RunWeliveExportOptions): Promise<
options.signal?.addEventListener('abort', abort)
const handleEvent = (event: WeliveExportEvent) => {
lastEvent = event
const eventSummary = summarizeEvent(event)
if (eventSummary) {
eventTrail.push(eventSummary)
if (eventTrail.length > 8) eventTrail.shift()
}
options.onEvent?.(event)
if (event.type === 'session_error') {
const sessionId = String(event.session_id || '').trim()
@@ -224,14 +264,35 @@ export async function runWeliveExport(options: RunWeliveExportOptions): Promise<
sessionOutputPaths,
rawSessionOutputPaths,
rawResult,
error: error.message
error: error.message,
diagnostics: {
exe,
platform: process.platform,
arch: process.arch,
pid: child.pid,
lastEvent: summarizeEvent(lastEvent),
eventTrail
}
})
})
child.on('exit', (code) => {
child.on('exit', (code, signal) => {
drainStdout()
const failedSessionIds = Object.keys(failedSessionErrors)
const success = code === 0 && rawResult?.type === 'result' && rawResult.success !== false
const diagnostics = {
exit: formatExitCode(code, signal),
code,
signal,
exe,
platform: process.platform,
arch: process.arch,
pid: child.pid,
lastEvent: summarizeEvent(lastEvent),
eventTrail
}
const diagnosticText = success ? '' : formatDiagnostics(diagnostics)
const errorText = stderr.trim() || `WeLive 导出引擎退出码: ${formatExitCode(code, signal)}`
finish({
success,
successCount: Number((rawResult as any)?.success_count ?? Object.keys(sessionOutputPaths).length),
@@ -241,7 +302,8 @@ export async function runWeliveExport(options: RunWeliveExportOptions): Promise<
sessionOutputPaths,
rawSessionOutputPaths,
rawResult,
error: success ? undefined : stderr.trim() || `WeLive 导出引擎退出码: ${code}`
diagnostics,
error: success ? undefined : [errorText, diagnosticText].filter(Boolean).join('\n')
})
})