U:调整后端和前端队列

This commit is contained in:
jimmy committed 2025-11-16 22:57:00 +08:00
1 parent c42628f38d
commit 98feb62cde
7 files changed
+385 -39

No files matched your search

+42
View File
@@ -0,0 +1,42 @@
# Repository Guidelines
This document is the top‑level guide for working in the `go-argus/src` monorepo. Project‑specific details live in `rear/AGENTS.md` and `front/argus-front/AGENTS.md`.
## Project Structure & Organization
- `rear/` – Go backend API and worker services (main entry: `rear/main.go`).
- `front/argus-front/` – Vue 3 + Vite frontend UI.
- `service/` – Experimental Python services and AI demos (not in the main build).
- `design/` – HTML/UX prototypes; treat files here as reference only.
- `codex.md`, `todo.md` – Architecture notes and ongoing work items.
## Build, Test & Development Commands
- Backend: `cd rear && go mod download && go run main.go` to run, `go test ./...` for Go tests, `go build -o rear.exe .` for a local binary.
- Frontend: `cd front/argus-front && pnpm install && pnpm dev` for dev, `pnpm build` for production build, `pnpm test:unit` for unit tests.
- Python experiments: run only in isolation from `service/src-python-v1` after setting up a virtualenv; keep any new dependencies local and documented.
## Coding Style & Naming Conventions
- Backend: follow Go conventions and `rear/AGENTS.md` (use `gofmt`, small lowercase package names, clear separation of API, services, and repositories).
- Frontend: follow ESLint + Prettier rules defined in `front/argus-front` (2‑space indent, TypeScript, Vue `<script setup>` where appropriate).
- Python: prefer explicit, typed code and standard tooling (e.g., `black`, `ruff`) if introduced; keep modules small and focused.
## Testing Guidelines
- For backend changes, add or update Go tests near the code (`*_test.go`) and use focused runs like `go test ./internal/...`.
- For frontend changes, cover behavior with Vitest tests close to components or in `__tests__/`.
- Python and design prototypes do not require strict test coverage but should remain manually verifiable.
## Commit & Pull Request Guidelines
- Use clear, conventional commit messages where possible, e.g., `feat(front): add timeline filters`, `fix(rear): handle missing exif`.
- Keep each commit logically scoped; avoid mixing backend, frontend, and service changes unless necessary.
- PRs should describe motivation, key changes, affected areas (`rear`, `front`, `service`), and how to test (commands, URLs, sample requests or screenshots).
## Agent-Specific Instructions
- Before changing code, read `codex.md` and any `AGENTS.md` in the relevant subdirectory.
- Prefer extending existing modules and patterns over introducing new top‑level folders or frameworks.
- Do not upgrade core dependencies, change build pipelines, or modify external tool usage without an explicit user request and matching documentation updates.
@@ -330,6 +330,32 @@ const isDevelopment = import.meta.env.DEV
// 文件系统 API 集成
const fileSystemAPI = useFileSystemAPI()
const THUMBNAIL_SIZES = {
grid: 300,
list: 200
}
const MAX_VISIBLE_THUMBNAILS = 160
const MAX_THUMBNAIL_CONCURRENCY = 6
interface ThumbnailRequest {
key: string
path: string
size: number
}
const thumbnailCache = reactive<Record<string, string>>({})
const thumbnailQueue: ThumbnailRequest[] = []
const inFlightThumbnailKeys = new Set<string>()
const thumbnailControllers = new Map<string, AbortController>()
let activeThumbnailWorkers = 0
const THUMBNAIL_BASE_STYLE = {
backgroundColor: '#f3f4f6',
backgroundSize: 'cover',
backgroundPosition: 'center',
backgroundRepeat: 'no-repeat'
} as const
const props = defineProps<{
paneId: PaneId
viewMode: ViewMode
@@ -581,6 +607,18 @@ const renderItems = computed(() => {
: gridVirtual.visibleItems.value
})
watch(
() => ({
items: renderItems.value,
mode: props.viewMode
}),
({ items, mode }) => {
const targetSize = mode === 'list' ? THUMBNAIL_SIZES.list : THUMBNAIL_SIZES.grid
scheduleVisibleThumbnails(items ?? [], targetSize)
},
{ immediate: true }
)
// 虚拟滚动容器样式
const virtualContainerStyle = computed(() => {
if (!shouldUseVirtualScroll.value) {
@@ -1600,6 +1638,8 @@ onUnmounted(() => {
observer.disconnect()
}
}
cleanupThumbnailResources()
})
// 键盘导航
@@ -1922,20 +1962,133 @@ function handlePreviewClose() {
function getThumbnailStyle(filePath: string, isListView = false) {
if (!filePath) return {}
const size = isListView ? 200 : 300 // 列表视图用小尺寸,网格视图用大尺寸
const size = isListView ? THUMBNAIL_SIZES.list : THUMBNAIL_SIZES.grid
const cacheKey = buildThumbnailCacheKey(filePath, size)
const cachedUrl = thumbnailCache[cacheKey]
// 使用 httpClient 的 baseURL,确保与其他 API 请求保持一致
const baseURL = httpClient.getAxiosInstance().defaults.baseURL || 'http://127.0.0.1:9484/api'
const thumbnailUrl = `${baseURL}/v1/photo/preview?path=${encodeURIComponent(filePath)}&size=${size}`
if (!cachedUrl) {
enqueueThumbnailRequest(filePath, size, cacheKey)
return THUMBNAIL_BASE_STYLE
}
return {
backgroundImage: `url("${thumbnailUrl}")`,
backgroundSize: 'cover',
backgroundPosition: 'center',
backgroundRepeat: 'no-repeat'
...THUMBNAIL_BASE_STYLE,
backgroundImage: `url("${cachedUrl}")`
}
}
function scheduleVisibleThumbnails(items: FileItem[], size: number) {
if (!items || items.length === 0) {
return
}
const candidates = items
.filter(item => item.type === 'photo' && item.path)
.slice(0, MAX_VISIBLE_THUMBNAILS)
if (!candidates.length) {
return
}
const allowedKeys = new Set<string>()
candidates.forEach(item => {
const key = buildThumbnailCacheKey(item.path!, size)
allowedKeys.add(key)
enqueueThumbnailRequest(item.path!, size, key)
})
prunePendingThumbnailQueue(allowedKeys)
}
function enqueueThumbnailRequest(path: string, size: number, cacheKey?: string) {
if (!path) return
const key = cacheKey ?? buildThumbnailCacheKey(path, size)
if (thumbnailCache[key] || inFlightThumbnailKeys.has(key)) {
return
}
thumbnailQueue.push({ key, path, size })
processThumbnailQueue()
}
function processThumbnailQueue() {
while (activeThumbnailWorkers < MAX_THUMBNAIL_CONCURRENCY && thumbnailQueue.length > 0) {
const task = thumbnailQueue.shift()
if (!task) break
activeThumbnailWorkers++
inFlightThumbnailKeys.add(task.key)
const controller = new AbortController()
thumbnailControllers.set(task.key, controller)
fetchThumbnailForPath(task, controller)
.catch(error => {
if (!controller.signal.aborted) {
console.warn('[FilePane] 缩略图加载失败', task.path, error)
}
})
.finally(() => {
activeThumbnailWorkers--
inFlightThumbnailKeys.delete(task.key)
thumbnailControllers.delete(task.key)
processThumbnailQueue()
})
}
}
async function fetchThumbnailForPath(task: ThumbnailRequest, controller: AbortController): Promise<void> {
const baseURL = httpClient.getAxiosInstance().defaults.baseURL || 'http://127.0.0.1:9484/api'
const encodedPath = encodeURIComponent(task.path)
const requestUrl = `${baseURL}/v1/photo/preview?path=${encodedPath}&size=${task.size}`
const response = await fetch(requestUrl, { signal: controller.signal })
if (!response.ok) {
throw new Error(`HTTP ${response.status}`)
}
const blob = await response.blob()
const objectUrl = URL.createObjectURL(blob)
setThumbnailCacheValue(task.key, objectUrl)
}
function prunePendingThumbnailQueue(allowedKeys: Set<string>) {
for (let i = thumbnailQueue.length - 1; i >= 0; i -= 1) {
if (!allowedKeys.has(thumbnailQueue[i].key)) {
thumbnailQueue.splice(i, 1)
}
}
}
function setThumbnailCacheValue(key: string, url: string) {
const previous = thumbnailCache[key]
if (previous && previous.startsWith('blob:') && previous !== url) {
URL.revokeObjectURL(previous)
}
thumbnailCache[key] = url
}
function buildThumbnailCacheKey(path: string, size: number) {
return `${path}|${size}`
}
function cleanupThumbnailResources() {
thumbnailQueue.length = 0
inFlightThumbnailKeys.clear()
thumbnailControllers.forEach(controller => controller.abort())
thumbnailControllers.clear()
Object.keys(thumbnailCache).forEach(key => {
const url = thumbnailCache[key]
if (url && url.startsWith('blob:')) {
URL.revokeObjectURL(url)
}
delete thumbnailCache[key]
})
}
// 暴露方法给父组件
// 获取选中的项目
function getSelectedItems(): FileItem[] {
@@ -2672,4 +2825,4 @@ defineExpose({
.debug-toggle:active {
transform: scale(0.95);
}
</style>
</style>
+6 -5
View File
@@ -130,8 +130,9 @@ func (api *ImageAPI) GetExif() error {
5. 如果是缩略图,则判断是否存在对应宽度的缩略图,如果存在则返回该路径,如果不存在则生成缩略图并返回(缩略图宽度会是固定的值)
6. 如果传入的参数很大,比如超过原图的宽度,则返回原图路径
*/
func (api *ImageAPI) GetImagePath(longSide int) (string, error) {
ctx := context.Background()
func (api *ImageAPI) GetImagePath(ctx context.Context, longSide int) (string, error) {
// 避免上层请求过早取消导致外部工具失败,同时保留上下文的值
ctx = context.WithoutCancel(ctx)
// 小于等于0获取原图
if longSide <= 0 {
@@ -522,7 +523,7 @@ func (api *ImageAPI) generateAllThumbnails(ctx context.Context) error {
thumbnailSizes := api.getThumbnailSizes()
for _, size := range thumbnailSizes {
thumbnailPath, err := api.GetImagePath(size)
thumbnailPath, err := api.GetImagePath(ctx, size)
if err != nil {
logger.Warn("生成缩略图失败",
zap.String("path", api.path),
@@ -551,7 +552,7 @@ func (api *ImageAPI) generateSmartThumbnails(ctx context.Context, fileSize int64
zap.Ints("sizes", thumbnailSizes))
for _, size := range thumbnailSizes {
thumbnailPath, err := api.GetImagePath(size)
thumbnailPath, err := api.GetImagePath(ctx, size)
if err != nil {
logger.Warn("生成缩略图失败",
zap.String("path", api.path),
@@ -669,7 +670,7 @@ func (api *ImageAPI) HasTiles() bool {
// CompressedImg 压缩图片获取路径
func (api *ImageAPI) CompressedImg(longSide int) (string, error) {
return api.GetImagePath(longSide)
return api.GetImagePath(context.Background(), longSide)
}
// ReadFileWithLimit 读取文件内容(限制大小)
@@ -78,7 +78,7 @@ func (h *BatchThumbnailHandler) GetBatchThumbnails(c *gin.Context) {
mw.SetBoundary(boundary)
defer mw.Close()
ctx := context.Background()
ctx := c.Request.Context()
successCount := 0
errorCount := 0
@@ -142,7 +142,7 @@ func (h *BatchThumbnailHandler) processSingleThumbnail(
}
// 获取缩略图路径
thumbnailPath, err := imageAPI.GetImagePath(size)
thumbnailPath, err := imageAPI.GetImagePath(ctx, size)
if err != nil {
return h.writeThumbnailError(mw, mu, index, filePath, err)
}
+108 -14
View File
@@ -1,8 +1,11 @@
package handler
import (
"context"
"errors"
"fmt"
"net/http"
"path/filepath"
"rear/internal/api"
"rear/internal/config"
"rear/internal/container"
@@ -11,6 +14,7 @@ import (
"rear/internal/repositories"
"strconv"
"strings"
"sync"
"time"
"rear/pkg/logger"
@@ -24,6 +28,25 @@ type PhotoHandler struct {
imgContain *container.TaskContainer
}
const defaultPreviewConcurrency = 6
var (
previewLimiter = make(chan struct{}, defaultPreviewConcurrency)
previewInflightMu sync.Mutex
previewInflightMap = make(map[string]*previewCall)
)
type previewCall struct {
done chan struct{}
result previewResult
}
type previewResult struct {
imagePath string
format string
err error
}
func NewPhotoHandler(container *container.DbContainer, imgContain *container.TaskContainer) *PhotoHandler {
return &PhotoHandler{
container: container,
@@ -189,6 +212,7 @@ func (h *PhotoHandler) GetPhoto(c *gin.Context) {
}
var imagePath string
ctx := c.Request.Context()
switch format {
case "thumbnail":
@@ -203,7 +227,7 @@ func (h *PhotoHandler) GetPhoto(c *gin.Context) {
}
// 使用ImageAPI获取缩略图
imagePath, err = imageAPI.GetImagePath(size)
imagePath, err = imageAPI.GetImagePath(ctx, size)
if err != nil {
logger.Error("获取缩略图失败", zap.String("hash", hash), zap.Error(err))
c.JSON(http.StatusInternalServerError, model.Response{
@@ -553,20 +577,14 @@ func (h *PhotoHandler) GetPhotoPreview(c *gin.Context) {
return
}
// 创建 ImageAPI(会自动计算 Hash 并检测格式)
imageAPI, err := api.NewImageAPI(path)
if err != nil {
logger.Error("创建ImageAPI失败", zap.String("path", path), zap.Error(err))
c.JSON(http.StatusInternalServerError, model.Response{
Code: http.StatusInternalServerError,
Message: "Failed to process image",
})
return
}
// 获取图片路径(ImageAPI 内部会处理格式转换和缩略图生成)
imagePath, err := imageAPI.GetImagePath(size)
reqCtx := c.Request.Context()
imagePath, format, err := h.getPreviewWithDedupe(reqCtx, path, size)
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
logger.Warn("预览请求取消", zap.String("path", path), zap.Int("size", size))
return
}
logger.Error("获取图片路径失败", zap.String("path", path), zap.Error(err))
c.JSON(http.StatusInternalServerError, model.Response{
Code: http.StatusInternalServerError,
@@ -576,7 +594,7 @@ func (h *PhotoHandler) GetPhotoPreview(c *gin.Context) {
}
// 设置响应头并返回文件
h.setImageHeaders(c, imagePath, imageAPI.GetFormat())
h.setImageHeaders(c, imagePath, format)
c.File(imagePath)
}
@@ -606,3 +624,79 @@ func (h *PhotoHandler) setImageHeaders(c *gin.Context, imagePath, format string)
fileName := fmt.Sprintf("image.%s", format)
c.Header("Content-Disposition", fmt.Sprintf(`inline; filename="%s"`, fileName))
}
func (h *PhotoHandler) getPreviewWithDedupe(ctx context.Context, filePath string, size int) (string, string, error) {
cacheKey := buildPreviewCacheKey(filePath, size)
return getOrCreatePreview(ctx, cacheKey, func(execCtx context.Context) (string, string, error) {
if err := acquirePreviewSlot(execCtx); err != nil {
return "", "", err
}
defer releasePreviewSlot()
imageAPI, err := api.NewImageAPI(filePath)
if err != nil {
return "", "", err
}
imagePath, err := imageAPI.GetImagePath(execCtx, size)
if err != nil {
return "", "", err
}
return imagePath, imageAPI.GetFormat(), nil
})
}
func acquirePreviewSlot(ctx context.Context) error {
select {
case previewLimiter <- struct{}{}:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func releasePreviewSlot() {
select {
case <-previewLimiter:
default:
}
}
func buildPreviewCacheKey(path string, size int) string {
normalized := strings.ToLower(filepath.ToSlash(filepath.Clean(path)))
return fmt.Sprintf("%s|%d", normalized, size)
}
func getOrCreatePreview(ctx context.Context, key string, fn func(context.Context) (string, string, error)) (string, string, error) {
previewInflightMu.Lock()
if call, ok := previewInflightMap[key]; ok {
previewInflightMu.Unlock()
select {
case <-call.done:
return call.result.imagePath, call.result.format, call.result.err
case <-ctx.Done():
return "", "", ctx.Err()
}
}
call := &previewCall{
done: make(chan struct{}),
}
previewInflightMap[key] = call
previewInflightMu.Unlock()
imagePath, format, err := fn(ctx)
call.result = previewResult{
imagePath: imagePath,
format: format,
err: err,
}
previewInflightMu.Lock()
delete(previewInflightMap, key)
previewInflightMu.Unlock()
close(call.done)
return imagePath, format, err
}
+55 -9
View File
@@ -359,7 +359,7 @@ func (pt *PictureTask) setStatus(s TaskStatus) {
func (pt *PictureTask) saveToDatabase(imageAPI *api.ImageAPI) error {
exifData := imageAPI.GetExifData()
hash := imageAPI.GetHash()
if hash == "" {
return fmt.Errorf("图像哈希值为空")
}
@@ -372,7 +372,7 @@ func (pt *PictureTask) saveToDatabase(imageAPI *api.ImageAPI) error {
if exifData != nil && !isExifRepoNil(pt.exifRepo) {
// 创建PhotoExif表记录
photoExif := tables.NewPhotoExifFromParsed(hash, exifData)
// 保存到数据库
err := pt.exifRepo.CreateOrUpdate(photoExif)
if err != nil {
@@ -396,7 +396,7 @@ func (pt *PictureTask) saveToDatabase(imageAPI *api.ImageAPI) error {
// 2. 保存Photo基础信息到photos表
if !isPhotoRepoNil(pt.photoRepo) {
photo := pt.photoRepo.CreatePhotoFromImageAPI(hash, pt.Path, exifData)
err := pt.photoRepo.CreateOrUpdate(photo)
if err != nil {
logger.Error("保存Photo信息失败",
@@ -405,13 +405,13 @@ func (pt *PictureTask) saveToDatabase(imageAPI *api.ImageAPI) error {
zap.Error(err))
return fmt.Errorf("保存Photo信息失败: %w", err)
}
logger.Info("Photo信息保存成功",
zap.String("hash", hash),
zap.String("path", pt.Path),
zap.Int("width", photo.Width),
zap.Int("height", photo.Height))
// 记录时间相关信息
if photo.TakenAt != nil {
logger.Info("照片拍摄时间",
@@ -446,6 +446,8 @@ func (pt *PictureTask) performIntelligentAnalysis(imageAPI *api.ImageAPI) {
type ImgTaskManager struct {
tasks map[string]*PictureTask
queue chan *PictureTask
highQueue chan *PictureTask
pathIndex map[string]string
workerPool chan struct{}
workerLimit int
mu sync.RWMutex
@@ -468,6 +470,8 @@ func NewImgTaskManager(concurrency int, exifRepo *repositories.ExifRepository, p
tm := &ImgTaskManager{
tasks: make(map[string]*PictureTask),
queue: make(chan *PictureTask, taskConfig.QueueCapacity),
highQueue: make(chan *PictureTask, taskConfig.QueueCapacity),
pathIndex: make(map[string]string),
workerPool: make(chan struct{}, concurrency),
workerLimit: concurrency,
globalPause: make(chan struct{}, 1),
@@ -574,15 +578,33 @@ func (tm *ImgTaskManager) SetConcurrency(n int) {
tm.mu.Unlock()
}
func (tm *ImgTaskManager) AddTask(path string) string {
task := NewPictureTask(path, tm.exifRepo, tm.photoRepo)
// AddTaskWithPriority 添加任务,可指定是否为高优先级
func (tm *ImgTaskManager) AddTaskWithPriority(path string, highPriority bool) string {
tm.mu.Lock()
if existingID, ok := tm.pathIndex[path]; ok {
if existingTask, exists := tm.tasks[existingID]; exists && existingTask.Status != StatusDone && existingTask.Status != StatusFailed {
tm.mu.Unlock()
return existingID
}
}
task := NewPictureTask(path, tm.exifRepo, tm.photoRepo)
tm.tasks[task.ID] = task
tm.pathIndex[path] = task.ID
tm.mu.Unlock()
tm.queue <- task
if highPriority {
tm.highQueue <- task
} else {
tm.queue <- task
}
return task.ID
}
// AddTask 添加任务,默认普通优先级(兼容旧调用)
func (tm *ImgTaskManager) AddTask(path string) string {
return tm.AddTaskWithPriority(path, false)
}
func (tm *ImgTaskManager) PauseAll() {
select {
case tm.globalPause <- struct{}{}:
@@ -611,7 +633,17 @@ func (tm *ImgTaskManager) applyGlobalPauseResume(task *PictureTask) {
}
func (tm *ImgTaskManager) run() {
for task := range tm.queue {
for {
var task *PictureTask
select {
case task = <-tm.highQueue:
case task = <-tm.queue:
}
if task == nil {
continue
}
tm.workerPool <- struct{}{}
tm.applyGlobalPauseResume(task)
go func(t *PictureTask) {
@@ -621,6 +653,10 @@ func (tm *ImgTaskManager) run() {
if t.Status == StatusDone {
tm.doneCount++
}
if t.Status == StatusDone || t.Status == StatusFailed {
// 允许后续同路径任务再次入队
delete(tm.pathIndex, t.Path)
}
tm.mu.Unlock()
}(task)
}
@@ -1149,6 +1185,16 @@ func AddImageTask(imagePath string) string {
return taskManager.AddTask(imagePath)
}
// AddImageTaskWithPriority 添加图像处理任务并指定是否高优先级
func AddImageTaskWithPriority(imagePath string, highPriority bool) string {
taskManager := GetGlobalTaskManager()
if taskManager == nil {
logger.Error("任务管理器未初始化")
return ""
}
return taskManager.AddTaskWithPriority(imagePath, highPriority)
}
// AddImageTasks 批量添加图像处理任务 (全局便捷函数)
func AddImageTasks(imagePaths []string) []string {
taskManager := GetGlobalTaskManager()
+10
View File
@@ -157,3 +157,13 @@
{"level":"info","time":"2025-11-16 14:43:16.424","caller":"config/config.go:90","msg":"加载配置文件成功","path":"config.yaml"}
{"level":"info","time":"2025-11-16 17:26:19.223","caller":"config/config.go:90","msg":"加载配置文件成功","path":"config.yaml"}
{"level":"info","time":"2025-11-16 17:32:50.846","caller":"config/config.go:90","msg":"加载配置文件成功","path":"config.yaml"}
{"level":"info","time":"2025-11-16 18:19:32.861","caller":"config/config.go:90","msg":"加载配置文件成功","path":"config.yaml"}
{"level":"info","time":"2025-11-16 18:26:28.326","caller":"config/config.go:90","msg":"加载配置文件成功","path":"config.yaml"}
{"level":"info","time":"2025-11-16 19:25:56.732","caller":"config/config.go:90","msg":"加载配置文件成功","path":"config.yaml"}
{"level":"info","time":"2025-11-16 19:26:02.076","caller":"config/config.go:90","msg":"加载配置文件成功","path":"config.yaml"}
{"level":"info","time":"2025-11-16 19:38:50.665","caller":"config/config.go:90","msg":"加载配置文件成功","path":"config.yaml"}
{"level":"info","time":"2025-11-16 20:16:03.635","caller":"config/config.go:90","msg":"加载配置文件成功","path":"config.yaml"}
{"level":"info","time":"2025-11-16 20:52:59.290","caller":"config/config.go:90","msg":"加载配置文件成功","path":"config.yaml"}
{"level":"info","time":"2025-11-16 20:59:09.711","caller":"config/config.go:90","msg":"加载配置文件成功","path":"config.yaml"}
{"level":"info","time":"2025-11-16 22:14:43.808","caller":"config/config.go:90","msg":"加载配置文件成功","path":"config.yaml"}
{"level":"info","time":"2025-11-16 22:19:31.382","caller":"config/config.go:90","msg":"加载配置文件成功","path":"config.yaml"}