A:数据库初始化，并增加User表示例
This commit is contained in:
jimmy committed
1 parent
da2222f2d9
commit
b8dc199cff
15 files changed
+1459
-108
No files matched your search
@@ -0,0 +1,400 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os/exec"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Task 任务接口
|
||||
type Task interface {
|
||||
// Execute 执行任务
|
||||
Execute(ctx context.Context) error
|
||||
// ID 获取任务ID
|
||||
ID() string
|
||||
// Type 获取任务类型
|
||||
Type() string
|
||||
|
||||
/*
|
||||
紧急任务：优先级 8-10
|
||||
正常任务：优先级 4-7
|
||||
后台任务：优先级 1-3
|
||||
*/
|
||||
// Priority 获取任务优先级 (数字越大优先级越高)
|
||||
Priority() int
|
||||
}
|
||||
|
||||
// TaskResult 任务执行结果
|
||||
type TaskResult struct {
|
||||
TaskID string
|
||||
TaskType string
|
||||
Success bool
|
||||
Error error
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
// TaskScheduler 任务调度器
|
||||
type TaskScheduler struct {
|
||||
workers int
|
||||
taskQueue chan Task
|
||||
resultQueue chan TaskResult
|
||||
workerPool chan struct{}
|
||||
wg sync.WaitGroup
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
|
||||
// 统计信息
|
||||
totalTasks int64
|
||||
completedTasks int64
|
||||
failedTasks int64
|
||||
|
||||
// 任务类型统计
|
||||
taskStats map[string]*TaskTypeStats
|
||||
statsMutex sync.RWMutex
|
||||
}
|
||||
|
||||
// TaskTypeStats 任务类型统计
|
||||
type TaskTypeStats struct {
|
||||
Total int64
|
||||
Completed int64
|
||||
Failed int64
|
||||
AvgTime time.Duration
|
||||
}
|
||||
|
||||
// NewTaskScheduler 创建任务调度器
|
||||
func NewTaskScheduler(workers int, queueSize int) *TaskScheduler {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
ts := &TaskScheduler{
|
||||
workers: workers,
|
||||
taskQueue: make(chan Task, queueSize),
|
||||
resultQueue: make(chan TaskResult, workers),
|
||||
workerPool: make(chan struct{}, workers),
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
taskStats: make(map[string]*TaskTypeStats),
|
||||
}
|
||||
|
||||
// 初始化worker池
|
||||
for i := 0; i < workers; i++ {
|
||||
ts.workerPool <- struct{}{}
|
||||
}
|
||||
|
||||
// 启动worker
|
||||
ts.startWorkers()
|
||||
|
||||
// 启动结果处理器
|
||||
go ts.resultProcessor()
|
||||
|
||||
return ts
|
||||
}
|
||||
|
||||
// startWorkers 启动worker
|
||||
func (ts *TaskScheduler) startWorkers() {
|
||||
for i := 0; i < ts.workers; i++ {
|
||||
ts.wg.Add(1)
|
||||
go ts.worker(i)
|
||||
}
|
||||
}
|
||||
|
||||
// worker 工作协程
|
||||
func (ts *TaskScheduler) worker(id int) {
|
||||
defer ts.wg.Done()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ts.ctx.Done():
|
||||
return
|
||||
case task, ok := <-ts.taskQueue:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// 获取worker令牌
|
||||
<-ts.workerPool
|
||||
|
||||
// 执行任务
|
||||
start := time.Now()
|
||||
err := task.Execute(ts.ctx)
|
||||
duration := time.Since(start)
|
||||
|
||||
// 发送结果
|
||||
ts.resultQueue <- TaskResult{
|
||||
TaskID: task.ID(),
|
||||
TaskType: task.Type(),
|
||||
Success: err == nil,
|
||||
Error: err,
|
||||
Duration: duration,
|
||||
}
|
||||
|
||||
// 归还worker令牌
|
||||
ts.workerPool <- struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// resultProcessor 结果处理器
|
||||
func (ts *TaskScheduler) resultProcessor() {
|
||||
for result := range ts.resultQueue {
|
||||
atomic.AddInt64(&ts.completedTasks, 1)
|
||||
|
||||
if !result.Success {
|
||||
atomic.AddInt64(&ts.failedTasks, 1)
|
||||
log.Printf("Task %s failed: %v", result.TaskID, result.Error)
|
||||
}
|
||||
|
||||
// 更新统计信息
|
||||
ts.updateStats(result)
|
||||
}
|
||||
}
|
||||
|
||||
// updateStats 更新统计信息
|
||||
func (ts *TaskScheduler) updateStats(result TaskResult) {
|
||||
ts.statsMutex.Lock()
|
||||
defer ts.statsMutex.Unlock()
|
||||
|
||||
stats, exists := ts.taskStats[result.TaskType]
|
||||
if !exists {
|
||||
stats = &TaskTypeStats{}
|
||||
ts.taskStats[result.TaskType] = stats
|
||||
}
|
||||
|
||||
stats.Total++
|
||||
if result.Success {
|
||||
stats.Completed++
|
||||
} else {
|
||||
stats.Failed++
|
||||
}
|
||||
|
||||
// 更新平均执行时间
|
||||
if stats.Completed > 0 {
|
||||
stats.AvgTime = time.Duration(
|
||||
(int64(stats.AvgTime)*(stats.Completed-1) + int64(result.Duration)) / stats.Completed,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Submit 提交任务
|
||||
func (ts *TaskScheduler) Submit(task Task) error {
|
||||
select {
|
||||
case <-ts.ctx.Done():
|
||||
return fmt.Errorf("scheduler is shutting down")
|
||||
case ts.taskQueue <- task:
|
||||
atomic.AddInt64(&ts.totalTasks, 1)
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("task queue is full")
|
||||
}
|
||||
}
|
||||
|
||||
// SubmitBatch 批量提交任务
|
||||
func (ts *TaskScheduler) SubmitBatch(tasks []Task) error {
|
||||
for _, task := range tasks {
|
||||
if err := ts.Submit(task); err != nil {
|
||||
return fmt.Errorf("failed to submit task %s: %w", task.ID(), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetStats 获取统计信息
|
||||
func (ts *TaskScheduler) GetStats() map[string]interface{} {
|
||||
ts.statsMutex.RLock()
|
||||
defer ts.statsMutex.RUnlock()
|
||||
|
||||
stats := make(map[string]interface{})
|
||||
stats["total_tasks"] = atomic.LoadInt64(&ts.totalTasks)
|
||||
stats["completed_tasks"] = atomic.LoadInt64(&ts.completedTasks)
|
||||
stats["failed_tasks"] = atomic.LoadInt64(&ts.failedTasks)
|
||||
stats["pending_tasks"] = len(ts.taskQueue)
|
||||
stats["active_workers"] = ts.workers - len(ts.workerPool)
|
||||
|
||||
// 复制任务类型统计
|
||||
typeStats := make(map[string]*TaskTypeStats)
|
||||
for k, v := range ts.taskStats {
|
||||
typeStats[k] = &TaskTypeStats{
|
||||
Total: v.Total,
|
||||
Completed: v.Completed,
|
||||
Failed: v.Failed,
|
||||
AvgTime: v.AvgTime,
|
||||
}
|
||||
}
|
||||
stats["type_stats"] = typeStats
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// Shutdown 关闭调度器
|
||||
func (ts *TaskScheduler) Shutdown() {
|
||||
ts.cancel()
|
||||
close(ts.taskQueue)
|
||||
ts.wg.Wait()
|
||||
close(ts.resultQueue)
|
||||
}
|
||||
|
||||
// --- 具体任务实现示例 ---
|
||||
|
||||
// ImageCompressTask 图片压缩任务
|
||||
type ImageCompressTask struct {
|
||||
id string
|
||||
filePath string
|
||||
quality int
|
||||
}
|
||||
|
||||
func NewImageCompressTask(id, filePath string, quality int) *ImageCompressTask {
|
||||
return &ImageCompressTask{
|
||||
id: id,
|
||||
filePath: filePath,
|
||||
quality: quality,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ImageCompressTask) Execute(ctx context.Context) error {
|
||||
// 这里是实际的图片压缩逻辑
|
||||
// 可以使用 imaging 库或调用外部工具
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
// 模拟压缩操作
|
||||
// 实际实现中，这里会调用图片处理库
|
||||
log.Printf("Compressing image: %s with quality %d", t.filePath, t.quality)
|
||||
time.Sleep(100 * time.Millisecond) // 模拟耗时操作
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ImageCompressTask) ID() string { return t.id }
|
||||
func (t *ImageCompressTask) Type() string { return "image_compress" }
|
||||
func (t *ImageCompressTask) Priority() int { return 5 }
|
||||
|
||||
// ExifReadTask EXIF读取任务
|
||||
type ExifReadTask struct {
|
||||
id string
|
||||
filePath string
|
||||
exifData map[string]interface{}
|
||||
}
|
||||
|
||||
func NewExifReadTask(id, filePath string) *ExifReadTask {
|
||||
return &ExifReadTask{
|
||||
id: id,
|
||||
filePath: filePath,
|
||||
exifData: make(map[string]interface{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ExifReadTask) Execute(ctx context.Context) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
// 使用 exiftool 命令行工具
|
||||
cmd := exec.CommandContext(ctx, "exiftool", "-j", t.filePath)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return fmt.Errorf("exiftool failed: %w", err)
|
||||
}
|
||||
|
||||
// 这里应该解析 JSON 输出到 t.exifData
|
||||
log.Printf("Read EXIF from: %s, data size: %d", t.filePath, len(output))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ExifReadTask) ID() string { return t.id }
|
||||
func (t *ExifReadTask) Type() string { return "exif_read" }
|
||||
func (t *ExifReadTask) Priority() int { return 3 }
|
||||
|
||||
// PriorityTaskScheduler 支持优先级的任务调度器
|
||||
type PriorityTaskScheduler struct {
|
||||
*TaskScheduler
|
||||
priorityQueue *PriorityQueue
|
||||
queueMutex sync.Mutex
|
||||
}
|
||||
|
||||
// PriorityQueue 优先级队列实现
|
||||
type PriorityQueue struct {
|
||||
tasks []Task
|
||||
}
|
||||
|
||||
func (pq *PriorityQueue) Push(task Task) {
|
||||
pq.tasks = append(pq.tasks, task)
|
||||
// 简单的插入排序，实际可以使用堆
|
||||
for i := len(pq.tasks) - 1; i > 0; i-- {
|
||||
if pq.tasks[i].Priority() > pq.tasks[i-1].Priority() {
|
||||
pq.tasks[i], pq.tasks[i-1] = pq.tasks[i-1], pq.tasks[i]
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (pq *PriorityQueue) Pop() Task {
|
||||
if len(pq.tasks) == 0 {
|
||||
return nil
|
||||
}
|
||||
task := pq.tasks[0]
|
||||
pq.tasks = pq.tasks[1:]
|
||||
return task
|
||||
}
|
||||
|
||||
func (pq *PriorityQueue) Len() int {
|
||||
return len(pq.tasks)
|
||||
}
|
||||
|
||||
// NewPriorityTaskScheduler 创建支持优先级的调度器
|
||||
func NewPriorityTaskScheduler(workers int) *PriorityTaskScheduler {
|
||||
pts := &PriorityTaskScheduler{
|
||||
TaskScheduler: NewTaskScheduler(workers, 0), // 不使用内置队列
|
||||
priorityQueue: &PriorityQueue{tasks: make([]Task, 0)},
|
||||
}
|
||||
|
||||
// 启动优先级调度协程
|
||||
go pts.priorityDispatcher()
|
||||
|
||||
return pts
|
||||
}
|
||||
|
||||
// priorityDispatcher 优先级调度器
|
||||
func (pts *PriorityTaskScheduler) priorityDispatcher() {
|
||||
ticker := time.NewTicker(10 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-pts.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
pts.queueMutex.Lock()
|
||||
if pts.priorityQueue.Len() > 0 && len(pts.taskQueue) < cap(pts.taskQueue) {
|
||||
task := pts.priorityQueue.Pop()
|
||||
select {
|
||||
case pts.taskQueue <- task:
|
||||
// 成功发送
|
||||
default:
|
||||
// 队列满了，放回优先级队列
|
||||
pts.priorityQueue.Push(task)
|
||||
}
|
||||
}
|
||||
pts.queueMutex.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SubmitWithPriority 提交带优先级的任务
|
||||
func (pts *PriorityTaskScheduler) SubmitWithPriority(task Task) error {
|
||||
select {
|
||||
case <-pts.ctx.Done():
|
||||
return fmt.Errorf("scheduler is shutting down")
|
||||
default:
|
||||
pts.queueMutex.Lock()
|
||||
pts.priorityQueue.Push(task)
|
||||
atomic.AddInt64(&pts.totalTasks, 1)
|
||||
pts.queueMutex.Unlock()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 示例：批量处理图片任务
|
||||
func processImages(scheduler *TaskScheduler) {
|
||||
// 假设有10000张图片需要处理
|
||||
imagePaths := make([]string, 10000)
|
||||
for i := 0; i < 10000; i++ {
|
||||
imagePaths[i] = fmt.Sprintf("/path/to/image_%d.jpg", i)
|
||||
}
|
||||
|
||||
// 批量提交压缩任务
|
||||
log.Println("提交图片压缩任务...")
|
||||
for i, path := range imagePaths {
|
||||
task := NewImageCompressTask(
|
||||
fmt.Sprintf("compress_%d", i),
|
||||
path,
|
||||
85, // 压缩质量
|
||||
)
|
||||
|
||||
if err := scheduler.Submit(task); err != nil {
|
||||
log.Printf("Failed to submit task: %v", err)
|
||||
// 可以实现重试逻辑
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
scheduler.Submit(task) // 重试一次
|
||||
}
|
||||
}
|
||||
|
||||
// 同时提交EXIF读取任务
|
||||
log.Println("提交EXIF读取任务...")
|
||||
for i, path := range imagePaths[:1000] { // 只处理前1000张
|
||||
task := NewExifReadTask(
|
||||
fmt.Sprintf("exif_%d", i),
|
||||
path,
|
||||
)
|
||||
scheduler.Submit(task)
|
||||
}
|
||||
}
|
||||
|
||||
// 高级任务管理器 - 支持任务组和依赖
|
||||
type TaskManager struct {
|
||||
scheduler *PriorityTaskScheduler
|
||||
taskGroups map[string]*TaskGroup
|
||||
}
|
||||
|
||||
type TaskGroup struct {
|
||||
ID string
|
||||
Tasks []Task
|
||||
Status string
|
||||
Progress float64
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
}
|
||||
|
||||
func NewTaskManager(workers int) *TaskManager {
|
||||
return &TaskManager{
|
||||
scheduler: NewPriorityTaskScheduler(workers),
|
||||
taskGroups: make(map[string]*TaskGroup),
|
||||
}
|
||||
}
|
||||
|
||||
// CreateImageProcessingGroup 创建图片处理任务组
|
||||
func (tm *TaskManager) CreateImageProcessingGroup(groupID string, imagePaths []string) error {
|
||||
group := &TaskGroup{
|
||||
ID: groupID,
|
||||
Tasks: make([]Task, 0),
|
||||
Status: "pending",
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
|
||||
// 为每张图片创建压缩和EXIF读取任务
|
||||
for i, path := range imagePaths {
|
||||
// 压缩任务
|
||||
compressTask := NewImageCompressTask(
|
||||
fmt.Sprintf("%s_compress_%d", groupID, i),
|
||||
path,
|
||||
85,
|
||||
)
|
||||
group.Tasks = append(group.Tasks, compressTask)
|
||||
|
||||
// EXIF任务
|
||||
exifTask := NewExifReadTask(
|
||||
fmt.Sprintf("%s_exif_%d", groupID, i),
|
||||
path,
|
||||
)
|
||||
group.Tasks = append(group.Tasks, exifTask)
|
||||
}
|
||||
|
||||
tm.taskGroups[groupID] = group
|
||||
|
||||
// 提交所有任务
|
||||
for _, task := range group.Tasks {
|
||||
if err := tm.scheduler.SubmitWithPriority(task); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
group.Status = "running"
|
||||
return nil
|
||||
}
|
||||
|
||||
// 动态Worker调整器
|
||||
type DynamicWorkerManager struct {
|
||||
scheduler *TaskScheduler
|
||||
minWorkers int
|
||||
maxWorkers int
|
||||
checkInterval time.Duration
|
||||
scaleThreshold float64 // 队列使用率阈值
|
||||
}
|
||||
|
||||
func NewDynamicWorkerManager(scheduler *TaskScheduler, min, max int) *DynamicWorkerManager {
|
||||
return &DynamicWorkerManager{
|
||||
scheduler: scheduler,
|
||||
minWorkers: min,
|
||||
maxWorkers: max,
|
||||
checkInterval: 5 * time.Second,
|
||||
scaleThreshold: 0.8,
|
||||
}
|
||||
}
|
||||
|
||||
func (dwm *DynamicWorkerManager) Start(ctx context.Context) {
|
||||
ticker := time.NewTicker(dwm.checkInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
dwm.adjustWorkers()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dwm *DynamicWorkerManager) adjustWorkers() {
|
||||
stats := dwm.scheduler.GetStats()
|
||||
pending := stats["pending_tasks"].(int)
|
||||
//active := stats["active_workers"].(int)
|
||||
|
||||
// 计算队列使用率
|
||||
queueUsage := float64(pending) / float64(cap(dwm.scheduler.taskQueue))
|
||||
|
||||
if queueUsage > dwm.scaleThreshold && dwm.scheduler.workers < dwm.maxWorkers {
|
||||
// 增加worker
|
||||
log.Printf("Scaling up workers: queue usage %.2f%%", queueUsage*100)
|
||||
// 这里需要实现动态增加worker的逻辑
|
||||
} else if queueUsage < 0.2 && dwm.scheduler.workers > dwm.minWorkers {
|
||||
// 减少worker
|
||||
log.Printf("Scaling down workers: queue usage %.2f%%", queueUsage*100)
|
||||
// 这里需要实现动态减少worker的逻辑
|
||||
}
|
||||
}
|
||||
|
||||
// 任务监控器
|
||||
type TaskMonitor struct {
|
||||
scheduler *TaskScheduler
|
||||
}
|
||||
|
||||
func NewTaskMonitor(scheduler *TaskScheduler) *TaskMonitor {
|
||||
return &TaskMonitor{scheduler: scheduler}
|
||||
}
|
||||
|
||||
func (tm *TaskMonitor) Start(ctx context.Context) {
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
tm.printStats()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (tm *TaskMonitor) printStats() {
|
||||
stats := tm.scheduler.GetStats()
|
||||
|
||||
fmt.Println("\n========== Task Scheduler Stats ==========")
|
||||
fmt.Printf("Total Tasks: %d\n", stats["total_tasks"])
|
||||
fmt.Printf("Completed: %d\n", stats["completed_tasks"])
|
||||
fmt.Printf("Failed: %d\n", stats["failed_tasks"])
|
||||
fmt.Printf("Pending: %d\n", stats["pending_tasks"])
|
||||
fmt.Printf("Active Workers: %d\n", stats["active_workers"])
|
||||
|
||||
if typeStats, ok := stats["type_stats"].(map[string]*TaskTypeStats); ok {
|
||||
fmt.Println("\nTask Type Statistics:")
|
||||
for taskType, stat := range typeStats {
|
||||
fmt.Printf(" %s: Total=%d, Completed=%d, Failed=%d, AvgTime=%v\n",
|
||||
taskType, stat.Total, stat.Completed, stat.Failed, stat.AvgTime)
|
||||
}
|
||||
}
|
||||
fmt.Println("==========================================\n")
|
||||
}
|
||||
|
||||
// 主函数示例
|
||||
func main() {
|
||||
// 创建调度器 - 50个worker，队列容量10000
|
||||
scheduler := NewTaskScheduler(50, 10000)
|
||||
defer scheduler.Shutdown()
|
||||
|
||||
// 创建任务管理器
|
||||
taskManager := NewTaskManager(50)
|
||||
|
||||
// 创建监控器
|
||||
monitor := NewTaskMonitor(scheduler)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go monitor.Start(ctx)
|
||||
|
||||
// 场景1：处理第一批10000张图片
|
||||
log.Println("开始处理第一批图片...")
|
||||
imagePaths1 := make([]string, 10000)
|
||||
for i := 0; i < 10000; i++ {
|
||||
imagePaths1[i] = fmt.Sprintf("/batch1/image_%d.jpg", i)
|
||||
}
|
||||
taskManager.CreateImageProcessingGroup("batch1", imagePaths1)
|
||||
|
||||
// 模拟过一段时间后又来了新任务
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
// 场景2：又来了10000张图片
|
||||
log.Println("开始处理第二批图片...")
|
||||
imagePaths2 := make([]string, 10000)
|
||||
for i := 0; i < 10000; i++ {
|
||||
imagePaths2[i] = fmt.Sprintf("/batch2/image_%d.jpg", i)
|
||||
}
|
||||
taskManager.CreateImageProcessingGroup("batch2", imagePaths2)
|
||||
|
||||
// 场景3：批量解析EXIF（优先级更高）
|
||||
log.Println("开始批量解析EXIF...")
|
||||
exifPaths := make([]string, 1000)
|
||||
for i := 0; i < 1000; i++ {
|
||||
exifPaths[i] = fmt.Sprintf("/priority/image_%d.jpg", i)
|
||||
}
|
||||
// 这些任务会有更高的优先级
|
||||
for i, path := range exifPaths {
|
||||
task := &HighPriorityExifTask{
|
||||
ExifReadTask: *NewExifReadTask(fmt.Sprintf("priority_exif_%d", i), path),
|
||||
}
|
||||
taskManager.scheduler.SubmitWithPriority(task)
|
||||
}
|
||||
|
||||
// 等待一段时间查看执行情况
|
||||
time.Sleep(30 * time.Second)
|
||||
|
||||
// 打印最终统计
|
||||
monitor.printStats()
|
||||
}
|
||||
|
||||
// HighPriorityExifTask 高优先级EXIF任务
|
||||
type HighPriorityExifTask struct {
|
||||
ExifReadTask
|
||||
}
|
||||
|
||||
func (t *HighPriorityExifTask) Priority() int {
|
||||
return 10 // 更高的优先级
|
||||
}
|
||||
|
||||
// 任务重试装饰器
|
||||
type RetryableTask struct {
|
||||
task Task
|
||||
maxRetries int
|
||||
retries int
|
||||
}
|
||||
|
||||
func NewRetryableTask(task Task, maxRetries int) *RetryableTask {
|
||||
return &RetryableTask{
|
||||
task: task,
|
||||
maxRetries: maxRetries,
|
||||
retries: 0,
|
||||
}
|
||||
}
|
||||
|
||||
func (rt *RetryableTask) Execute(ctx context.Context) error {
|
||||
var lastErr error
|
||||
|
||||
for rt.retries <= rt.maxRetries {
|
||||
err := rt.task.Execute(ctx)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
lastErr = err
|
||||
rt.retries++
|
||||
|
||||
if rt.retries <= rt.maxRetries {
|
||||
// 指数退避
|
||||
backoff := time.Duration(rt.retries) * time.Second
|
||||
log.Printf("Task %s failed, retrying in %v (attempt %d/%d)",
|
||||
rt.task.ID(), backoff, rt.retries, rt.maxRetries)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
// 继续重试
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("task failed after %d retries: %w", rt.maxRetries, lastErr)
|
||||
}
|
||||
|
||||
func (rt *RetryableTask) ID() string { return rt.task.ID() }
|
||||
func (rt *RetryableTask) Type() string { return rt.task.Type() }
|
||||
func (rt *RetryableTask) Priority() int { return rt.task.Priority() }
|
||||
|
||||
// 批处理优化 - 将多个小任务合并为一个大任务
|
||||
type BatchImageCompressTask struct {
|
||||
id string
|
||||
filePaths []string
|
||||
quality int
|
||||
}
|
||||
|
||||
func NewBatchImageCompressTask(id string, filePaths []string, quality int) *BatchImageCompressTask {
|
||||
return &BatchImageCompressTask{
|
||||
id: id,
|
||||
filePaths: filePaths,
|
||||
quality: quality,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *BatchImageCompressTask) Execute(ctx context.Context) error {
|
||||
// 批量处理多张图片，减少任务切换开销
|
||||
for _, path := range t.filePaths {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
// 处理单张图片
|
||||
log.Printf("Batch compressing: %s", path)
|
||||
// 实际的压缩逻辑
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *BatchImageCompressTask) ID() string { return t.id }
|
||||
func (t *BatchImageCompressTask) Type() string { return "batch_image_compress" }
|
||||
func (t *BatchImageCompressTask) Priority() int { return 5 }
|
||||
Reference in new issue
Block a user