A:数据库初始化，并增加User表示例
This commit is contained in:
jimmy committed
1 parent
da2222f2d9
commit
b8dc199cff
15 files changed
+1459
-108
No files matched your search
@@ -0,0 +1,348 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 示例：批量处理图片任务
|
||||
func processImages(scheduler *TaskScheduler) {
|
||||
// 假设有10000张图片需要处理
|
||||
imagePaths := make([]string, 10000)
|
||||
for i := 0; i < 10000; i++ {
|
||||
imagePaths[i] = fmt.Sprintf("/path/to/image_%d.jpg", i)
|
||||
}
|
||||
|
||||
// 批量提交压缩任务
|
||||
log.Println("提交图片压缩任务...")
|
||||
for i, path := range imagePaths {
|
||||
task := NewImageCompressTask(
|
||||
fmt.Sprintf("compress_%d", i),
|
||||
path,
|
||||
85, // 压缩质量
|
||||
)
|
||||
|
||||
if err := scheduler.Submit(task); err != nil {
|
||||
log.Printf("Failed to submit task: %v", err)
|
||||
// 可以实现重试逻辑
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
scheduler.Submit(task) // 重试一次
|
||||
}
|
||||
}
|
||||
|
||||
// 同时提交EXIF读取任务
|
||||
log.Println("提交EXIF读取任务...")
|
||||
for i, path := range imagePaths[:1000] { // 只处理前1000张
|
||||
task := NewExifReadTask(
|
||||
fmt.Sprintf("exif_%d", i),
|
||||
path,
|
||||
)
|
||||
scheduler.Submit(task)
|
||||
}
|
||||
}
|
||||
|
||||
// 高级任务管理器 - 支持任务组和依赖
|
||||
type TaskManager struct {
|
||||
scheduler *PriorityTaskScheduler
|
||||
taskGroups map[string]*TaskGroup
|
||||
}
|
||||
|
||||
type TaskGroup struct {
|
||||
ID string
|
||||
Tasks []Task
|
||||
Status string
|
||||
Progress float64
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
}
|
||||
|
||||
func NewTaskManager(workers int) *TaskManager {
|
||||
return &TaskManager{
|
||||
scheduler: NewPriorityTaskScheduler(workers),
|
||||
taskGroups: make(map[string]*TaskGroup),
|
||||
}
|
||||
}
|
||||
|
||||
// CreateImageProcessingGroup 创建图片处理任务组
|
||||
func (tm *TaskManager) CreateImageProcessingGroup(groupID string, imagePaths []string) error {
|
||||
group := &TaskGroup{
|
||||
ID: groupID,
|
||||
Tasks: make([]Task, 0),
|
||||
Status: "pending",
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
|
||||
// 为每张图片创建压缩和EXIF读取任务
|
||||
for i, path := range imagePaths {
|
||||
// 压缩任务
|
||||
compressTask := NewImageCompressTask(
|
||||
fmt.Sprintf("%s_compress_%d", groupID, i),
|
||||
path,
|
||||
85,
|
||||
)
|
||||
group.Tasks = append(group.Tasks, compressTask)
|
||||
|
||||
// EXIF任务
|
||||
exifTask := NewExifReadTask(
|
||||
fmt.Sprintf("%s_exif_%d", groupID, i),
|
||||
path,
|
||||
)
|
||||
group.Tasks = append(group.Tasks, exifTask)
|
||||
}
|
||||
|
||||
tm.taskGroups[groupID] = group
|
||||
|
||||
// 提交所有任务
|
||||
for _, task := range group.Tasks {
|
||||
if err := tm.scheduler.SubmitWithPriority(task); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
group.Status = "running"
|
||||
return nil
|
||||
}
|
||||
|
||||
// 动态Worker调整器
|
||||
type DynamicWorkerManager struct {
|
||||
scheduler *TaskScheduler
|
||||
minWorkers int
|
||||
maxWorkers int
|
||||
checkInterval time.Duration
|
||||
scaleThreshold float64 // 队列使用率阈值
|
||||
}
|
||||
|
||||
func NewDynamicWorkerManager(scheduler *TaskScheduler, min, max int) *DynamicWorkerManager {
|
||||
return &DynamicWorkerManager{
|
||||
scheduler: scheduler,
|
||||
minWorkers: min,
|
||||
maxWorkers: max,
|
||||
checkInterval: 5 * time.Second,
|
||||
scaleThreshold: 0.8,
|
||||
}
|
||||
}
|
||||
|
||||
func (dwm *DynamicWorkerManager) Start(ctx context.Context) {
|
||||
ticker := time.NewTicker(dwm.checkInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
dwm.adjustWorkers()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dwm *DynamicWorkerManager) adjustWorkers() {
|
||||
stats := dwm.scheduler.GetStats()
|
||||
pending := stats["pending_tasks"].(int)
|
||||
//active := stats["active_workers"].(int)
|
||||
|
||||
// 计算队列使用率
|
||||
queueUsage := float64(pending) / float64(cap(dwm.scheduler.taskQueue))
|
||||
|
||||
if queueUsage > dwm.scaleThreshold && dwm.scheduler.workers < dwm.maxWorkers {
|
||||
// 增加worker
|
||||
log.Printf("Scaling up workers: queue usage %.2f%%", queueUsage*100)
|
||||
// 这里需要实现动态增加worker的逻辑
|
||||
} else if queueUsage < 0.2 && dwm.scheduler.workers > dwm.minWorkers {
|
||||
// 减少worker
|
||||
log.Printf("Scaling down workers: queue usage %.2f%%", queueUsage*100)
|
||||
// 这里需要实现动态减少worker的逻辑
|
||||
}
|
||||
}
|
||||
|
||||
// 任务监控器
|
||||
type TaskMonitor struct {
|
||||
scheduler *TaskScheduler
|
||||
}
|
||||
|
||||
func NewTaskMonitor(scheduler *TaskScheduler) *TaskMonitor {
|
||||
return &TaskMonitor{scheduler: scheduler}
|
||||
}
|
||||
|
||||
func (tm *TaskMonitor) Start(ctx context.Context) {
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
tm.printStats()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (tm *TaskMonitor) printStats() {
|
||||
stats := tm.scheduler.GetStats()
|
||||
|
||||
fmt.Println("\n========== Task Scheduler Stats ==========")
|
||||
fmt.Printf("Total Tasks: %d\n", stats["total_tasks"])
|
||||
fmt.Printf("Completed: %d\n", stats["completed_tasks"])
|
||||
fmt.Printf("Failed: %d\n", stats["failed_tasks"])
|
||||
fmt.Printf("Pending: %d\n", stats["pending_tasks"])
|
||||
fmt.Printf("Active Workers: %d\n", stats["active_workers"])
|
||||
|
||||
if typeStats, ok := stats["type_stats"].(map[string]*TaskTypeStats); ok {
|
||||
fmt.Println("\nTask Type Statistics:")
|
||||
for taskType, stat := range typeStats {
|
||||
fmt.Printf(" %s: Total=%d, Completed=%d, Failed=%d, AvgTime=%v\n",
|
||||
taskType, stat.Total, stat.Completed, stat.Failed, stat.AvgTime)
|
||||
}
|
||||
}
|
||||
fmt.Println("==========================================\n")
|
||||
}
|
||||
|
||||
// 主函数示例
|
||||
func main() {
|
||||
// 创建调度器 - 50个worker，队列容量10000
|
||||
scheduler := NewTaskScheduler(50, 10000)
|
||||
defer scheduler.Shutdown()
|
||||
|
||||
// 创建任务管理器
|
||||
taskManager := NewTaskManager(50)
|
||||
|
||||
// 创建监控器
|
||||
monitor := NewTaskMonitor(scheduler)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go monitor.Start(ctx)
|
||||
|
||||
// 场景1：处理第一批10000张图片
|
||||
log.Println("开始处理第一批图片...")
|
||||
imagePaths1 := make([]string, 10000)
|
||||
for i := 0; i < 10000; i++ {
|
||||
imagePaths1[i] = fmt.Sprintf("/batch1/image_%d.jpg", i)
|
||||
}
|
||||
taskManager.CreateImageProcessingGroup("batch1", imagePaths1)
|
||||
|
||||
// 模拟过一段时间后又来了新任务
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
// 场景2：又来了10000张图片
|
||||
log.Println("开始处理第二批图片...")
|
||||
imagePaths2 := make([]string, 10000)
|
||||
for i := 0; i < 10000; i++ {
|
||||
imagePaths2[i] = fmt.Sprintf("/batch2/image_%d.jpg", i)
|
||||
}
|
||||
taskManager.CreateImageProcessingGroup("batch2", imagePaths2)
|
||||
|
||||
// 场景3：批量解析EXIF（优先级更高）
|
||||
log.Println("开始批量解析EXIF...")
|
||||
exifPaths := make([]string, 1000)
|
||||
for i := 0; i < 1000; i++ {
|
||||
exifPaths[i] = fmt.Sprintf("/priority/image_%d.jpg", i)
|
||||
}
|
||||
// 这些任务会有更高的优先级
|
||||
for i, path := range exifPaths {
|
||||
task := &HighPriorityExifTask{
|
||||
ExifReadTask: *NewExifReadTask(fmt.Sprintf("priority_exif_%d", i), path),
|
||||
}
|
||||
taskManager.scheduler.SubmitWithPriority(task)
|
||||
}
|
||||
|
||||
// 等待一段时间查看执行情况
|
||||
time.Sleep(30 * time.Second)
|
||||
|
||||
// 打印最终统计
|
||||
monitor.printStats()
|
||||
}
|
||||
|
||||
// HighPriorityExifTask 高优先级EXIF任务
|
||||
type HighPriorityExifTask struct {
|
||||
ExifReadTask
|
||||
}
|
||||
|
||||
func (t *HighPriorityExifTask) Priority() int {
|
||||
return 10 // 更高的优先级
|
||||
}
|
||||
|
||||
// 任务重试装饰器
|
||||
type RetryableTask struct {
|
||||
task Task
|
||||
maxRetries int
|
||||
retries int
|
||||
}
|
||||
|
||||
func NewRetryableTask(task Task, maxRetries int) *RetryableTask {
|
||||
return &RetryableTask{
|
||||
task: task,
|
||||
maxRetries: maxRetries,
|
||||
retries: 0,
|
||||
}
|
||||
}
|
||||
|
||||
func (rt *RetryableTask) Execute(ctx context.Context) error {
|
||||
var lastErr error
|
||||
|
||||
for rt.retries <= rt.maxRetries {
|
||||
err := rt.task.Execute(ctx)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
lastErr = err
|
||||
rt.retries++
|
||||
|
||||
if rt.retries <= rt.maxRetries {
|
||||
// 指数退避
|
||||
backoff := time.Duration(rt.retries) * time.Second
|
||||
log.Printf("Task %s failed, retrying in %v (attempt %d/%d)",
|
||||
rt.task.ID(), backoff, rt.retries, rt.maxRetries)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
// 继续重试
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("task failed after %d retries: %w", rt.maxRetries, lastErr)
|
||||
}
|
||||
|
||||
func (rt *RetryableTask) ID() string { return rt.task.ID() }
|
||||
func (rt *RetryableTask) Type() string { return rt.task.Type() }
|
||||
func (rt *RetryableTask) Priority() int { return rt.task.Priority() }
|
||||
|
||||
// 批处理优化 - 将多个小任务合并为一个大任务
|
||||
type BatchImageCompressTask struct {
|
||||
id string
|
||||
filePaths []string
|
||||
quality int
|
||||
}
|
||||
|
||||
func NewBatchImageCompressTask(id string, filePaths []string, quality int) *BatchImageCompressTask {
|
||||
return &BatchImageCompressTask{
|
||||
id: id,
|
||||
filePaths: filePaths,
|
||||
quality: quality,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *BatchImageCompressTask) Execute(ctx context.Context) error {
|
||||
// 批量处理多张图片，减少任务切换开销
|
||||
for _, path := range t.filePaths {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
// 处理单张图片
|
||||
log.Printf("Batch compressing: %s", path)
|
||||
// 实际的压缩逻辑
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *BatchImageCompressTask) ID() string { return t.id }
|
||||
func (t *BatchImageCompressTask) Type() string { return "batch_image_compress" }
|
||||
func (t *BatchImageCompressTask) Priority() int { return 5 }
|
||||
Reference in new issue
Block a user