A:提供按天合并函数

This commit is contained in:
jimmy committed 2025-08-07 13:12:11 +08:00
1 parent 2f54037836
commit c999440973
5 files changed
+1137 -33

No files matched your search

@@ -3,6 +3,7 @@ using System.Threading.Channels;
using Common.Logger;
using Data.Models;
using Data.Services;
using Dapper;
namespace Data.Core;
@@ -11,10 +12,21 @@ namespace Data.Core;
/// </summary>
public enum DbOperationType
{
/// <summary>
/// App 信息
/// </summary>
SaveAppInfo,
SaveAppNetwork,
SaveGlobalNetwork,
/// <summary>
/// 总流量数据
/// </summary>
AggregateData,
/// <summary>
/// 清理老数据
/// </summary>
CleanupOldData
}
@@ -338,8 +350,17 @@ public class DatabaseWriteManager : IDisposable
// 先刷新所有批处理数据
await FlushAllBatchesAsync();
// TODO: 实现数据聚合逻辑
// 这里会实现按小时、天的数据聚合
var currentTimestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
var oneDayAgo = currentTimestamp - 86400; // 1天前
var oneWeekAgo = currentTimestamp - 604800; // 1周前
var oneMonthAgo = currentTimestamp - 2592000; // 1个月前 (30天)
var threeDaysAgo = currentTimestamp - 259200; // 3天前
// 聚合应用网络数据
await AggregateAppNetworkDataAsync(oneDayAgo, oneWeekAgo, oneMonthAgo);
// 聚合全局网络数据
await AggregateGlobalNetworkDataAsync(threeDaysAgo, oneWeekAgo);
Log.Information("数据聚合任务完成");
return true;
@@ -351,6 +372,668 @@ public class DatabaseWriteManager : IDisposable
}
}
/// <summary>
/// 聚合应用网络数据
/// </summary>
private async Task AggregateAppNetworkDataAsync(long oneDayAgo, long oneWeekAgo, long oneMonthAgo)
{
try
{
Log.Information($"开始聚合应用网络数据 - 时间戳: 1天前={oneDayAgo}, 1周前={oneWeekAgo}, 1月前={oneMonthAgo}");
// 获取所有应用ID
var connection = _dataService.Networks.GetType()
.GetField("_context",
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
?.GetValue(_dataService.Networks);
var contextConnection =
connection?.GetType().GetProperty("Connection")?.GetValue(connection) as System.Data.IDbConnection;
if (contextConnection == null)
{
Log.Warning("无法获取数据库连接,跳过应用网络数据聚合");
return;
}
// 先查看总数据量
var totalRecords = await contextConnection.QuerySingleAsync<int>("SELECT COUNT(*) FROM AppNetwork");
Log.Information($"AppNetwork表总记录数: {totalRecords}");
// 获取需要聚合的应用ID列表 - 修改查询条件以包含更多数据进行测试
var appIds = await contextConnection.QueryAsync<string>(
"SELECT DISTINCT AppId FROM AppNetwork", new { });
Log.Information($"找到 {appIds.Count()} 个需要聚合的应用");
if (!appIds.Any())
{
Log.Warning("没有找到任何应用数据,跳过聚合");
return;
}
foreach (var appId in appIds)
{
Log.Information($"开始处理应用: {appId}");
// 第一步:从原始数据聚合到小时数据
Log.Information($"为应用 {appId} 聚合小时数据");
await AggregateAppNetworkHourlyAsync(appId);
// 第二步:从小时数据聚合到天数据(超过1天的数据)
Log.Information($"为应用 {appId} 聚合天数据");
await AggregateAppNetworkDailyAsync(appId, oneDayAgo);
// 第三步:从天数据聚合到周数据(超过1周的数据)
Log.Information($"为应用 {appId} 聚合周数据");
await AggregateAppNetworkWeeklyAsync(appId, oneWeekAgo);
// 第四步:从周数据聚合到月数据(超过1月的数据)
Log.Information($"为应用 {appId} 聚合月数据");
await AggregateAppNetworkMonthlyAsync(appId, oneMonthAgo);
}
Log.Information($"应用网络数据聚合完成,处理了 {appIds.Count()} 个应用");
}
catch (Exception ex)
{
Log.Error(ex, "聚合应用网络数据失败");
}
}
/// <summary>
/// 聚合全局网络数据
/// </summary>
private async Task AggregateGlobalNetworkDataAsync(long threeDaysAgo, long oneWeekAgo)
{
try
{
Log.Information("开始聚合全局网络数据");
var connection = _dataService.Networks.GetType()
.GetField("_context",
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
?.GetValue(_dataService.Networks);
var contextConnection =
connection?.GetType().GetProperty("Connection")?.GetValue(connection) as System.Data.IDbConnection;
if (contextConnection == null)
{
Log.Warning("无法获取数据库连接,跳过全局网络数据聚合");
return;
}
// 先查看总数据量
var totalRecords = await contextConnection.QuerySingleAsync<int>("SELECT COUNT(*) FROM GlobalNetwork");
Log.Information($"GlobalNetwork表总记录数: {totalRecords}");
// 获取需要聚合的网卡GUID列表
var networkCardGuids = await contextConnection.QueryAsync<string>(
"SELECT DISTINCT NetworkCardGuid FROM GlobalNetwork", new { });
Log.Information($"找到 {networkCardGuids.Count()} 个需要聚合的网卡");
if (!networkCardGuids.Any())
{
Log.Warning("没有找到任何全局网络数据,跳过聚合");
return;
}
foreach (var networkCardGuid in networkCardGuids)
{
Log.Information($"开始处理网卡: {networkCardGuid}");
// 第一步:从原始数据聚合到小时数据
Log.Information($"为网卡 {networkCardGuid} 聚合小时数据");
await AggregateGlobalNetworkHourlyAsync(networkCardGuid);
// 第二步:从小时数据聚合到天数据
Log.Information($"为网卡 {networkCardGuid} 聚合天数据");
await AggregateGlobalNetworkDailyAsync(networkCardGuid, threeDaysAgo);
// 第三步:从天数据聚合到周数据
Log.Information($"为网卡 {networkCardGuid} 聚合周数据");
await AggregateGlobalNetworkWeeklyAsync(networkCardGuid, oneWeekAgo);
}
Log.Information($"全局网络数据聚合完成,处理了 {networkCardGuids.Count()} 个网卡");
}
catch (Exception ex)
{
Log.Error(ex, "聚合全局网络数据失败");
}
}
/// <summary>
/// 按小时聚合应用网络数据
/// </summary>
private async Task AggregateAppNetworkHourlyAsync(string appId)
{
try
{
var hourSeconds = 3600;
Log.Information($"聚合应用 {appId} 的小时数据");
var connection = _dataService.Networks.GetType()
.GetField("_context",
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
?.GetValue(_dataService.Networks);
var contextConnection =
connection?.GetType().GetProperty("Connection")?.GetValue(connection) as System.Data.IDbConnection;
if (contextConnection == null)
{
Log.Warning("无法获取数据库连接");
return;
}
string sql = @"
SELECT
(Timestamp / @HourSeconds) * @HourSeconds as HourTimestamp,
SUM(UploadBytes) as TotalUpload,
SUM(DownloadBytes) as TotalDownload,
COUNT(*) as ConnectionCount,
COUNT(DISTINCT RemoteIP) as UniqueRemoteIPs,
COUNT(DISTINCT RemotePort) as UniqueRemotePorts
FROM AppNetwork
WHERE AppId = @AppId
GROUP BY (Timestamp / @HourSeconds)
ORDER BY HourTimestamp";
var parameters = new { AppId = appId, HourSeconds = hourSeconds };
var aggregatedData = await contextConnection.QueryAsync(sql, parameters);
Log.Information($"查询到 {aggregatedData.Count()} 条小时聚合数据");
int savedCount = 0;
int skippedCount = 0;
foreach (dynamic data in aggregatedData)
{
var hourTimestamp = (long)data.HourTimestamp;
var totalUpload = (long)data.TotalUpload;
var totalDownload = (long)data.TotalDownload;
var connectionCount = (int)data.ConnectionCount;
var uniqueIPs = (int)data.UniqueRemoteIPs;
var uniquePorts = (int)data.UniqueRemotePorts;
bool exists = await _dataService.Networks.AppNetworkHourlyExistsAsync(appId, hourTimestamp);
if (!exists)
{
var hourlyData = new AppNetworkHourly
{
AppId = appId,
HourTimestamp = hourTimestamp,
TotalUploadBytes = totalUpload,
TotalDownloadBytes = totalDownload,
ConnectionCount = connectionCount,
UniqueRemoteIPs = uniqueIPs,
UniqueRemotePorts = uniquePorts,
CreatedTimestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds()
};
await _dataService.Networks.SaveAppNetworkHourlyAsync(hourlyData);
savedCount++;
}
else
{
skippedCount++;
}
}
Log.Information($"应用 {appId} 的小时聚合完成,保存了 {savedCount} 条新数据,跳过了 {skippedCount} 条已存在数据");
}
catch (Exception ex)
{
Log.Error(ex, $"聚合应用 {appId} 的小时数据失败");
}
}
/// <summary>
/// 按天聚合应用网络数据(从小时数据聚合)
/// </summary>
private async Task AggregateAppNetworkDailyAsync(string appId, long oneDayAgo)
{
try
{
Log.Information($"聚合应用 {appId} 的天数据,从小时数据聚合");
var connection = _dataService.Networks.GetType()
.GetField("_context",
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
?.GetValue(_dataService.Networks);
var contextConnection =
connection?.GetType().GetProperty("Connection")?.GetValue(connection) as System.Data.IDbConnection;
if (contextConnection == null) return;
var daySeconds = 86400;
string sql = @"
SELECT
(HourTimestamp / @DaySeconds) * @DaySeconds as DayTimestamp,
SUM(TotalUploadBytes) as TotalUpload,
SUM(TotalDownloadBytes) as TotalDownload,
SUM(ConnectionCount) as ConnectionCount,
MAX(UniqueRemoteIPs) as UniqueRemoteIPs,
MAX(UniqueRemotePorts) as UniqueRemotePorts
FROM AppNetworkHourly
WHERE AppId = @AppId AND HourTimestamp < @OneDayAgo
GROUP BY (HourTimestamp / @DaySeconds)
ORDER BY DayTimestamp";
var parameters = new { AppId = appId, OneDayAgo = oneDayAgo, DaySeconds = daySeconds };
var aggregatedData = await contextConnection.QueryAsync(sql, parameters);
int savedCount = 0;
foreach (dynamic data in aggregatedData)
{
var dayTimestamp = (long)data.DayTimestamp;
bool exists = await _dataService.Networks.AppNetworkDailyExistsAsync(appId, dayTimestamp);
if (!exists)
{
var dailyData = new AppNetworkDaily
{
AppId = appId,
DayTimestamp = dayTimestamp,
TotalUploadBytes = (long)data.TotalUpload,
TotalDownloadBytes = (long)data.TotalDownload,
ConnectionCount = (int)data.ConnectionCount,
UniqueRemoteIPs = (int)data.UniqueRemoteIPs,
UniqueRemotePorts = (int)data.UniqueRemotePorts,
CreatedTimestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds()
};
await _dataService.Networks.SaveAppNetworkDailyAsync(dailyData);
savedCount++;
}
}
Log.Information($"应用 {appId} 的天聚合完成,保存了 {savedCount} 条新数据");
}
catch (Exception ex)
{
Log.Error(ex, $"聚合应用 {appId} 的天数据失败");
}
}
/// <summary>
/// 按周聚合应用网络数据(从天数据聚合)
/// </summary>
private async Task AggregateAppNetworkWeeklyAsync(string appId, long oneWeekAgo)
{
try
{
Log.Information($"聚合应用 {appId} 的周数据,从天数据聚合");
var connection = _dataService.Networks.GetType()
.GetField("_context",
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
?.GetValue(_dataService.Networks);
var contextConnection =
connection?.GetType().GetProperty("Connection")?.GetValue(connection) as System.Data.IDbConnection;
if (contextConnection == null) return;
var weekSeconds = 604800;
string sql = @"
SELECT
(DayTimestamp / @WeekSeconds) * @WeekSeconds as WeekTimestamp,
SUM(TotalUploadBytes) as TotalUpload,
SUM(TotalDownloadBytes) as TotalDownload,
SUM(ConnectionCount) as ConnectionCount,
MAX(UniqueRemoteIPs) as UniqueRemoteIPs,
MAX(UniqueRemotePorts) as UniqueRemotePorts
FROM AppNetworkDaily
WHERE AppId = @AppId AND DayTimestamp < @OneWeekAgo
GROUP BY (DayTimestamp / @WeekSeconds)
ORDER BY WeekTimestamp";
var parameters = new { AppId = appId, OneWeekAgo = oneWeekAgo, WeekSeconds = weekSeconds };
var aggregatedData = await contextConnection.QueryAsync(sql, parameters);
int savedCount = 0;
foreach (dynamic data in aggregatedData)
{
var weekTimestamp = (long)data.WeekTimestamp;
bool exists = await _dataService.Networks.AppNetworkWeeklyExistsAsync(appId, weekTimestamp);
if (!exists)
{
var weeklyData = new AppNetworkWeekly
{
AppId = appId,
WeekTimestamp = weekTimestamp,
TotalUploadBytes = (long)data.TotalUpload,
TotalDownloadBytes = (long)data.TotalDownload,
ConnectionCount = (int)data.ConnectionCount,
UniqueRemoteIPs = (int)data.UniqueRemoteIPs,
UniqueRemotePorts = (int)data.UniqueRemotePorts,
CreatedTimestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds()
};
await _dataService.Networks.SaveAppNetworkWeeklyAsync(weeklyData);
savedCount++;
}
}
Log.Information($"应用 {appId} 的周聚合完成,保存了 {savedCount} 条新数据");
}
catch (Exception ex)
{
Log.Error(ex, $"聚合应用 {appId} 的周数据失败");
}
}
/// <summary>
/// 按月聚合应用网络数据(从周数据聚合)
/// </summary>
private async Task AggregateAppNetworkMonthlyAsync(string appId, long oneMonthAgo)
{
try
{
Log.Information($"聚合应用 {appId} 的月数据,从周数据聚合");
var connection = _dataService.Networks.GetType()
.GetField("_context",
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
?.GetValue(_dataService.Networks);
var contextConnection =
connection?.GetType().GetProperty("Connection")?.GetValue(connection) as System.Data.IDbConnection;
if (contextConnection == null) return;
var monthSeconds = 2592000;
string sql = @"
SELECT
(WeekTimestamp / @MonthSeconds) * @MonthSeconds as MonthTimestamp,
SUM(TotalUploadBytes) as TotalUpload,
SUM(TotalDownloadBytes) as TotalDownload,
SUM(ConnectionCount) as ConnectionCount,
MAX(UniqueRemoteIPs) as UniqueRemoteIPs,
MAX(UniqueRemotePorts) as UniqueRemotePorts
FROM AppNetworkWeekly
WHERE AppId = @AppId AND WeekTimestamp < @OneMonthAgo
GROUP BY (WeekTimestamp / @MonthSeconds)
ORDER BY MonthTimestamp";
var parameters = new { AppId = appId, OneMonthAgo = oneMonthAgo, MonthSeconds = monthSeconds };
var aggregatedData = await contextConnection.QueryAsync(sql, parameters);
int savedCount = 0;
foreach (dynamic data in aggregatedData)
{
var monthTimestamp = (long)data.MonthTimestamp;
bool exists = await _dataService.Networks.AppNetworkMonthlyExistsAsync(appId, monthTimestamp);
if (!exists)
{
var monthlyData = new AppNetworkMonthly
{
AppId = appId,
MonthTimestamp = monthTimestamp,
TotalUploadBytes = (long)data.TotalUpload,
TotalDownloadBytes = (long)data.TotalDownload,
ConnectionCount = (int)data.ConnectionCount,
UniqueRemoteIPs = (int)data.UniqueRemoteIPs,
UniqueRemotePorts = (int)data.UniqueRemotePorts,
CreatedTimestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds()
};
await _dataService.Networks.SaveAppNetworkMonthlyAsync(monthlyData);
savedCount++;
}
}
Log.Information($"应用 {appId} 的月聚合完成,保存了 {savedCount} 条新数据");
}
catch (Exception ex)
{
Log.Error(ex, $"聚合应用 {appId} 的月数据失败");
}
}
/// <summary>
/// 按小时聚合全局网络数据
/// </summary>
private async Task AggregateGlobalNetworkHourlyAsync(string networkCardGuid)
{
try
{
var hourSeconds = 3600;
Log.Information($"聚合网卡 {networkCardGuid} 的小时数据");
var connection = _dataService.Networks.GetType()
.GetField("_context",
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
?.GetValue(_dataService.Networks);
var contextConnection =
connection?.GetType().GetProperty("Connection")?.GetValue(connection) as System.Data.IDbConnection;
if (contextConnection == null)
{
Log.Warning("无法获取数据库连接");
return;
}
string sql = @"
SELECT
(Timestep / @HourSeconds) * @HourSeconds as HourTimestamp,
SUM(Upload) as TotalUpload,
SUM(Download) as TotalDownload,
AVG(Upload) as AvgUpload,
AVG(Download) as AvgDownload,
MAX(Upload) as MaxUpload,
MAX(Download) as MaxDownload,
COUNT(*) as RecordCount
FROM GlobalNetwork
WHERE NetworkCardGuid = @NetworkCardGuid
GROUP BY (Timestep / @HourSeconds)
ORDER BY HourTimestamp";
var parameters = new { NetworkCardGuid = networkCardGuid, HourSeconds = hourSeconds };
var aggregatedData = await contextConnection.QueryAsync(sql, parameters);
Log.Information($"查询到 {aggregatedData.Count()} 条小时聚合数据");
int savedCount = 0;
int skippedCount = 0;
foreach (dynamic data in aggregatedData)
{
var hourTimestamp = (long)data.HourTimestamp;
var totalUpload = (long)data.TotalUpload;
var totalDownload = (long)data.TotalDownload;
var avgUpload = (long)data.AvgUpload;
var avgDownload = (long)data.AvgDownload;
var maxUpload = (long)data.MaxUpload;
var maxDownload = (long)data.MaxDownload;
var recordCount = (int)data.RecordCount;
bool exists =
await _dataService.Networks.GlobalNetworkHourlyExistsAsync(networkCardGuid, hourTimestamp);
if (!exists)
{
var hourlyData = new GlobalNetworkHourly
{
NetworkCardGuid = networkCardGuid,
HourTimestamp = hourTimestamp,
TotalUpload = totalUpload,
TotalDownload = totalDownload,
AvgUpload = avgUpload,
AvgDownload = avgDownload,
MaxUpload = maxUpload,
MaxDownload = maxDownload,
RecordCount = recordCount,
CreatedTimestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds()
};
await _dataService.Networks.SaveGlobalNetworkHourlyAsync(hourlyData);
savedCount++;
}
else
{
skippedCount++;
}
}
Log.Information($"网卡 {networkCardGuid} 的小时聚合完成,保存了 {savedCount} 条新数据,跳过了 {skippedCount} 条已存在数据");
}
catch (Exception ex)
{
Log.Error(ex, $"聚合网卡 {networkCardGuid} 的小时数据失败");
}
}
/// <summary>
/// 按天聚合全局网络数据(从小时数据聚合)
/// </summary>
private async Task AggregateGlobalNetworkDailyAsync(string networkCardGuid, long threeDaysAgo)
{
try
{
Log.Information($"聚合网卡 {networkCardGuid} 的天数据,从小时数据聚合");
var connection = _dataService.Networks.GetType()
.GetField("_context",
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
?.GetValue(_dataService.Networks);
var contextConnection =
connection?.GetType().GetProperty("Connection")?.GetValue(connection) as System.Data.IDbConnection;
if (contextConnection == null) return;
var daySeconds = 86400;
string sql = @"
SELECT
(HourTimestamp / @DaySeconds) * @DaySeconds as DayTimestamp,
SUM(TotalUpload) as TotalUpload,
SUM(TotalDownload) as TotalDownload,
AVG(AvgUpload) as AvgUpload,
AVG(AvgDownload) as AvgDownload,
MAX(MaxUpload) as MaxUpload,
MAX(MaxDownload) as MaxDownload,
SUM(RecordCount) as RecordCount
FROM GlobalNetworkHourly
WHERE NetworkCardGuid = @NetworkCardGuid AND HourTimestamp < @ThreeDaysAgo
GROUP BY (HourTimestamp / @DaySeconds)
ORDER BY DayTimestamp";
var parameters = new
{ NetworkCardGuid = networkCardGuid, ThreeDaysAgo = threeDaysAgo, DaySeconds = daySeconds };
var aggregatedData = await contextConnection.QueryAsync(sql, parameters);
int savedCount = 0;
foreach (dynamic data in aggregatedData)
{
var dayTimestamp = (long)data.DayTimestamp;
bool exists = await _dataService.Networks.GlobalNetworkDailyExistsAsync(networkCardGuid, dayTimestamp);
if (!exists)
{
var dailyData = new GlobalNetworkDaily
{
NetworkCardGuid = networkCardGuid,
DayTimestamp = dayTimestamp,
TotalUpload = (long)data.TotalUpload,
TotalDownload = (long)data.TotalDownload,
AvgUpload = (long)data.AvgUpload,
AvgDownload = (long)data.AvgDownload,
MaxUpload = (long)data.MaxUpload,
MaxDownload = (long)data.MaxDownload,
RecordCount = (int)data.RecordCount,
CreatedTimestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds()
};
await _dataService.Networks.SaveGlobalNetworkDailyAsync(dailyData);
savedCount++;
}
}
Log.Information($"网卡 {networkCardGuid} 的天聚合完成,保存了 {savedCount} 条新数据");
}
catch (Exception ex)
{
Log.Error(ex, $"聚合网卡 {networkCardGuid} 的天数据失败");
}
}
/// <summary>
/// 按周聚合全局网络数据(从天数据聚合)
/// </summary>
private async Task AggregateGlobalNetworkWeeklyAsync(string networkCardGuid, long oneWeekAgo)
{
try
{
Log.Information($"聚合网卡 {networkCardGuid} 的周数据,从天数据聚合");
var connection = _dataService.Networks.GetType()
.GetField("_context",
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
?.GetValue(_dataService.Networks);
var contextConnection =
connection?.GetType().GetProperty("Connection")?.GetValue(connection) as System.Data.IDbConnection;
if (contextConnection == null) return;
var weekSeconds = 604800;
string sql = @"
SELECT
(DayTimestamp / @WeekSeconds) * @WeekSeconds as WeekTimestamp,
SUM(TotalUpload) as TotalUpload,
SUM(TotalDownload) as TotalDownload,
AVG(AvgUpload) as AvgUpload,
AVG(AvgDownload) as AvgDownload,
MAX(MaxUpload) as MaxUpload,
MAX(MaxDownload) as MaxDownload,
SUM(RecordCount) as RecordCount
FROM GlobalNetworkDaily
WHERE NetworkCardGuid = @NetworkCardGuid AND DayTimestamp < @OneWeekAgo
GROUP BY (DayTimestamp / @WeekSeconds)
ORDER BY WeekTimestamp";
var parameters = new
{ NetworkCardGuid = networkCardGuid, OneWeekAgo = oneWeekAgo, WeekSeconds = weekSeconds };
var aggregatedData = await contextConnection.QueryAsync(sql, parameters);
int savedCount = 0;
foreach (dynamic data in aggregatedData)
{
var weekTimestamp = (long)data.WeekTimestamp;
bool exists =
await _dataService.Networks.GlobalNetworkWeeklyExistsAsync(networkCardGuid, weekTimestamp);
if (!exists)
{
var weeklyData = new GlobalNetworkWeekly
{
NetworkCardGuid = networkCardGuid,
WeekTimestamp = weekTimestamp,
TotalUpload = (long)data.TotalUpload,
TotalDownload = (long)data.TotalDownload,
AvgUpload = (long)data.AvgUpload,
AvgDownload = (long)data.AvgDownload,
MaxUpload = (long)data.MaxUpload,
MaxDownload = (long)data.MaxDownload,
RecordCount = (int)data.RecordCount,
CreatedTimestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds()
};
await _dataService.Networks.SaveGlobalNetworkWeeklyAsync(weeklyData);
savedCount++;
}
}
Log.Information($"网卡 {networkCardGuid} 的周聚合完成,保存了 {savedCount} 条新数据");
}
catch (Exception ex)
{
Log.Error(ex, $"聚合网卡 {networkCardGuid} 的周数据失败");
}
}
/// <summary>
/// 执行数据清理
/// </summary>
@@ -360,8 +1043,17 @@ public class DatabaseWriteManager : IDisposable
{
Log.Information("开始执行数据清理任务");
// TODO: 实现数据清理逻辑
// 删除过期的原始数据
var currentTimestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
var oneDayAgo = currentTimestamp - 86400; // 1天前
var oneWeekAgo = currentTimestamp - 604800; // 1周前
var oneMonthAgo = currentTimestamp - 2592000; // 1个月前 (30天)
var threeDaysAgo = currentTimestamp - 259200; // 3天前
// 清理应用网络数据
await CleanupAppNetworkDataAsync(oneDayAgo, oneWeekAgo, oneMonthAgo);
// 清理全局网络数据
await CleanupGlobalNetworkDataAsync(threeDaysAgo, oneWeekAgo);
Log.Information("数据清理任务完成");
return true;
@@ -373,6 +1065,61 @@ public class DatabaseWriteManager : IDisposable
}
}
/// <summary>
/// 清理应用网络数据
/// </summary>
private async Task CleanupAppNetworkDataAsync(long oneDayAgo, long oneWeekAgo, long oneMonthAgo)
{
try
{
Log.Information("开始清理应用网络数据");
// 删除超过1天的原始数据(已聚合到天级数据)
var deletedOldData = await _dataService.Networks.DeleteOldAppNetworkDataAsync(oneDayAgo);
Log.Information($"删除了 {deletedOldData} 条超过1天的应用网络原始数据");
// 删除超过1周的天级聚合数据(已聚合到周级数据)
var deletedWeeklyData = await _dataService.Networks.DeleteAppNetworkDailyBeforeAsync(oneWeekAgo);
Log.Information($"删除了 {deletedWeeklyData} 条超过1周的应用网络天级聚合数据");
// 删除超过1个月的周级聚合数据(已聚合到月级数据)
var deletedMonthlyData = await _dataService.Networks.DeleteAppNetworkWeeklyBeforeAsync(oneMonthAgo);
Log.Information($"删除了 {deletedMonthlyData} 条超过1个月的应用网络周级聚合数据");
Log.Information("应用网络数据清理完成");
}
catch (Exception ex)
{
Log.Error(ex, "清理应用网络数据失败");
}
}
/// <summary>
/// 清理全局网络数据
/// </summary>
private async Task CleanupGlobalNetworkDataAsync(long threeDaysAgo, long oneWeekAgo)
{
try
{
Log.Information("开始清理全局网络数据");
// 删除超过3天的原始数据(已聚合到周级数据)
var deletedOldData = await _dataService.Networks.DeleteOldGlobalNetworkDataAsync(threeDaysAgo);
Log.Information($"删除了 {deletedOldData} 条超过3天的全局网络原始数据");
// 删除超过1周的周级聚合数据(已聚合到月级数据)
var deletedMonthlyData = await _dataService.Networks.DeleteGlobalNetworkWeeklyBeforeAsync(oneWeekAgo);
Log.Information($"删除了 {deletedMonthlyData} 条超过1周的全局网络周级聚合数据");
Log.Information("全局网络数据清理完成");
}
catch (Exception ex)
{
Log.Error(ex, "清理全局网络数据失败");
}
}
/// <summary>
/// 报告统计信息
/// </summary>
+107 -28
View File
@@ -29,6 +29,7 @@ public class NetVizorDbContext : IDisposable
_connection.Execute("PRAGMA synchronous=NORMAL;");
_connection.Execute("PRAGMA cache_size=10000;");
_connection.Execute("PRAGMA temp_store=memory;");
_connection.Execute("PRAGMA auto_vacuum = INCREMENTAL;");
}
return _connection;
@@ -96,65 +97,132 @@ public class NetVizorDbContext : IDisposable
NetworkCardGuid TEXT NOT NULL
)");
// 创建聚合数据表 - 按小时聚合的应用网络数据
// 创建应用网络聚合数据表 - 按小时聚合
await connection.ExecuteAsync(@"
CREATE TABLE IF NOT EXISTS AppNetworkHourly (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
AppId TEXT NOT NULL,
HourTimestamp INTEGER NOT NULL,
RemoteIP TEXT NOT NULL,
RemotePort INTEGER NOT NULL,
Protocol TEXT NOT NULL,
TotalUpload INTEGER DEFAULT 0,
TotalDownload INTEGER DEFAULT 0,
TotalUploadBytes INTEGER DEFAULT 0,
TotalDownloadBytes INTEGER DEFAULT 0,
ConnectionCount INTEGER DEFAULT 0,
FirstSeen INTEGER NOT NULL,
LastSeen INTEGER NOT NULL,
UNIQUE(AppId, HourTimestamp, RemoteIP, RemotePort, Protocol)
UniqueRemoteIPs INTEGER DEFAULT 0,
UniqueRemotePorts INTEGER DEFAULT 0,
CreatedTimestamp INTEGER NOT NULL,
UNIQUE(AppId, HourTimestamp)
)");
// 创建聚合数据表 - 按天聚合的应用网络数据
// 创建应用网络聚合数据表 - 按天聚合
await connection.ExecuteAsync(@"
CREATE TABLE IF NOT EXISTS AppNetworkDaily (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
AppId TEXT NOT NULL,
DayTimestamp INTEGER NOT NULL,
TotalUpload INTEGER DEFAULT 0,
TotalDownload INTEGER DEFAULT 0,
TotalUploadBytes INTEGER DEFAULT 0,
TotalDownloadBytes INTEGER DEFAULT 0,
ConnectionCount INTEGER DEFAULT 0,
TopRemoteIP TEXT,
TopRemotePort INTEGER,
FirstSeen INTEGER NOT NULL,
LastSeen INTEGER NOT NULL,
UniqueRemoteIPs INTEGER DEFAULT 0,
UniqueRemotePorts INTEGER DEFAULT 0,
CreatedTimestamp INTEGER NOT NULL,
UNIQUE(AppId, DayTimestamp)
)");
// 创建全局网络按小时聚合表
// 创建应用网络聚合数据表 - 按周聚合
await connection.ExecuteAsync(@"
CREATE TABLE IF NOT EXISTS AppNetworkWeekly (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
AppId TEXT NOT NULL,
WeekTimestamp INTEGER NOT NULL,
TotalUploadBytes INTEGER DEFAULT 0,
TotalDownloadBytes INTEGER DEFAULT 0,
ConnectionCount INTEGER DEFAULT 0,
UniqueRemoteIPs INTEGER DEFAULT 0,
UniqueRemotePorts INTEGER DEFAULT 0,
CreatedTimestamp INTEGER NOT NULL,
UNIQUE(AppId, WeekTimestamp)
)");
// 创建应用网络聚合数据表 - 按月聚合
await connection.ExecuteAsync(@"
CREATE TABLE IF NOT EXISTS AppNetworkMonthly (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
AppId TEXT NOT NULL,
MonthTimestamp INTEGER NOT NULL,
TotalUploadBytes INTEGER DEFAULT 0,
TotalDownloadBytes INTEGER DEFAULT 0,
ConnectionCount INTEGER DEFAULT 0,
UniqueRemoteIPs INTEGER DEFAULT 0,
UniqueRemotePorts INTEGER DEFAULT 0,
CreatedTimestamp INTEGER NOT NULL,
UNIQUE(AppId, MonthTimestamp)
)");
// 创建全局网络聚合数据表 - 按小时聚合
await connection.ExecuteAsync(@"
CREATE TABLE IF NOT EXISTS GlobalNetworkHourly (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
HourTimestamp INTEGER NOT NULL,
NetworkCardGuid TEXT NOT NULL,
HourTimestamp INTEGER NOT NULL,
TotalUpload INTEGER DEFAULT 0,
TotalDownload INTEGER DEFAULT 0,
MaxUploadSpeed INTEGER DEFAULT 0,
MaxDownloadSpeed INTEGER DEFAULT 0,
UNIQUE(HourTimestamp, NetworkCardGuid)
AvgUpload INTEGER DEFAULT 0,
AvgDownload INTEGER DEFAULT 0,
MaxUpload INTEGER DEFAULT 0,
MaxDownload INTEGER DEFAULT 0,
RecordCount INTEGER DEFAULT 0,
CreatedTimestamp INTEGER NOT NULL,
UNIQUE(NetworkCardGuid, HourTimestamp)
)");
// 创建全局网络按天聚合表
// 创建全局网络聚合数据表 - 按天聚合
await connection.ExecuteAsync(@"
CREATE TABLE IF NOT EXISTS GlobalNetworkDaily (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
DayTimestamp INTEGER NOT NULL,
NetworkCardGuid TEXT NOT NULL,
DayTimestamp INTEGER NOT NULL,
TotalUpload INTEGER DEFAULT 0,
TotalDownload INTEGER DEFAULT 0,
AvgUploadSpeed INTEGER DEFAULT 0,
AvgDownloadSpeed INTEGER DEFAULT 0,
PeakUploadSpeed INTEGER DEFAULT 0,
PeakDownloadSpeed INTEGER DEFAULT 0,
UNIQUE(DayTimestamp, NetworkCardGuid)
AvgUpload INTEGER DEFAULT 0,
AvgDownload INTEGER DEFAULT 0,
MaxUpload INTEGER DEFAULT 0,
MaxDownload INTEGER DEFAULT 0,
RecordCount INTEGER DEFAULT 0,
CreatedTimestamp INTEGER NOT NULL,
UNIQUE(NetworkCardGuid, DayTimestamp)
)");
// 创建全局网络聚合数据表 - 按周聚合
await connection.ExecuteAsync(@"
CREATE TABLE IF NOT EXISTS GlobalNetworkWeekly (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
NetworkCardGuid TEXT NOT NULL,
WeekTimestamp INTEGER NOT NULL,
TotalUpload INTEGER DEFAULT 0,
TotalDownload INTEGER DEFAULT 0,
AvgUpload INTEGER DEFAULT 0,
AvgDownload INTEGER DEFAULT 0,
MaxUpload INTEGER DEFAULT 0,
MaxDownload INTEGER DEFAULT 0,
RecordCount INTEGER DEFAULT 0,
CreatedTimestamp INTEGER NOT NULL,
UNIQUE(NetworkCardGuid, WeekTimestamp)
)");
// 创建全局网络聚合数据表 - 按月聚合
await connection.ExecuteAsync(@"
CREATE TABLE IF NOT EXISTS GlobalNetworkMonthly (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
NetworkCardGuid TEXT NOT NULL,
MonthTimestamp INTEGER NOT NULL,
TotalUpload INTEGER DEFAULT 0,
TotalDownload INTEGER DEFAULT 0,
AvgUpload INTEGER DEFAULT 0,
AvgDownload INTEGER DEFAULT 0,
MaxUpload INTEGER DEFAULT 0,
MaxDownload INTEGER DEFAULT 0,
RecordCount INTEGER DEFAULT 0,
CreatedTimestamp INTEGER NOT NULL,
UNIQUE(NetworkCardGuid, MonthTimestamp)
)");
// 创建索引以提高查询性能
@@ -169,14 +237,25 @@ public class NetVizorDbContext : IDisposable
await connection.ExecuteAsync(
"CREATE INDEX IF NOT EXISTS idx_globalnetwork_guid ON GlobalNetwork(NetworkCardGuid)");
// 应用网络聚合表索引
await connection.ExecuteAsync(
"CREATE INDEX IF NOT EXISTS idx_appnetwork_hourly_timestamp ON AppNetworkHourly(HourTimestamp)");
await connection.ExecuteAsync(
"CREATE INDEX IF NOT EXISTS idx_appnetwork_daily_timestamp ON AppNetworkDaily(DayTimestamp)");
await connection.ExecuteAsync(
"CREATE INDEX IF NOT EXISTS idx_appnetwork_weekly_timestamp ON AppNetworkWeekly(WeekTimestamp)");
await connection.ExecuteAsync(
"CREATE INDEX IF NOT EXISTS idx_appnetwork_monthly_timestamp ON AppNetworkMonthly(MonthTimestamp)");
// 全局网络聚合表索引
await connection.ExecuteAsync(
"CREATE INDEX IF NOT EXISTS idx_globalnetwork_hourly_timestamp ON GlobalNetworkHourly(HourTimestamp)");
await connection.ExecuteAsync(
"CREATE INDEX IF NOT EXISTS idx_globalnetwork_daily_timestamp ON GlobalNetworkDaily(DayTimestamp)");
await connection.ExecuteAsync(
"CREATE INDEX IF NOT EXISTS idx_globalnetwork_weekly_timestamp ON GlobalNetworkWeekly(WeekTimestamp)");
await connection.ExecuteAsync(
"CREATE INDEX IF NOT EXISTS idx_globalnetwork_monthly_timestamp ON GlobalNetworkMonthly(MonthTimestamp)");
}
private async Task MigrateDatabaseAsync()
@@ -42,6 +42,38 @@ public interface INetworkRepository
// 数据维护
Task<int> DeleteOldAppNetworkDataAsync(long beforeTimestamp);
Task<int> DeleteOldGlobalNetworkDataAsync(long beforeTimestamp);
// 应用网络聚合数据操作
Task<int> SaveAppNetworkHourlyAsync(AppNetworkHourly hourly);
Task<int> SaveAppNetworkDailyAsync(AppNetworkDaily daily);
Task<int> SaveAppNetworkWeeklyAsync(AppNetworkWeekly weekly);
Task<int> SaveAppNetworkMonthlyAsync(AppNetworkMonthly monthly);
// 全局网络聚合数据操作
Task<int> SaveGlobalNetworkHourlyAsync(GlobalNetworkHourly hourly);
Task<int> SaveGlobalNetworkDailyAsync(GlobalNetworkDaily daily);
Task<int> SaveGlobalNetworkWeeklyAsync(GlobalNetworkWeekly weekly);
Task<int> SaveGlobalNetworkMonthlyAsync(GlobalNetworkMonthly monthly);
// 数据存在性检查
Task<bool> AppNetworkHourlyExistsAsync(string appId, long hourTimestamp);
Task<bool> AppNetworkDailyExistsAsync(string appId, long dayTimestamp);
Task<bool> AppNetworkWeeklyExistsAsync(string appId, long weekTimestamp);
Task<bool> AppNetworkMonthlyExistsAsync(string appId, long monthTimestamp);
Task<bool> GlobalNetworkHourlyExistsAsync(string networkCardGuid, long hourTimestamp);
Task<bool> GlobalNetworkDailyExistsAsync(string networkCardGuid, long dayTimestamp);
Task<bool> GlobalNetworkWeeklyExistsAsync(string networkCardGuid, long weekTimestamp);
Task<bool> GlobalNetworkMonthlyExistsAsync(string networkCardGuid, long monthTimestamp);
// 聚合数据清理
Task<int> DeleteAppNetworkHourlyBeforeAsync(long beforeTimestamp);
Task<int> DeleteAppNetworkDailyBeforeAsync(long beforeTimestamp);
Task<int> DeleteAppNetworkWeeklyBeforeAsync(long beforeTimestamp);
Task<int> DeleteGlobalNetworkHourlyBeforeAsync(long beforeTimestamp);
Task<int> DeleteGlobalNetworkDailyBeforeAsync(long beforeTimestamp);
Task<int> DeleteGlobalNetworkWeeklyBeforeAsync(long beforeTimestamp);
}
public class NetworkRepository : INetworkRepository
@@ -424,6 +456,201 @@ public class NetworkRepository : INetworkRepository
}
#endregion
#region 应用网络聚合数据操作
public async Task<int> SaveAppNetworkHourlyAsync(AppNetworkHourly hourly)
{
const string sql = @"
INSERT INTO AppNetworkHourly (AppId, HourTimestamp, TotalUploadBytes, TotalDownloadBytes, ConnectionCount, UniqueRemoteIPs, UniqueRemotePorts, CreatedTimestamp)
VALUES (@AppId, @HourTimestamp, @TotalUploadBytes, @TotalDownloadBytes, @ConnectionCount, @UniqueRemoteIPs, @UniqueRemotePorts, @CreatedTimestamp)";
return await _context.Connection.ExecuteAsync(sql, hourly);
}
public async Task<int> SaveAppNetworkDailyAsync(AppNetworkDaily daily)
{
const string sql = @"
INSERT INTO AppNetworkDaily (AppId, DayTimestamp, TotalUploadBytes, TotalDownloadBytes, ConnectionCount, UniqueRemoteIPs, UniqueRemotePorts, CreatedTimestamp)
VALUES (@AppId, @DayTimestamp, @TotalUploadBytes, @TotalDownloadBytes, @ConnectionCount, @UniqueRemoteIPs, @UniqueRemotePorts, @CreatedTimestamp)";
return await _context.Connection.ExecuteAsync(sql, daily);
}
public async Task<int> SaveAppNetworkWeeklyAsync(AppNetworkWeekly weekly)
{
const string sql = @"
INSERT INTO AppNetworkWeekly (AppId, WeekTimestamp, TotalUploadBytes, TotalDownloadBytes, ConnectionCount, UniqueRemoteIPs, UniqueRemotePorts, CreatedTimestamp)
VALUES (@AppId, @WeekTimestamp, @TotalUploadBytes, @TotalDownloadBytes, @ConnectionCount, @UniqueRemoteIPs, @UniqueRemotePorts, @CreatedTimestamp)";
return await _context.Connection.ExecuteAsync(sql, weekly);
}
public async Task<int> SaveAppNetworkMonthlyAsync(AppNetworkMonthly monthly)
{
const string sql = @"
INSERT INTO AppNetworkMonthly (AppId, MonthTimestamp, TotalUploadBytes, TotalDownloadBytes, ConnectionCount, UniqueRemoteIPs, UniqueRemotePorts, CreatedTimestamp)
VALUES (@AppId, @MonthTimestamp, @TotalUploadBytes, @TotalDownloadBytes, @ConnectionCount, @UniqueRemoteIPs, @UniqueRemotePorts, @CreatedTimestamp)";
return await _context.Connection.ExecuteAsync(sql, monthly);
}
#endregion
#region 全局网络聚合数据操作
public async Task<int> SaveGlobalNetworkHourlyAsync(GlobalNetworkHourly hourly)
{
const string sql = @"
INSERT INTO GlobalNetworkHourly (NetworkCardGuid, HourTimestamp, TotalUpload, TotalDownload, AvgUpload, AvgDownload, MaxUpload, MaxDownload, RecordCount, CreatedTimestamp)
VALUES (@NetworkCardGuid, @HourTimestamp, @TotalUpload, @TotalDownload, @AvgUpload, @AvgDownload, @MaxUpload, @MaxDownload, @RecordCount, @CreatedTimestamp)";
return await _context.Connection.ExecuteAsync(sql, hourly);
}
public async Task<int> SaveGlobalNetworkDailyAsync(GlobalNetworkDaily daily)
{
const string sql = @"
INSERT INTO GlobalNetworkDaily (NetworkCardGuid, DayTimestamp, TotalUpload, TotalDownload, AvgUpload, AvgDownload, MaxUpload, MaxDownload, RecordCount, CreatedTimestamp)
VALUES (@NetworkCardGuid, @DayTimestamp, @TotalUpload, @TotalDownload, @AvgUpload, @AvgDownload, @MaxUpload, @MaxDownload, @RecordCount, @CreatedTimestamp)";
return await _context.Connection.ExecuteAsync(sql, daily);
}
public async Task<int> SaveGlobalNetworkWeeklyAsync(GlobalNetworkWeekly weekly)
{
const string sql = @"
INSERT INTO GlobalNetworkWeekly (NetworkCardGuid, WeekTimestamp, TotalUpload, TotalDownload, AvgUpload, AvgDownload, MaxUpload, MaxDownload, RecordCount, CreatedTimestamp)
VALUES (@NetworkCardGuid, @WeekTimestamp, @TotalUpload, @TotalDownload, @AvgUpload, @AvgDownload, @MaxUpload, @MaxDownload, @RecordCount, @CreatedTimestamp)";
return await _context.Connection.ExecuteAsync(sql, weekly);
}
public async Task<int> SaveGlobalNetworkMonthlyAsync(GlobalNetworkMonthly monthly)
{
const string sql = @"
INSERT INTO GlobalNetworkMonthly (NetworkCardGuid, MonthTimestamp, TotalUpload, TotalDownload, AvgUpload, AvgDownload, MaxUpload, MaxDownload, RecordCount, CreatedTimestamp)
VALUES (@NetworkCardGuid, @MonthTimestamp, @TotalUpload, @TotalDownload, @AvgUpload, @AvgDownload, @MaxUpload, @MaxDownload, @RecordCount, @CreatedTimestamp)";
return await _context.Connection.ExecuteAsync(sql, monthly);
}
#endregion
#region 数据存在性检查
public async Task<bool> AppNetworkHourlyExistsAsync(string appId, long hourTimestamp)
{
const string sql =
"SELECT COUNT(1) FROM AppNetworkHourly WHERE AppId = @AppId AND HourTimestamp = @HourTimestamp";
var count = await _context.Connection.QuerySingleAsync<int>(sql,
new { AppId = appId, HourTimestamp = hourTimestamp });
return count > 0;
}
public async Task<bool> AppNetworkDailyExistsAsync(string appId, long dayTimestamp)
{
const string sql = "SELECT COUNT(1) FROM AppNetworkDaily WHERE AppId = @AppId AND DayTimestamp = @DayTimestamp";
var count = await _context.Connection.QuerySingleAsync<int>(sql,
new { AppId = appId, DayTimestamp = dayTimestamp });
return count > 0;
}
public async Task<bool> AppNetworkWeeklyExistsAsync(string appId, long weekTimestamp)
{
const string sql =
"SELECT COUNT(1) FROM AppNetworkWeekly WHERE AppId = @AppId AND WeekTimestamp = @WeekTimestamp";
var count = await _context.Connection.QuerySingleAsync<int>(sql,
new { AppId = appId, WeekTimestamp = weekTimestamp });
return count > 0;
}
public async Task<bool> AppNetworkMonthlyExistsAsync(string appId, long monthTimestamp)
{
const string sql =
"SELECT COUNT(1) FROM AppNetworkMonthly WHERE AppId = @AppId AND MonthTimestamp = @MonthTimestamp";
var count = await _context.Connection.QuerySingleAsync<int>(sql,
new { AppId = appId, MonthTimestamp = monthTimestamp });
return count > 0;
}
public async Task<bool> GlobalNetworkHourlyExistsAsync(string networkCardGuid, long hourTimestamp)
{
const string sql =
"SELECT COUNT(1) FROM GlobalNetworkHourly WHERE NetworkCardGuid = @NetworkCardGuid AND HourTimestamp = @HourTimestamp";
var count = await _context.Connection.QuerySingleAsync<int>(sql,
new { NetworkCardGuid = networkCardGuid, HourTimestamp = hourTimestamp });
return count > 0;
}
public async Task<bool> GlobalNetworkDailyExistsAsync(string networkCardGuid, long dayTimestamp)
{
const string sql =
"SELECT COUNT(1) FROM GlobalNetworkDaily WHERE NetworkCardGuid = @NetworkCardGuid AND DayTimestamp = @DayTimestamp";
var count = await _context.Connection.QuerySingleAsync<int>(sql,
new { NetworkCardGuid = networkCardGuid, DayTimestamp = dayTimestamp });
return count > 0;
}
public async Task<bool> GlobalNetworkWeeklyExistsAsync(string networkCardGuid, long weekTimestamp)
{
const string sql =
"SELECT COUNT(1) FROM GlobalNetworkWeekly WHERE NetworkCardGuid = @NetworkCardGuid AND WeekTimestamp = @WeekTimestamp";
var count = await _context.Connection.QuerySingleAsync<int>(sql,
new { NetworkCardGuid = networkCardGuid, WeekTimestamp = weekTimestamp });
return count > 0;
}
public async Task<bool> GlobalNetworkMonthlyExistsAsync(string networkCardGuid, long monthTimestamp)
{
const string sql =
"SELECT COUNT(1) FROM GlobalNetworkMonthly WHERE NetworkCardGuid = @NetworkCardGuid AND MonthTimestamp = @MonthTimestamp";
var count = await _context.Connection.QuerySingleAsync<int>(sql,
new { NetworkCardGuid = networkCardGuid, MonthTimestamp = monthTimestamp });
return count > 0;
}
#endregion
#region 聚合数据清理
public async Task<int> DeleteAppNetworkHourlyBeforeAsync(long beforeTimestamp)
{
const string sql = "DELETE FROM AppNetworkHourly WHERE HourTimestamp < @BeforeTimestamp";
return await _context.Connection.ExecuteAsync(sql, new { BeforeTimestamp = beforeTimestamp });
}
public async Task<int> DeleteAppNetworkDailyBeforeAsync(long beforeTimestamp)
{
const string sql = "DELETE FROM AppNetworkDaily WHERE DayTimestamp < @BeforeTimestamp";
return await _context.Connection.ExecuteAsync(sql, new { BeforeTimestamp = beforeTimestamp });
}
public async Task<int> DeleteAppNetworkWeeklyBeforeAsync(long beforeTimestamp)
{
const string sql = "DELETE FROM AppNetworkWeekly WHERE WeekTimestamp < @BeforeTimestamp";
return await _context.Connection.ExecuteAsync(sql, new { BeforeTimestamp = beforeTimestamp });
}
public async Task<int> DeleteGlobalNetworkHourlyBeforeAsync(long beforeTimestamp)
{
const string sql = "DELETE FROM GlobalNetworkHourly WHERE HourTimestamp < @BeforeTimestamp";
return await _context.Connection.ExecuteAsync(sql, new { BeforeTimestamp = beforeTimestamp });
}
public async Task<int> DeleteGlobalNetworkDailyBeforeAsync(long beforeTimestamp)
{
const string sql = "DELETE FROM GlobalNetworkDaily WHERE DayTimestamp < @BeforeTimestamp";
return await _context.Connection.ExecuteAsync(sql, new { BeforeTimestamp = beforeTimestamp });
}
public async Task<int> DeleteGlobalNetworkWeeklyBeforeAsync(long beforeTimestamp)
{
const string sql = "DELETE FROM GlobalNetworkWeekly WHERE WeekTimestamp < @BeforeTimestamp";
return await _context.Connection.ExecuteAsync(sql, new { BeforeTimestamp = beforeTimestamp });
}
#endregion
}
#region 查询结果模型
@@ -20,6 +20,7 @@ public class NetworkDataCollectionService : IDisposable
private readonly Timer _etwNetworkTimer;
private readonly Timer _networkInterfaceUpdateTimer;
private readonly Timer _batchFlushTimer;
private readonly Timer _dataCleanupTimer;
// 网卡信息缓存
private readonly ConcurrentDictionary<string, BasicNetworkMonitor.BasicNetworkInterface> _networkInterfaces = new();
@@ -48,6 +49,10 @@ public class NetworkDataCollectionService : IDisposable
_batchFlushTimer = new Timer(async _ => await _writeManager.FlushAllBatchesAsync(), null,
TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30));
// 每小时检测清理并合并旧数据
_dataCleanupTimer = new Timer(TriggerDataCleanupAndAggregation, null,
TimeSpan.FromHours(1), TimeSpan.FromHours(1));
Log.Information("网络数据收集服务已启动");
}
@@ -220,7 +225,7 @@ public class NetworkDataCollectionService : IDisposable
_lastNetworkInterfaceUpdate = DateTime.UtcNow;
Log.Information($"更新网络接口列表,共找到 {interfaces.Count} 个活跃接口");
// Log.Information($"更新网络接口列表,共找到 {interfaces.Count} 个活跃接口");
}
catch (Exception ex)
{
@@ -286,6 +291,48 @@ public class NetworkDataCollectionService : IDisposable
};
}
/// <summary>
/// 触发数据清理和聚合任务
/// </summary>
private async void TriggerDataCleanupAndAggregation(object state)
{
try
{
Log.Information("Start 开始执行定时数据清理和聚合任务");
// 先触发数据聚合
Log.Information("触发数据聚合任务");
var aggregationResult = await _writeManager.TriggerDataAggregationAsync();
if (aggregationResult)
{
Log.Information("数据聚合任务完成,开始触发数据清理任务");
// 聚合成功后进行数据清理
var cleanupResult = await _writeManager.TriggerDataCleanupAsync();
if (cleanupResult)
{
Log.Information("数据清理任务完成");
}
else
{
Log.Warning("数据清理任务执行失败");
}
}
else
{
Log.Warning("数据聚合任务执行失败,跳过清理任务");
}
Log.Information("定时数据清理和聚合任务执行完成");
}
catch (Exception ex)
{
Log.Error(ex, "定时数据清理和聚合任务执行时发生错误");
}
}
public void Dispose()
{
Log.Information("正在关闭网络数据收集服务...");
@@ -294,6 +341,7 @@ public class NetworkDataCollectionService : IDisposable
_etwNetworkTimer?.Dispose();
_networkInterfaceUpdateTimer?.Dispose();
_batchFlushTimer?.Dispose();
_dataCleanupTimer?.Dispose();
// 强制刷新所有批处理数据
try
@@ -37,6 +37,7 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ANetworkInterfaceType_002Ecs_002Fl_003AC_0021_003FUsers_003Fyllyo_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F92cb821cac5045cbbf918dc41b2ea2ca26920_003Fa4_003F57cbd2f9_003FNetworkInterfaceType_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ANetworkInterface_002Ecs_002Fl_003AC_0021_003FUsers_003Fcharlatans_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F9c72a65889eb40ae9d8e0f130d1572e0268a0_003F34_003F49f5e6ff_003FNetworkInterface_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ANetworkInterface_002Ecs_002Fl_003AC_0021_003FUsers_003Fyllyo_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F92cb821cac5045cbbf918dc41b2ea2ca26920_003F35_003Fdb05c251_003FNetworkInterface_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AObject_002Ecs_002Fl_003AC_0021_003FUsers_003Fyllyo_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F5aeb6167299d4cfba30c86624130edbcc90938_003F1d_003F3ef0ca46_003FObject_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AProcessModule_002Ecs_002Fl_003AC_0021_003FUsers_003Fyllyo_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fa70eaad10e4f42d4a1f24ff3865c6c6252928_003Fec_003F0881d57a_003FProcessModule_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AProcess_002Ecs_002Fl_003AC_0021_003FUsers_003Fcharlatans_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F0d82aa51d3914b469b94baa8bc2312c052910_003F11_003Fa7f7fe71_003FProcess_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AProcess_002Ecs_002Fl_003AC_0021_003FUsers_003Fcharlatans_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F4a26a752ec6249c4a090905d31c0cdfa528a0_003F8f_003F7037e590_003FProcess_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
@@ -49,11 +50,13 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ASemaphoreSlim_002Ecs_002Fl_003AC_0021_003FUsers_003Fcharlatans_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F211e6f3b24fa438a92f1815153647ce2c8f908_003Fe9_003F4d4110fd_003FSemaphoreSlim_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002Ecs_002Fl_003AC_0021_003FUsers_003Fcharlatans_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F211e6f3b24fa438a92f1815153647ce2c8f908_003Ff1_003F7b5a67e3_003FString_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AString_002Ecs_002Fl_003AC_0021_003FUsers_003Fyllyo_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F5aeb6167299d4cfba30c86624130edbcc90938_003Fac_003F9cba3bcf_003FString_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATaskCompletionSource_005FT_002Ecs_002Fl_003AC_0021_003FUsers_003Fyllyo_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FSourcesCache_003F5eaf6f9dd55429069729b5d6a6b99c1571fb63cf5689bed62ef241e99e88d7_003FTaskCompletionSource_005FT_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATask_002Ecs_002Fl_003AC_0021_003FUsers_003Fyllyo_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FSourcesCache_003Fa77b231cea33ff338c41dc7869d6c493d4dae137ff51e342173efa61d933_003FTask_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATcpIpConnectTraceData_002Ecs_002Fl_003AC_0021_003FUsers_003Fcharlatans_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F25b12134b1124ffbab73641db2e54e5b310830_003F93_003Fd6e6c065_003FTcpIpConnectTraceData_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATcpIpSendTraceData_002Ecs_002Fl_003AC_0021_003FUsers_003Fcharlatans_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F25b12134b1124ffbab73641db2e54e5b310830_003F28_003Fce7c9cfa_003FTcpIpSendTraceData_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATimer_002Ecs_002Fl_003AC_0021_003FUsers_003Fcharlatans_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F211e6f3b24fa438a92f1815153647ce2c8f908_003Fa8_003F04476824_003FTimer_002Ecs_002Fz_003A2_002D1/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATimer_002Ecs_002Fl_003AC_0021_003FUsers_003Fcharlatans_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Ffe2ae574758f4da99bb3bfcba69a0a40b6928_003F7f_003Fb5cb1e21_003FTimer_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATimeSpan_002Ecs_002Fl_003AC_0021_003FUsers_003Fyllyo_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FSourcesCache_003F4045ad381dc69b9a8c076afe4c122961b5a86158d8dd0c76c117243ce72b_003FTimeSpan_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATraceEventLevel_002Ecs_002Fl_003AC_0021_003FUsers_003Fcharlatans_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F25b12134b1124ffbab73641db2e54e5b310830_003F3e_003F5dae014e_003FTraceEventLevel_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATraceEventNativeMethods_002Ecs_002Fl_003AC_0021_003FUsers_003Fcharlatans_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F25b12134b1124ffbab73641db2e54e5b310830_003F03_003Fcb4b3163_003FTraceEventNativeMethods_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ATraceEventSession_002Ecs_002Fl_003AC_0021_003FUsers_003Fcharlatans_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E1_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F25b12134b1124ffbab73641db2e54e5b310830_003F4e_003F99d3e433_003FTraceEventSession_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>