using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; using SqrtSpace.SpaceTime.Core; using SqrtSpace.SpaceTime.EntityFramework; using SqrtSpace.SpaceTime.Linq; using SampleWebApi.Data; using SampleWebApi.Models; using System.Diagnostics; namespace SampleWebApi.Services; public interface IOrderAnalyticsService { Task> GetRevenueByCategoryAsync(DateTime? startDate, DateTime? endDate); Task> GetTopCustomersAsync(int top, DateTime? since); IAsyncEnumerable StreamRealTimeAnalyticsAsync(CancellationToken cancellationToken); Task GenerateComplexReportAsync(ReportRequest request, string reportId, ReportState? previousState, CheckpointManager? checkpoint); Task AnalyzeOrderPatternsAsync(PatternAnalysisRequest request); MemoryStatistics GetMemoryStatistics(); } public class OrderAnalyticsService : IOrderAnalyticsService { private readonly SampleDbContext _context; private readonly ILogger _logger; private readonly MemoryOptions _memoryOptions; private static readonly MemoryStatistics _memoryStats = new(); public OrderAnalyticsService( SampleDbContext context, ILogger logger, IOptions memoryOptions) { _context = context; _logger = logger; _memoryOptions = memoryOptions.Value; } public async Task> GetRevenueByCategoryAsync(DateTime? startDate, DateTime? endDate) { var query = _context.OrderItems .Include(oi => oi.Product) .Include(oi => oi.Order) .AsQueryable(); if (startDate.HasValue) query = query.Where(oi => oi.Order.OrderDate >= startDate.Value); if (endDate.HasValue) query = query.Where(oi => oi.Order.OrderDate <= endDate.Value); var itemCount = await query.CountAsync(); _logger.LogInformation("Processing revenue for {count} order items", itemCount); // Use external grouping for large datasets if (itemCount > 50000) { _logger.LogInformation("Using external grouping for revenue calculation"); _memoryStats.ExternalSortOperations++; var categoryRevenue = new Dictionary(); // Process in memory-efficient batches await foreach (var batch in query.BatchBySqrtNAsync()) { foreach (var item in batch) { var category = item.Product.Category; if (!categoryRevenue.ContainsKey(category)) { categoryRevenue[category] = (0, 0); } var current = categoryRevenue[category]; categoryRevenue[category] = (current.revenue + item.TotalPrice, current.count + 1); } } return categoryRevenue.Select(kvp => new CategoryRevenue { Category = kvp.Key, TotalRevenue = kvp.Value.revenue, OrderCount = kvp.Value.count, AverageOrderValue = kvp.Value.count > 0 ? kvp.Value.revenue / kvp.Value.count : 0 }).OrderByDescending(c => c.TotalRevenue); } else { // Use in-memory grouping for smaller datasets var grouped = await query .GroupBy(oi => oi.Product.Category) .Select(g => new CategoryRevenue { Category = g.Key, TotalRevenue = g.Sum(oi => oi.TotalPrice), OrderCount = g.Select(oi => oi.OrderId).Distinct().Count(), AverageOrderValue = g.Average(oi => oi.TotalPrice) }) .OrderByDescending(c => c.TotalRevenue) .ToListAsync(); return grouped; } } public async Task> GetTopCustomersAsync(int top, DateTime? since) { var query = _context.Orders.AsQueryable(); if (since.HasValue) query = query.Where(o => o.OrderDate >= since.Value); var orderCount = await query.CountAsync(); _logger.LogInformation("Finding top {top} customers from {count} orders", top, orderCount); // For large datasets, use external sorting if (orderCount > 100000) { _logger.LogInformation("Using external sorting for top customers"); _memoryStats.ExternalSortOperations++; var customerData = new Dictionary(); // Aggregate customer data in batches await foreach (var batch in query.BatchBySqrtNAsync()) { foreach (var order in batch) { if (!customerData.ContainsKey(order.CustomerId)) { customerData[order.CustomerId] = (0, 0, order.OrderDate, order.OrderDate); } var current = customerData[order.CustomerId]; customerData[order.CustomerId] = ( current.total + order.TotalAmount, current.count + 1, order.OrderDate < current.first ? order.OrderDate : current.first, order.OrderDate > current.last ? order.OrderDate : current.last ); } } // Get customer details var customerIds = customerData.Keys.ToList(); var customers = await _context.Customers .Where(c => customerIds.Contains(c.Id)) .ToDictionaryAsync(c => c.Id, c => c.Name); // Sort and take top N return customerData .OrderByDescending(kvp => kvp.Value.total) .Take(top) .Select(kvp => new CustomerSummary { CustomerId = kvp.Key, CustomerName = customers.GetValueOrDefault(kvp.Key, "Unknown"), TotalOrders = kvp.Value.count, TotalSpent = kvp.Value.total, AverageOrderValue = kvp.Value.total / kvp.Value.count, FirstOrderDate = kvp.Value.first, LastOrderDate = kvp.Value.last }); } else { // Use in-memory processing for smaller datasets var topCustomers = await query .GroupBy(o => o.CustomerId) .Select(g => new { CustomerId = g.Key, TotalSpent = g.Sum(o => o.TotalAmount), OrderCount = g.Count(), FirstOrder = g.Min(o => o.OrderDate), LastOrder = g.Max(o => o.OrderDate) }) .OrderByDescending(c => c.TotalSpent) .Take(top) .ToListAsync(); var customerIds = topCustomers.Select(c => c.CustomerId).ToList(); var customers = await _context.Customers .Where(c => customerIds.Contains(c.Id)) .ToDictionaryAsync(c => c.Id, c => c.Name); return topCustomers.Select(c => new CustomerSummary { CustomerId = c.CustomerId, CustomerName = customers.GetValueOrDefault(c.CustomerId, "Unknown"), TotalOrders = c.OrderCount, TotalSpent = c.TotalSpent, AverageOrderValue = c.TotalSpent / c.OrderCount, FirstOrderDate = c.FirstOrder, LastOrderDate = c.LastOrder }); } } public async IAsyncEnumerable StreamRealTimeAnalyticsAsync( [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { var now = DateTime.UtcNow; var hourAgo = now.AddHours(-1); // Get orders from last hour var recentOrders = await _context.Orders .Where(o => o.OrderDate >= hourAgo) .Include(o => o.Items) .ThenInclude(oi => oi.Product) .ToListAsync(cancellationToken); // Calculate analytics var analytics = new RealTimeAnalytics { Timestamp = now, OrdersLastHour = recentOrders.Count, RevenueLastHour = recentOrders.Sum(o => o.TotalAmount), ActiveCustomers = recentOrders.Select(o => o.CustomerId).Distinct().Count(), OrdersPerMinute = recentOrders.Count / 60.0 }; // Get top products analytics.TopProductsLastHour = recentOrders .SelectMany(o => o.Items) .GroupBy(oi => oi.Product.Name) .OrderByDescending(g => g.Sum(oi => oi.Quantity)) .Take(5) .ToDictionary(g => g.Key, g => g.Sum(oi => oi.Quantity)); yield return analytics; // Update memory stats var process = Process.GetCurrentProcess(); _memoryStats.CurrentMemoryUsageMB = process.WorkingSet64 / (1024 * 1024); _memoryStats.PeakMemoryUsageMB = Math.Max(_memoryStats.PeakMemoryUsageMB, _memoryStats.CurrentMemoryUsageMB); await Task.Delay(1000, cancellationToken); // Wait before next update } } public async Task GenerateComplexReportAsync( ReportRequest request, string reportId, ReportState? previousState, CheckpointManager? checkpoint) { var stopwatch = Stopwatch.StartNew(); var state = previousState ?? new ReportState { ReportId = reportId }; var result = new ReportResult { ReportId = reportId, GeneratedAt = DateTime.UtcNow, Metrics = state.PartialResults }; try { // Step 1: Calculate total revenue (0-25%) if (state.ProgressPercent < 25) { var revenue = await CalculateTotalRevenueAsync(request.StartDate, request.EndDate); result.Metrics["totalRevenue"] = revenue; state.ProgressPercent = 25; if (checkpoint?.ShouldCheckpoint() == true) { state.PartialResults = result.Metrics; await checkpoint.CreateCheckpointAsync(state); _memoryStats.CheckpointsSaved++; } } // Step 2: Calculate category breakdown (25-50%) if (state.ProgressPercent < 50) { var categoryRevenue = await GetRevenueByCategoryAsync(request.StartDate, request.EndDate); result.Metrics["categoryBreakdown"] = categoryRevenue; state.ProgressPercent = 50; if (checkpoint?.ShouldCheckpoint() == true) { state.PartialResults = result.Metrics; await checkpoint.CreateCheckpointAsync(state); _memoryStats.CheckpointsSaved++; } } // Step 3: Customer analytics (50-75%) if (state.ProgressPercent < 75) { var topCustomers = await GetTopCustomersAsync(100, request.StartDate); result.Metrics["topCustomers"] = topCustomers; state.ProgressPercent = 75; if (checkpoint?.ShouldCheckpoint() == true) { state.PartialResults = result.Metrics; await checkpoint.CreateCheckpointAsync(state); _memoryStats.CheckpointsSaved++; } } // Step 4: Product performance (75-100%) if (state.ProgressPercent < 100) { var productStats = await CalculateProductPerformanceAsync(request.StartDate, request.EndDate); result.Metrics["productPerformance"] = productStats; state.ProgressPercent = 100; } result.Completed = true; result.ProgressPercent = 100; result.ProcessingTimeMs = stopwatch.ElapsedMilliseconds; result.MemoryUsedMB = _memoryStats.CurrentMemoryUsageMB; _logger.LogInformation("Report {reportId} completed in {time}ms", reportId, result.ProcessingTimeMs); return result; } catch (Exception ex) { _logger.LogError(ex, "Error generating report {reportId}", reportId); // Save checkpoint on error if (checkpoint != null) { state.PartialResults = result.Metrics; await checkpoint.CreateCheckpointAsync(state); } throw; } } public async Task AnalyzeOrderPatternsAsync(PatternAnalysisRequest request) { var stopwatch = Stopwatch.StartNew(); var result = new PatternAnalysisResult(); // Limit the analysis scope var orders = await _context.Orders .OrderByDescending(o => o.OrderDate) .Take(request.MaxOrdersToAnalyze) .Include(o => o.Items) .ToListAsync(); result.RecordsProcessed = orders.Count; // Analyze order patterns result.OrderPatterns["averageOrderValue"] = orders.Average(o => (double)o.TotalAmount); result.OrderPatterns["ordersPerDay"] = orders .GroupBy(o => o.OrderDate.Date) .Average(g => g.Count()); // Customer segmentation if (request.IncludeCustomerSegmentation) { var customerGroups = orders .GroupBy(o => o.CustomerId) .Select(g => new { CustomerId = g.Key, OrderCount = g.Count(), TotalSpent = g.Sum(o => o.TotalAmount), AverageOrder = g.Average(o => o.TotalAmount) }) .ToList(); // Simple segmentation based on spending result.CustomerSegments = new List { new CustomerSegment { SegmentName = "High Value", CustomerCount = customerGroups.Count(c => c.TotalSpent > 1000), Characteristics = new Dictionary { ["averageOrderValue"] = customerGroups.Where(c => c.TotalSpent > 1000).Average(c => (double)c.AverageOrder), ["ordersPerCustomer"] = customerGroups.Where(c => c.TotalSpent > 1000).Average(c => c.OrderCount) } }, new CustomerSegment { SegmentName = "Regular", CustomerCount = customerGroups.Count(c => c.TotalSpent >= 100 && c.TotalSpent <= 1000), Characteristics = new Dictionary { ["averageOrderValue"] = customerGroups.Where(c => c.TotalSpent >= 100 && c.TotalSpent <= 1000).Average(c => (double)c.AverageOrder), ["ordersPerCustomer"] = customerGroups.Where(c => c.TotalSpent >= 100 && c.TotalSpent <= 1000).Average(c => c.OrderCount) } } }; } // Seasonal analysis if (request.IncludeSeasonalAnalysis) { result.SeasonalAnalysis = new SeasonalAnalysis { MonthlySalesPattern = orders .GroupBy(o => o.OrderDate.Month) .ToDictionary(g => g.Key.ToString(), g => (double)g.Sum(o => o.TotalAmount)), WeeklySalesPattern = orders .GroupBy(o => o.OrderDate.DayOfWeek) .ToDictionary(g => g.Key.ToString(), g => (double)g.Sum(o => o.TotalAmount)), PeakPeriods = orders .GroupBy(o => o.OrderDate.Date) .OrderByDescending(g => g.Sum(o => o.TotalAmount)) .Take(5) .Select(g => g.Key.ToString("yyyy-MM-dd")) .ToList() }; } result.AnalysisTimeMs = stopwatch.ElapsedMilliseconds; result.MemoryUsedMB = _memoryStats.CurrentMemoryUsageMB; return result; } public MemoryStatistics GetMemoryStatistics() { var process = Process.GetCurrentProcess(); _memoryStats.CurrentMemoryUsageMB = process.WorkingSet64 / (1024 * 1024); // Determine memory pressure var usagePercent = (_memoryStats.CurrentMemoryUsageMB * 100) / _memoryOptions.MaxMemoryMB; _memoryStats.CurrentMemoryPressure = usagePercent switch { < 50 => "Low", < 80 => "Medium", _ => "High" }; return _memoryStats; } private async Task CalculateTotalRevenueAsync(DateTime startDate, DateTime endDate) { var revenue = await _context.Orders .Where(o => o.OrderDate >= startDate && o.OrderDate <= endDate) .SumAsync(o => o.TotalAmount); return revenue; } private async Task CalculateProductPerformanceAsync(DateTime startDate, DateTime endDate) { var query = _context.OrderItems .Include(oi => oi.Product) .Include(oi => oi.Order) .Where(oi => oi.Order.OrderDate >= startDate && oi.Order.OrderDate <= endDate); var productPerformance = await query .GroupBy(oi => new { oi.ProductId, oi.Product.Name }) .Select(g => new { ProductId = g.Key.ProductId, ProductName = g.Key.Name, UnitsSold = g.Sum(oi => oi.Quantity), Revenue = g.Sum(oi => oi.TotalPrice), OrderCount = g.Select(oi => oi.OrderId).Distinct().Count() }) .OrderByDescending(p => p.Revenue) .Take(50) .ToListAsync(); return productPerformance; } }