Initial push

This commit is contained in:
2025-07-20 03:41:39 -04:00
commit d315f5d26e
118 changed files with 25819 additions and 0 deletions

View File

@@ -0,0 +1,172 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.ChangeTracking;
using Microsoft.EntityFrameworkCore.Infrastructure;
using SqrtSpace.SpaceTime.Core;
namespace SqrtSpace.SpaceTime.EntityFramework;
/// <summary>
/// Factory for creating SpaceTime-optimized change trackers
/// </summary>
internal interface IChangeTrackerFactory
{
ChangeTracker CreateChangeTracker(DbContext context);
}
/// <summary>
/// Implementation of SpaceTime change tracker factory
/// </summary>
internal class SpaceTimeChangeTrackerFactory : IChangeTrackerFactory
{
private readonly SpaceTimeOptions _options;
public SpaceTimeChangeTrackerFactory(SpaceTimeOptions options)
{
_options = options;
}
public ChangeTracker CreateChangeTracker(DbContext context)
{
// In practice, we'd need to hook into EF Core's internal change tracking
// For now, return the standard change tracker with optimizations applied
var changeTracker = context.ChangeTracker;
if (_options.EnableSqrtNChangeTracking)
{
ConfigureSqrtNTracking(changeTracker);
}
return changeTracker;
}
private void ConfigureSqrtNTracking(ChangeTracker changeTracker)
{
// Configure change tracker for √n memory usage
changeTracker.AutoDetectChangesEnabled = false;
changeTracker.LazyLoadingEnabled = false;
changeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTrackingWithIdentityResolution;
}
}
/// <summary>
/// Query processor with SpaceTime optimizations
/// </summary>
internal interface IQueryProcessor
{
IQueryable<T> OptimizeQuery<T>(IQueryable<T> query) where T : class;
Task<List<T>> ExecuteOptimizedAsync<T>(IQueryable<T> query, CancellationToken cancellationToken = default) where T : class;
}
/// <summary>
/// Implementation of SpaceTime query processor
/// </summary>
internal class SpaceTimeQueryProcessor : IQueryProcessor
{
private readonly SpaceTimeOptions _options;
private readonly CheckpointManager? _checkpointManager;
public SpaceTimeQueryProcessor(SpaceTimeOptions options)
{
_options = options;
if (_options.EnableQueryCheckpointing)
{
_checkpointManager = new CheckpointManager(_options.CheckpointDirectory);
}
}
public IQueryable<T> OptimizeQuery<T>(IQueryable<T> query) where T : class
{
// Apply optimizations to the query
if (_options.EnableBatchSizeOptimization)
{
// This would need integration with the query provider
// For demonstration, we'll just return the query
}
return query;
}
public async Task<List<T>> ExecuteOptimizedAsync<T>(IQueryable<T> query, CancellationToken cancellationToken = default) where T : class
{
if (_checkpointManager != null)
{
// Try to restore from checkpoint
var checkpoint = await _checkpointManager.RestoreLatestCheckpointAsync<List<T>>();
if (checkpoint != null)
{
return checkpoint;
}
}
var results = new List<T>();
var batchSize = _options.MaxTrackedEntities ?? SpaceTimeCalculator.CalculateSqrtInterval(10000);
// Execute in batches
var processed = 0;
while (true)
{
var batch = await query
.Skip(processed)
.Take(batchSize)
.ToListAsync(cancellationToken);
if (!batch.Any())
break;
results.AddRange(batch);
processed += batch.Count;
// Checkpoint if needed
if (_checkpointManager != null && _checkpointManager.ShouldCheckpoint())
{
await _checkpointManager.CreateCheckpointAsync(results);
}
}
return results;
}
}
/// <summary>
/// Extension methods for SpaceTime query optimization
/// </summary>
public static class SpaceTimeQueryableExtensions
{
/// <summary>
/// Executes the query with √n memory optimization
/// </summary>
public static async Task<List<T>> ToListWithSqrtNMemoryAsync<T>(
this IQueryable<T> query,
CancellationToken cancellationToken = default) where T : class
{
var context = GetDbContext(query);
if (context == null)
{
return await query.ToListAsync(cancellationToken);
}
var processor = context.GetService<IQueryProcessor>();
if (processor == null)
{
return await query.ToListAsync(cancellationToken);
}
return await processor.ExecuteOptimizedAsync(query, cancellationToken);
}
/// <summary>
/// Applies no-tracking with √n identity resolution
/// </summary>
public static IQueryable<T> AsNoTrackingWithSqrtNIdentityResolution<T>(this IQueryable<T> query) where T : class
{
return query.AsNoTrackingWithIdentityResolution();
}
private static DbContext? GetDbContext<T>(IQueryable<T> query)
{
var provider = query.Provider;
var contextProperty = provider.GetType().GetProperty("Context");
return contextProperty?.GetValue(provider) as DbContext;
}
}

View File

@@ -0,0 +1,145 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.Extensions.DependencyInjection;
using SqrtSpace.SpaceTime.Core;
namespace SqrtSpace.SpaceTime.EntityFramework;
/// <summary>
/// Extension methods for configuring SpaceTime optimizations in Entity Framework Core
/// </summary>
public static class SpaceTimeDbContextOptionsExtensions
{
/// <summary>
/// Configures the context to use SpaceTime optimizations
/// </summary>
public static DbContextOptionsBuilder UseSpaceTimeOptimizer(
this DbContextOptionsBuilder optionsBuilder,
Action<SpaceTimeOptions>? configureOptions = null)
{
var options = new SpaceTimeOptions();
configureOptions?.Invoke(options);
var extension = new SpaceTimeOptionsExtension(options);
((IDbContextOptionsBuilderInfrastructure)optionsBuilder).AddOrUpdateExtension(extension);
return optionsBuilder;
}
/// <summary>
/// Configures the context to use SpaceTime optimizations
/// </summary>
public static DbContextOptionsBuilder<TContext> UseSpaceTimeOptimizer<TContext>(
this DbContextOptionsBuilder<TContext> optionsBuilder,
Action<SpaceTimeOptions>? configureOptions = null) where TContext : DbContext
{
return (DbContextOptionsBuilder<TContext>)UseSpaceTimeOptimizer(
(DbContextOptionsBuilder)optionsBuilder, configureOptions);
}
}
/// <summary>
/// Options for SpaceTime optimizations
/// </summary>
public class SpaceTimeOptions
{
/// <summary>
/// Enable √n change tracking optimization
/// </summary>
public bool EnableSqrtNChangeTracking { get; set; } = true;
/// <summary>
/// Enable query result checkpointing
/// </summary>
public bool EnableQueryCheckpointing { get; set; } = true;
/// <summary>
/// Maximum entities to track before spilling to external storage
/// </summary>
public int? MaxTrackedEntities { get; set; }
/// <summary>
/// Buffer pool strategy
/// </summary>
public BufferPoolStrategy BufferPoolStrategy { get; set; } = BufferPoolStrategy.SqrtN;
/// <summary>
/// Checkpoint directory for query results
/// </summary>
public string? CheckpointDirectory { get; set; }
/// <summary>
/// Enable automatic batch size optimization
/// </summary>
public bool EnableBatchSizeOptimization { get; set; } = true;
}
/// <summary>
/// Buffer pool strategies
/// </summary>
public enum BufferPoolStrategy
{
/// <summary>Default EF Core behavior</summary>
Default,
/// <summary>Use √n of available memory</summary>
SqrtN,
/// <summary>Fixed size buffer pool</summary>
Fixed,
/// <summary>Adaptive based on workload</summary>
Adaptive
}
/// <summary>
/// Internal extension for EF Core
/// </summary>
internal class SpaceTimeOptionsExtension : IDbContextOptionsExtension
{
private readonly SpaceTimeOptions _options;
private DbContextOptionsExtensionInfo? _info;
public SpaceTimeOptionsExtension(SpaceTimeOptions options)
{
_options = options;
}
public DbContextOptionsExtensionInfo Info => _info ??= new ExtensionInfo(this);
public void ApplyServices(IServiceCollection services)
{
services.AddSingleton(_options);
services.AddScoped<IChangeTrackerFactory, SpaceTimeChangeTrackerFactory>();
services.AddScoped<IQueryProcessor, SpaceTimeQueryProcessor>();
}
public void Validate(IDbContextOptions options)
{
// Validation logic if needed
}
private sealed class ExtensionInfo : DbContextOptionsExtensionInfo
{
private readonly SpaceTimeOptionsExtension _extension;
public ExtensionInfo(SpaceTimeOptionsExtension extension) : base(extension)
{
_extension = extension;
}
public override bool IsDatabaseProvider => false;
public override string LogFragment => "SpaceTimeOptimizer";
public override int GetServiceProviderHashCode() => _extension._options.GetHashCode();
public override void PopulateDebugInfo(IDictionary<string, string> debugInfo)
{
debugInfo["SpaceTime:SqrtNChangeTracking"] = _extension._options.EnableSqrtNChangeTracking.ToString();
debugInfo["SpaceTime:QueryCheckpointing"] = _extension._options.EnableQueryCheckpointing.ToString();
}
public override bool ShouldUseSameServiceProvider(DbContextOptionsExtensionInfo other)
{
return other is ExtensionInfo;
}
}
}

View File

@@ -0,0 +1,276 @@
using System.Linq.Expressions;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Query;
using SqrtSpace.SpaceTime.Core;
namespace SqrtSpace.SpaceTime.EntityFramework;
/// <summary>
/// Extended query extensions for SpaceTime optimizations
/// </summary>
public static class SpaceTimeQueryExtensions
{
/// <summary>
/// Configures the query to use external sorting for large datasets
/// </summary>
public static IQueryable<T> UseExternalSorting<T>(this IQueryable<T> query) where T : class
{
// Mark the query for external sorting
return query.TagWith("SpaceTime:UseExternalSorting");
}
/// <summary>
/// Streams query results asynchronously for memory efficiency
/// </summary>
public static async IAsyncEnumerable<T> StreamQueryResultsAsync<T>(
this IQueryable<T> query,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) where T : class
{
var context = GetDbContext(query);
if (context == null)
{
// Fallback to regular async enumeration
await foreach (var item in query.AsAsyncEnumerable().WithCancellation(cancellationToken))
{
yield return item;
}
yield break;
}
// Get total count for batch size calculation
var totalCount = await query.CountAsync(cancellationToken);
var batchSize = SpaceTimeCalculator.CalculateSqrtInterval(totalCount);
// Stream in batches
for (int offset = 0; offset < totalCount; offset += batchSize)
{
var batch = await query
.Skip(offset)
.Take(batchSize)
.ToListAsync(cancellationToken);
foreach (var item in batch)
{
yield return item;
}
// Clear change tracker to prevent memory buildup
context.ChangeTracker.Clear();
}
}
/// <summary>
/// Processes query results in √n-sized batches with checkpoint support
/// </summary>
public static async IAsyncEnumerable<IReadOnlyList<T>> BatchBySqrtNAsync<T>(
this IQueryable<T> query,
string? checkpointId = null,
bool resumeFromCheckpoint = false,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) where T : class
{
var context = GetDbContext(query);
var options = context?.GetService<SpaceTimeOptions>();
CheckpointManager? checkpointManager = null;
if (!string.IsNullOrEmpty(checkpointId) && options?.EnableQueryCheckpointing == true)
{
checkpointManager = new CheckpointManager(options.CheckpointDirectory);
}
try
{
var totalCount = await query.CountAsync(cancellationToken);
var batchSize = SpaceTimeCalculator.CalculateSqrtInterval(totalCount);
var startOffset = 0;
// Resume from checkpoint if requested
if (resumeFromCheckpoint && checkpointManager != null)
{
var checkpoint = await checkpointManager.RestoreCheckpointAsync<QueryCheckpoint>(checkpointId!);
if (checkpoint != null)
{
startOffset = checkpoint.ProcessedCount;
}
}
for (int offset = startOffset; offset < totalCount; offset += batchSize)
{
var batch = await query
.Skip(offset)
.Take(batchSize)
.ToListAsync(cancellationToken);
if (batch.Count == 0)
yield break;
yield return batch;
// Save checkpoint
if (checkpointManager != null && !string.IsNullOrEmpty(checkpointId))
{
await checkpointManager.CreateCheckpointAsync(new QueryCheckpoint
{
ProcessedCount = offset + batch.Count,
TotalCount = totalCount
}, checkpointId);
}
// Clear change tracker if enabled
if (context != null && options?.EnableSqrtNChangeTracking == true)
{
var trackedCount = context.ChangeTracker.Entries().Count();
if (trackedCount > (options.MaxTrackedEntities ?? batchSize))
{
context.ChangeTracker.Clear();
}
}
}
}
finally
{
checkpointManager?.Dispose();
}
}
private static DbContext? GetDbContext<T>(IQueryable<T> query)
{
if (query.Provider is IInfrastructure<IServiceProvider> infrastructure)
{
var context = infrastructure.Instance.GetService(typeof(DbContext)) as DbContext;
return context;
}
// Fallback: try reflection
var provider = query.Provider;
var contextProperty = provider.GetType().GetProperty("Context");
return contextProperty?.GetValue(provider) as DbContext;
}
private class QueryCheckpoint
{
public int ProcessedCount { get; set; }
public int TotalCount { get; set; }
}
}
/// <summary>
/// Extension methods for DbContext bulk operations
/// </summary>
public static class SpaceTimeDbContextExtensions
{
/// <summary>
/// Performs bulk insert with √n buffering for memory efficiency
/// </summary>
public static async Task BulkInsertWithSqrtNBufferingAsync<T>(
this DbContext context,
IEnumerable<T> entities,
CancellationToken cancellationToken = default) where T : class
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(entities);
var options = context.GetService<SpaceTimeOptions>();
var entityList = entities as IList<T> ?? entities.ToList();
var totalCount = entityList.Count;
var batchSize = SpaceTimeCalculator.CalculateSqrtInterval(totalCount);
// Disable auto-detect changes for performance
var originalAutoDetectChanges = context.ChangeTracker.AutoDetectChangesEnabled;
context.ChangeTracker.AutoDetectChangesEnabled = false;
try
{
for (int i = 0; i < totalCount; i += batchSize)
{
var batch = entityList.Skip(i).Take(batchSize);
await context.AddRangeAsync(batch, cancellationToken);
await context.SaveChangesAsync(cancellationToken);
// Clear change tracker after each batch to prevent memory buildup
context.ChangeTracker.Clear();
}
}
finally
{
context.ChangeTracker.AutoDetectChangesEnabled = originalAutoDetectChanges;
}
}
/// <summary>
/// Performs bulk update with √n buffering for memory efficiency
/// </summary>
public static async Task BulkUpdateWithSqrtNBufferingAsync<T>(
this DbContext context,
IEnumerable<T> entities,
CancellationToken cancellationToken = default) where T : class
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(entities);
var entityList = entities as IList<T> ?? entities.ToList();
var totalCount = entityList.Count;
var batchSize = SpaceTimeCalculator.CalculateSqrtInterval(totalCount);
// Disable auto-detect changes for performance
var originalAutoDetectChanges = context.ChangeTracker.AutoDetectChangesEnabled;
context.ChangeTracker.AutoDetectChangesEnabled = false;
try
{
for (int i = 0; i < totalCount; i += batchSize)
{
var batch = entityList.Skip(i).Take(batchSize);
context.UpdateRange(batch);
await context.SaveChangesAsync(cancellationToken);
// Clear change tracker after each batch
context.ChangeTracker.Clear();
}
}
finally
{
context.ChangeTracker.AutoDetectChangesEnabled = originalAutoDetectChanges;
}
}
/// <summary>
/// Performs bulk delete with √n buffering for memory efficiency
/// </summary>
public static async Task BulkDeleteWithSqrtNBufferingAsync<T>(
this DbContext context,
IEnumerable<T> entities,
CancellationToken cancellationToken = default) where T : class
{
ArgumentNullException.ThrowIfNull(context);
ArgumentNullException.ThrowIfNull(entities);
var entityList = entities as IList<T> ?? entities.ToList();
var totalCount = entityList.Count;
var batchSize = SpaceTimeCalculator.CalculateSqrtInterval(totalCount);
// Disable auto-detect changes for performance
var originalAutoDetectChanges = context.ChangeTracker.AutoDetectChangesEnabled;
context.ChangeTracker.AutoDetectChangesEnabled = false;
try
{
for (int i = 0; i < totalCount; i += batchSize)
{
var batch = entityList.Skip(i).Take(batchSize);
context.RemoveRange(batch);
await context.SaveChangesAsync(cancellationToken);
// Clear change tracker after each batch
context.ChangeTracker.Clear();
}
}
finally
{
context.ChangeTracker.AutoDetectChangesEnabled = originalAutoDetectChanges;
}
}
}

View File

@@ -0,0 +1,26 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<Description>Entity Framework Core optimizations using √n space-time tradeoffs</Description>
<PackageId>SqrtSpace.SpaceTime.EntityFramework</PackageId>
<IsPackable>true</IsPackable>
<Authors>David H. Friedel Jr</Authors>
<Company>MarketAlly LLC</Company>
<Copyright>Copyright © 2025 MarketAlly LLC</Copyright>
<PackageLicenseExpression>MIT</PackageLicenseExpression>
<PackageProjectUrl>https://github.com/sqrtspace/sqrtspace-dotnet</PackageProjectUrl>
<RepositoryUrl>https://www.sqrtspace.dev</RepositoryUrl>
<RepositoryType>git</RepositoryType>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\SqrtSpace.SpaceTime.Core\SqrtSpace.SpaceTime.Core.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.7" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="9.0.7" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.7" />
</ItemGroup>
</Project>