Initial

examples/laravel-app/README.md

@@ -0,0 +1,375 @@
+# SqrtSpace SpaceTime Laravel Sample Application
+
+This sample demonstrates how to integrate SqrtSpace SpaceTime with a Laravel application to build memory-efficient, scalable web applications.
+
+## Features Demonstrated
+
+### 1. **Large Dataset API Endpoints**
+- Streaming JSON responses for large datasets
+- Paginated queries with automatic memory management
+- CSV export without memory bloat
+
+### 2. **Background Job Processing**
+- Memory-aware queue workers
+- Checkpointed long-running jobs
+- Batch processing with progress tracking
+
+### 3. **Caching with SpaceTime**
+- Hot/cold cache tiers
+- Automatic memory pressure handling
+- Cache warming strategies
+
+### 4. **Real-World Use Cases**
+- User activity log processing
+- Sales report generation
+- Product catalog management
+- Real-time analytics
+
+## Installation
+
+1. **Install dependencies:**
+```bash
+composer install
+```
+
+2. **Configure environment:**
+```bash
+cp .env.example .env
+php artisan key:generate
+```
+
+3. **Configure SpaceTime in `.env`:**
+```
+SPACETIME_MEMORY_LIMIT=256M
+SPACETIME_EXTERNAL_STORAGE=/tmp/spacetime
+SPACETIME_CHUNK_STRATEGY=sqrt_n
+SPACETIME_ENABLE_CHECKPOINTING=true
+```
+
+4. **Run migrations:**
+```bash
+php artisan migrate
+php artisan db:seed
+```
+
+## Project Structure
+
+```
+laravel-app/
+├── app/
+│   ├── Http/
+│   │   ├── Controllers/
+│   │   │   ├── ProductController.php      # Streaming APIs
+│   │   │   ├── AnalyticsController.php    # Real-time analytics
+│   │   │   └── ReportController.php       # Large report generation
+│   │   └── Middleware/
+│   │       └── SpaceTimeMiddleware.php    # Memory monitoring
+│   ├── Jobs/
+│   │   ├── ProcessLargeDataset.php       # Checkpointed job
+│   │   ├── GenerateReport.php            # Batch processing job
+│   │   └── ImportProducts.php            # CSV import job
+│   ├── Services/
+│   │   ├── ProductService.php            # Business logic
+│   │   ├── AnalyticsService.php         # Analytics processing
+│   │   └── SpaceTimeCache.php           # Cache wrapper
+│   └── Providers/
+│       └── SpaceTimeServiceProvider.php  # Service registration
+├── config/
+│   └── spacetime.php                     # Configuration
+├── routes/
+│   ├── api.php                          # API routes
+│   └── web.php                          # Web routes
+└── tests/
+    └── Feature/
+        └── SpaceTimeTest.php            # Integration tests
+```
+
+## Usage Examples
+
+### 1. Streaming Large Datasets
+
+```php
+// ProductController.php
+public function stream()
+{
+    return response()->stream(function () {
+        $products = SpaceTimeStream::fromQuery(
+            Product::query()->orderBy('id')
+        );
+        
+        echo "[";
+        $first = true;
+        
+        foreach ($products->chunk(100) as $chunk) {
+            foreach ($chunk as $product) {
+                if (!$first) echo ",";
+                echo $product->toJson();
+                $first = false;
+            }
+            
+            // Flush output buffer
+            ob_flush();
+            flush();
+        }
+        
+        echo "]";
+    }, 200, [
+        'Content-Type' => 'application/json',
+        'X-Accel-Buffering' => 'no'
+    ]);
+}
+```
+
+### 2. Memory-Efficient CSV Export
+
+```php
+// ReportController.php
+public function exportCsv()
+{
+    $filename = 'products_' . date('Y-m-d') . '.csv';
+    
+    return response()->streamDownload(function () {
+        $exporter = new CsvExporter('php://output');
+        $exporter->writeHeaders(['ID', 'Name', 'Price', 'Stock']);
+        
+        Product::query()
+            ->orderBy('id')
+            ->chunkById(1000, function ($products) use ($exporter) {
+                foreach ($products as $product) {
+                    $exporter->writeRow([
+                        $product->id,
+                        $product->name,
+                        $product->price,
+                        $product->stock
+                    ]);
+                }
+            });
+    }, $filename, [
+        'Content-Type' => 'text/csv',
+    ]);
+}
+```
+
+### 3. Checkpointed Background Job
+
+```php
+// ProcessLargeDataset.php
+class ProcessLargeDataset implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+    use SpaceTimeCheckpointable;
+    
+    public function handle()
+    {
+        $checkpoint = $this->getCheckpoint();
+        $lastId = $checkpoint['last_id'] ?? 0;
+        
+        Order::where('id', '>', $lastId)
+            ->orderBy('id')
+            ->chunkById(100, function ($orders) {
+                foreach ($orders as $order) {
+                    // Process order
+                    $this->processOrder($order);
+                    
+                    // Save checkpoint every 100 orders
+                    if ($order->id % 100 === 0) {
+                        $this->saveCheckpoint([
+                            'last_id' => $order->id,
+                            'processed' => $this->processed,
+                        ]);
+                    }
+                }
+            });
+    }
+}
+```
+
+### 4. Real-Time Analytics
+
+```php
+// AnalyticsController.php
+public function realtime()
+{
+    return response()->stream(function () {
+        $monitor = new MemoryPressureMonitor('100M');
+        
+        while (true) {
+            $stats = $this->analyticsService->getCurrentStats();
+            
+            // Send as Server-Sent Event
+            echo "data: " . json_encode($stats) . "\n\n";
+            ob_flush();
+            flush();
+            
+            // Check memory pressure
+            if ($monitor->check() !== MemoryPressureLevel::NONE) {
+                $this->analyticsService->compact();
+            }
+            
+            sleep(1);
+        }
+    }, 200, [
+        'Content-Type' => 'text/event-stream',
+        'Cache-Control' => 'no-cache',
+        'X-Accel-Buffering' => 'no'
+    ]);
+}
+```
+
+### 5. Memory-Aware Caching
+
+```php
+// SpaceTimeCache.php
+class SpaceTimeCache
+{
+    private SpaceTimeDict $hot;
+    private CacheInterface $cold;
+    private MemoryPressureMonitor $monitor;
+    
+    public function get($key)
+    {
+        // Check hot cache first
+        if (isset($this->hot[$key])) {
+            return $this->hot[$key];
+        }
+        
+        // Check cold storage
+        $value = $this->cold->get($key);
+        if ($value !== null) {
+            // Promote to hot cache if memory allows
+            if ($this->monitor->canAllocate(strlen($value))) {
+                $this->hot[$key] = $value;
+            }
+        }
+        
+        return $value;
+    }
+}
+```
+
+## API Endpoints
+
+### Products API
+
+- `GET /api/products` - Paginated list
+- `GET /api/products/stream` - Stream all products as NDJSON
+- `GET /api/products/export/csv` - Export as CSV
+- `POST /api/products/bulk-update` - Bulk update with checkpointing
+- `POST /api/products/import` - Import CSV with progress
+
+### Analytics API
+
+- `GET /api/analytics/summary` - Get summary statistics
+- `GET /api/analytics/realtime` - Real-time SSE stream
+- `POST /api/analytics/report` - Generate large report
+- `GET /api/analytics/top-products` - Top products with external sorting
+
+### Reports API
+
+- `POST /api/reports/generate` - Generate report (queued)
+- `GET /api/reports/{id}/status` - Check generation status
+- `GET /api/reports/{id}/download` - Download completed report
+
+## Testing
+
+Run the test suite:
+
+```bash
+php artisan test
+```
+
+Example test:
+
+```php
+public function test_can_stream_large_dataset()
+{
+    // Seed test data
+    Product::factory()->count(10000)->create();
+    
+    // Make streaming request
+    $response = $this->getJson('/api/products/stream');
+    
+    $response->assertStatus(200);
+    $response->assertHeader('Content-Type', 'application/json');
+    
+    // Verify memory usage stayed low
+    $this->assertLessThan(50 * 1024 * 1024, memory_get_peak_usage());
+}
+```
+
+## Performance Tips
+
+1. **Configure memory limits** based on your server capacity
+2. **Use streaming responses** for large datasets
+3. **Enable checkpointing** for long-running jobs
+4. **Monitor memory pressure** in production
+5. **Use external storage** on fast SSDs
+6. **Configure queue workers** with appropriate memory limits
+
+## Deployment
+
+### Nginx Configuration
+
+```nginx
+location /api/products/stream {
+    proxy_pass http://backend;
+    proxy_buffering off;
+    proxy_read_timeout 3600;
+}
+
+location /api/analytics/realtime {
+    proxy_pass http://backend;
+    proxy_buffering off;
+    proxy_read_timeout 0;
+    proxy_http_version 1.1;
+}
+```
+
+### Supervisor Configuration
+
+```ini
+[program:spacetime-worker]
+command=php /path/to/artisan queue:work --memory=256
+numprocs=4
+autostart=true
+autorestart=true
+```
+
+## Monitoring
+
+Add to your monitoring:
+
+```php
+// app/Console/Commands/MonitorSpaceTime.php
+$stats = [
+    'memory_usage' => memory_get_usage(true),
+    'peak_memory' => memory_get_peak_usage(true),
+    'external_files' => count(glob(config('spacetime.external_storage') . '/*')),
+    'cache_size' => $this->cache->size(),
+];
+
+Log::channel('metrics')->info('spacetime.stats', $stats);
+```
+
+## Troubleshooting
+
+### High Memory Usage
+- Check `SPACETIME_MEMORY_LIMIT` setting
+- Enable more aggressive spillover
+- Use smaller chunk sizes
+
+### Slow Performance
+- Ensure external storage is on SSD
+- Increase memory limit if possible
+- Use compression for large values
+
+### Failed Checkpoints
+- Check storage permissions
+- Ensure sufficient disk space
+- Verify checkpoint directory exists
+
+## Learn More
+
+- [SqrtSpace SpaceTime Documentation](https://github.com/MarketAlly/Ubiquity)
+- [Laravel Documentation](https://laravel.com/docs)
+- [Memory-Efficient PHP Patterns](https://example.com/patterns)

examples/laravel-app/app/Http/Controllers/ProductController.php

@@ -0,0 +1,194 @@
+<?php
+
+namespace App\Http\Controllers;
+
+use App\Models\Product;
+use App\Services\ProductService;
+use Illuminate\Http\Request;
+use Illuminate\Support\Facades\DB;
+use SqrtSpace\SpaceTime\Streams\SpaceTimeStream;
+use SqrtSpace\SpaceTime\File\CsvExporter;
+use SqrtSpace\SpaceTime\Checkpoint\CheckpointManager;
+
+class ProductController extends Controller
+{
+    private ProductService $productService;
+    
+    public function __construct(ProductService $productService)
+    {
+        $this->productService = $productService;
+    }
+    
+    /**
+     * Get paginated products
+     */
+    public function index(Request $request)
+    {
+        $perPage = min($request->get('per_page', 50), 100);
+        
+        return Product::query()
+            ->when($request->get('category'), function ($query, $category) {
+                $query->where('category', $category);
+            })
+            ->when($request->get('min_price'), function ($query, $minPrice) {
+                $query->where('price', '>=', $minPrice);
+            })
+            ->orderBy('id')
+            ->paginate($perPage);
+    }
+    
+    /**
+     * Stream all products as NDJSON
+     */
+    public function stream(Request $request)
+    {
+        return response()->stream(function () use ($request) {
+            $query = Product::query()
+                ->when($request->get('category'), function ($query, $category) {
+                    $query->where('category', $category);
+                })
+                ->orderBy('id');
+            
+            $stream = SpaceTimeStream::fromQuery($query, 100);
+            
+            foreach ($stream as $product) {
+                echo $product->toJson() . "\n";
+                ob_flush();
+                flush();
+            }
+        }, 200, [
+            'Content-Type' => 'application/x-ndjson',
+            'X-Accel-Buffering' => 'no',
+            'Cache-Control' => 'no-cache'
+        ]);
+    }
+    
+    /**
+     * Export products as CSV
+     */
+    public function exportCsv(Request $request)
+    {
+        $filename = 'products_' . date('Y-m-d_His') . '.csv';
+        
+        return response()->streamDownload(function () use ($request) {
+            $exporter = new CsvExporter('php://output');
+            $exporter->writeHeaders([
+                'ID', 'Name', 'SKU', 'Category', 'Price', 
+                'Stock', 'Description', 'Created At'
+            ]);
+            
+            Product::query()
+                ->when($request->get('category'), function ($query, $category) {
+                    $query->where('category', $category);
+                })
+                ->orderBy('id')
+                ->chunkById(1000, function ($products) use ($exporter) {
+                    foreach ($products as $product) {
+                        $exporter->writeRow([
+                            $product->id,
+                            $product->name,
+                            $product->sku,
+                            $product->category,
+                            $product->price,
+                            $product->stock,
+                            $product->description,
+                            $product->created_at
+                        ]);
+                    }
+                });
+        }, $filename, [
+            'Content-Type' => 'text/csv',
+        ]);
+    }
+    
+    /**
+     * Bulk update product prices with checkpointing
+     */
+    public function bulkUpdatePrices(Request $request)
+    {
+        $request->validate([
+            'category' => 'required|string',
+            'adjustment_type' => 'required|in:percentage,fixed',
+            'adjustment_value' => 'required|numeric'
+        ]);
+        
+        $jobId = 'price_update_' . uniqid();
+        $checkpointManager = app(CheckpointManager::class);
+        
+        // Check for existing checkpoint
+        $checkpoint = $checkpointManager->restore($jobId);
+        $lastId = $checkpoint['last_id'] ?? 0;
+        $updated = $checkpoint['updated'] ?? 0;
+        
+        DB::beginTransaction();
+        
+        try {
+            Product::where('category', $request->category)
+                ->where('id', '>', $lastId)
+                ->orderBy('id')
+                ->chunkById(100, function ($products) use ($request, &$updated, $jobId, $checkpointManager) {
+                    foreach ($products as $product) {
+                        if ($request->adjustment_type === 'percentage') {
+                            $product->price *= (1 + $request->adjustment_value / 100);
+                        } else {
+                            $product->price += $request->adjustment_value;
+                        }
+                        $product->save();
+                        $updated++;
+                        
+                        // Checkpoint every 100 updates
+                        if ($updated % 100 === 0) {
+                            $checkpointManager->save($jobId, [
+                                'last_id' => $product->id,
+                                'updated' => $updated
+                            ]);
+                        }
+                    }
+                });
+            
+            DB::commit();
+            $checkpointManager->delete($jobId);
+            
+            return response()->json([
+                'success' => true,
+                'updated' => $updated,
+                'job_id' => $jobId
+            ]);
+            
+        } catch (\Exception $e) {
+            DB::rollBack();
+            
+            return response()->json([
+                'success' => false,
+                'error' => $e->getMessage(),
+                'job_id' => $jobId,
+                'can_resume' => true
+            ], 500);
+        }
+    }
+    
+    /**
+     * Search products with memory-efficient sorting
+     */
+    public function search(Request $request)
+    {
+        $request->validate([
+            'q' => 'required|string|min:2',
+            'sort_by' => 'in:relevance,price_asc,price_desc,name'
+        ]);
+        
+        return $this->productService->searchProducts(
+            $request->get('q'),
+            $request->get('sort_by', 'relevance'),
+            $request->get('limit', 100)
+        );
+    }
+    
+    /**
+     * Get product statistics
+     */
+    public function statistics()
+    {
+        return $this->productService->getStatistics();
+    }
+}

examples/laravel-app/app/Jobs/ProcessLargeDataset.php

@@ -0,0 +1,239 @@
+<?php
+
+namespace App\Jobs;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use App\Models\Order;
+use App\Models\OrderAnalytics;
+use SqrtSpace\SpaceTime\Checkpoint\CheckpointManager;
+use SqrtSpace\SpaceTime\Memory\MemoryPressureMonitor;
+use SqrtSpace\SpaceTime\Collections\SpaceTimeArray;
+
+class ProcessLargeDataset implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    private string $jobId;
+    private CheckpointManager $checkpointManager;
+    private MemoryPressureMonitor $memoryMonitor;
+    
+    public function __construct(string $jobId = null)
+    {
+        $this->jobId = $jobId ?? 'process_dataset_' . uniqid();
+    }
+    
+    public function handle()
+    {
+        $this->checkpointManager = app(CheckpointManager::class);
+        $this->memoryMonitor = new MemoryPressureMonitor('64M');
+        
+        // Restore checkpoint if exists
+        $checkpoint = $this->checkpointManager->restore($this->jobId);
+        $state = $checkpoint ?? [
+            'last_order_id' => 0,
+            'processed_count' => 0,
+            'analytics' => [
+                'total_revenue' => 0,
+                'order_count' => 0,
+                'customers' => new SpaceTimeArray(1000),
+                'products' => new SpaceTimeArray(1000),
+                'daily_stats' => []
+            ]
+        ];
+        
+        $this->processOrders($state);
+        
+        // Clean up checkpoint after successful completion
+        $this->checkpointManager->delete($this->jobId);
+        
+        // Save final analytics
+        $this->saveAnalytics($state['analytics']);
+    }
+    
+    private function processOrders(array &$state)
+    {
+        $lastOrderId = $state['last_order_id'];
+        
+        Order::where('id', '>', $lastOrderId)
+            ->with(['customer', 'items.product'])
+            ->orderBy('id')
+            ->chunkById(100, function ($orders) use (&$state) {
+                foreach ($orders as $order) {
+                    // Process order
+                    $this->processOrder($order, $state['analytics']);
+                    
+                    $state['processed_count']++;
+                    $state['last_order_id'] = $order->id;
+                    
+                    // Checkpoint every 100 orders
+                    if ($state['processed_count'] % 100 === 0) {
+                        $this->saveCheckpoint($state);
+                        
+                        // Check memory pressure
+                        if ($this->memoryMonitor->shouldCleanup()) {
+                            // Flush some analytics to database
+                            $this->flushPartialAnalytics($state['analytics']);
+                        }
+                    }
+                }
+            });
+    }
+    
+    private function processOrder(Order $order, array &$analytics)
+    {
+        // Update totals
+        $analytics['total_revenue'] += $order->total_amount;
+        $analytics['order_count']++;
+        
+        // Track customer spending
+        $customerId = $order->customer_id;
+        if (!isset($analytics['customers'][$customerId])) {
+            $analytics['customers'][$customerId] = [
+                'total_spent' => 0,
+                'order_count' => 0,
+                'last_order_date' => null
+            ];
+        }
+        
+        $analytics['customers'][$customerId]['total_spent'] += $order->total_amount;
+        $analytics['customers'][$customerId]['order_count']++;
+        $analytics['customers'][$customerId]['last_order_date'] = $order->created_at;
+        
+        // Track product sales
+        foreach ($order->items as $item) {
+            $productId = $item->product_id;
+            if (!isset($analytics['products'][$productId])) {
+                $analytics['products'][$productId] = [
+                    'quantity_sold' => 0,
+                    'revenue' => 0,
+                    'order_count' => 0
+                ];
+            }
+            
+            $analytics['products'][$productId]['quantity_sold'] += $item->quantity;
+            $analytics['products'][$productId]['revenue'] += $item->total_price;
+            $analytics['products'][$productId]['order_count']++;
+        }
+        
+        // Daily statistics
+        $date = $order->created_at->format('Y-m-d');
+        if (!isset($analytics['daily_stats'][$date])) {
+            $analytics['daily_stats'][$date] = [
+                'revenue' => 0,
+                'orders' => 0,
+                'unique_customers' => []
+            ];
+        }
+        
+        $analytics['daily_stats'][$date]['revenue'] += $order->total_amount;
+        $analytics['daily_stats'][$date]['orders']++;
+        $analytics['daily_stats'][$date]['unique_customers'][$customerId] = true;
+    }
+    
+    private function saveCheckpoint(array $state)
+    {
+        $this->checkpointManager->save($this->jobId, [
+            'last_order_id' => $state['last_order_id'],
+            'processed_count' => $state['processed_count'],
+            'analytics' => [
+                'total_revenue' => $state['analytics']['total_revenue'],
+                'order_count' => $state['analytics']['order_count'],
+                'customers' => $state['analytics']['customers'],
+                'products' => $state['analytics']['products'],
+                'daily_stats' => $state['analytics']['daily_stats']
+            ]
+        ]);
+        
+        \Log::info("Checkpoint saved", [
+            'job_id' => $this->jobId,
+            'processed' => $state['processed_count']
+        ]);
+    }
+    
+    private function flushPartialAnalytics(array &$analytics)
+    {
+        // Save top customers to database
+        $topCustomers = $this->getTopItems($analytics['customers'], 'total_spent', 100);
+        foreach ($topCustomers as $customerId => $data) {
+            OrderAnalytics::updateOrCreate(
+                ['type' => 'customer', 'entity_id' => $customerId],
+                ['data' => json_encode($data)]
+            );
+        }
+        
+        // Save top products
+        $topProducts = $this->getTopItems($analytics['products'], 'revenue', 100);
+        foreach ($topProducts as $productId => $data) {
+            OrderAnalytics::updateOrCreate(
+                ['type' => 'product', 'entity_id' => $productId],
+                ['data' => json_encode($data)]
+            );
+        }
+        
+        // Clear processed items from memory
+        $analytics['customers'] = new SpaceTimeArray(1000);
+        $analytics['products'] = new SpaceTimeArray(1000);
+        
+        gc_collect_cycles();
+    }
+    
+    private function getTopItems($items, $sortKey, $limit)
+    {
+        $sorted = [];
+        foreach ($items as $id => $data) {
+            $sorted[$id] = $data[$sortKey];
+        }
+        
+        arsort($sorted);
+        $topIds = array_slice(array_keys($sorted), 0, $limit);
+        
+        $result = [];
+        foreach ($topIds as $id) {
+            $result[$id] = $items[$id];
+        }
+        
+        return $result;
+    }
+    
+    private function saveAnalytics(array $analytics)
+    {
+        // Save summary
+        OrderAnalytics::updateOrCreate(
+            ['type' => 'summary', 'entity_id' => 'global'],
+            [
+                'data' => json_encode([
+                    'total_revenue' => $analytics['total_revenue'],
+                    'order_count' => $analytics['order_count'],
+                    'avg_order_value' => $analytics['total_revenue'] / $analytics['order_count'],
+                    'unique_customers' => count($analytics['customers']),
+                    'unique_products' => count($analytics['products']),
+                    'processed_at' => now()
+                ])
+            ]
+        );
+        
+        // Save daily stats
+        foreach ($analytics['daily_stats'] as $date => $stats) {
+            OrderAnalytics::updateOrCreate(
+                ['type' => 'daily', 'entity_id' => $date],
+                [
+                    'data' => json_encode([
+                        'revenue' => $stats['revenue'],
+                        'orders' => $stats['orders'],
+                        'unique_customers' => count($stats['unique_customers']),
+                        'avg_order_value' => $stats['revenue'] / $stats['orders']
+                    ])
+                ]
+            );
+        }
+        
+        \Log::info("Analytics processing completed", [
+            'job_id' => $this->jobId,
+            'total_processed' => $analytics['order_count']
+        ]);
+    }
+}

examples/laravel-app/app/Services/ProductService.php

@@ -0,0 +1,224 @@
+<?php
+
+namespace App\Services;
+
+use App\Models\Product;
+use Illuminate\Support\Collection;
+use SqrtSpace\SpaceTime\Algorithms\ExternalSort;
+use SqrtSpace\SpaceTime\Algorithms\ExternalGroupBy;
+use SqrtSpace\SpaceTime\Collections\SpaceTimeArray;
+use SqrtSpace\SpaceTime\Memory\MemoryPressureMonitor;
+
+class ProductService
+{
+    private MemoryPressureMonitor $memoryMonitor;
+    
+    public function __construct()
+    {
+        $this->memoryMonitor = new MemoryPressureMonitor(
+            config('spacetime.memory_limit', '128M')
+        );
+    }
+    
+    /**
+     * Search products with memory-efficient sorting
+     */
+    public function searchProducts(string $query, string $sortBy, int $limit): Collection
+    {
+        // Get all matching products
+        $products = Product::where('name', 'like', "%{$query}%")
+            ->orWhere('description', 'like', "%{$query}%")
+            ->get()
+            ->map(function ($product) use ($query) {
+                // Calculate relevance score
+                $nameScore = $this->calculateRelevance($product->name, $query) * 2;
+                $descScore = $this->calculateRelevance($product->description, $query);
+                $product->relevance_score = $nameScore + $descScore;
+                return $product;
+            });
+        
+        // Use external sort for large result sets
+        if ($products->count() > 1000) {
+            $sorted = $this->externalSort($products, $sortBy);
+        } else {
+            $sorted = $this->inMemorySort($products, $sortBy);
+        }
+        
+        return collect($sorted)->take($limit);
+    }
+    
+    /**
+     * Get product statistics using external grouping
+     */
+    public function getStatistics(): array
+    {
+        $stats = [
+            'total_products' => Product::count(),
+            'total_value' => 0,
+            'by_category' => [],
+            'price_ranges' => [],
+            'stock_alerts' => []
+        ];
+        
+        // Use SpaceTimeArray for memory efficiency
+        $allProducts = new SpaceTimeArray(1000);
+        
+        Product::chunk(1000, function ($products) use (&$allProducts) {
+            foreach ($products as $product) {
+                $allProducts[] = [
+                    'category' => $product->category,
+                    'price' => $product->price,
+                    'stock' => $product->stock,
+                    'value' => $product->price * $product->stock
+                ];
+            }
+        });
+        
+        // Calculate total value
+        $stats['total_value'] = array_sum(array_column($allProducts->toArray(), 'value'));
+        
+        // Group by category using external grouping
+        $byCategory = ExternalGroupBy::groupBySum(
+            $allProducts->toArray(),
+            fn($p) => $p['category'],
+            fn($p) => $p['value']
+        );
+        $stats['by_category'] = $byCategory;
+        
+        // Price range distribution
+        $priceRanges = [
+            '0-50' => 0,
+            '50-100' => 0,
+            '100-500' => 0,
+            '500+' => 0
+        ];
+        
+        foreach ($allProducts as $product) {
+            if ($product['price'] < 50) {
+                $priceRanges['0-50']++;
+            } elseif ($product['price'] < 100) {
+                $priceRanges['50-100']++;
+            } elseif ($product['price'] < 500) {
+                $priceRanges['100-500']++;
+            } else {
+                $priceRanges['500+']++;
+            }
+            
+            // Low stock alerts
+            if ($product['stock'] < 10) {
+                $stats['stock_alerts'][] = [
+                    'category' => $product['category'],
+                    'stock' => $product['stock']
+                ];
+            }
+        }
+        
+        $stats['price_ranges'] = $priceRanges;
+        $stats['memory_usage'] = $this->memoryMonitor->getMemoryInfo();
+        
+        return $stats;
+    }
+    
+    /**
+     * Import products from CSV with progress tracking
+     */
+    public function importFromCsv(string $filePath, callable $progressCallback = null): array
+    {
+        $imported = 0;
+        $errors = [];
+        $batchSize = 100;
+        $batch = [];
+        
+        $handle = fopen($filePath, 'r');
+        $headers = fgetcsv($handle); // Skip headers
+        
+        while (($row = fgetcsv($handle)) !== false) {
+            try {
+                $batch[] = [
+                    'name' => $row[0],
+                    'sku' => $row[1],
+                    'category' => $row[2],
+                    'price' => (float)$row[3],
+                    'stock' => (int)$row[4],
+                    'description' => $row[5] ?? '',
+                    'created_at' => now(),
+                    'updated_at' => now()
+                ];
+                
+                if (count($batch) >= $batchSize) {
+                    Product::insert($batch);
+                    $imported += count($batch);
+                    $batch = [];
+                    
+                    if ($progressCallback) {
+                        $progressCallback($imported);
+                    }
+                    
+                    // Check memory pressure
+                    if ($this->memoryMonitor->shouldCleanup()) {
+                        gc_collect_cycles();
+                    }
+                }
+            } catch (\Exception $e) {
+                $errors[] = "Row " . ($imported + 1) . ": " . $e->getMessage();
+            }
+        }
+        
+        // Insert remaining batch
+        if (!empty($batch)) {
+            Product::insert($batch);
+            $imported += count($batch);
+        }
+        
+        fclose($handle);
+        
+        return [
+            'imported' => $imported,
+            'errors' => $errors
+        ];
+    }
+    
+    private function calculateRelevance(string $text, string $query): float
+    {
+        $text = strtolower($text);
+        $query = strtolower($query);
+        
+        // Exact match
+        if (strpos($text, $query) !== false) {
+            return 1.0;
+        }
+        
+        // Word match
+        $words = explode(' ', $query);
+        $matches = 0;
+        foreach ($words as $word) {
+            if (strpos($text, $word) !== false) {
+                $matches++;
+            }
+        }
+        
+        return $matches / count($words);
+    }
+    
+    private function externalSort(Collection $products, string $sortBy): array
+    {
+        $sortKey = match($sortBy) {
+            'price_asc' => fn($p) => $p->price,
+            'price_desc' => fn($p) => -$p->price,
+            'name' => fn($p) => $p->name,
+            default => fn($p) => -$p->relevance_score
+        };
+        
+        return ExternalSort::sortBy($products->toArray(), $sortKey);
+    }
+    
+    private function inMemorySort(Collection $products, string $sortBy): Collection
+    {
+        return match($sortBy) {
+            'price_asc' => $products->sortBy('price'),
+            'price_desc' => $products->sortByDesc('price'),
+            'name' => $products->sortBy('name'),
+            default => $products->sortByDesc('relevance_score')
+        };
+    }
+}