Initial
This commit is contained in:
206
examples/comprehensive_example.php
Normal file
206
examples/comprehensive_example.php
Normal file
@@ -0,0 +1,206 @@
|
||||
<?php
|
||||
|
||||
require_once __DIR__ . '/../vendor/autoload.php';
|
||||
|
||||
use SqrtSpace\SpaceTime\SpaceTimeConfig;
|
||||
use SqrtSpace\SpaceTime\Collections\SpaceTimeArray;
|
||||
use SqrtSpace\SpaceTime\Algorithms\ExternalSort;
|
||||
use SqrtSpace\SpaceTime\Algorithms\ExternalGroupBy;
|
||||
use SqrtSpace\SpaceTime\Streams\SpaceTimeStream;
|
||||
use SqrtSpace\SpaceTime\File\CsvReader;
|
||||
use SqrtSpace\SpaceTime\File\CsvExporter;
|
||||
use SqrtSpace\SpaceTime\File\JsonLinesProcessor;
|
||||
use SqrtSpace\SpaceTime\Batch\BatchProcessor;
|
||||
use SqrtSpace\SpaceTime\Memory\MemoryPressureMonitor;
|
||||
use SqrtSpace\SpaceTime\Memory\MemoryPressureLevel;
|
||||
use SqrtSpace\SpaceTime\Memory\Handlers\LoggingHandler;
|
||||
use SqrtSpace\SpaceTime\Checkpoint\CheckpointManager;
|
||||
|
||||
// Configure SpaceTime
|
||||
SpaceTimeConfig::configure([
|
||||
'memory_limit' => '256M',
|
||||
'external_storage_path' => __DIR__ . '/temp',
|
||||
'chunk_strategy' => 'sqrt_n',
|
||||
'enable_checkpointing' => true,
|
||||
'compression' => true,
|
||||
]);
|
||||
|
||||
echo "=== Ubiquity SpaceTime PHP Examples ===\n\n";
|
||||
|
||||
// Example 1: Memory-Efficient Array
|
||||
echo "1. SpaceTimeArray Example\n";
|
||||
$array = new SpaceTimeArray(1000); // Spill to disk after 1000 items
|
||||
|
||||
// Add 10,000 items
|
||||
for ($i = 0; $i < 10000; $i++) {
|
||||
$array["key_$i"] = "value_$i";
|
||||
}
|
||||
|
||||
echo " - Created array with " . count($array) . " items\n";
|
||||
echo " - Memory usage: " . number_format(memory_get_usage(true) / 1024 / 1024, 2) . " MB\n\n";
|
||||
|
||||
// Example 2: External Sorting
|
||||
echo "2. External Sort Example\n";
|
||||
$unsorted = [];
|
||||
for ($i = 0; $i < 50000; $i++) {
|
||||
$unsorted[] = [
|
||||
'id' => $i,
|
||||
'value' => mt_rand(1, 1000000),
|
||||
'name' => 'Item ' . $i
|
||||
];
|
||||
}
|
||||
|
||||
$sorted = ExternalSort::sortBy($unsorted, fn($item) => $item['value']);
|
||||
echo " - Sorted " . count($sorted) . " items by value\n";
|
||||
echo " - First item value: " . $sorted[0]['value'] . "\n";
|
||||
echo " - Last item value: " . $sorted[count($sorted) - 1]['value'] . "\n\n";
|
||||
|
||||
// Example 3: External GroupBy
|
||||
echo "3. External GroupBy Example\n";
|
||||
$orders = [];
|
||||
for ($i = 0; $i < 10000; $i++) {
|
||||
$orders[] = [
|
||||
'customer_id' => mt_rand(1, 100),
|
||||
'amount' => mt_rand(10, 1000),
|
||||
'category' => ['Electronics', 'Clothing', 'Food', 'Books'][mt_rand(0, 3)]
|
||||
];
|
||||
}
|
||||
|
||||
$byCategory = ExternalGroupBy::groupBySum(
|
||||
$orders,
|
||||
fn($order) => $order['category'],
|
||||
fn($order) => $order['amount']
|
||||
);
|
||||
|
||||
foreach ($byCategory as $category => $total) {
|
||||
echo " - $category: $" . number_format($total, 2) . "\n";
|
||||
}
|
||||
echo "\n";
|
||||
|
||||
// Example 4: Stream Processing
|
||||
echo "4. Stream Processing Example\n";
|
||||
|
||||
// Create sample CSV file
|
||||
$csvFile = __DIR__ . '/sample.csv';
|
||||
$csv = fopen($csvFile, 'w');
|
||||
fputcsv($csv, ['id', 'name', 'price', 'quantity']);
|
||||
for ($i = 1; $i <= 1000; $i++) {
|
||||
fputcsv($csv, [$i, "Product $i", mt_rand(10, 100), mt_rand(1, 50)]);
|
||||
}
|
||||
fclose($csv);
|
||||
|
||||
// Process CSV stream
|
||||
$totalRevenue = SpaceTimeStream::fromCsv($csvFile)
|
||||
->map(fn($row) => [
|
||||
'id' => $row['id'],
|
||||
'revenue' => (float)$row['price'] * (int)$row['quantity']
|
||||
])
|
||||
->reduce(fn($total, $row) => $total + $row['revenue'], 0);
|
||||
|
||||
echo " - Total revenue from 1000 products: $" . number_format($totalRevenue, 2) . "\n\n";
|
||||
|
||||
// Example 5: Memory Pressure Monitoring
|
||||
echo "5. Memory Pressure Monitoring Example\n";
|
||||
$monitor = new MemoryPressureMonitor('100M');
|
||||
|
||||
// Simulate memory usage
|
||||
$data = [];
|
||||
for ($i = 0; $i < 100; $i++) {
|
||||
$data[] = str_repeat('x', 100000); // 100KB per item
|
||||
|
||||
$level = $monitor->check();
|
||||
if ($level !== MemoryPressureLevel::NONE) {
|
||||
echo " - Memory pressure detected: " . $level->value . "\n";
|
||||
$info = $monitor->getMemoryInfo();
|
||||
echo " - Memory usage: " . round($info['percentage'], 2) . "%\n";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up
|
||||
unset($data);
|
||||
$monitor->forceCleanup();
|
||||
echo "\n";
|
||||
|
||||
// Example 6: Batch Processing with Checkpoints
|
||||
echo "6. Batch Processing Example\n";
|
||||
|
||||
$processor = new BatchProcessor([
|
||||
'batch_size' => 100,
|
||||
'checkpoint_enabled' => true,
|
||||
'progress_callback' => function($batch, $size, $result) {
|
||||
echo " - Processing batch $batch ($size items)\n";
|
||||
}
|
||||
]);
|
||||
|
||||
$items = range(1, 500);
|
||||
$result = $processor->process($items, function($batch) {
|
||||
$processed = [];
|
||||
foreach ($batch as $key => $value) {
|
||||
// Simulate processing
|
||||
$processed[$key] = $value * 2;
|
||||
}
|
||||
return $processed;
|
||||
}, 'example_job');
|
||||
|
||||
echo " - Processed: " . $result->getSuccessCount() . " items\n";
|
||||
echo " - Execution time: " . round($result->getExecutionTime(), 2) . " seconds\n\n";
|
||||
|
||||
// Example 7: CSV Export with Streaming
|
||||
echo "7. CSV Export Example\n";
|
||||
|
||||
$exportFile = __DIR__ . '/export.csv';
|
||||
$exporter = new CsvExporter($exportFile);
|
||||
|
||||
$exporter->writeHeaders(['ID', 'Name', 'Email', 'Created']);
|
||||
|
||||
// Simulate exporting user data
|
||||
$exporter->writeInChunks(function() {
|
||||
for ($i = 1; $i <= 1000; $i++) {
|
||||
yield [
|
||||
'id' => $i,
|
||||
'name' => "User $i",
|
||||
'email' => "user$i@example.com",
|
||||
'created' => date('Y-m-d H:i:s')
|
||||
];
|
||||
}
|
||||
}());
|
||||
|
||||
echo " - Exported to: $exportFile\n";
|
||||
echo " - File size: " . number_format(filesize($exportFile) / 1024, 2) . " KB\n\n";
|
||||
|
||||
// Example 8: JSON Lines Processing
|
||||
echo "8. JSON Lines Processing Example\n";
|
||||
|
||||
$jsonlFile = __DIR__ . '/events.jsonl';
|
||||
$events = [];
|
||||
for ($i = 0; $i < 100; $i++) {
|
||||
$events[] = [
|
||||
'id' => $i,
|
||||
'type' => ['click', 'view', 'purchase'][mt_rand(0, 2)],
|
||||
'timestamp' => time() - mt_rand(0, 86400),
|
||||
'user_id' => mt_rand(1, 50)
|
||||
];
|
||||
}
|
||||
|
||||
JsonLinesProcessor::write($events, $jsonlFile);
|
||||
|
||||
// Process and filter
|
||||
$filtered = __DIR__ . '/purchases.jsonl';
|
||||
$count = JsonLinesProcessor::filter(
|
||||
$jsonlFile,
|
||||
$filtered,
|
||||
fn($event) => $event['type'] === 'purchase'
|
||||
);
|
||||
|
||||
echo " - Created JSONL with 100 events\n";
|
||||
echo " - Filtered $count purchase events\n\n";
|
||||
|
||||
// Clean up example files
|
||||
echo "Cleaning up example files...\n";
|
||||
unlink($csvFile);
|
||||
unlink($exportFile);
|
||||
unlink($jsonlFile);
|
||||
unlink($filtered);
|
||||
|
||||
echo "\n=== Examples Complete ===\n";
|
||||
375
examples/laravel-app/README.md
Normal file
375
examples/laravel-app/README.md
Normal file
@@ -0,0 +1,375 @@
|
||||
# SqrtSpace SpaceTime Laravel Sample Application
|
||||
|
||||
This sample demonstrates how to integrate SqrtSpace SpaceTime with a Laravel application to build memory-efficient, scalable web applications.
|
||||
|
||||
## Features Demonstrated
|
||||
|
||||
### 1. **Large Dataset API Endpoints**
|
||||
- Streaming JSON responses for large datasets
|
||||
- Paginated queries with automatic memory management
|
||||
- CSV export without memory bloat
|
||||
|
||||
### 2. **Background Job Processing**
|
||||
- Memory-aware queue workers
|
||||
- Checkpointed long-running jobs
|
||||
- Batch processing with progress tracking
|
||||
|
||||
### 3. **Caching with SpaceTime**
|
||||
- Hot/cold cache tiers
|
||||
- Automatic memory pressure handling
|
||||
- Cache warming strategies
|
||||
|
||||
### 4. **Real-World Use Cases**
|
||||
- User activity log processing
|
||||
- Sales report generation
|
||||
- Product catalog management
|
||||
- Real-time analytics
|
||||
|
||||
## Installation
|
||||
|
||||
1. **Install dependencies:**
|
||||
```bash
|
||||
composer install
|
||||
```
|
||||
|
||||
2. **Configure environment:**
|
||||
```bash
|
||||
cp .env.example .env
|
||||
php artisan key:generate
|
||||
```
|
||||
|
||||
3. **Configure SpaceTime in `.env`:**
|
||||
```
|
||||
SPACETIME_MEMORY_LIMIT=256M
|
||||
SPACETIME_EXTERNAL_STORAGE=/tmp/spacetime
|
||||
SPACETIME_CHUNK_STRATEGY=sqrt_n
|
||||
SPACETIME_ENABLE_CHECKPOINTING=true
|
||||
```
|
||||
|
||||
4. **Run migrations:**
|
||||
```bash
|
||||
php artisan migrate
|
||||
php artisan db:seed
|
||||
```
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
laravel-app/
|
||||
├── app/
|
||||
│ ├── Http/
|
||||
│ │ ├── Controllers/
|
||||
│ │ │ ├── ProductController.php # Streaming APIs
|
||||
│ │ │ ├── AnalyticsController.php # Real-time analytics
|
||||
│ │ │ └── ReportController.php # Large report generation
|
||||
│ │ └── Middleware/
|
||||
│ │ └── SpaceTimeMiddleware.php # Memory monitoring
|
||||
│ ├── Jobs/
|
||||
│ │ ├── ProcessLargeDataset.php # Checkpointed job
|
||||
│ │ ├── GenerateReport.php # Batch processing job
|
||||
│ │ └── ImportProducts.php # CSV import job
|
||||
│ ├── Services/
|
||||
│ │ ├── ProductService.php # Business logic
|
||||
│ │ ├── AnalyticsService.php # Analytics processing
|
||||
│ │ └── SpaceTimeCache.php # Cache wrapper
|
||||
│ └── Providers/
|
||||
│ └── SpaceTimeServiceProvider.php # Service registration
|
||||
├── config/
|
||||
│ └── spacetime.php # Configuration
|
||||
├── routes/
|
||||
│ ├── api.php # API routes
|
||||
│ └── web.php # Web routes
|
||||
└── tests/
|
||||
└── Feature/
|
||||
└── SpaceTimeTest.php # Integration tests
|
||||
```
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### 1. Streaming Large Datasets
|
||||
|
||||
```php
|
||||
// ProductController.php
|
||||
public function stream()
|
||||
{
|
||||
return response()->stream(function () {
|
||||
$products = SpaceTimeStream::fromQuery(
|
||||
Product::query()->orderBy('id')
|
||||
);
|
||||
|
||||
echo "[";
|
||||
$first = true;
|
||||
|
||||
foreach ($products->chunk(100) as $chunk) {
|
||||
foreach ($chunk as $product) {
|
||||
if (!$first) echo ",";
|
||||
echo $product->toJson();
|
||||
$first = false;
|
||||
}
|
||||
|
||||
// Flush output buffer
|
||||
ob_flush();
|
||||
flush();
|
||||
}
|
||||
|
||||
echo "]";
|
||||
}, 200, [
|
||||
'Content-Type' => 'application/json',
|
||||
'X-Accel-Buffering' => 'no'
|
||||
]);
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Memory-Efficient CSV Export
|
||||
|
||||
```php
|
||||
// ReportController.php
|
||||
public function exportCsv()
|
||||
{
|
||||
$filename = 'products_' . date('Y-m-d') . '.csv';
|
||||
|
||||
return response()->streamDownload(function () {
|
||||
$exporter = new CsvExporter('php://output');
|
||||
$exporter->writeHeaders(['ID', 'Name', 'Price', 'Stock']);
|
||||
|
||||
Product::query()
|
||||
->orderBy('id')
|
||||
->chunkById(1000, function ($products) use ($exporter) {
|
||||
foreach ($products as $product) {
|
||||
$exporter->writeRow([
|
||||
$product->id,
|
||||
$product->name,
|
||||
$product->price,
|
||||
$product->stock
|
||||
]);
|
||||
}
|
||||
});
|
||||
}, $filename, [
|
||||
'Content-Type' => 'text/csv',
|
||||
]);
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Checkpointed Background Job
|
||||
|
||||
```php
|
||||
// ProcessLargeDataset.php
|
||||
class ProcessLargeDataset implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
use SpaceTimeCheckpointable;
|
||||
|
||||
public function handle()
|
||||
{
|
||||
$checkpoint = $this->getCheckpoint();
|
||||
$lastId = $checkpoint['last_id'] ?? 0;
|
||||
|
||||
Order::where('id', '>', $lastId)
|
||||
->orderBy('id')
|
||||
->chunkById(100, function ($orders) {
|
||||
foreach ($orders as $order) {
|
||||
// Process order
|
||||
$this->processOrder($order);
|
||||
|
||||
// Save checkpoint every 100 orders
|
||||
if ($order->id % 100 === 0) {
|
||||
$this->saveCheckpoint([
|
||||
'last_id' => $order->id,
|
||||
'processed' => $this->processed,
|
||||
]);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 4. Real-Time Analytics
|
||||
|
||||
```php
|
||||
// AnalyticsController.php
|
||||
public function realtime()
|
||||
{
|
||||
return response()->stream(function () {
|
||||
$monitor = new MemoryPressureMonitor('100M');
|
||||
|
||||
while (true) {
|
||||
$stats = $this->analyticsService->getCurrentStats();
|
||||
|
||||
// Send as Server-Sent Event
|
||||
echo "data: " . json_encode($stats) . "\n\n";
|
||||
ob_flush();
|
||||
flush();
|
||||
|
||||
// Check memory pressure
|
||||
if ($monitor->check() !== MemoryPressureLevel::NONE) {
|
||||
$this->analyticsService->compact();
|
||||
}
|
||||
|
||||
sleep(1);
|
||||
}
|
||||
}, 200, [
|
||||
'Content-Type' => 'text/event-stream',
|
||||
'Cache-Control' => 'no-cache',
|
||||
'X-Accel-Buffering' => 'no'
|
||||
]);
|
||||
}
|
||||
```
|
||||
|
||||
### 5. Memory-Aware Caching
|
||||
|
||||
```php
|
||||
// SpaceTimeCache.php
|
||||
class SpaceTimeCache
|
||||
{
|
||||
private SpaceTimeDict $hot;
|
||||
private CacheInterface $cold;
|
||||
private MemoryPressureMonitor $monitor;
|
||||
|
||||
public function get($key)
|
||||
{
|
||||
// Check hot cache first
|
||||
if (isset($this->hot[$key])) {
|
||||
return $this->hot[$key];
|
||||
}
|
||||
|
||||
// Check cold storage
|
||||
$value = $this->cold->get($key);
|
||||
if ($value !== null) {
|
||||
// Promote to hot cache if memory allows
|
||||
if ($this->monitor->canAllocate(strlen($value))) {
|
||||
$this->hot[$key] = $value;
|
||||
}
|
||||
}
|
||||
|
||||
return $value;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### Products API
|
||||
|
||||
- `GET /api/products` - Paginated list
|
||||
- `GET /api/products/stream` - Stream all products as NDJSON
|
||||
- `GET /api/products/export/csv` - Export as CSV
|
||||
- `POST /api/products/bulk-update` - Bulk update with checkpointing
|
||||
- `POST /api/products/import` - Import CSV with progress
|
||||
|
||||
### Analytics API
|
||||
|
||||
- `GET /api/analytics/summary` - Get summary statistics
|
||||
- `GET /api/analytics/realtime` - Real-time SSE stream
|
||||
- `POST /api/analytics/report` - Generate large report
|
||||
- `GET /api/analytics/top-products` - Top products with external sorting
|
||||
|
||||
### Reports API
|
||||
|
||||
- `POST /api/reports/generate` - Generate report (queued)
|
||||
- `GET /api/reports/{id}/status` - Check generation status
|
||||
- `GET /api/reports/{id}/download` - Download completed report
|
||||
|
||||
## Testing
|
||||
|
||||
Run the test suite:
|
||||
|
||||
```bash
|
||||
php artisan test
|
||||
```
|
||||
|
||||
Example test:
|
||||
|
||||
```php
|
||||
public function test_can_stream_large_dataset()
|
||||
{
|
||||
// Seed test data
|
||||
Product::factory()->count(10000)->create();
|
||||
|
||||
// Make streaming request
|
||||
$response = $this->getJson('/api/products/stream');
|
||||
|
||||
$response->assertStatus(200);
|
||||
$response->assertHeader('Content-Type', 'application/json');
|
||||
|
||||
// Verify memory usage stayed low
|
||||
$this->assertLessThan(50 * 1024 * 1024, memory_get_peak_usage());
|
||||
}
|
||||
```
|
||||
|
||||
## Performance Tips
|
||||
|
||||
1. **Configure memory limits** based on your server capacity
|
||||
2. **Use streaming responses** for large datasets
|
||||
3. **Enable checkpointing** for long-running jobs
|
||||
4. **Monitor memory pressure** in production
|
||||
5. **Use external storage** on fast SSDs
|
||||
6. **Configure queue workers** with appropriate memory limits
|
||||
|
||||
## Deployment
|
||||
|
||||
### Nginx Configuration
|
||||
|
||||
```nginx
|
||||
location /api/products/stream {
|
||||
proxy_pass http://backend;
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 3600;
|
||||
}
|
||||
|
||||
location /api/analytics/realtime {
|
||||
proxy_pass http://backend;
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 0;
|
||||
proxy_http_version 1.1;
|
||||
}
|
||||
```
|
||||
|
||||
### Supervisor Configuration
|
||||
|
||||
```ini
|
||||
[program:spacetime-worker]
|
||||
command=php /path/to/artisan queue:work --memory=256
|
||||
numprocs=4
|
||||
autostart=true
|
||||
autorestart=true
|
||||
```
|
||||
|
||||
## Monitoring
|
||||
|
||||
Add to your monitoring:
|
||||
|
||||
```php
|
||||
// app/Console/Commands/MonitorSpaceTime.php
|
||||
$stats = [
|
||||
'memory_usage' => memory_get_usage(true),
|
||||
'peak_memory' => memory_get_peak_usage(true),
|
||||
'external_files' => count(glob(config('spacetime.external_storage') . '/*')),
|
||||
'cache_size' => $this->cache->size(),
|
||||
];
|
||||
|
||||
Log::channel('metrics')->info('spacetime.stats', $stats);
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### High Memory Usage
|
||||
- Check `SPACETIME_MEMORY_LIMIT` setting
|
||||
- Enable more aggressive spillover
|
||||
- Use smaller chunk sizes
|
||||
|
||||
### Slow Performance
|
||||
- Ensure external storage is on SSD
|
||||
- Increase memory limit if possible
|
||||
- Use compression for large values
|
||||
|
||||
### Failed Checkpoints
|
||||
- Check storage permissions
|
||||
- Ensure sufficient disk space
|
||||
- Verify checkpoint directory exists
|
||||
|
||||
## Learn More
|
||||
|
||||
- [SqrtSpace SpaceTime Documentation](https://github.com/MarketAlly/Ubiquity)
|
||||
- [Laravel Documentation](https://laravel.com/docs)
|
||||
- [Memory-Efficient PHP Patterns](https://example.com/patterns)
|
||||
194
examples/laravel-app/app/Http/Controllers/ProductController.php
Normal file
194
examples/laravel-app/app/Http/Controllers/ProductController.php
Normal file
@@ -0,0 +1,194 @@
|
||||
<?php
|
||||
|
||||
namespace App\Http\Controllers;
|
||||
|
||||
use App\Models\Product;
|
||||
use App\Services\ProductService;
|
||||
use Illuminate\Http\Request;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use SqrtSpace\SpaceTime\Streams\SpaceTimeStream;
|
||||
use SqrtSpace\SpaceTime\File\CsvExporter;
|
||||
use SqrtSpace\SpaceTime\Checkpoint\CheckpointManager;
|
||||
|
||||
class ProductController extends Controller
|
||||
{
|
||||
private ProductService $productService;
|
||||
|
||||
public function __construct(ProductService $productService)
|
||||
{
|
||||
$this->productService = $productService;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get paginated products
|
||||
*/
|
||||
public function index(Request $request)
|
||||
{
|
||||
$perPage = min($request->get('per_page', 50), 100);
|
||||
|
||||
return Product::query()
|
||||
->when($request->get('category'), function ($query, $category) {
|
||||
$query->where('category', $category);
|
||||
})
|
||||
->when($request->get('min_price'), function ($query, $minPrice) {
|
||||
$query->where('price', '>=', $minPrice);
|
||||
})
|
||||
->orderBy('id')
|
||||
->paginate($perPage);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream all products as NDJSON
|
||||
*/
|
||||
public function stream(Request $request)
|
||||
{
|
||||
return response()->stream(function () use ($request) {
|
||||
$query = Product::query()
|
||||
->when($request->get('category'), function ($query, $category) {
|
||||
$query->where('category', $category);
|
||||
})
|
||||
->orderBy('id');
|
||||
|
||||
$stream = SpaceTimeStream::fromQuery($query, 100);
|
||||
|
||||
foreach ($stream as $product) {
|
||||
echo $product->toJson() . "\n";
|
||||
ob_flush();
|
||||
flush();
|
||||
}
|
||||
}, 200, [
|
||||
'Content-Type' => 'application/x-ndjson',
|
||||
'X-Accel-Buffering' => 'no',
|
||||
'Cache-Control' => 'no-cache'
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Export products as CSV
|
||||
*/
|
||||
public function exportCsv(Request $request)
|
||||
{
|
||||
$filename = 'products_' . date('Y-m-d_His') . '.csv';
|
||||
|
||||
return response()->streamDownload(function () use ($request) {
|
||||
$exporter = new CsvExporter('php://output');
|
||||
$exporter->writeHeaders([
|
||||
'ID', 'Name', 'SKU', 'Category', 'Price',
|
||||
'Stock', 'Description', 'Created At'
|
||||
]);
|
||||
|
||||
Product::query()
|
||||
->when($request->get('category'), function ($query, $category) {
|
||||
$query->where('category', $category);
|
||||
})
|
||||
->orderBy('id')
|
||||
->chunkById(1000, function ($products) use ($exporter) {
|
||||
foreach ($products as $product) {
|
||||
$exporter->writeRow([
|
||||
$product->id,
|
||||
$product->name,
|
||||
$product->sku,
|
||||
$product->category,
|
||||
$product->price,
|
||||
$product->stock,
|
||||
$product->description,
|
||||
$product->created_at
|
||||
]);
|
||||
}
|
||||
});
|
||||
}, $filename, [
|
||||
'Content-Type' => 'text/csv',
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bulk update product prices with checkpointing
|
||||
*/
|
||||
public function bulkUpdatePrices(Request $request)
|
||||
{
|
||||
$request->validate([
|
||||
'category' => 'required|string',
|
||||
'adjustment_type' => 'required|in:percentage,fixed',
|
||||
'adjustment_value' => 'required|numeric'
|
||||
]);
|
||||
|
||||
$jobId = 'price_update_' . uniqid();
|
||||
$checkpointManager = app(CheckpointManager::class);
|
||||
|
||||
// Check for existing checkpoint
|
||||
$checkpoint = $checkpointManager->restore($jobId);
|
||||
$lastId = $checkpoint['last_id'] ?? 0;
|
||||
$updated = $checkpoint['updated'] ?? 0;
|
||||
|
||||
DB::beginTransaction();
|
||||
|
||||
try {
|
||||
Product::where('category', $request->category)
|
||||
->where('id', '>', $lastId)
|
||||
->orderBy('id')
|
||||
->chunkById(100, function ($products) use ($request, &$updated, $jobId, $checkpointManager) {
|
||||
foreach ($products as $product) {
|
||||
if ($request->adjustment_type === 'percentage') {
|
||||
$product->price *= (1 + $request->adjustment_value / 100);
|
||||
} else {
|
||||
$product->price += $request->adjustment_value;
|
||||
}
|
||||
$product->save();
|
||||
$updated++;
|
||||
|
||||
// Checkpoint every 100 updates
|
||||
if ($updated % 100 === 0) {
|
||||
$checkpointManager->save($jobId, [
|
||||
'last_id' => $product->id,
|
||||
'updated' => $updated
|
||||
]);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
DB::commit();
|
||||
$checkpointManager->delete($jobId);
|
||||
|
||||
return response()->json([
|
||||
'success' => true,
|
||||
'updated' => $updated,
|
||||
'job_id' => $jobId
|
||||
]);
|
||||
|
||||
} catch (\Exception $e) {
|
||||
DB::rollBack();
|
||||
|
||||
return response()->json([
|
||||
'success' => false,
|
||||
'error' => $e->getMessage(),
|
||||
'job_id' => $jobId,
|
||||
'can_resume' => true
|
||||
], 500);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search products with memory-efficient sorting
|
||||
*/
|
||||
public function search(Request $request)
|
||||
{
|
||||
$request->validate([
|
||||
'q' => 'required|string|min:2',
|
||||
'sort_by' => 'in:relevance,price_asc,price_desc,name'
|
||||
]);
|
||||
|
||||
return $this->productService->searchProducts(
|
||||
$request->get('q'),
|
||||
$request->get('sort_by', 'relevance'),
|
||||
$request->get('limit', 100)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get product statistics
|
||||
*/
|
||||
public function statistics()
|
||||
{
|
||||
return $this->productService->getStatistics();
|
||||
}
|
||||
}
|
||||
239
examples/laravel-app/app/Jobs/ProcessLargeDataset.php
Normal file
239
examples/laravel-app/app/Jobs/ProcessLargeDataset.php
Normal file
@@ -0,0 +1,239 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use App\Models\Order;
|
||||
use App\Models\OrderAnalytics;
|
||||
use SqrtSpace\SpaceTime\Checkpoint\CheckpointManager;
|
||||
use SqrtSpace\SpaceTime\Memory\MemoryPressureMonitor;
|
||||
use SqrtSpace\SpaceTime\Collections\SpaceTimeArray;
|
||||
|
||||
class ProcessLargeDataset implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
private string $jobId;
|
||||
private CheckpointManager $checkpointManager;
|
||||
private MemoryPressureMonitor $memoryMonitor;
|
||||
|
||||
public function __construct(string $jobId = null)
|
||||
{
|
||||
$this->jobId = $jobId ?? 'process_dataset_' . uniqid();
|
||||
}
|
||||
|
||||
public function handle()
|
||||
{
|
||||
$this->checkpointManager = app(CheckpointManager::class);
|
||||
$this->memoryMonitor = new MemoryPressureMonitor('64M');
|
||||
|
||||
// Restore checkpoint if exists
|
||||
$checkpoint = $this->checkpointManager->restore($this->jobId);
|
||||
$state = $checkpoint ?? [
|
||||
'last_order_id' => 0,
|
||||
'processed_count' => 0,
|
||||
'analytics' => [
|
||||
'total_revenue' => 0,
|
||||
'order_count' => 0,
|
||||
'customers' => new SpaceTimeArray(1000),
|
||||
'products' => new SpaceTimeArray(1000),
|
||||
'daily_stats' => []
|
||||
]
|
||||
];
|
||||
|
||||
$this->processOrders($state);
|
||||
|
||||
// Clean up checkpoint after successful completion
|
||||
$this->checkpointManager->delete($this->jobId);
|
||||
|
||||
// Save final analytics
|
||||
$this->saveAnalytics($state['analytics']);
|
||||
}
|
||||
|
||||
private function processOrders(array &$state)
|
||||
{
|
||||
$lastOrderId = $state['last_order_id'];
|
||||
|
||||
Order::where('id', '>', $lastOrderId)
|
||||
->with(['customer', 'items.product'])
|
||||
->orderBy('id')
|
||||
->chunkById(100, function ($orders) use (&$state) {
|
||||
foreach ($orders as $order) {
|
||||
// Process order
|
||||
$this->processOrder($order, $state['analytics']);
|
||||
|
||||
$state['processed_count']++;
|
||||
$state['last_order_id'] = $order->id;
|
||||
|
||||
// Checkpoint every 100 orders
|
||||
if ($state['processed_count'] % 100 === 0) {
|
||||
$this->saveCheckpoint($state);
|
||||
|
||||
// Check memory pressure
|
||||
if ($this->memoryMonitor->shouldCleanup()) {
|
||||
// Flush some analytics to database
|
||||
$this->flushPartialAnalytics($state['analytics']);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private function processOrder(Order $order, array &$analytics)
|
||||
{
|
||||
// Update totals
|
||||
$analytics['total_revenue'] += $order->total_amount;
|
||||
$analytics['order_count']++;
|
||||
|
||||
// Track customer spending
|
||||
$customerId = $order->customer_id;
|
||||
if (!isset($analytics['customers'][$customerId])) {
|
||||
$analytics['customers'][$customerId] = [
|
||||
'total_spent' => 0,
|
||||
'order_count' => 0,
|
||||
'last_order_date' => null
|
||||
];
|
||||
}
|
||||
|
||||
$analytics['customers'][$customerId]['total_spent'] += $order->total_amount;
|
||||
$analytics['customers'][$customerId]['order_count']++;
|
||||
$analytics['customers'][$customerId]['last_order_date'] = $order->created_at;
|
||||
|
||||
// Track product sales
|
||||
foreach ($order->items as $item) {
|
||||
$productId = $item->product_id;
|
||||
if (!isset($analytics['products'][$productId])) {
|
||||
$analytics['products'][$productId] = [
|
||||
'quantity_sold' => 0,
|
||||
'revenue' => 0,
|
||||
'order_count' => 0
|
||||
];
|
||||
}
|
||||
|
||||
$analytics['products'][$productId]['quantity_sold'] += $item->quantity;
|
||||
$analytics['products'][$productId]['revenue'] += $item->total_price;
|
||||
$analytics['products'][$productId]['order_count']++;
|
||||
}
|
||||
|
||||
// Daily statistics
|
||||
$date = $order->created_at->format('Y-m-d');
|
||||
if (!isset($analytics['daily_stats'][$date])) {
|
||||
$analytics['daily_stats'][$date] = [
|
||||
'revenue' => 0,
|
||||
'orders' => 0,
|
||||
'unique_customers' => []
|
||||
];
|
||||
}
|
||||
|
||||
$analytics['daily_stats'][$date]['revenue'] += $order->total_amount;
|
||||
$analytics['daily_stats'][$date]['orders']++;
|
||||
$analytics['daily_stats'][$date]['unique_customers'][$customerId] = true;
|
||||
}
|
||||
|
||||
private function saveCheckpoint(array $state)
|
||||
{
|
||||
$this->checkpointManager->save($this->jobId, [
|
||||
'last_order_id' => $state['last_order_id'],
|
||||
'processed_count' => $state['processed_count'],
|
||||
'analytics' => [
|
||||
'total_revenue' => $state['analytics']['total_revenue'],
|
||||
'order_count' => $state['analytics']['order_count'],
|
||||
'customers' => $state['analytics']['customers'],
|
||||
'products' => $state['analytics']['products'],
|
||||
'daily_stats' => $state['analytics']['daily_stats']
|
||||
]
|
||||
]);
|
||||
|
||||
\Log::info("Checkpoint saved", [
|
||||
'job_id' => $this->jobId,
|
||||
'processed' => $state['processed_count']
|
||||
]);
|
||||
}
|
||||
|
||||
private function flushPartialAnalytics(array &$analytics)
|
||||
{
|
||||
// Save top customers to database
|
||||
$topCustomers = $this->getTopItems($analytics['customers'], 'total_spent', 100);
|
||||
foreach ($topCustomers as $customerId => $data) {
|
||||
OrderAnalytics::updateOrCreate(
|
||||
['type' => 'customer', 'entity_id' => $customerId],
|
||||
['data' => json_encode($data)]
|
||||
);
|
||||
}
|
||||
|
||||
// Save top products
|
||||
$topProducts = $this->getTopItems($analytics['products'], 'revenue', 100);
|
||||
foreach ($topProducts as $productId => $data) {
|
||||
OrderAnalytics::updateOrCreate(
|
||||
['type' => 'product', 'entity_id' => $productId],
|
||||
['data' => json_encode($data)]
|
||||
);
|
||||
}
|
||||
|
||||
// Clear processed items from memory
|
||||
$analytics['customers'] = new SpaceTimeArray(1000);
|
||||
$analytics['products'] = new SpaceTimeArray(1000);
|
||||
|
||||
gc_collect_cycles();
|
||||
}
|
||||
|
||||
private function getTopItems($items, $sortKey, $limit)
|
||||
{
|
||||
$sorted = [];
|
||||
foreach ($items as $id => $data) {
|
||||
$sorted[$id] = $data[$sortKey];
|
||||
}
|
||||
|
||||
arsort($sorted);
|
||||
$topIds = array_slice(array_keys($sorted), 0, $limit);
|
||||
|
||||
$result = [];
|
||||
foreach ($topIds as $id) {
|
||||
$result[$id] = $items[$id];
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
private function saveAnalytics(array $analytics)
|
||||
{
|
||||
// Save summary
|
||||
OrderAnalytics::updateOrCreate(
|
||||
['type' => 'summary', 'entity_id' => 'global'],
|
||||
[
|
||||
'data' => json_encode([
|
||||
'total_revenue' => $analytics['total_revenue'],
|
||||
'order_count' => $analytics['order_count'],
|
||||
'avg_order_value' => $analytics['total_revenue'] / $analytics['order_count'],
|
||||
'unique_customers' => count($analytics['customers']),
|
||||
'unique_products' => count($analytics['products']),
|
||||
'processed_at' => now()
|
||||
])
|
||||
]
|
||||
);
|
||||
|
||||
// Save daily stats
|
||||
foreach ($analytics['daily_stats'] as $date => $stats) {
|
||||
OrderAnalytics::updateOrCreate(
|
||||
['type' => 'daily', 'entity_id' => $date],
|
||||
[
|
||||
'data' => json_encode([
|
||||
'revenue' => $stats['revenue'],
|
||||
'orders' => $stats['orders'],
|
||||
'unique_customers' => count($stats['unique_customers']),
|
||||
'avg_order_value' => $stats['revenue'] / $stats['orders']
|
||||
])
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
\Log::info("Analytics processing completed", [
|
||||
'job_id' => $this->jobId,
|
||||
'total_processed' => $analytics['order_count']
|
||||
]);
|
||||
}
|
||||
}
|
||||
224
examples/laravel-app/app/Services/ProductService.php
Normal file
224
examples/laravel-app/app/Services/ProductService.php
Normal file
@@ -0,0 +1,224 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services;
|
||||
|
||||
use App\Models\Product;
|
||||
use Illuminate\Support\Collection;
|
||||
use SqrtSpace\SpaceTime\Algorithms\ExternalSort;
|
||||
use SqrtSpace\SpaceTime\Algorithms\ExternalGroupBy;
|
||||
use SqrtSpace\SpaceTime\Collections\SpaceTimeArray;
|
||||
use SqrtSpace\SpaceTime\Memory\MemoryPressureMonitor;
|
||||
|
||||
class ProductService
|
||||
{
|
||||
private MemoryPressureMonitor $memoryMonitor;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->memoryMonitor = new MemoryPressureMonitor(
|
||||
config('spacetime.memory_limit', '128M')
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Search products with memory-efficient sorting
|
||||
*/
|
||||
public function searchProducts(string $query, string $sortBy, int $limit): Collection
|
||||
{
|
||||
// Get all matching products
|
||||
$products = Product::where('name', 'like', "%{$query}%")
|
||||
->orWhere('description', 'like', "%{$query}%")
|
||||
->get()
|
||||
->map(function ($product) use ($query) {
|
||||
// Calculate relevance score
|
||||
$nameScore = $this->calculateRelevance($product->name, $query) * 2;
|
||||
$descScore = $this->calculateRelevance($product->description, $query);
|
||||
$product->relevance_score = $nameScore + $descScore;
|
||||
return $product;
|
||||
});
|
||||
|
||||
// Use external sort for large result sets
|
||||
if ($products->count() > 1000) {
|
||||
$sorted = $this->externalSort($products, $sortBy);
|
||||
} else {
|
||||
$sorted = $this->inMemorySort($products, $sortBy);
|
||||
}
|
||||
|
||||
return collect($sorted)->take($limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get product statistics using external grouping
|
||||
*/
|
||||
public function getStatistics(): array
|
||||
{
|
||||
$stats = [
|
||||
'total_products' => Product::count(),
|
||||
'total_value' => 0,
|
||||
'by_category' => [],
|
||||
'price_ranges' => [],
|
||||
'stock_alerts' => []
|
||||
];
|
||||
|
||||
// Use SpaceTimeArray for memory efficiency
|
||||
$allProducts = new SpaceTimeArray(1000);
|
||||
|
||||
Product::chunk(1000, function ($products) use (&$allProducts) {
|
||||
foreach ($products as $product) {
|
||||
$allProducts[] = [
|
||||
'category' => $product->category,
|
||||
'price' => $product->price,
|
||||
'stock' => $product->stock,
|
||||
'value' => $product->price * $product->stock
|
||||
];
|
||||
}
|
||||
});
|
||||
|
||||
// Calculate total value
|
||||
$stats['total_value'] = array_sum(array_column($allProducts->toArray(), 'value'));
|
||||
|
||||
// Group by category using external grouping
|
||||
$byCategory = ExternalGroupBy::groupBySum(
|
||||
$allProducts->toArray(),
|
||||
fn($p) => $p['category'],
|
||||
fn($p) => $p['value']
|
||||
);
|
||||
$stats['by_category'] = $byCategory;
|
||||
|
||||
// Price range distribution
|
||||
$priceRanges = [
|
||||
'0-50' => 0,
|
||||
'50-100' => 0,
|
||||
'100-500' => 0,
|
||||
'500+' => 0
|
||||
];
|
||||
|
||||
foreach ($allProducts as $product) {
|
||||
if ($product['price'] < 50) {
|
||||
$priceRanges['0-50']++;
|
||||
} elseif ($product['price'] < 100) {
|
||||
$priceRanges['50-100']++;
|
||||
} elseif ($product['price'] < 500) {
|
||||
$priceRanges['100-500']++;
|
||||
} else {
|
||||
$priceRanges['500+']++;
|
||||
}
|
||||
|
||||
// Low stock alerts
|
||||
if ($product['stock'] < 10) {
|
||||
$stats['stock_alerts'][] = [
|
||||
'category' => $product['category'],
|
||||
'stock' => $product['stock']
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
$stats['price_ranges'] = $priceRanges;
|
||||
$stats['memory_usage'] = $this->memoryMonitor->getMemoryInfo();
|
||||
|
||||
return $stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Import products from CSV with progress tracking
|
||||
*/
|
||||
public function importFromCsv(string $filePath, callable $progressCallback = null): array
|
||||
{
|
||||
$imported = 0;
|
||||
$errors = [];
|
||||
$batchSize = 100;
|
||||
$batch = [];
|
||||
|
||||
$handle = fopen($filePath, 'r');
|
||||
$headers = fgetcsv($handle); // Skip headers
|
||||
|
||||
while (($row = fgetcsv($handle)) !== false) {
|
||||
try {
|
||||
$batch[] = [
|
||||
'name' => $row[0],
|
||||
'sku' => $row[1],
|
||||
'category' => $row[2],
|
||||
'price' => (float)$row[3],
|
||||
'stock' => (int)$row[4],
|
||||
'description' => $row[5] ?? '',
|
||||
'created_at' => now(),
|
||||
'updated_at' => now()
|
||||
];
|
||||
|
||||
if (count($batch) >= $batchSize) {
|
||||
Product::insert($batch);
|
||||
$imported += count($batch);
|
||||
$batch = [];
|
||||
|
||||
if ($progressCallback) {
|
||||
$progressCallback($imported);
|
||||
}
|
||||
|
||||
// Check memory pressure
|
||||
if ($this->memoryMonitor->shouldCleanup()) {
|
||||
gc_collect_cycles();
|
||||
}
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
$errors[] = "Row " . ($imported + 1) . ": " . $e->getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
// Insert remaining batch
|
||||
if (!empty($batch)) {
|
||||
Product::insert($batch);
|
||||
$imported += count($batch);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
|
||||
return [
|
||||
'imported' => $imported,
|
||||
'errors' => $errors
|
||||
];
|
||||
}
|
||||
|
||||
private function calculateRelevance(string $text, string $query): float
|
||||
{
|
||||
$text = strtolower($text);
|
||||
$query = strtolower($query);
|
||||
|
||||
// Exact match
|
||||
if (strpos($text, $query) !== false) {
|
||||
return 1.0;
|
||||
}
|
||||
|
||||
// Word match
|
||||
$words = explode(' ', $query);
|
||||
$matches = 0;
|
||||
foreach ($words as $word) {
|
||||
if (strpos($text, $word) !== false) {
|
||||
$matches++;
|
||||
}
|
||||
}
|
||||
|
||||
return $matches / count($words);
|
||||
}
|
||||
|
||||
private function externalSort(Collection $products, string $sortBy): array
|
||||
{
|
||||
$sortKey = match($sortBy) {
|
||||
'price_asc' => fn($p) => $p->price,
|
||||
'price_desc' => fn($p) => -$p->price,
|
||||
'name' => fn($p) => $p->name,
|
||||
default => fn($p) => -$p->relevance_score
|
||||
};
|
||||
|
||||
return ExternalSort::sortBy($products->toArray(), $sortKey);
|
||||
}
|
||||
|
||||
private function inMemorySort(Collection $products, string $sortBy): Collection
|
||||
{
|
||||
return match($sortBy) {
|
||||
'price_asc' => $products->sortBy('price'),
|
||||
'price_desc' => $products->sortByDesc('price'),
|
||||
'name' => $products->sortBy('name'),
|
||||
default => $products->sortByDesc('relevance_score')
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user