Initial
This commit is contained in:
194
examples/laravel-app/app/Http/Controllers/ProductController.php
Normal file
194
examples/laravel-app/app/Http/Controllers/ProductController.php
Normal file
@@ -0,0 +1,194 @@
|
||||
<?php
|
||||
|
||||
namespace App\Http\Controllers;
|
||||
|
||||
use App\Models\Product;
|
||||
use App\Services\ProductService;
|
||||
use Illuminate\Http\Request;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use SqrtSpace\SpaceTime\Streams\SpaceTimeStream;
|
||||
use SqrtSpace\SpaceTime\File\CsvExporter;
|
||||
use SqrtSpace\SpaceTime\Checkpoint\CheckpointManager;
|
||||
|
||||
class ProductController extends Controller
|
||||
{
|
||||
private ProductService $productService;
|
||||
|
||||
public function __construct(ProductService $productService)
|
||||
{
|
||||
$this->productService = $productService;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get paginated products
|
||||
*/
|
||||
public function index(Request $request)
|
||||
{
|
||||
$perPage = min($request->get('per_page', 50), 100);
|
||||
|
||||
return Product::query()
|
||||
->when($request->get('category'), function ($query, $category) {
|
||||
$query->where('category', $category);
|
||||
})
|
||||
->when($request->get('min_price'), function ($query, $minPrice) {
|
||||
$query->where('price', '>=', $minPrice);
|
||||
})
|
||||
->orderBy('id')
|
||||
->paginate($perPage);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream all products as NDJSON
|
||||
*/
|
||||
public function stream(Request $request)
|
||||
{
|
||||
return response()->stream(function () use ($request) {
|
||||
$query = Product::query()
|
||||
->when($request->get('category'), function ($query, $category) {
|
||||
$query->where('category', $category);
|
||||
})
|
||||
->orderBy('id');
|
||||
|
||||
$stream = SpaceTimeStream::fromQuery($query, 100);
|
||||
|
||||
foreach ($stream as $product) {
|
||||
echo $product->toJson() . "\n";
|
||||
ob_flush();
|
||||
flush();
|
||||
}
|
||||
}, 200, [
|
||||
'Content-Type' => 'application/x-ndjson',
|
||||
'X-Accel-Buffering' => 'no',
|
||||
'Cache-Control' => 'no-cache'
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Export products as CSV
|
||||
*/
|
||||
public function exportCsv(Request $request)
|
||||
{
|
||||
$filename = 'products_' . date('Y-m-d_His') . '.csv';
|
||||
|
||||
return response()->streamDownload(function () use ($request) {
|
||||
$exporter = new CsvExporter('php://output');
|
||||
$exporter->writeHeaders([
|
||||
'ID', 'Name', 'SKU', 'Category', 'Price',
|
||||
'Stock', 'Description', 'Created At'
|
||||
]);
|
||||
|
||||
Product::query()
|
||||
->when($request->get('category'), function ($query, $category) {
|
||||
$query->where('category', $category);
|
||||
})
|
||||
->orderBy('id')
|
||||
->chunkById(1000, function ($products) use ($exporter) {
|
||||
foreach ($products as $product) {
|
||||
$exporter->writeRow([
|
||||
$product->id,
|
||||
$product->name,
|
||||
$product->sku,
|
||||
$product->category,
|
||||
$product->price,
|
||||
$product->stock,
|
||||
$product->description,
|
||||
$product->created_at
|
||||
]);
|
||||
}
|
||||
});
|
||||
}, $filename, [
|
||||
'Content-Type' => 'text/csv',
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bulk update product prices with checkpointing
|
||||
*/
|
||||
public function bulkUpdatePrices(Request $request)
|
||||
{
|
||||
$request->validate([
|
||||
'category' => 'required|string',
|
||||
'adjustment_type' => 'required|in:percentage,fixed',
|
||||
'adjustment_value' => 'required|numeric'
|
||||
]);
|
||||
|
||||
$jobId = 'price_update_' . uniqid();
|
||||
$checkpointManager = app(CheckpointManager::class);
|
||||
|
||||
// Check for existing checkpoint
|
||||
$checkpoint = $checkpointManager->restore($jobId);
|
||||
$lastId = $checkpoint['last_id'] ?? 0;
|
||||
$updated = $checkpoint['updated'] ?? 0;
|
||||
|
||||
DB::beginTransaction();
|
||||
|
||||
try {
|
||||
Product::where('category', $request->category)
|
||||
->where('id', '>', $lastId)
|
||||
->orderBy('id')
|
||||
->chunkById(100, function ($products) use ($request, &$updated, $jobId, $checkpointManager) {
|
||||
foreach ($products as $product) {
|
||||
if ($request->adjustment_type === 'percentage') {
|
||||
$product->price *= (1 + $request->adjustment_value / 100);
|
||||
} else {
|
||||
$product->price += $request->adjustment_value;
|
||||
}
|
||||
$product->save();
|
||||
$updated++;
|
||||
|
||||
// Checkpoint every 100 updates
|
||||
if ($updated % 100 === 0) {
|
||||
$checkpointManager->save($jobId, [
|
||||
'last_id' => $product->id,
|
||||
'updated' => $updated
|
||||
]);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
DB::commit();
|
||||
$checkpointManager->delete($jobId);
|
||||
|
||||
return response()->json([
|
||||
'success' => true,
|
||||
'updated' => $updated,
|
||||
'job_id' => $jobId
|
||||
]);
|
||||
|
||||
} catch (\Exception $e) {
|
||||
DB::rollBack();
|
||||
|
||||
return response()->json([
|
||||
'success' => false,
|
||||
'error' => $e->getMessage(),
|
||||
'job_id' => $jobId,
|
||||
'can_resume' => true
|
||||
], 500);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search products with memory-efficient sorting
|
||||
*/
|
||||
public function search(Request $request)
|
||||
{
|
||||
$request->validate([
|
||||
'q' => 'required|string|min:2',
|
||||
'sort_by' => 'in:relevance,price_asc,price_desc,name'
|
||||
]);
|
||||
|
||||
return $this->productService->searchProducts(
|
||||
$request->get('q'),
|
||||
$request->get('sort_by', 'relevance'),
|
||||
$request->get('limit', 100)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get product statistics
|
||||
*/
|
||||
public function statistics()
|
||||
{
|
||||
return $this->productService->getStatistics();
|
||||
}
|
||||
}
|
||||
239
examples/laravel-app/app/Jobs/ProcessLargeDataset.php
Normal file
239
examples/laravel-app/app/Jobs/ProcessLargeDataset.php
Normal file
@@ -0,0 +1,239 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use App\Models\Order;
|
||||
use App\Models\OrderAnalytics;
|
||||
use SqrtSpace\SpaceTime\Checkpoint\CheckpointManager;
|
||||
use SqrtSpace\SpaceTime\Memory\MemoryPressureMonitor;
|
||||
use SqrtSpace\SpaceTime\Collections\SpaceTimeArray;
|
||||
|
||||
class ProcessLargeDataset implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
private string $jobId;
|
||||
private CheckpointManager $checkpointManager;
|
||||
private MemoryPressureMonitor $memoryMonitor;
|
||||
|
||||
public function __construct(string $jobId = null)
|
||||
{
|
||||
$this->jobId = $jobId ?? 'process_dataset_' . uniqid();
|
||||
}
|
||||
|
||||
public function handle()
|
||||
{
|
||||
$this->checkpointManager = app(CheckpointManager::class);
|
||||
$this->memoryMonitor = new MemoryPressureMonitor('64M');
|
||||
|
||||
// Restore checkpoint if exists
|
||||
$checkpoint = $this->checkpointManager->restore($this->jobId);
|
||||
$state = $checkpoint ?? [
|
||||
'last_order_id' => 0,
|
||||
'processed_count' => 0,
|
||||
'analytics' => [
|
||||
'total_revenue' => 0,
|
||||
'order_count' => 0,
|
||||
'customers' => new SpaceTimeArray(1000),
|
||||
'products' => new SpaceTimeArray(1000),
|
||||
'daily_stats' => []
|
||||
]
|
||||
];
|
||||
|
||||
$this->processOrders($state);
|
||||
|
||||
// Clean up checkpoint after successful completion
|
||||
$this->checkpointManager->delete($this->jobId);
|
||||
|
||||
// Save final analytics
|
||||
$this->saveAnalytics($state['analytics']);
|
||||
}
|
||||
|
||||
private function processOrders(array &$state)
|
||||
{
|
||||
$lastOrderId = $state['last_order_id'];
|
||||
|
||||
Order::where('id', '>', $lastOrderId)
|
||||
->with(['customer', 'items.product'])
|
||||
->orderBy('id')
|
||||
->chunkById(100, function ($orders) use (&$state) {
|
||||
foreach ($orders as $order) {
|
||||
// Process order
|
||||
$this->processOrder($order, $state['analytics']);
|
||||
|
||||
$state['processed_count']++;
|
||||
$state['last_order_id'] = $order->id;
|
||||
|
||||
// Checkpoint every 100 orders
|
||||
if ($state['processed_count'] % 100 === 0) {
|
||||
$this->saveCheckpoint($state);
|
||||
|
||||
// Check memory pressure
|
||||
if ($this->memoryMonitor->shouldCleanup()) {
|
||||
// Flush some analytics to database
|
||||
$this->flushPartialAnalytics($state['analytics']);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private function processOrder(Order $order, array &$analytics)
|
||||
{
|
||||
// Update totals
|
||||
$analytics['total_revenue'] += $order->total_amount;
|
||||
$analytics['order_count']++;
|
||||
|
||||
// Track customer spending
|
||||
$customerId = $order->customer_id;
|
||||
if (!isset($analytics['customers'][$customerId])) {
|
||||
$analytics['customers'][$customerId] = [
|
||||
'total_spent' => 0,
|
||||
'order_count' => 0,
|
||||
'last_order_date' => null
|
||||
];
|
||||
}
|
||||
|
||||
$analytics['customers'][$customerId]['total_spent'] += $order->total_amount;
|
||||
$analytics['customers'][$customerId]['order_count']++;
|
||||
$analytics['customers'][$customerId]['last_order_date'] = $order->created_at;
|
||||
|
||||
// Track product sales
|
||||
foreach ($order->items as $item) {
|
||||
$productId = $item->product_id;
|
||||
if (!isset($analytics['products'][$productId])) {
|
||||
$analytics['products'][$productId] = [
|
||||
'quantity_sold' => 0,
|
||||
'revenue' => 0,
|
||||
'order_count' => 0
|
||||
];
|
||||
}
|
||||
|
||||
$analytics['products'][$productId]['quantity_sold'] += $item->quantity;
|
||||
$analytics['products'][$productId]['revenue'] += $item->total_price;
|
||||
$analytics['products'][$productId]['order_count']++;
|
||||
}
|
||||
|
||||
// Daily statistics
|
||||
$date = $order->created_at->format('Y-m-d');
|
||||
if (!isset($analytics['daily_stats'][$date])) {
|
||||
$analytics['daily_stats'][$date] = [
|
||||
'revenue' => 0,
|
||||
'orders' => 0,
|
||||
'unique_customers' => []
|
||||
];
|
||||
}
|
||||
|
||||
$analytics['daily_stats'][$date]['revenue'] += $order->total_amount;
|
||||
$analytics['daily_stats'][$date]['orders']++;
|
||||
$analytics['daily_stats'][$date]['unique_customers'][$customerId] = true;
|
||||
}
|
||||
|
||||
private function saveCheckpoint(array $state)
|
||||
{
|
||||
$this->checkpointManager->save($this->jobId, [
|
||||
'last_order_id' => $state['last_order_id'],
|
||||
'processed_count' => $state['processed_count'],
|
||||
'analytics' => [
|
||||
'total_revenue' => $state['analytics']['total_revenue'],
|
||||
'order_count' => $state['analytics']['order_count'],
|
||||
'customers' => $state['analytics']['customers'],
|
||||
'products' => $state['analytics']['products'],
|
||||
'daily_stats' => $state['analytics']['daily_stats']
|
||||
]
|
||||
]);
|
||||
|
||||
\Log::info("Checkpoint saved", [
|
||||
'job_id' => $this->jobId,
|
||||
'processed' => $state['processed_count']
|
||||
]);
|
||||
}
|
||||
|
||||
private function flushPartialAnalytics(array &$analytics)
|
||||
{
|
||||
// Save top customers to database
|
||||
$topCustomers = $this->getTopItems($analytics['customers'], 'total_spent', 100);
|
||||
foreach ($topCustomers as $customerId => $data) {
|
||||
OrderAnalytics::updateOrCreate(
|
||||
['type' => 'customer', 'entity_id' => $customerId],
|
||||
['data' => json_encode($data)]
|
||||
);
|
||||
}
|
||||
|
||||
// Save top products
|
||||
$topProducts = $this->getTopItems($analytics['products'], 'revenue', 100);
|
||||
foreach ($topProducts as $productId => $data) {
|
||||
OrderAnalytics::updateOrCreate(
|
||||
['type' => 'product', 'entity_id' => $productId],
|
||||
['data' => json_encode($data)]
|
||||
);
|
||||
}
|
||||
|
||||
// Clear processed items from memory
|
||||
$analytics['customers'] = new SpaceTimeArray(1000);
|
||||
$analytics['products'] = new SpaceTimeArray(1000);
|
||||
|
||||
gc_collect_cycles();
|
||||
}
|
||||
|
||||
private function getTopItems($items, $sortKey, $limit)
|
||||
{
|
||||
$sorted = [];
|
||||
foreach ($items as $id => $data) {
|
||||
$sorted[$id] = $data[$sortKey];
|
||||
}
|
||||
|
||||
arsort($sorted);
|
||||
$topIds = array_slice(array_keys($sorted), 0, $limit);
|
||||
|
||||
$result = [];
|
||||
foreach ($topIds as $id) {
|
||||
$result[$id] = $items[$id];
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
private function saveAnalytics(array $analytics)
|
||||
{
|
||||
// Save summary
|
||||
OrderAnalytics::updateOrCreate(
|
||||
['type' => 'summary', 'entity_id' => 'global'],
|
||||
[
|
||||
'data' => json_encode([
|
||||
'total_revenue' => $analytics['total_revenue'],
|
||||
'order_count' => $analytics['order_count'],
|
||||
'avg_order_value' => $analytics['total_revenue'] / $analytics['order_count'],
|
||||
'unique_customers' => count($analytics['customers']),
|
||||
'unique_products' => count($analytics['products']),
|
||||
'processed_at' => now()
|
||||
])
|
||||
]
|
||||
);
|
||||
|
||||
// Save daily stats
|
||||
foreach ($analytics['daily_stats'] as $date => $stats) {
|
||||
OrderAnalytics::updateOrCreate(
|
||||
['type' => 'daily', 'entity_id' => $date],
|
||||
[
|
||||
'data' => json_encode([
|
||||
'revenue' => $stats['revenue'],
|
||||
'orders' => $stats['orders'],
|
||||
'unique_customers' => count($stats['unique_customers']),
|
||||
'avg_order_value' => $stats['revenue'] / $stats['orders']
|
||||
])
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
\Log::info("Analytics processing completed", [
|
||||
'job_id' => $this->jobId,
|
||||
'total_processed' => $analytics['order_count']
|
||||
]);
|
||||
}
|
||||
}
|
||||
224
examples/laravel-app/app/Services/ProductService.php
Normal file
224
examples/laravel-app/app/Services/ProductService.php
Normal file
@@ -0,0 +1,224 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services;
|
||||
|
||||
use App\Models\Product;
|
||||
use Illuminate\Support\Collection;
|
||||
use SqrtSpace\SpaceTime\Algorithms\ExternalSort;
|
||||
use SqrtSpace\SpaceTime\Algorithms\ExternalGroupBy;
|
||||
use SqrtSpace\SpaceTime\Collections\SpaceTimeArray;
|
||||
use SqrtSpace\SpaceTime\Memory\MemoryPressureMonitor;
|
||||
|
||||
class ProductService
|
||||
{
|
||||
private MemoryPressureMonitor $memoryMonitor;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->memoryMonitor = new MemoryPressureMonitor(
|
||||
config('spacetime.memory_limit', '128M')
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Search products with memory-efficient sorting
|
||||
*/
|
||||
public function searchProducts(string $query, string $sortBy, int $limit): Collection
|
||||
{
|
||||
// Get all matching products
|
||||
$products = Product::where('name', 'like', "%{$query}%")
|
||||
->orWhere('description', 'like', "%{$query}%")
|
||||
->get()
|
||||
->map(function ($product) use ($query) {
|
||||
// Calculate relevance score
|
||||
$nameScore = $this->calculateRelevance($product->name, $query) * 2;
|
||||
$descScore = $this->calculateRelevance($product->description, $query);
|
||||
$product->relevance_score = $nameScore + $descScore;
|
||||
return $product;
|
||||
});
|
||||
|
||||
// Use external sort for large result sets
|
||||
if ($products->count() > 1000) {
|
||||
$sorted = $this->externalSort($products, $sortBy);
|
||||
} else {
|
||||
$sorted = $this->inMemorySort($products, $sortBy);
|
||||
}
|
||||
|
||||
return collect($sorted)->take($limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get product statistics using external grouping
|
||||
*/
|
||||
public function getStatistics(): array
|
||||
{
|
||||
$stats = [
|
||||
'total_products' => Product::count(),
|
||||
'total_value' => 0,
|
||||
'by_category' => [],
|
||||
'price_ranges' => [],
|
||||
'stock_alerts' => []
|
||||
];
|
||||
|
||||
// Use SpaceTimeArray for memory efficiency
|
||||
$allProducts = new SpaceTimeArray(1000);
|
||||
|
||||
Product::chunk(1000, function ($products) use (&$allProducts) {
|
||||
foreach ($products as $product) {
|
||||
$allProducts[] = [
|
||||
'category' => $product->category,
|
||||
'price' => $product->price,
|
||||
'stock' => $product->stock,
|
||||
'value' => $product->price * $product->stock
|
||||
];
|
||||
}
|
||||
});
|
||||
|
||||
// Calculate total value
|
||||
$stats['total_value'] = array_sum(array_column($allProducts->toArray(), 'value'));
|
||||
|
||||
// Group by category using external grouping
|
||||
$byCategory = ExternalGroupBy::groupBySum(
|
||||
$allProducts->toArray(),
|
||||
fn($p) => $p['category'],
|
||||
fn($p) => $p['value']
|
||||
);
|
||||
$stats['by_category'] = $byCategory;
|
||||
|
||||
// Price range distribution
|
||||
$priceRanges = [
|
||||
'0-50' => 0,
|
||||
'50-100' => 0,
|
||||
'100-500' => 0,
|
||||
'500+' => 0
|
||||
];
|
||||
|
||||
foreach ($allProducts as $product) {
|
||||
if ($product['price'] < 50) {
|
||||
$priceRanges['0-50']++;
|
||||
} elseif ($product['price'] < 100) {
|
||||
$priceRanges['50-100']++;
|
||||
} elseif ($product['price'] < 500) {
|
||||
$priceRanges['100-500']++;
|
||||
} else {
|
||||
$priceRanges['500+']++;
|
||||
}
|
||||
|
||||
// Low stock alerts
|
||||
if ($product['stock'] < 10) {
|
||||
$stats['stock_alerts'][] = [
|
||||
'category' => $product['category'],
|
||||
'stock' => $product['stock']
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
$stats['price_ranges'] = $priceRanges;
|
||||
$stats['memory_usage'] = $this->memoryMonitor->getMemoryInfo();
|
||||
|
||||
return $stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Import products from CSV with progress tracking
|
||||
*/
|
||||
public function importFromCsv(string $filePath, callable $progressCallback = null): array
|
||||
{
|
||||
$imported = 0;
|
||||
$errors = [];
|
||||
$batchSize = 100;
|
||||
$batch = [];
|
||||
|
||||
$handle = fopen($filePath, 'r');
|
||||
$headers = fgetcsv($handle); // Skip headers
|
||||
|
||||
while (($row = fgetcsv($handle)) !== false) {
|
||||
try {
|
||||
$batch[] = [
|
||||
'name' => $row[0],
|
||||
'sku' => $row[1],
|
||||
'category' => $row[2],
|
||||
'price' => (float)$row[3],
|
||||
'stock' => (int)$row[4],
|
||||
'description' => $row[5] ?? '',
|
||||
'created_at' => now(),
|
||||
'updated_at' => now()
|
||||
];
|
||||
|
||||
if (count($batch) >= $batchSize) {
|
||||
Product::insert($batch);
|
||||
$imported += count($batch);
|
||||
$batch = [];
|
||||
|
||||
if ($progressCallback) {
|
||||
$progressCallback($imported);
|
||||
}
|
||||
|
||||
// Check memory pressure
|
||||
if ($this->memoryMonitor->shouldCleanup()) {
|
||||
gc_collect_cycles();
|
||||
}
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
$errors[] = "Row " . ($imported + 1) . ": " . $e->getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
// Insert remaining batch
|
||||
if (!empty($batch)) {
|
||||
Product::insert($batch);
|
||||
$imported += count($batch);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
|
||||
return [
|
||||
'imported' => $imported,
|
||||
'errors' => $errors
|
||||
];
|
||||
}
|
||||
|
||||
private function calculateRelevance(string $text, string $query): float
|
||||
{
|
||||
$text = strtolower($text);
|
||||
$query = strtolower($query);
|
||||
|
||||
// Exact match
|
||||
if (strpos($text, $query) !== false) {
|
||||
return 1.0;
|
||||
}
|
||||
|
||||
// Word match
|
||||
$words = explode(' ', $query);
|
||||
$matches = 0;
|
||||
foreach ($words as $word) {
|
||||
if (strpos($text, $word) !== false) {
|
||||
$matches++;
|
||||
}
|
||||
}
|
||||
|
||||
return $matches / count($words);
|
||||
}
|
||||
|
||||
private function externalSort(Collection $products, string $sortBy): array
|
||||
{
|
||||
$sortKey = match($sortBy) {
|
||||
'price_asc' => fn($p) => $p->price,
|
||||
'price_desc' => fn($p) => -$p->price,
|
||||
'name' => fn($p) => $p->name,
|
||||
default => fn($p) => -$p->relevance_score
|
||||
};
|
||||
|
||||
return ExternalSort::sortBy($products->toArray(), $sortKey);
|
||||
}
|
||||
|
||||
private function inMemorySort(Collection $products, string $sortBy): Collection
|
||||
{
|
||||
return match($sortBy) {
|
||||
'price_asc' => $products->sortBy('price'),
|
||||
'price_desc' => $products->sortByDesc('price'),
|
||||
'name' => $products->sortBy('name'),
|
||||
default => $products->sortByDesc('relevance_score')
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user