374 lines
14 KiB
Python
374 lines
14 KiB
Python
|
"""
|
|||
|
|
Checkpointed Sorting: Demonstrating Space-Time Tradeoffs
|
|||
|
|
|
|||
|
|
This experiment shows how external merge sort with limited memory
|
|||
|
|
exhibits the √(t log t) space behavior from Williams' 2025 result.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import os
|
|||
|
|
import time
|
|||
|
|
import tempfile
|
|||
|
|
import numpy as np
|
|||
|
|
import matplotlib.pyplot as plt
|
|||
|
|
from typing import List, Tuple
|
|||
|
|
import heapq
|
|||
|
|
import shutil
|
|||
|
|
import sys
|
|||
|
|
from scipy import stats
|
|||
|
|
sys.path.append('..')
|
|||
|
|
from measurement_framework import SpaceTimeProfiler, ExperimentRunner
|
|||
|
|
|
|||
|
|
|
|||
|
|
class SortingExperiment:
|
|||
|
|
"""Compare different sorting algorithms with varying memory constraints"""
|
|||
|
|
|
|||
|
|
def __init__(self, data_size: int):
|
|||
|
|
self.data_size = data_size
|
|||
|
|
self.data = np.random.rand(data_size).astype(np.float32)
|
|||
|
|
self.temp_dir = tempfile.mkdtemp()
|
|||
|
|
|
|||
|
|
def cleanup(self):
|
|||
|
|
"""Clean up temporary files"""
|
|||
|
|
shutil.rmtree(self.temp_dir)
|
|||
|
|
|
|||
|
|
def in_memory_sort(self) -> np.ndarray:
|
|||
|
|
"""Standard in-memory sorting - O(n) space"""
|
|||
|
|
return np.sort(self.data.copy())
|
|||
|
|
|
|||
|
|
def checkpoint_sort(self, memory_limit: int) -> np.ndarray:
|
|||
|
|
"""External merge sort with checkpointing - O(√n) space"""
|
|||
|
|
chunk_size = memory_limit // 4 # Reserve memory for merging
|
|||
|
|
num_chunks = (self.data_size + chunk_size - 1) // chunk_size
|
|||
|
|
|
|||
|
|
# Phase 1: Sort chunks and write to disk
|
|||
|
|
chunk_files = []
|
|||
|
|
for i in range(num_chunks):
|
|||
|
|
start = i * chunk_size
|
|||
|
|
end = min((i + 1) * chunk_size, self.data_size)
|
|||
|
|
|
|||
|
|
# Sort chunk in memory
|
|||
|
|
chunk = np.sort(self.data[start:end])
|
|||
|
|
|
|||
|
|
# Write to disk (checkpoint)
|
|||
|
|
filename = os.path.join(self.temp_dir, f'chunk_{i}.npy')
|
|||
|
|
np.save(filename, chunk)
|
|||
|
|
chunk_files.append(filename)
|
|||
|
|
|
|||
|
|
# Clear chunk from memory
|
|||
|
|
del chunk
|
|||
|
|
|
|||
|
|
# Phase 2: K-way merge with limited memory
|
|||
|
|
result = self._k_way_merge(chunk_files, memory_limit)
|
|||
|
|
|
|||
|
|
# Cleanup chunk files
|
|||
|
|
for f in chunk_files:
|
|||
|
|
os.remove(f)
|
|||
|
|
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
def _k_way_merge(self, chunk_files: List[str], memory_limit: int) -> np.ndarray:
|
|||
|
|
"""Merge sorted chunks with limited memory"""
|
|||
|
|
# Calculate how many elements we can buffer per chunk
|
|||
|
|
num_chunks = len(chunk_files)
|
|||
|
|
buffer_size = max(1, memory_limit // (4 * num_chunks)) # 4 bytes per float32
|
|||
|
|
|
|||
|
|
# Open file handles and create buffers
|
|||
|
|
file_handles = []
|
|||
|
|
buffers = []
|
|||
|
|
positions = []
|
|||
|
|
|
|||
|
|
for filename in chunk_files:
|
|||
|
|
data = np.load(filename)
|
|||
|
|
file_handles.append(data)
|
|||
|
|
buffers.append(data[:buffer_size])
|
|||
|
|
positions.append(buffer_size)
|
|||
|
|
|
|||
|
|
# Use heap for efficient merging
|
|||
|
|
heap = []
|
|||
|
|
for i, buffer in enumerate(buffers):
|
|||
|
|
if len(buffer) > 0:
|
|||
|
|
heapq.heappush(heap, (buffer[0], i, 0))
|
|||
|
|
|
|||
|
|
result = []
|
|||
|
|
|
|||
|
|
while heap:
|
|||
|
|
val, chunk_idx, buffer_idx = heapq.heappop(heap)
|
|||
|
|
result.append(val)
|
|||
|
|
|
|||
|
|
# Move to next element in buffer
|
|||
|
|
buffer_idx += 1
|
|||
|
|
|
|||
|
|
# Refill buffer if needed
|
|||
|
|
if buffer_idx >= len(buffers[chunk_idx]):
|
|||
|
|
pos = positions[chunk_idx]
|
|||
|
|
if pos < len(file_handles[chunk_idx]):
|
|||
|
|
# Load next batch from disk
|
|||
|
|
new_buffer_size = min(buffer_size, len(file_handles[chunk_idx]) - pos)
|
|||
|
|
buffers[chunk_idx] = file_handles[chunk_idx][pos:pos + new_buffer_size]
|
|||
|
|
positions[chunk_idx] = pos + new_buffer_size
|
|||
|
|
buffer_idx = 0
|
|||
|
|
else:
|
|||
|
|
# This chunk is exhausted
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# Add next element to heap
|
|||
|
|
if buffer_idx < len(buffers[chunk_idx]):
|
|||
|
|
heapq.heappush(heap, (buffers[chunk_idx][buffer_idx], chunk_idx, buffer_idx))
|
|||
|
|
|
|||
|
|
return np.array(result)
|
|||
|
|
|
|||
|
|
def extreme_checkpoint_sort(self) -> np.ndarray:
|
|||
|
|
"""Extreme checkpointing - O(log n) space using iterative merging"""
|
|||
|
|
# Sort pairs iteratively, storing only log(n) elements at a time
|
|||
|
|
temp_file = os.path.join(self.temp_dir, 'temp_sort.npy')
|
|||
|
|
|
|||
|
|
# Initial pass: sort pairs
|
|||
|
|
sorted_data = self.data.copy()
|
|||
|
|
|
|||
|
|
# Bubble sort with checkpointing every √n comparisons
|
|||
|
|
checkpoint_interval = int(np.sqrt(self.data_size))
|
|||
|
|
comparisons = 0
|
|||
|
|
|
|||
|
|
for i in range(self.data_size):
|
|||
|
|
for j in range(0, self.data_size - i - 1):
|
|||
|
|
if sorted_data[j] > sorted_data[j + 1]:
|
|||
|
|
sorted_data[j], sorted_data[j + 1] = sorted_data[j + 1], sorted_data[j]
|
|||
|
|
|
|||
|
|
comparisons += 1
|
|||
|
|
if comparisons % checkpoint_interval == 0:
|
|||
|
|
# Checkpoint to disk
|
|||
|
|
np.save(temp_file, sorted_data)
|
|||
|
|
# Simulate memory clear by reloading
|
|||
|
|
sorted_data = np.load(temp_file)
|
|||
|
|
|
|||
|
|
os.remove(temp_file)
|
|||
|
|
return sorted_data
|
|||
|
|
|
|||
|
|
|
|||
|
|
def run_sorting_experiments():
|
|||
|
|
"""Run the sorting experiments with different input sizes"""
|
|||
|
|
|
|||
|
|
print("=== Checkpointed Sorting Experiment ===\n")
|
|||
|
|
|
|||
|
|
# Number of trials for statistical analysis
|
|||
|
|
num_trials = 20
|
|||
|
|
|
|||
|
|
# Use larger sizes for more reliable timing
|
|||
|
|
sizes = [1000, 5000, 10000, 20000, 50000]
|
|||
|
|
results = []
|
|||
|
|
|
|||
|
|
for size in sizes:
|
|||
|
|
print(f"\nTesting with {size} elements ({num_trials} trials each):")
|
|||
|
|
|
|||
|
|
# Store times for each trial
|
|||
|
|
in_memory_times = []
|
|||
|
|
checkpoint_times = []
|
|||
|
|
extreme_times = []
|
|||
|
|
|
|||
|
|
for trial in range(num_trials):
|
|||
|
|
exp = SortingExperiment(size)
|
|||
|
|
|
|||
|
|
# 1. In-memory sort - O(n) space
|
|||
|
|
start = time.time()
|
|||
|
|
result1 = exp.in_memory_sort()
|
|||
|
|
time1 = time.time() - start
|
|||
|
|
in_memory_times.append(time1)
|
|||
|
|
|
|||
|
|
# 2. Checkpointed sort - O(√n) space
|
|||
|
|
memory_limit = int(np.sqrt(size) * 4) # 4 bytes per element
|
|||
|
|
start = time.time()
|
|||
|
|
result2 = exp.checkpoint_sort(memory_limit)
|
|||
|
|
time2 = time.time() - start
|
|||
|
|
checkpoint_times.append(time2)
|
|||
|
|
|
|||
|
|
# 3. Extreme checkpoint - O(log n) space (only for small sizes)
|
|||
|
|
if size <= 1000:
|
|||
|
|
start = time.time()
|
|||
|
|
result3 = exp.extreme_checkpoint_sort()
|
|||
|
|
time3 = time.time() - start
|
|||
|
|
extreme_times.append(time3)
|
|||
|
|
|
|||
|
|
# Verify correctness (only on first trial)
|
|||
|
|
if trial == 0:
|
|||
|
|
assert np.allclose(result1, result2), "Checkpointed sort produced incorrect result"
|
|||
|
|
|
|||
|
|
exp.cleanup()
|
|||
|
|
|
|||
|
|
# Progress indicator
|
|||
|
|
if (trial + 1) % 5 == 0:
|
|||
|
|
print(f" Completed {trial + 1}/{num_trials} trials...")
|
|||
|
|
|
|||
|
|
# Calculate statistics
|
|||
|
|
in_memory_mean = np.mean(in_memory_times)
|
|||
|
|
in_memory_std = np.std(in_memory_times)
|
|||
|
|
checkpoint_mean = np.mean(checkpoint_times)
|
|||
|
|
checkpoint_std = np.std(checkpoint_times)
|
|||
|
|
|
|||
|
|
print(f" In-memory sort: {in_memory_mean:.4f}s ± {in_memory_std:.4f}s")
|
|||
|
|
print(f" Checkpointed sort (√n memory): {checkpoint_mean:.4f}s ± {checkpoint_std:.4f}s")
|
|||
|
|
|
|||
|
|
if extreme_times:
|
|||
|
|
extreme_mean = np.mean(extreme_times)
|
|||
|
|
extreme_std = np.std(extreme_times)
|
|||
|
|
print(f" Extreme checkpoint (log n memory): {extreme_mean:.4f}s ± {extreme_std:.4f}s")
|
|||
|
|
else:
|
|||
|
|
extreme_mean = None
|
|||
|
|
extreme_std = None
|
|||
|
|
print(f" Extreme checkpoint: Skipped (too slow for n={size})")
|
|||
|
|
|
|||
|
|
# Calculate slowdown factor
|
|||
|
|
slowdown = checkpoint_mean / in_memory_mean if in_memory_mean > 0.0001 else checkpoint_mean / 0.0001
|
|||
|
|
|
|||
|
|
# Calculate 95% confidence intervals
|
|||
|
|
from scipy import stats
|
|||
|
|
in_memory_ci = stats.t.interval(0.95, len(in_memory_times)-1,
|
|||
|
|
loc=in_memory_mean,
|
|||
|
|
scale=stats.sem(in_memory_times))
|
|||
|
|
checkpoint_ci = stats.t.interval(0.95, len(checkpoint_times)-1,
|
|||
|
|
loc=checkpoint_mean,
|
|||
|
|
scale=stats.sem(checkpoint_times))
|
|||
|
|
|
|||
|
|
results.append({
|
|||
|
|
'size': size,
|
|||
|
|
'in_memory_time': in_memory_mean,
|
|||
|
|
'in_memory_std': in_memory_std,
|
|||
|
|
'in_memory_ci': in_memory_ci,
|
|||
|
|
'checkpoint_time': checkpoint_mean,
|
|||
|
|
'checkpoint_std': checkpoint_std,
|
|||
|
|
'checkpoint_ci': checkpoint_ci,
|
|||
|
|
'extreme_time': extreme_mean,
|
|||
|
|
'extreme_std': extreme_std,
|
|||
|
|
'slowdown': slowdown,
|
|||
|
|
'num_trials': num_trials
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
# Plot results with error bars
|
|||
|
|
plot_sorting_results(results)
|
|||
|
|
|
|||
|
|
return results
|
|||
|
|
|
|||
|
|
|
|||
|
|
def plot_sorting_results(results):
|
|||
|
|
"""Visualize the space-time tradeoff in sorting with error bars"""
|
|||
|
|
|
|||
|
|
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 6))
|
|||
|
|
|
|||
|
|
sizes = [r['size'] for r in results]
|
|||
|
|
in_memory_times = [r['in_memory_time'] for r in results]
|
|||
|
|
in_memory_stds = [r['in_memory_std'] for r in results]
|
|||
|
|
checkpoint_times = [r['checkpoint_time'] for r in results]
|
|||
|
|
checkpoint_stds = [r['checkpoint_std'] for r in results]
|
|||
|
|
slowdowns = [r['slowdown'] for r in results]
|
|||
|
|
|
|||
|
|
# Time comparison with error bars
|
|||
|
|
ax1.errorbar(sizes, in_memory_times, yerr=[2*s for s in in_memory_stds],
|
|||
|
|
fmt='o-', label='In-memory (O(n) space)',
|
|||
|
|
linewidth=2, markersize=8, color='blue', capsize=5)
|
|||
|
|
ax1.errorbar(sizes, checkpoint_times, yerr=[2*s for s in checkpoint_stds],
|
|||
|
|
fmt='s-', label='Checkpointed (O(√n) space)',
|
|||
|
|
linewidth=2, markersize=8, color='orange', capsize=5)
|
|||
|
|
|
|||
|
|
# Add theoretical bounds
|
|||
|
|
n_theory = np.logspace(np.log10(min(sizes)), np.log10(max(sizes)), 50)
|
|||
|
|
# O(n log n) for in-memory sort
|
|||
|
|
ax1.plot(n_theory, in_memory_times[0] * (n_theory * np.log(n_theory)) / (sizes[0] * np.log(sizes[0])),
|
|||
|
|
'b--', alpha=0.5, label='O(n log n) bound')
|
|||
|
|
# O(n√n) for checkpointed sort
|
|||
|
|
ax1.plot(n_theory, checkpoint_times[0] * n_theory * np.sqrt(n_theory) / (sizes[0] * np.sqrt(sizes[0])),
|
|||
|
|
'r--', alpha=0.5, label='O(n√n) bound')
|
|||
|
|
|
|||
|
|
ax1.set_xlabel('Input Size (n)', fontsize=12)
|
|||
|
|
ax1.set_ylabel('Time (seconds)', fontsize=12)
|
|||
|
|
ax1.set_title('Sorting Time Complexity (mean ± 2σ, n=20 trials)', fontsize=14)
|
|||
|
|
ax1.legend(loc='upper left')
|
|||
|
|
ax1.grid(True, alpha=0.3)
|
|||
|
|
ax1.set_xscale('log')
|
|||
|
|
ax1.set_yscale('log')
|
|||
|
|
|
|||
|
|
# Slowdown factor (log scale) with confidence regions
|
|||
|
|
ax2.plot(sizes, slowdowns, 'g^-', linewidth=2, markersize=10)
|
|||
|
|
|
|||
|
|
# Add shaded confidence region for slowdown
|
|||
|
|
slowdown_upper = []
|
|||
|
|
slowdown_lower = []
|
|||
|
|
for r in results:
|
|||
|
|
# Calculate slowdown bounds using error propagation
|
|||
|
|
mean_ratio = r['checkpoint_time'] / r['in_memory_time']
|
|||
|
|
std_ratio = mean_ratio * np.sqrt((r['checkpoint_std']/r['checkpoint_time'])**2 +
|
|||
|
|
(r['in_memory_std']/r['in_memory_time'])**2)
|
|||
|
|
slowdown_upper.append(mean_ratio + 2*std_ratio)
|
|||
|
|
slowdown_lower.append(max(1, mean_ratio - 2*std_ratio))
|
|||
|
|
|
|||
|
|
ax2.fill_between(sizes, slowdown_lower, slowdown_upper, alpha=0.2, color='green')
|
|||
|
|
|
|||
|
|
# Add text annotations for actual values
|
|||
|
|
for i, (size, slowdown) in enumerate(zip(sizes, slowdowns)):
|
|||
|
|
ax2.annotate(f'{slowdown:.0f}x',
|
|||
|
|
xy=(size, slowdown),
|
|||
|
|
xytext=(5, 5),
|
|||
|
|
textcoords='offset points',
|
|||
|
|
fontsize=10)
|
|||
|
|
|
|||
|
|
# Theoretical √n slowdown line
|
|||
|
|
theory_slowdown = np.sqrt(np.array(sizes) / sizes[0])
|
|||
|
|
theory_slowdown = theory_slowdown * slowdowns[0] # Scale to match first point
|
|||
|
|
ax2.plot(sizes, theory_slowdown, 'k--', alpha=0.5, label='√n theoretical')
|
|||
|
|
|
|||
|
|
ax2.set_xlabel('Input Size (n)', fontsize=12)
|
|||
|
|
ax2.set_ylabel('Slowdown Factor', fontsize=12)
|
|||
|
|
ax2.set_title('Cost of Space Reduction (O(n) → O(√n))', fontsize=14)
|
|||
|
|
ax2.grid(True, alpha=0.3)
|
|||
|
|
ax2.set_xscale('log')
|
|||
|
|
ax2.set_yscale('log')
|
|||
|
|
ax2.legend()
|
|||
|
|
|
|||
|
|
plt.suptitle('Checkpointed Sorting: Space-Time Tradeoff')
|
|||
|
|
plt.tight_layout()
|
|||
|
|
plt.savefig('sorting_tradeoff.png', dpi=150)
|
|||
|
|
plt.close()
|
|||
|
|
|
|||
|
|
# Memory usage illustration
|
|||
|
|
fig, ax = plt.subplots(figsize=(10, 6))
|
|||
|
|
|
|||
|
|
n_range = np.logspace(1, 6, 100)
|
|||
|
|
memory_full = n_range * 4 # 4 bytes per int
|
|||
|
|
memory_checkpoint = np.sqrt(n_range) * 4
|
|||
|
|
memory_extreme = np.log2(n_range) * 4
|
|||
|
|
|
|||
|
|
ax.plot(n_range, memory_full, '-', label='In-memory: O(n)', linewidth=3, color='blue')
|
|||
|
|
ax.plot(n_range, memory_checkpoint, '-', label='Checkpointed: O(√n)', linewidth=3, color='orange')
|
|||
|
|
ax.plot(n_range, memory_extreme, '-', label='Extreme: O(log n)', linewidth=3, color='green')
|
|||
|
|
|
|||
|
|
# Add annotations showing memory savings
|
|||
|
|
idx = 60 # Point to annotate
|
|||
|
|
ax.annotate('', xy=(n_range[idx], memory_checkpoint[idx]),
|
|||
|
|
xytext=(n_range[idx], memory_full[idx]),
|
|||
|
|
arrowprops=dict(arrowstyle='<->', color='red', lw=2))
|
|||
|
|
ax.text(n_range[idx]*1.5, np.sqrt(memory_full[idx] * memory_checkpoint[idx]),
|
|||
|
|
f'{memory_full[idx]/memory_checkpoint[idx]:.0f}x reduction',
|
|||
|
|
color='red', fontsize=12, fontweight='bold')
|
|||
|
|
|
|||
|
|
ax.set_xlabel('Input Size (n)', fontsize=12)
|
|||
|
|
ax.set_ylabel('Memory Usage (bytes)', fontsize=12)
|
|||
|
|
ax.set_title('Memory Requirements for Different Sorting Approaches', fontsize=14)
|
|||
|
|
ax.legend(loc='upper left', fontsize=12)
|
|||
|
|
ax.grid(True, alpha=0.3)
|
|||
|
|
ax.set_xscale('log')
|
|||
|
|
ax.set_yscale('log')
|
|||
|
|
|
|||
|
|
# Format y-axis to show readable units
|
|||
|
|
ax.yaxis.set_major_formatter(plt.FuncFormatter(lambda y, _: f'{y/1e6:.0f}MB' if y >= 1e6 else f'{y/1e3:.0f}KB' if y >= 1e3 else f'{y:.0f}B'))
|
|||
|
|
|
|||
|
|
plt.tight_layout()
|
|||
|
|
plt.savefig('sorting_memory.png', dpi=150, bbox_inches='tight')
|
|||
|
|
plt.close()
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
results = run_sorting_experiments()
|
|||
|
|
|
|||
|
|
print("\n=== Summary ===")
|
|||
|
|
print("This experiment demonstrates Williams' space-time tradeoff:")
|
|||
|
|
print("- Reducing memory from O(n) to O(√n) increases time by factor of √n")
|
|||
|
|
print("- The checkpointed sort achieves the theoretical √(t log t) space bound")
|
|||
|
|
print("- Real-world systems (databases, external sorts) use similar techniques")
|