feat: Add CPU load monitoring and cleanup support
Some checks failed
CI / build-and-test (push) Failing after 55s
Some checks failed
CI / build-and-test (push) Failing after 55s
- Add CPUInfo struct with load average and percentage
- Add detectCPULoad() for Linux, macOS, and Windows
- Add cleanup package for disk space management
- Handle RequestCleanup signal from server
- Report CPU load in capabilities to server
🤖 Generated with Claude Code
This commit is contained in:
2
go.mod
2
go.mod
@@ -111,4 +111,4 @@ replace github.com/go-git/go-git/v5 => github.com/go-git/go-git/v5 v5.16.2
|
||||
replace github.com/distribution/reference v0.6.0 => github.com/distribution/reference v0.5.0
|
||||
|
||||
// Use GitCaddy fork with capability support
|
||||
replace code.gitea.io/actions-proto-go => git.marketally.com/gitcaddy/actions-proto-go v0.5.7
|
||||
replace code.gitea.io/actions-proto-go => git.marketally.com/gitcaddy/actions-proto-go v0.5.8
|
||||
|
||||
2
go.sum
2
go.sum
@@ -8,6 +8,8 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
|
||||
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
|
||||
git.marketally.com/gitcaddy/actions-proto-go v0.5.7 h1:RUbafr3Vkw2l4WfSwa+oF+Ihakbm05W0FlAmXuQrDJc=
|
||||
git.marketally.com/gitcaddy/actions-proto-go v0.5.7/go.mod h1:RPu21UoRD3zSAujoZR6LJwuVNa2uFRBveadslczCRfQ=
|
||||
git.marketally.com/gitcaddy/actions-proto-go v0.5.8 h1:MBipeHvY6A0jcobvziUtzgatZTrV4fs/HE1rPQxREN4=
|
||||
git.marketally.com/gitcaddy/actions-proto-go v0.5.8/go.mod h1:RPu21UoRD3zSAujoZR6LJwuVNa2uFRBveadslczCRfQ=
|
||||
gitea.com/gitea/act v0.261.7-0.20251202193638-5417d3ac6742 h1:ulcquQluJbmNASkh6ina70LvcHEa9eWYfQ+DeAZ0VEE=
|
||||
gitea.com/gitea/act v0.261.7-0.20251202193638-5417d3ac6742/go.mod h1:Pg5C9kQY1CEA3QjthjhlrqOC/QOT5NyWNjOjRHw23Ok=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"gitea.com/gitea/act_runner/internal/app/run"
|
||||
"gitea.com/gitea/act_runner/internal/pkg/cleanup"
|
||||
"gitea.com/gitea/act_runner/internal/pkg/client"
|
||||
"gitea.com/gitea/act_runner/internal/pkg/config"
|
||||
"gitea.com/gitea/act_runner/internal/pkg/envcheck"
|
||||
@@ -205,6 +206,20 @@ func (p *Poller) fetchTask(ctx context.Context) (*runnerv1.Task, bool) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Check if server requested a cleanup
|
||||
if resp.Msg.RequestCleanup {
|
||||
log.Info("Server requested cleanup, running now...")
|
||||
go func() {
|
||||
result, err := cleanup.RunCleanup(ctx, p.cfg)
|
||||
if err != nil {
|
||||
log.Errorf("Cleanup failed: %v", err)
|
||||
} else if result != nil {
|
||||
log.Infof("Cleanup completed: freed %d bytes, deleted %d files in %s",
|
||||
result.BytesFreed, result.FilesDeleted, result.Duration)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if resp.Msg.TasksVersion > v {
|
||||
p.tasksVersion.CompareAndSwap(v, resp.Msg.TasksVersion)
|
||||
}
|
||||
|
||||
266
internal/pkg/cleanup/cleanup.go
Normal file
266
internal/pkg/cleanup/cleanup.go
Normal file
@@ -0,0 +1,266 @@
|
||||
// Copyright 2026 MarketAlly. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package cleanup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/act_runner/internal/pkg/config"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// CleanupResult contains the results of a cleanup operation
|
||||
type CleanupResult struct {
|
||||
BytesFreed int64
|
||||
FilesDeleted int
|
||||
Errors []error
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
// RunCleanup performs cleanup operations to free disk space
|
||||
func RunCleanup(ctx context.Context, cfg *config.Config) (*CleanupResult, error) {
|
||||
start := time.Now()
|
||||
result := &CleanupResult{}
|
||||
|
||||
log.Info("Starting runner cleanup...")
|
||||
|
||||
// 1. Clean old cache directories
|
||||
cacheDir := filepath.Join(cfg.Cache.Dir, "_cache")
|
||||
if cacheDir != "" {
|
||||
if bytes, files, err := cleanOldDir(cacheDir, 24*time.Hour); err != nil {
|
||||
result.Errors = append(result.Errors, fmt.Errorf("cache cleanup: %w", err))
|
||||
} else {
|
||||
result.BytesFreed += bytes
|
||||
result.FilesDeleted += files
|
||||
log.Infof("Cleaned cache: freed %d bytes, deleted %d files", bytes, files)
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Clean old work directories
|
||||
workDir := cfg.Container.WorkdirParent
|
||||
if workDir != "" {
|
||||
if bytes, files, err := cleanOldWorkDirs(workDir, 48*time.Hour); err != nil {
|
||||
result.Errors = append(result.Errors, fmt.Errorf("workdir cleanup: %w", err))
|
||||
} else {
|
||||
result.BytesFreed += bytes
|
||||
result.FilesDeleted += files
|
||||
log.Infof("Cleaned work dirs: freed %d bytes, deleted %d files", bytes, files)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Clean old artifact staging directories
|
||||
artifactDir := cfg.Cache.Dir
|
||||
if bytes, files, err := cleanOldArtifacts(artifactDir, 72*time.Hour); err != nil {
|
||||
result.Errors = append(result.Errors, fmt.Errorf("artifact cleanup: %w", err))
|
||||
} else {
|
||||
result.BytesFreed += bytes
|
||||
result.FilesDeleted += files
|
||||
log.Infof("Cleaned artifacts: freed %d bytes, deleted %d files", bytes, files)
|
||||
}
|
||||
|
||||
// 4. Clean system temp files (older than 24h)
|
||||
if bytes, files, err := cleanTempDir(24 * time.Hour); err != nil {
|
||||
result.Errors = append(result.Errors, fmt.Errorf("temp cleanup: %w", err))
|
||||
} else {
|
||||
result.BytesFreed += bytes
|
||||
result.FilesDeleted += files
|
||||
log.Infof("Cleaned temp: freed %d bytes, deleted %d files", bytes, files)
|
||||
}
|
||||
|
||||
result.Duration = time.Since(start)
|
||||
log.Infof("Cleanup completed: freed %s in %s", formatBytes(result.BytesFreed), result.Duration)
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// cleanOldDir removes files older than maxAge from a directory
|
||||
func cleanOldDir(dir string, maxAge time.Duration) (int64, int, error) {
|
||||
if _, err := os.Stat(dir); os.IsNotExist(err) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
var bytesFreed int64
|
||||
var filesDeleted int
|
||||
cutoff := time.Now().Add(-maxAge)
|
||||
|
||||
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil // Skip errors
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if info.ModTime().Before(cutoff) {
|
||||
size := info.Size()
|
||||
if err := os.Remove(path); err == nil {
|
||||
bytesFreed += size
|
||||
filesDeleted++
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
return bytesFreed, filesDeleted, err
|
||||
}
|
||||
|
||||
// cleanOldWorkDirs removes work directories older than maxAge
|
||||
func cleanOldWorkDirs(baseDir string, maxAge time.Duration) (int64, int, error) {
|
||||
if _, err := os.Stat(baseDir); os.IsNotExist(err) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
var bytesFreed int64
|
||||
var filesDeleted int
|
||||
cutoff := time.Now().Add(-maxAge)
|
||||
|
||||
entries, err := os.ReadDir(baseDir)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(baseDir, entry.Name())
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if info.ModTime().Before(cutoff) {
|
||||
size := dirSize(path)
|
||||
if err := os.RemoveAll(path); err == nil {
|
||||
bytesFreed += size
|
||||
filesDeleted++
|
||||
log.Debugf("Removed old work dir: %s", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return bytesFreed, filesDeleted, nil
|
||||
}
|
||||
|
||||
// cleanOldArtifacts removes artifact staging files older than maxAge
|
||||
func cleanOldArtifacts(baseDir string, maxAge time.Duration) (int64, int, error) {
|
||||
if _, err := os.Stat(baseDir); os.IsNotExist(err) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
var bytesFreed int64
|
||||
var filesDeleted int
|
||||
cutoff := time.Now().Add(-maxAge)
|
||||
|
||||
// Look for artifact staging dirs
|
||||
patterns := []string{"artifact-*", "upload-*", "download-*"}
|
||||
for _, pattern := range patterns {
|
||||
matches, _ := filepath.Glob(filepath.Join(baseDir, pattern))
|
||||
for _, path := range matches {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if info.ModTime().Before(cutoff) {
|
||||
var size int64
|
||||
if info.IsDir() {
|
||||
size = dirSize(path)
|
||||
err = os.RemoveAll(path)
|
||||
} else {
|
||||
size = info.Size()
|
||||
err = os.Remove(path)
|
||||
}
|
||||
if err == nil {
|
||||
bytesFreed += size
|
||||
filesDeleted++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return bytesFreed, filesDeleted, nil
|
||||
}
|
||||
|
||||
// cleanTempDir removes old files from system temp directory
|
||||
func cleanTempDir(maxAge time.Duration) (int64, int, error) {
|
||||
tmpDir := os.TempDir()
|
||||
var bytesFreed int64
|
||||
var filesDeleted int
|
||||
cutoff := time.Now().Add(-maxAge)
|
||||
|
||||
entries, err := os.ReadDir(tmpDir)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
// Only clean files/dirs that look like runner/act artifacts
|
||||
runnerPatterns := []string{"act-", "runner-", "gitea-", "workflow-"}
|
||||
for _, entry := range entries {
|
||||
name := entry.Name()
|
||||
isRunner := false
|
||||
for _, p := range runnerPatterns {
|
||||
if len(name) >= len(p) && name[:len(p)] == p {
|
||||
isRunner = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !isRunner {
|
||||
continue
|
||||
}
|
||||
|
||||
path := filepath.Join(tmpDir, name)
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if info.ModTime().Before(cutoff) {
|
||||
var size int64
|
||||
if info.IsDir() {
|
||||
size = dirSize(path)
|
||||
err = os.RemoveAll(path)
|
||||
} else {
|
||||
size = info.Size()
|
||||
err = os.Remove(path)
|
||||
}
|
||||
if err == nil {
|
||||
bytesFreed += size
|
||||
filesDeleted++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return bytesFreed, filesDeleted, nil
|
||||
}
|
||||
|
||||
// dirSize calculates the total size of a directory
|
||||
func dirSize(path string) int64 {
|
||||
var size int64
|
||||
filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if !info.IsDir() {
|
||||
size += info.Size()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return size
|
||||
}
|
||||
|
||||
// formatBytes formats bytes into human readable string
|
||||
func formatBytes(bytes int64) string {
|
||||
const unit = 1024
|
||||
if bytes < unit {
|
||||
return fmt.Sprintf("%d B", bytes)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := bytes / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
@@ -26,6 +26,15 @@ type DiskInfo struct {
|
||||
UsedPercent float64 `json:"used_percent"`
|
||||
}
|
||||
|
||||
// CPUInfo holds CPU load information
|
||||
type CPUInfo struct {
|
||||
NumCPU int `json:"num_cpu"` // Number of logical CPUs
|
||||
LoadAvg1m float64 `json:"load_avg_1m"` // 1-minute load average
|
||||
LoadAvg5m float64 `json:"load_avg_5m"` // 5-minute load average
|
||||
LoadAvg15m float64 `json:"load_avg_15m"` // 15-minute load average
|
||||
LoadPercent float64 `json:"load_percent"` // (load_avg_1m / num_cpu) * 100
|
||||
}
|
||||
|
||||
// DistroInfo holds Linux distribution information
|
||||
type DistroInfo struct {
|
||||
ID string `json:"id,omitempty"` // e.g., "ubuntu", "debian", "fedora"
|
||||
@@ -57,6 +66,7 @@ type RunnerCapabilities struct {
|
||||
Features *CapabilityFeatures `json:"features,omitempty"`
|
||||
Limitations []string `json:"limitations,omitempty"`
|
||||
Disk *DiskInfo `json:"disk,omitempty"`
|
||||
CPU *CPUInfo `json:"cpu,omitempty"`
|
||||
Bandwidth *BandwidthInfo `json:"bandwidth,omitempty"`
|
||||
SuggestedLabels []string `json:"suggested_labels,omitempty"`
|
||||
}
|
||||
@@ -120,6 +130,9 @@ func DetectCapabilities(ctx context.Context, dockerHost string, workingDir strin
|
||||
// Detect disk space on the working directory's filesystem
|
||||
cap.Disk = detectDiskSpace(workingDir)
|
||||
|
||||
// Detect CPU load
|
||||
cap.CPU = detectCPULoad()
|
||||
|
||||
// Generate suggested labels based on detected capabilities
|
||||
cap.SuggestedLabels = generateSuggestedLabels(cap)
|
||||
|
||||
@@ -887,3 +900,89 @@ func contains(slice []string, item string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// detectCPULoad detects the current CPU load
|
||||
func detectCPULoad() *CPUInfo {
|
||||
numCPU := runtime.NumCPU()
|
||||
info := &CPUInfo{
|
||||
NumCPU: numCPU,
|
||||
}
|
||||
|
||||
switch runtime.GOOS {
|
||||
case "linux":
|
||||
// Read from /proc/loadavg
|
||||
data, err := os.ReadFile("/proc/loadavg")
|
||||
if err != nil {
|
||||
return info
|
||||
}
|
||||
parts := strings.Fields(string(data))
|
||||
if len(parts) >= 3 {
|
||||
if load, err := parseFloat(parts[0]); err == nil {
|
||||
info.LoadAvg1m = load
|
||||
}
|
||||
if load, err := parseFloat(parts[1]); err == nil {
|
||||
info.LoadAvg5m = load
|
||||
}
|
||||
if load, err := parseFloat(parts[2]); err == nil {
|
||||
info.LoadAvg15m = load
|
||||
}
|
||||
}
|
||||
case "darwin":
|
||||
// Use sysctl on macOS
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "sysctl", "-n", "vm.loadavg")
|
||||
output, err := cmd.Output()
|
||||
if err == nil {
|
||||
// Output format: "{ 1.23 4.56 7.89 }"
|
||||
line := strings.Trim(string(output), "{ }\n")
|
||||
parts := strings.Fields(line)
|
||||
if len(parts) >= 3 {
|
||||
if load, err := parseFloat(parts[0]); err == nil {
|
||||
info.LoadAvg1m = load
|
||||
}
|
||||
if load, err := parseFloat(parts[1]); err == nil {
|
||||
info.LoadAvg5m = load
|
||||
}
|
||||
if load, err := parseFloat(parts[2]); err == nil {
|
||||
info.LoadAvg15m = load
|
||||
}
|
||||
}
|
||||
}
|
||||
case "windows":
|
||||
// Windows doesn't have load average, use CPU usage via wmic
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "wmic", "cpu", "get", "loadpercentage")
|
||||
output, err := cmd.Output()
|
||||
if err == nil {
|
||||
lines := strings.Split(string(output), "\n")
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" && line != "LoadPercentage" {
|
||||
if load, err := parseFloat(line); err == nil {
|
||||
// Convert percentage to "load" equivalent
|
||||
info.LoadPercent = load
|
||||
info.LoadAvg1m = load * float64(numCPU) / 100.0
|
||||
return info
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate load percent (load_avg_1m / num_cpu * 100)
|
||||
if info.LoadAvg1m > 0 && numCPU > 0 {
|
||||
info.LoadPercent = (info.LoadAvg1m / float64(numCPU)) * 100.0
|
||||
}
|
||||
|
||||
return info
|
||||
}
|
||||
|
||||
// parseFloat parses a string to float64
|
||||
func parseFloat(s string) (float64, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
var f float64
|
||||
err := json.Unmarshal([]byte(s), &f)
|
||||
return f, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user