205 lines
6.6 KiB
Go
205 lines
6.6 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"log"
|
|
"path/filepath"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
const storageInterval = time.Hour
|
|
|
|
const storageScopeNotice = "Native Claude, Codex and OpenCode histories outside Pipelines data and known project directories are not included."
|
|
const storagePreviewLifetime = 10 * time.Minute
|
|
|
|
type storagePreview struct {
|
|
result protocol.StorageCleanupResult
|
|
expires time.Time
|
|
}
|
|
type storageManager struct {
|
|
scan func(context.Context, storageInventoryConfig) (storageInventory, error)
|
|
worker chan struct{}
|
|
roots *storageInventoryConfig
|
|
workspaceMu sync.RWMutex
|
|
mu sync.Mutex
|
|
runner *runner
|
|
store *pipeline.Store
|
|
previews map[string]storagePreview
|
|
}
|
|
|
|
func newStorageManager(r *runner) *storageManager {
|
|
return &storageManager{runner: r, store: r.store, previews: map[string]storagePreview{}, worker: make(chan struct{}, 1)}
|
|
}
|
|
|
|
// dispatch bounds slow storage work to one goroutine without blocking the
|
|
// shared socket read loop. The transport's write callback serializes replies.
|
|
func (m *storageManager) dispatch(ctx context.Context, request protocol.Message, write func(protocol.Message) error) {
|
|
select {
|
|
case m.worker <- struct{}{}:
|
|
go func() { defer func() { <-m.worker }(); _ = write(m.handle(ctx, request)) }()
|
|
default:
|
|
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "storage management is busy; retry shortly"})
|
|
}
|
|
}
|
|
func isStorageRequest(t string) bool {
|
|
return t == protocol.GetStorage || t == protocol.SetStoragePolicy || t == protocol.PreviewStorageCleanup || t == protocol.ApplyStorageCleanup
|
|
}
|
|
func (m *storageManager) handle(ctx context.Context, request protocol.Message) protocol.Message {
|
|
if !m.mu.TryLock() {
|
|
return protocol.Message{Type: protocol.Error, ID: request.ID, Error: "storage management is busy; retry shortly"}
|
|
}
|
|
defer m.mu.Unlock()
|
|
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
|
defer cancel()
|
|
response := protocol.Message{ID: request.ID}
|
|
var err error
|
|
switch request.Type {
|
|
case protocol.GetStorage:
|
|
response.Type = protocol.StorageReply
|
|
response.Storage, err = m.overview(ctx)
|
|
case protocol.SetStoragePolicy:
|
|
response.Type = protocol.StoragePolicySaved
|
|
if request.StoragePolicy == nil {
|
|
err = fmt.Errorf("storage_policy is required")
|
|
break
|
|
}
|
|
err = m.store.SaveStoragePolicy(ctx, *request.StoragePolicy)
|
|
if err == nil {
|
|
response.Storage, err = m.overview(ctx)
|
|
}
|
|
case protocol.PreviewStorageCleanup:
|
|
response.Type = protocol.StorageCleanupPreview
|
|
var policy protocol.StoragePolicy
|
|
policy, err = m.policy(ctx, request.StoragePolicy)
|
|
if err == nil {
|
|
response.StorageCleanup, err = m.preview(ctx, policy)
|
|
if err == nil {
|
|
response.StorageCleanup.PreviewToken = m.remember(*response.StorageCleanup)
|
|
}
|
|
}
|
|
case protocol.ApplyStorageCleanup:
|
|
response.Type = protocol.StorageCleanupApplied
|
|
var saved storagePreview
|
|
saved, err = m.consume(request.StorageCleanupToken, request.StoragePolicy)
|
|
if err == nil {
|
|
response.StorageCleanup, err = m.apply(ctx, saved.result)
|
|
}
|
|
}
|
|
if err != nil {
|
|
response.Type = protocol.Error
|
|
response.Error = err.Error()
|
|
}
|
|
return response
|
|
}
|
|
func (m *storageManager) policy(ctx context.Context, supplied *protocol.StoragePolicy) (protocol.StoragePolicy, error) {
|
|
if supplied != nil {
|
|
return *supplied, supplied.Validate()
|
|
}
|
|
return m.store.StoragePolicy(ctx)
|
|
}
|
|
func (m *storageManager) remember(result protocol.StorageCleanupResult) string {
|
|
for token, p := range m.previews {
|
|
if time.Now().After(p.expires) {
|
|
delete(m.previews, token)
|
|
}
|
|
}
|
|
// A preview is small and short lived; bound clients repeatedly requesting it.
|
|
if len(m.previews) >= 32 {
|
|
for token := range m.previews {
|
|
delete(m.previews, token)
|
|
break
|
|
}
|
|
}
|
|
var key [24]byte
|
|
if _, err := rand.Read(key[:]); err != nil {
|
|
return ""
|
|
}
|
|
token := hex.EncodeToString(key[:])
|
|
m.previews[token] = storagePreview{result: result, expires: time.Now().Add(storagePreviewLifetime)}
|
|
return token
|
|
}
|
|
func (m *storageManager) consume(token string, policy *protocol.StoragePolicy) (storagePreview, error) {
|
|
p, ok := m.previews[token]
|
|
if !ok || time.Now().After(p.expires) {
|
|
return p, fmt.Errorf("cleanup preview expired; preview cleanup again")
|
|
}
|
|
if policy == nil || *policy != p.result.Policy {
|
|
return p, fmt.Errorf("retention policy changed; preview cleanup again")
|
|
}
|
|
delete(m.previews, token)
|
|
return p, nil
|
|
}
|
|
func (m *storageManager) overview(ctx context.Context) (*protocol.StorageOverview, error) {
|
|
policy, err := m.store.StoragePolicy(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
inventory, err := m.inventory(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
result := &protocol.StorageOverview{Measurement: "logical_file_bytes", TotalBytes: inventory.TotalBytes, Policy: policy, MeasuredAt: time.Now().UTC().Format(time.RFC3339), Partial: inventory.Partial, Warnings: append([]string{storageScopeNotice}, inventory.Errors...), AutomaticIntervalSeconds: int(storageInterval / time.Second), Categories: []protocol.StorageCategory{}}
|
|
for _, c := range inventory.Categories {
|
|
result.Categories = append(result.Categories, protocol.StorageCategory{ID: c.ID, Label: c.Label, Path: c.Path, Bytes: c.Bytes, Files: c.Files, Partial: c.Partial, Error: c.Error})
|
|
}
|
|
return result, nil
|
|
}
|
|
func storageTerminal(status string) bool {
|
|
switch status {
|
|
case "succeeded", "failed", "canceled", "interrupted":
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
func (m *storageManager) loop(ctx context.Context) {
|
|
defer m.runner.schedulerWG.Done()
|
|
ticker := time.NewTicker(storageInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-m.store.Done():
|
|
return
|
|
case <-ticker.C:
|
|
if !m.mu.TryLock() {
|
|
continue
|
|
}
|
|
cycle, cancel := context.WithTimeout(ctx, 20*time.Second)
|
|
err := m.automaticPass(cycle)
|
|
cancel()
|
|
m.mu.Unlock()
|
|
if err != nil && ctx.Err() == nil {
|
|
log.Printf("storage retention: %v", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// canonicalStoragePath supplies the consistent key used by preview identities.
|
|
func canonicalStoragePath(path string) string { return filepath.Clean(path) }
|
|
|
|
// automaticPass is invoked only by the hourly tick, never at startup or save.
|
|
func (m *storageManager) automaticPass(ctx context.Context) error {
|
|
policy, err := m.store.StoragePolicy(ctx)
|
|
if err != nil || !policy.Enabled {
|
|
return err
|
|
}
|
|
preview, err := m.preview(ctx, policy)
|
|
if err != nil || preview.CandidateCount == 0 {
|
|
return err
|
|
}
|
|
result, err := m.apply(ctx, *preview)
|
|
if err == nil && len(result.Errors) > 0 {
|
|
return fmt.Errorf("retention incomplete: %s", result.Errors[0])
|
|
}
|
|
return err
|
|
}
|