Files

205 lines
6.6 KiB
Go

package server
import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"log"
"path/filepath"
"sync"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
const storageInterval = time.Hour
const storageScopeNotice = "Native Claude, Codex and OpenCode histories outside Pipelines data and known project directories are not included."
const storagePreviewLifetime = 10 * time.Minute
type storagePreview struct {
result protocol.StorageCleanupResult
expires time.Time
}
type storageManager struct {
scan func(context.Context, storageInventoryConfig) (storageInventory, error)
worker chan struct{}
roots *storageInventoryConfig
workspaceMu sync.RWMutex
mu sync.Mutex
runner *runner
store *pipeline.Store
previews map[string]storagePreview
}
func newStorageManager(r *runner) *storageManager {
return &storageManager{runner: r, store: r.store, previews: map[string]storagePreview{}, worker: make(chan struct{}, 1)}
}
// dispatch bounds slow storage work to one goroutine without blocking the
// shared socket read loop. The transport's write callback serializes replies.
func (m *storageManager) dispatch(ctx context.Context, request protocol.Message, write func(protocol.Message) error) {
select {
case m.worker <- struct{}{}:
go func() { defer func() { <-m.worker }(); _ = write(m.handle(ctx, request)) }()
default:
_ = write(protocol.Message{Type: protocol.Error, ID: request.ID, Error: "storage management is busy; retry shortly"})
}
}
func isStorageRequest(t string) bool {
return t == protocol.GetStorage || t == protocol.SetStoragePolicy || t == protocol.PreviewStorageCleanup || t == protocol.ApplyStorageCleanup
}
func (m *storageManager) handle(ctx context.Context, request protocol.Message) protocol.Message {
if !m.mu.TryLock() {
return protocol.Message{Type: protocol.Error, ID: request.ID, Error: "storage management is busy; retry shortly"}
}
defer m.mu.Unlock()
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
response := protocol.Message{ID: request.ID}
var err error
switch request.Type {
case protocol.GetStorage:
response.Type = protocol.StorageReply
response.Storage, err = m.overview(ctx)
case protocol.SetStoragePolicy:
response.Type = protocol.StoragePolicySaved
if request.StoragePolicy == nil {
err = fmt.Errorf("storage_policy is required")
break
}
err = m.store.SaveStoragePolicy(ctx, *request.StoragePolicy)
if err == nil {
response.Storage, err = m.overview(ctx)
}
case protocol.PreviewStorageCleanup:
response.Type = protocol.StorageCleanupPreview
var policy protocol.StoragePolicy
policy, err = m.policy(ctx, request.StoragePolicy)
if err == nil {
response.StorageCleanup, err = m.preview(ctx, policy)
if err == nil {
response.StorageCleanup.PreviewToken = m.remember(*response.StorageCleanup)
}
}
case protocol.ApplyStorageCleanup:
response.Type = protocol.StorageCleanupApplied
var saved storagePreview
saved, err = m.consume(request.StorageCleanupToken, request.StoragePolicy)
if err == nil {
response.StorageCleanup, err = m.apply(ctx, saved.result)
}
}
if err != nil {
response.Type = protocol.Error
response.Error = err.Error()
}
return response
}
func (m *storageManager) policy(ctx context.Context, supplied *protocol.StoragePolicy) (protocol.StoragePolicy, error) {
if supplied != nil {
return *supplied, supplied.Validate()
}
return m.store.StoragePolicy(ctx)
}
func (m *storageManager) remember(result protocol.StorageCleanupResult) string {
for token, p := range m.previews {
if time.Now().After(p.expires) {
delete(m.previews, token)
}
}
// A preview is small and short lived; bound clients repeatedly requesting it.
if len(m.previews) >= 32 {
for token := range m.previews {
delete(m.previews, token)
break
}
}
var key [24]byte
if _, err := rand.Read(key[:]); err != nil {
return ""
}
token := hex.EncodeToString(key[:])
m.previews[token] = storagePreview{result: result, expires: time.Now().Add(storagePreviewLifetime)}
return token
}
func (m *storageManager) consume(token string, policy *protocol.StoragePolicy) (storagePreview, error) {
p, ok := m.previews[token]
if !ok || time.Now().After(p.expires) {
return p, fmt.Errorf("cleanup preview expired; preview cleanup again")
}
if policy == nil || *policy != p.result.Policy {
return p, fmt.Errorf("retention policy changed; preview cleanup again")
}
delete(m.previews, token)
return p, nil
}
func (m *storageManager) overview(ctx context.Context) (*protocol.StorageOverview, error) {
policy, err := m.store.StoragePolicy(ctx)
if err != nil {
return nil, err
}
inventory, err := m.inventory(ctx)
if err != nil {
return nil, err
}
result := &protocol.StorageOverview{Measurement: "logical_file_bytes", TotalBytes: inventory.TotalBytes, Policy: policy, MeasuredAt: time.Now().UTC().Format(time.RFC3339), Partial: inventory.Partial, Warnings: append([]string{storageScopeNotice}, inventory.Errors...), AutomaticIntervalSeconds: int(storageInterval / time.Second), Categories: []protocol.StorageCategory{}}
for _, c := range inventory.Categories {
result.Categories = append(result.Categories, protocol.StorageCategory{ID: c.ID, Label: c.Label, Path: c.Path, Bytes: c.Bytes, Files: c.Files, Partial: c.Partial, Error: c.Error})
}
return result, nil
}
func storageTerminal(status string) bool {
switch status {
case "succeeded", "failed", "canceled", "interrupted":
return true
}
return false
}
func (m *storageManager) loop(ctx context.Context) {
defer m.runner.schedulerWG.Done()
ticker := time.NewTicker(storageInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-m.store.Done():
return
case <-ticker.C:
if !m.mu.TryLock() {
continue
}
cycle, cancel := context.WithTimeout(ctx, 20*time.Second)
err := m.automaticPass(cycle)
cancel()
m.mu.Unlock()
if err != nil && ctx.Err() == nil {
log.Printf("storage retention: %v", err)
}
}
}
}
// canonicalStoragePath supplies the consistent key used by preview identities.
func canonicalStoragePath(path string) string { return filepath.Clean(path) }
// automaticPass is invoked only by the hourly tick, never at startup or save.
func (m *storageManager) automaticPass(ctx context.Context) error {
policy, err := m.store.StoragePolicy(ctx)
if err != nil || !policy.Enabled {
return err
}
preview, err := m.preview(ctx, policy)
if err != nil || preview.CandidateCount == 0 {
return err
}
result, err := m.apply(ctx, *preview)
if err == nil && len(result.Errors) > 0 {
return fmt.Errorf("retention incomplete: %s", result.Errors[0])
}
return err
}