package server import ( "context" "database/sql" "fmt" "net/http/httptest" "os" "path/filepath" "strconv" "testing" "time" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" ) type storageFixture struct { m *storageManager definition *protocol.PipelineDetail workspace string } func newStorageFixture(t *testing.T) storageFixture { t.Helper() ctx := context.Background() root := t.TempDir() store, err := pipeline.Open(filepath.Join(root, "data", "storage.db")) if err != nil { if err := os.MkdirAll(filepath.Join(root, "data"), 0700); err != nil { t.Fatal(err) } store, err = pipeline.Open(filepath.Join(root, "data", "storage.db")) if err != nil { t.Fatal(err) } } t.Cleanup(func() { store.Close() }) if _, err := store.Submit(ctx, "storage-fixture", pipeline.Template([]string{"one"}, true)); err != nil { t.Fatal(err) } list, err := store.List(ctx) if err != nil { t.Fatal(err) } d, err := store.Get(ctx, list[0].ID, 1) if err != nil { t.Fatal(err) } r := &runner{ctx: ctx, store: store, artifactsRoot: filepath.Join(root, "artifacts"), active: map[int64]activeRun{}} m := newStorageManager(r) m.roots = &storageInventoryConfig{AttachmentsRoot: filepath.Join(root, "attachments"), SessionsRoot: filepath.Join(root, "sessions")} workspace := filepath.Join(root, "workspace") if err := os.MkdirAll(workspace, 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(workspace, "source.txt"), []byte("source is preserved"), 0600); err != nil { t.Fatal(err) } return storageFixture{m, d, workspace} } func (f storageFixture) run(t *testing.T, key, status string, created, finished time.Time) int64 { t.Helper() ctx := context.Background() id, _, err := f.m.store.CreateRun(ctx, f.definition, key, f.workspace, false, nil, map[string]any{"prompt": "fixture"}) if err != nil { t.Fatal(err) } if storageTerminal(status) { if err := f.m.store.FinishRun(ctx, id, status, "success", "", nil); err != nil { t.Fatal(err) } } db, err := sql.Open("sqlite", f.m.store.DatabasePath()) if err != nil { t.Fatal(err) } defer db.Close() if _, err := db.Exec(`UPDATE pipeline_runs SET created_at=?,finished_at=? WHERE id=?`, created.UTC().Format(time.RFC3339), finished.UTC().Format(time.RFC3339), id); err != nil { t.Fatal(err) } path := filepath.Join(f.m.runner.artifactsRoot, strconv.FormatInt(id, 10)) if err := os.MkdirAll(path, 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(path, "report.txt"), make([]byte, 4096), 0600); err != nil { t.Fatal(err) } return id } func TestStorageRetentionPreviewBoundApplyAndCompletionAge(t *testing.T) { f := newStorageFixture(t) now := time.Now() old := now.Add(-40 * 24 * time.Hour) expired := f.run(t, "old", "succeeded", old, old) recent := f.run(t, "long-but-recent", "succeeded", old, now) active := f.run(t, "active", "running", old, old) policy := protocol.StoragePolicy{MaxAgeDays: 7} response := f.m.handle(context.Background(), protocol.Message{Type: protocol.PreviewStorageCleanup, StoragePolicy: &policy}) if response.Type != protocol.StorageCleanupPreview { t.Fatalf("preview %+v", response) } p := response.StorageCleanup if p.CandidateCount != 1 || p.ProtectedCount != 1 || p.ReclaimableBytes != 4096 || p.PreviewToken == "" { t.Fatalf("preview %+v", p) } // A new eligible run that appears after confirmation is never added to apply. newerCandidate := f.run(t, "new-candidate", "succeeded", old, old) applied := f.m.handle(context.Background(), protocol.Message{Type: protocol.ApplyStorageCleanup, StoragePolicy: &policy, StorageCleanupToken: p.PreviewToken}) if applied.Type != protocol.StorageCleanupApplied || applied.StorageCleanup.RemovedCount != 1 { t.Fatalf("apply %+v", applied) } runs, err := f.m.store.ListRuns(context.Background()) if err != nil { t.Fatal(err) } seen := map[int64]bool{} for _, r := range runs { seen[r.ID] = true } if seen[expired] || !seen[recent] || !seen[active] || !seen[newerCandidate] { t.Fatalf("remaining %+v", seen) } if _, err := os.Stat(filepath.Join(f.workspace, "source.txt")); err != nil { t.Fatal("workspace was removed") } // Remembered roots retain total inventory coverage after their last run goes. paths, err := f.m.store.StorageWorkspacePaths(context.Background()) if err != nil || len(paths) == 0 { t.Fatalf("roots %v %v", paths, err) } again := f.m.handle(context.Background(), protocol.Message{Type: protocol.ApplyStorageCleanup, StoragePolicy: &policy, StorageCleanupToken: p.PreviewToken}) if again.Type != protocol.Error { t.Fatal("preview token reused") } } func TestStorageTokensPolicyValidationAndSizeTarget(t *testing.T) { f := newStorageFixture(t) old := time.Now().Add(-40 * 24 * time.Hour) f.run(t, "old", "succeeded", old, old) policy := protocol.StoragePolicy{MaxBytes: 1} response := f.m.handle(context.Background(), protocol.Message{Type: protocol.PreviewStorageCleanup, StoragePolicy: &policy}) p := response.StorageCleanup if p == nil || p.TargetReached || p.CandidateCount != 1 { t.Fatalf("size preview %+v", response) } changed := policy changed.MaxBytes = 2 if got := f.m.handle(context.Background(), protocol.Message{Type: protocol.ApplyStorageCleanup, StoragePolicy: &changed, StorageCleanupToken: p.PreviewToken}); got.Type != protocol.Error { t.Fatal("changed policy accepted") } saved := f.m.previews[p.PreviewToken] saved.expires = time.Now().Add(-time.Second) f.m.previews[p.PreviewToken] = saved if got := f.m.handle(context.Background(), protocol.Message{Type: protocol.ApplyStorageCleanup, StoragePolicy: &policy, StorageCleanupToken: p.PreviewToken}); got.Type != protocol.Error { t.Fatal("expired token accepted") } bad := protocol.StoragePolicy{Enabled: true} if got := f.m.handle(context.Background(), protocol.Message{Type: protocol.SetStoragePolicy, StoragePolicy: &bad}); got.Type != protocol.Error { t.Fatal("invalid automatic policy accepted") } overview := f.m.handle(context.Background(), protocol.Message{Type: protocol.GetStorage}) if overview.Storage == nil || overview.Storage.Policy.Enabled || overview.Storage.Measurement != "logical_file_bytes" { t.Fatalf("overview %+v", overview) } } func TestStorageBusyDispatchDoesNotWaitForMaintenance(t *testing.T) { f := newStorageFixture(t) f.m.mu.Lock() replies := make(chan protocol.Message, 1) f.m.dispatch(context.Background(), protocol.Message{Type: protocol.GetStorage, ID: "busy"}, func(m protocol.Message) error { replies <- m; return nil }) select { case reply := <-replies: if reply.Type != protocol.Error { t.Fatalf("reply %+v", reply) } case <-time.After(time.Second): t.Fatal("busy storage request blocked") } f.m.mu.Unlock() } func TestStorageWebsocketProtocolAndNoStartupCleanup(t *testing.T) { // Isolate attachment and harness cache roots; this test never inventories the // operator's real home, chats or repositories. t.Setenv("HOME", t.TempDir()) t.Setenv("PIPELINES_CHAT_ATTACHMENT_DIR", t.TempDir()) ctx, cancel := context.WithCancel(context.Background()) defer cancel() f := newStorageFixture(t) old := time.Now().Add(-40 * 24 * time.Hour) id := f.run(t, "old", "succeeded", old, old) policy := protocol.StoragePolicy{Enabled: true, MaxAgeDays: 7} if err := f.m.store.SaveStoragePolicy(ctx, policy); err != nil { t.Fatal(err) } handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: f.m.runner.artifactsRoot}, f.m.store) defer func() { cancel(); WaitRuns(handler) }() srv := httptest.NewServer(handler) defer srv.Close() conn := dialAckClient(t, ctx, srv.URL) reply := ackExchange(t, ctx, conn, protocol.Message{Type: protocol.GetStorage, ID: "storage"}) if reply.Type != protocol.StorageReply || reply.ID != "storage" || reply.Storage == nil || !reply.Storage.Policy.Enabled { t.Fatalf("wire %+v", reply) } if reply.Storage.Partial { t.Fatal("a scope notice must not mark a complete inventory partial") } foundNotice := false for _, warning := range reply.Storage.Warnings { if warning == storageScopeNotice { foundNotice = true } } if !foundNotice { t.Fatalf("wire reply omitted inventory scope notice: %+v", reply.Storage.Warnings) } foundSessions := false for _, category := range reply.Storage.Categories { if category.ID == "sessions" { foundSessions = true if category.Label != "Managed Pi session files" { t.Fatalf("ambiguous session category: %q", category.Label) } } } if !foundSessions { t.Fatal("wire reply omitted managed Pi session category") } if _, err := f.m.store.GetRun(ctx, id); err != nil { t.Fatal("startup pruned run", err) } invalid := ackExchange(t, ctx, conn, protocol.Message{Type: protocol.SetStoragePolicy, ID: "invalid"}) if invalid.Type != protocol.Error { t.Fatalf("invalid %+v", invalid) } ping := ackExchange(t, ctx, conn, protocol.Message{Type: protocol.Ping, ID: "ping"}) if ping.Type != protocol.Pong { t.Fatal(fmt.Sprint(ping)) } } func TestStorageAutomaticPolicyOptInAndActiveProtection(t *testing.T) { f := newStorageFixture(t) ctx := context.Background() old := time.Now().Add(-40 * 24 * time.Hour) expired := f.run(t, "automatic-old", "succeeded", old, old) active := f.run(t, "automatic-active", "running", old, old) if err := f.m.automaticPass(ctx); err != nil { t.Fatal(err) } if _, err := f.m.store.GetRun(ctx, expired); err != nil { t.Fatal("disabled automatic policy deleted history") } if err := f.m.store.SaveStoragePolicy(ctx, protocol.StoragePolicy{Enabled: true, MaxAgeDays: 7}); err != nil { t.Fatal(err) } if err := f.m.automaticPass(ctx); err != nil { t.Fatal(err) } if _, err := f.m.store.GetRun(ctx, expired); err == nil { t.Fatal("enabled policy did not remove expired run") } if _, err := f.m.store.GetRun(ctx, active); err != nil { t.Fatal("active run removed", err) } }