Files

267 lines
9.7 KiB
Go

package server
import (
"context"
"database/sql"
"fmt"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
type storageFixture struct {
m *storageManager
definition *protocol.PipelineDetail
workspace string
}
func newStorageFixture(t *testing.T) storageFixture {
t.Helper()
ctx := context.Background()
root := t.TempDir()
store, err := pipeline.Open(filepath.Join(root, "data", "storage.db"))
if err != nil {
if err := os.MkdirAll(filepath.Join(root, "data"), 0700); err != nil {
t.Fatal(err)
}
store, err = pipeline.Open(filepath.Join(root, "data", "storage.db"))
if err != nil {
t.Fatal(err)
}
}
t.Cleanup(func() { store.Close() })
if _, err := store.Submit(ctx, "storage-fixture", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
list, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
d, err := store.Get(ctx, list[0].ID, 1)
if err != nil {
t.Fatal(err)
}
r := &runner{ctx: ctx, store: store, artifactsRoot: filepath.Join(root, "artifacts"), active: map[int64]activeRun{}}
m := newStorageManager(r)
m.roots = &storageInventoryConfig{AttachmentsRoot: filepath.Join(root, "attachments"), SessionsRoot: filepath.Join(root, "sessions")}
workspace := filepath.Join(root, "workspace")
if err := os.MkdirAll(workspace, 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(workspace, "source.txt"), []byte("source is preserved"), 0600); err != nil {
t.Fatal(err)
}
return storageFixture{m, d, workspace}
}
func (f storageFixture) run(t *testing.T, key, status string, created, finished time.Time) int64 {
t.Helper()
ctx := context.Background()
id, _, err := f.m.store.CreateRun(ctx, f.definition, key, f.workspace, false, nil, map[string]any{"prompt": "fixture"})
if err != nil {
t.Fatal(err)
}
if storageTerminal(status) {
if err := f.m.store.FinishRun(ctx, id, status, "success", "", nil); err != nil {
t.Fatal(err)
}
}
db, err := sql.Open("sqlite", f.m.store.DatabasePath())
if err != nil {
t.Fatal(err)
}
defer db.Close()
if _, err := db.Exec(`UPDATE pipeline_runs SET created_at=?,finished_at=? WHERE id=?`, created.UTC().Format(time.RFC3339), finished.UTC().Format(time.RFC3339), id); err != nil {
t.Fatal(err)
}
path := filepath.Join(f.m.runner.artifactsRoot, strconv.FormatInt(id, 10))
if err := os.MkdirAll(path, 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(path, "report.txt"), make([]byte, 4096), 0600); err != nil {
t.Fatal(err)
}
return id
}
func TestStorageRetentionPreviewBoundApplyAndCompletionAge(t *testing.T) {
f := newStorageFixture(t)
now := time.Now()
old := now.Add(-40 * 24 * time.Hour)
expired := f.run(t, "old", "succeeded", old, old)
recent := f.run(t, "long-but-recent", "succeeded", old, now)
active := f.run(t, "active", "running", old, old)
policy := protocol.StoragePolicy{MaxAgeDays: 7}
response := f.m.handle(context.Background(), protocol.Message{Type: protocol.PreviewStorageCleanup, StoragePolicy: &policy})
if response.Type != protocol.StorageCleanupPreview {
t.Fatalf("preview %+v", response)
}
p := response.StorageCleanup
if p.CandidateCount != 1 || p.ProtectedCount != 1 || p.ReclaimableBytes != 4096 || p.PreviewToken == "" {
t.Fatalf("preview %+v", p)
}
// A new eligible run that appears after confirmation is never added to apply.
newerCandidate := f.run(t, "new-candidate", "succeeded", old, old)
applied := f.m.handle(context.Background(), protocol.Message{Type: protocol.ApplyStorageCleanup, StoragePolicy: &policy, StorageCleanupToken: p.PreviewToken})
if applied.Type != protocol.StorageCleanupApplied || applied.StorageCleanup.RemovedCount != 1 {
t.Fatalf("apply %+v", applied)
}
runs, err := f.m.store.ListRuns(context.Background())
if err != nil {
t.Fatal(err)
}
seen := map[int64]bool{}
for _, r := range runs {
seen[r.ID] = true
}
if seen[expired] || !seen[recent] || !seen[active] || !seen[newerCandidate] {
t.Fatalf("remaining %+v", seen)
}
if _, err := os.Stat(filepath.Join(f.workspace, "source.txt")); err != nil {
t.Fatal("workspace was removed")
}
// Remembered roots retain total inventory coverage after their last run goes.
paths, err := f.m.store.StorageWorkspacePaths(context.Background())
if err != nil || len(paths) == 0 {
t.Fatalf("roots %v %v", paths, err)
}
again := f.m.handle(context.Background(), protocol.Message{Type: protocol.ApplyStorageCleanup, StoragePolicy: &policy, StorageCleanupToken: p.PreviewToken})
if again.Type != protocol.Error {
t.Fatal("preview token reused")
}
}
func TestStorageTokensPolicyValidationAndSizeTarget(t *testing.T) {
f := newStorageFixture(t)
old := time.Now().Add(-40 * 24 * time.Hour)
f.run(t, "old", "succeeded", old, old)
policy := protocol.StoragePolicy{MaxBytes: 1}
response := f.m.handle(context.Background(), protocol.Message{Type: protocol.PreviewStorageCleanup, StoragePolicy: &policy})
p := response.StorageCleanup
if p == nil || p.TargetReached || p.CandidateCount != 1 {
t.Fatalf("size preview %+v", response)
}
changed := policy
changed.MaxBytes = 2
if got := f.m.handle(context.Background(), protocol.Message{Type: protocol.ApplyStorageCleanup, StoragePolicy: &changed, StorageCleanupToken: p.PreviewToken}); got.Type != protocol.Error {
t.Fatal("changed policy accepted")
}
saved := f.m.previews[p.PreviewToken]
saved.expires = time.Now().Add(-time.Second)
f.m.previews[p.PreviewToken] = saved
if got := f.m.handle(context.Background(), protocol.Message{Type: protocol.ApplyStorageCleanup, StoragePolicy: &policy, StorageCleanupToken: p.PreviewToken}); got.Type != protocol.Error {
t.Fatal("expired token accepted")
}
bad := protocol.StoragePolicy{Enabled: true}
if got := f.m.handle(context.Background(), protocol.Message{Type: protocol.SetStoragePolicy, StoragePolicy: &bad}); got.Type != protocol.Error {
t.Fatal("invalid automatic policy accepted")
}
overview := f.m.handle(context.Background(), protocol.Message{Type: protocol.GetStorage})
if overview.Storage == nil || overview.Storage.Policy.Enabled || overview.Storage.Measurement != "logical_file_bytes" {
t.Fatalf("overview %+v", overview)
}
}
func TestStorageBusyDispatchDoesNotWaitForMaintenance(t *testing.T) {
f := newStorageFixture(t)
f.m.mu.Lock()
replies := make(chan protocol.Message, 1)
f.m.dispatch(context.Background(), protocol.Message{Type: protocol.GetStorage, ID: "busy"}, func(m protocol.Message) error { replies <- m; return nil })
select {
case reply := <-replies:
if reply.Type != protocol.Error {
t.Fatalf("reply %+v", reply)
}
case <-time.After(time.Second):
t.Fatal("busy storage request blocked")
}
f.m.mu.Unlock()
}
func TestStorageWebsocketProtocolAndNoStartupCleanup(t *testing.T) {
// Isolate attachment and harness cache roots; this test never inventories the
// operator's real home, chats or repositories.
t.Setenv("HOME", t.TempDir())
t.Setenv("PIPELINES_CHAT_ATTACHMENT_DIR", t.TempDir())
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
f := newStorageFixture(t)
old := time.Now().Add(-40 * 24 * time.Hour)
id := f.run(t, "old", "succeeded", old, old)
policy := protocol.StoragePolicy{Enabled: true, MaxAgeDays: 7}
if err := f.m.store.SaveStoragePolicy(ctx, policy); err != nil {
t.Fatal(err)
}
handler := HandlerWithOptions(ctx, Options{ArtifactsRoot: f.m.runner.artifactsRoot}, f.m.store)
defer func() { cancel(); WaitRuns(handler) }()
srv := httptest.NewServer(handler)
defer srv.Close()
conn := dialAckClient(t, ctx, srv.URL)
reply := ackExchange(t, ctx, conn, protocol.Message{Type: protocol.GetStorage, ID: "storage"})
if reply.Type != protocol.StorageReply || reply.ID != "storage" || reply.Storage == nil || !reply.Storage.Policy.Enabled {
t.Fatalf("wire %+v", reply)
}
if reply.Storage.Partial {
t.Fatal("a scope notice must not mark a complete inventory partial")
}
foundNotice := false
for _, warning := range reply.Storage.Warnings {
if warning == storageScopeNotice {
foundNotice = true
}
}
if !foundNotice {
t.Fatalf("wire reply omitted inventory scope notice: %+v", reply.Storage.Warnings)
}
foundSessions := false
for _, category := range reply.Storage.Categories {
if category.ID == "sessions" {
foundSessions = true
if category.Label != "Managed Pi session files" {
t.Fatalf("ambiguous session category: %q", category.Label)
}
}
}
if !foundSessions {
t.Fatal("wire reply omitted managed Pi session category")
}
if _, err := f.m.store.GetRun(ctx, id); err != nil {
t.Fatal("startup pruned run", err)
}
invalid := ackExchange(t, ctx, conn, protocol.Message{Type: protocol.SetStoragePolicy, ID: "invalid"})
if invalid.Type != protocol.Error {
t.Fatalf("invalid %+v", invalid)
}
ping := ackExchange(t, ctx, conn, protocol.Message{Type: protocol.Ping, ID: "ping"})
if ping.Type != protocol.Pong {
t.Fatal(fmt.Sprint(ping))
}
}
func TestStorageAutomaticPolicyOptInAndActiveProtection(t *testing.T) {
f := newStorageFixture(t)
ctx := context.Background()
old := time.Now().Add(-40 * 24 * time.Hour)
expired := f.run(t, "automatic-old", "succeeded", old, old)
active := f.run(t, "automatic-active", "running", old, old)
if err := f.m.automaticPass(ctx); err != nil {
t.Fatal(err)
}
if _, err := f.m.store.GetRun(ctx, expired); err != nil {
t.Fatal("disabled automatic policy deleted history")
}
if err := f.m.store.SaveStoragePolicy(ctx, protocol.StoragePolicy{Enabled: true, MaxAgeDays: 7}); err != nil {
t.Fatal(err)
}
if err := f.m.automaticPass(ctx); err != nil {
t.Fatal(err)
}
if _, err := f.m.store.GetRun(ctx, expired); err == nil {
t.Fatal("enabled policy did not remove expired run")
}
if _, err := f.m.store.GetRun(ctx, active); err != nil {
t.Fatal("active run removed", err)
}
}