14175eba70
Add `pipelines retention report` (strictly read-only) and `pipelines retention apply` (explicit, approval-gated removal) with an age and/or count policy, protected-run rules, atomic single-transaction removal and honest database space reclaim. Nothing prunes automatically.
145 lines
4.4 KiB
Go
145 lines
4.4 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
)
|
|
|
|
// TestRetentionRemovesServedRunAndArtifact proves the artifact endpoint serves a
|
|
// live run's artifact, that an explicit retention apply removes the run and its
|
|
// artifact, and that the endpoint then 404s while a still-defined pipeline and
|
|
// version load as before.
|
|
func TestRetentionRemovesServedRunAndArtifact(t *testing.T) {
|
|
ctx := context.Background()
|
|
handler, step := runWithArtifact(t)
|
|
store := handler.runner.store
|
|
root := handler.runner.artifactsRoot
|
|
runID := runIDFromURL(step.Artifacts[0].URL)
|
|
|
|
srv := httptest.NewServer(handler)
|
|
defer srv.Close()
|
|
resp, err := http.Get(srv.URL + step.Artifacts[0].URL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("recorded artifact must serve before removal: %d", resp.StatusCode)
|
|
}
|
|
|
|
result, err := store.ApplyRetention(ctx, pipeline.RetentionPolicy{Keep: 0}, root, time.Now().UTC())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !result.Applied || result.RemovedRunCount != 1 {
|
|
t.Fatalf("retention did not remove the served run: %+v", result)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(root, fmt.Sprint(runID))); !os.IsNotExist(err) {
|
|
t.Fatalf("removed run artifacts must be gone: %v", err)
|
|
}
|
|
gone, err := http.Get(srv.URL + step.Artifacts[0].URL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gone.Body.Close()
|
|
if gone.StatusCode != http.StatusNotFound {
|
|
t.Fatalf("removed run's artifact endpoint must 404: %d", gone.StatusCode)
|
|
}
|
|
|
|
runs, err := store.ListRuns(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, run := range runs {
|
|
if run.ID == runID {
|
|
t.Fatalf("removed run still listed: %+v", run)
|
|
}
|
|
}
|
|
// The pipeline definition and its immutable version still load, and a run of
|
|
// another definition would be unaffected.
|
|
items, err := store.List(ctx)
|
|
if err != nil || len(items) == 0 {
|
|
t.Fatalf("definitions must survive retention: %v %v", items, err)
|
|
}
|
|
definition, err := store.Get(ctx, items[0].ID, 1)
|
|
if err != nil || definition.Source == "" {
|
|
t.Fatalf("pipeline version must still load after its run history is removed: %+v %v", definition, err)
|
|
}
|
|
}
|
|
|
|
// TestServerStartupAndListRunsNeverPrune pins the no-automatic-pruning rule:
|
|
// constructing the handler and issuing a list_runs leaves old runs untouched,
|
|
// and a run that is still active is protected from an explicit apply.
|
|
func TestServerStartupAndListRunsNeverPrune(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "server-retention.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "server-retention", pipeline.Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
definition, err := store.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 3; i++ {
|
|
id, _, err := store.CreateRun(ctx, definition, fmt.Sprintf("old-%d", i), "ws", false, nil, map[string]any{"prompt": "task"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.FinishRun(ctx, id, "succeeded", "success", "", map[string]any{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
handler := Handler(ctx, store).(*runHandler)
|
|
defer func() { cancel(); WaitRuns(handler) }()
|
|
|
|
// Startup plus a list_runs request must not prune the old runs.
|
|
runs, err := store.ListRuns(ctx)
|
|
if err != nil || len(runs) != 3 {
|
|
t.Fatalf("startup/list_runs pruned history: %d %v", len(runs), err)
|
|
}
|
|
|
|
// A run created after startup and left active is protected and kept.
|
|
active, _, err := store.CreateRun(ctx, definition, "active", "ws", false, nil, map[string]any{"prompt": "task"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
result, err := store.ApplyRetention(ctx, pipeline.RetentionPolicy{Keep: 0}, handler.runner.artifactsRoot, time.Now().UTC())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.RemovedRunCount != 3 {
|
|
t.Fatalf("only the finished runs may be removed: %d", result.RemovedRunCount)
|
|
}
|
|
found := false
|
|
for _, protected := range result.Protected {
|
|
if protected.ID == active {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("active run must be reported protected: %+v", result.Protected)
|
|
}
|
|
runs, err = store.ListRuns(ctx)
|
|
if err != nil || len(runs) != 1 || runs[0].ID != active {
|
|
t.Fatalf("active run must survive retention: %+v %v", runs, err)
|
|
}
|
|
}
|