Files
pipelines/internal/server/session_export_exhausted_test.go
alan 3bfb3d6b44 Fix decide-after-finish refusal and run size loop data race
A run stays in r.active after finishRunState persists its terminal status
until its goroutine finishes notifying, so a decision in that window was
refused as "already active". decide now refuses only runs whose terminal
state is not yet durable; a finished run gets ResolveDecision's
already-decided / not-waiting answer, and an executing run is still refused.

Tests overwrote runner.artifactsRoot after Handler/newRunner had started
background goroutines (sizeLoop, session export jobs) that read it. The
root is now fixed at construction through newRunnerAt, which
HandlerWithOptions uses for Options.ArtifactsRoot, and tests configure it
there instead of mutating the field afterwards.
2026-11-01 18:58:14 +00:00

146 lines
5.3 KiB
Go

package server
import (
"context"
"path/filepath"
"strings"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// truncatedGateExport mirrors the live failure: a native session larger than
// the export bound always renders a truncated, partial file.
func truncatedGateExport(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
result, err := validGateExport(ctx, req)
result.Status, result.Truncated = "partial", true
return result, err
}
// exhaustExportJob spends the whole recovery budget of visit 1 on the runner.
func exhaustExportJob(t *testing.T, r *runner, s *pipeline.Store, id int64) {
t.Helper()
ctx := context.Background()
for range pipeline.SessionExportJobAttempts {
job, err := s.SessionExportJob(ctx, id, 1)
if err != nil || job == nil {
t.Fatalf("missing recovery job: %v", err)
}
if worked, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
t.Fatalf("recovery attempt not run: %v %v", worked, err)
}
}
job, _ := s.SessionExportJob(ctx, id, 1)
if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "exceeds the 8 MiB export bound") {
t.Fatalf("job not exhausted with the truncation reason: %+v", job)
}
}
func workspaceFree(r *runner, workspace string) error {
r.mu.Lock()
defer r.mu.Unlock()
return r.checkWorkspaceFree(workspace)
}
func TestSessionExportGateExhaustedJobReleasesRunAndWorkspace(t *testing.T) {
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
h.runner.wg.Wait()
ctx := context.Background()
d, _ := s.GetRun(ctx, id)
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 {
t.Fatalf("partial export did not park the handoff: %+v", d)
}
if err := workspaceFree(h.runner, d.Workspace); err == nil {
t.Fatal("parked export did not reserve its workspace")
}
exhaustExportJob(t, h.runner, s, id)
if err := h.runner.resumeExportGates(ctx); err != nil {
t.Fatal(err)
}
h.runner.wg.Wait()
d, _ = s.GetRun(ctx, id)
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" || len(d.Steps) != 2 || d.ExportPendingSequence != 0 {
t.Fatalf("exhausted export did not continue the run: %+v (%d/%d)", d, first.Load(), second.Load())
}
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
t.Fatalf("gate survived: %v %v", blocked, err)
}
visit := d.Steps[0]
if visit.SessionExport == nil || visit.SessionExport.Status != "partial" || visit.SessionExportRecovery == nil || visit.SessionExportRecovery.Status != "exhausted" || visit.SessionExportRecovery.Diagnostic == "" {
t.Fatalf("partial evidence/diagnostic not retained: %+v %+v", visit.SessionExport, visit.SessionExportRecovery)
}
// The workspace is free for the next (for example scheduled) run.
if err := workspaceFree(h.runner, d.Workspace); err != nil {
t.Fatalf("workspace still reserved: %v", err)
}
// The operator can still repair the export later through the normal path.
h.runner.exportSession = validGateExport
repaired, err := h.runner.repairSessionExport(ctx, id, 1)
if err != nil {
t.Fatal(err)
}
repairs := repaired.Steps[0].SessionExportRepairs
if len(repairs) == 0 || repairs[len(repairs)-1].Status != "complete" {
t.Fatalf("later repair not recorded: %+v", repairs)
}
if _, err := s.ClaimSessionExportJob(ctx, time.Now(), time.Minute); err != nil {
t.Fatal(err)
}
if job, _ := s.SessionExportJob(ctx, id, 1); job.Status != "complete" {
t.Fatalf("successful repair did not complete the job: %+v", job)
}
}
// A run parked before the fix, whose job was already exhausted, is released by
// the restarted server's startup reconciliation without any manual edit.
func TestSessionExportGateStartupReleasesExistingExhaustedGate(t *testing.T) {
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
h.runner.wg.Wait()
exhaustExportJob(t, h.runner, s, id)
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
if err := s.Close(); err != nil {
t.Fatal(err)
}
reopened, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
parked, _ := reopened.GetRun(context.Background(), id)
if parked.Status != "waiting" || parked.ExportPendingSequence != 1 {
t.Fatalf("fixture is not a parked export: %+v", parked)
}
ctx, cancel := context.WithCancel(context.Background())
r, err := newRunnerAt(ctx, reopened, root)
if err != nil {
t.Fatal(err)
}
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
if def.Key != "two" {
t.Errorf("replayed owner: %s", def.Key)
}
second.Add(1)
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
}
deadline := time.Now().Add(10 * time.Second)
for {
d, _ := reopened.GetRun(context.Background(), id)
if d.Status == "succeeded" {
break
}
if time.Now().After(deadline) {
t.Fatalf("startup did not release the exhausted gate: %+v", d)
}
time.Sleep(20 * time.Millisecond)
}
r.wg.Wait()
if first.Load() != 1 || second.Load() != 1 {
t.Fatalf("startup release replayed or skipped execution: %d/%d", first.Load(), second.Load())
}
if err := workspaceFree(r, parked.Workspace); err != nil {
t.Fatalf("workspace still reserved after startup release: %v", err)
}
}