3bfb3d6b44
A run stays in r.active after finishRunState persists its terminal status until its goroutine finishes notifying, so a decision in that window was refused as "already active". decide now refuses only runs whose terminal state is not yet durable; a finished run gets ResolveDecision's already-decided / not-waiting answer, and an executing run is still refused. Tests overwrote runner.artifactsRoot after Handler/newRunner had started background goroutines (sizeLoop, session export jobs) that read it. The root is now fixed at construction through newRunnerAt, which HandlerWithOptions uses for Options.ArtifactsRoot, and tests configure it there instead of mutating the field afterwards.
146 lines
5.3 KiB
Go
146 lines
5.3 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// truncatedGateExport mirrors the live failure: a native session larger than
|
|
// the export bound always renders a truncated, partial file.
|
|
func truncatedGateExport(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
|
result, err := validGateExport(ctx, req)
|
|
result.Status, result.Truncated = "partial", true
|
|
return result, err
|
|
}
|
|
|
|
// exhaustExportJob spends the whole recovery budget of visit 1 on the runner.
|
|
func exhaustExportJob(t *testing.T, r *runner, s *pipeline.Store, id int64) {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
for range pipeline.SessionExportJobAttempts {
|
|
job, err := s.SessionExportJob(ctx, id, 1)
|
|
if err != nil || job == nil {
|
|
t.Fatalf("missing recovery job: %v", err)
|
|
}
|
|
if worked, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
|
|
t.Fatalf("recovery attempt not run: %v %v", worked, err)
|
|
}
|
|
}
|
|
job, _ := s.SessionExportJob(ctx, id, 1)
|
|
if job.Status != "exhausted" || !strings.Contains(job.Diagnostic, "exceeds the 8 MiB export bound") {
|
|
t.Fatalf("job not exhausted with the truncation reason: %+v", job)
|
|
}
|
|
}
|
|
|
|
func workspaceFree(r *runner, workspace string) error {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return r.checkWorkspaceFree(workspace)
|
|
}
|
|
|
|
func TestSessionExportGateExhaustedJobReleasesRunAndWorkspace(t *testing.T) {
|
|
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
|
|
h.runner.wg.Wait()
|
|
ctx := context.Background()
|
|
d, _ := s.GetRun(ctx, id)
|
|
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 {
|
|
t.Fatalf("partial export did not park the handoff: %+v", d)
|
|
}
|
|
if err := workspaceFree(h.runner, d.Workspace); err == nil {
|
|
t.Fatal("parked export did not reserve its workspace")
|
|
}
|
|
exhaustExportJob(t, h.runner, s, id)
|
|
if err := h.runner.resumeExportGates(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.runner.wg.Wait()
|
|
d, _ = s.GetRun(ctx, id)
|
|
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" || len(d.Steps) != 2 || d.ExportPendingSequence != 0 {
|
|
t.Fatalf("exhausted export did not continue the run: %+v (%d/%d)", d, first.Load(), second.Load())
|
|
}
|
|
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
|
|
t.Fatalf("gate survived: %v %v", blocked, err)
|
|
}
|
|
visit := d.Steps[0]
|
|
if visit.SessionExport == nil || visit.SessionExport.Status != "partial" || visit.SessionExportRecovery == nil || visit.SessionExportRecovery.Status != "exhausted" || visit.SessionExportRecovery.Diagnostic == "" {
|
|
t.Fatalf("partial evidence/diagnostic not retained: %+v %+v", visit.SessionExport, visit.SessionExportRecovery)
|
|
}
|
|
// The workspace is free for the next (for example scheduled) run.
|
|
if err := workspaceFree(h.runner, d.Workspace); err != nil {
|
|
t.Fatalf("workspace still reserved: %v", err)
|
|
}
|
|
// The operator can still repair the export later through the normal path.
|
|
h.runner.exportSession = validGateExport
|
|
repaired, err := h.runner.repairSessionExport(ctx, id, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
repairs := repaired.Steps[0].SessionExportRepairs
|
|
if len(repairs) == 0 || repairs[len(repairs)-1].Status != "complete" {
|
|
t.Fatalf("later repair not recorded: %+v", repairs)
|
|
}
|
|
if _, err := s.ClaimSessionExportJob(ctx, time.Now(), time.Minute); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if job, _ := s.SessionExportJob(ctx, id, 1); job.Status != "complete" {
|
|
t.Fatalf("successful repair did not complete the job: %+v", job)
|
|
}
|
|
}
|
|
|
|
// A run parked before the fix, whose job was already exhausted, is released by
|
|
// the restarted server's startup reconciliation without any manual edit.
|
|
func TestSessionExportGateStartupReleasesExistingExhaustedGate(t *testing.T) {
|
|
h, s, id, first, second := gateRunner(t, truncatedGateExport, false)
|
|
h.runner.wg.Wait()
|
|
exhaustExportJob(t, h.runner, s, id)
|
|
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
|
|
if err := s.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reopened, err := pipeline.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
parked, _ := reopened.GetRun(context.Background(), id)
|
|
if parked.Status != "waiting" || parked.ExportPendingSequence != 1 {
|
|
t.Fatalf("fixture is not a parked export: %+v", parked)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
r, err := newRunnerAt(ctx, reopened, root)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
|
|
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
if def.Key != "two" {
|
|
t.Errorf("replayed owner: %s", def.Key)
|
|
}
|
|
second.Add(1)
|
|
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
|
|
}
|
|
deadline := time.Now().Add(10 * time.Second)
|
|
for {
|
|
d, _ := reopened.GetRun(context.Background(), id)
|
|
if d.Status == "succeeded" {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("startup did not release the exhausted gate: %+v", d)
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
r.wg.Wait()
|
|
if first.Load() != 1 || second.Load() != 1 {
|
|
t.Fatalf("startup release replayed or skipped execution: %d/%d", first.Load(), second.Load())
|
|
}
|
|
if err := workspaceFree(r, parked.Workspace); err != nil {
|
|
t.Fatalf("workspace still reserved after startup release: %v", err)
|
|
}
|
|
}
|