runner: gate stage handoff on settled verified session export
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
# Session export handoff gate
|
||||
|
||||
Production model visits may publish their output to subsequent stages only after
|
||||
their exact recorded native session has settled and a complete, nonempty,
|
||||
nontruncated export file has passed normal-file and size verification. Ownership
|
||||
reads and final export attempts have bounded contexts independent of execution
|
||||
cancellation. Unconfirmed cleanup never exports or advances.
|
||||
|
||||
An export failure retains the computed step, outcome, target, output and file
|
||||
evidence. The run fails closed with an export-gate diagnostic, not an invented
|
||||
model failure. A separate `session_export_gates` checkpoint is committed with the
|
||||
step and its existing durable export job. Model retry is refused for that
|
||||
checkpoint: recovery reads only the original session. Missing exact ownership
|
||||
cannot be reconstructed or guessed and remains blocked.
|
||||
|
||||
The checkpoint reserves the workspace durably even though no agent is executing.
|
||||
`SessionExportReservations` supplies those reservations to launch admission;
|
||||
`PendingSessionExportGate(runID, sequence)` identifies the exact quiescent visit
|
||||
eligible for export-only repair. Cancellation releases the reservation without
|
||||
discarding or replaying the computation, and prevents automatic continuation.
|
||||
|
||||
After a successful automatic or manual repair, the worker re-verifies ownership
|
||||
(including backend/reference) and the actual file, then rebuilds memory, turn
|
||||
history and stage budgets from the recorded visits and follows the saved target.
|
||||
Starting the successor consumes the gate in the same transaction as `BeginStep`;
|
||||
terminal targets consume it with `FinishRun`. A restart before either transaction
|
||||
can resume the saved target, but never replays the completed owner. Cancellation,
|
||||
unconfirmed cleanup or an overlapping active workspace prevents continuation.
|
||||
|
||||
New repair files belong in the original private visit directory, not public
|
||||
handoff files. Historical public repair paths remain verifiable for compatibility.
|
||||
Only unowned injected test executors are exempt from native-session enforcement.
|
||||
An original computation failure routed directly to failure stays a failure even
|
||||
when its best-effort diagnostic export cannot be recovered.
|
||||
@@ -44,12 +44,18 @@ func (s *Store) BeginStep(ctx context.Context, id int64, sequence int, key strin
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err := affected(tx.ExecContext(ctx, "UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0", key, id)); err != nil {
|
||||
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',current_stage=? WHERE id=? AND status IN ('queued','running') AND cancel_requested=0
|
||||
AND NOT EXISTS(SELECT 1 FROM session_export_gates g LEFT JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND COALESCE(j.status,'')<>'complete')`, key, id)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, "INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES (?,?,?,?)", id, sequence, key, string(encoded)); err != nil {
|
||||
return err
|
||||
}
|
||||
// Consuming a recovered handoff and starting its successor are atomic. A
|
||||
// crash before this transaction leaves the gate available after restart.
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND sequence<?`, id, sequence); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
@@ -73,6 +79,16 @@ type stageVerdictEvidence struct {
|
||||
}
|
||||
|
||||
func (s *Store) FinishStep(ctx context.Context, id int64, step protocol.RunStep) error {
|
||||
return s.finishStep(ctx, id, step, false)
|
||||
}
|
||||
|
||||
// FinishStepExportBlocked retains the computed step and its routing target, but
|
||||
// checkpoints the handoff separately: the output is not yet eligible for reuse.
|
||||
func (s *Store) FinishStepExportBlocked(ctx context.Context, id int64, step protocol.RunStep) error {
|
||||
return s.finishStep(ctx, id, step, true)
|
||||
}
|
||||
|
||||
func (s *Store) finishStep(ctx context.Context, id int64, step protocol.RunStep, exportBlocked bool) error {
|
||||
output, err := json.Marshal(step.Output)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -148,6 +164,11 @@ WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome,
|
||||
if err := enqueueStepExportJob(ctx, tx, id, step.Sequence); err != nil {
|
||||
return err
|
||||
}
|
||||
if exportBlocked && !requested {
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO session_export_gates(run_id,sequence) VALUES(?,?)`, id, step.Sequence); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// The reported usage the stage already measured for budget accounting is
|
||||
// persisted as step-scoped evidence in the same transaction, mirroring the
|
||||
// other step evidence. It is written only when the stage actually reported
|
||||
@@ -302,7 +323,9 @@ func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',cancel_requested=1,diagnostic='Canceled while waiting for a decision',finished_at=CURRENT_TIMESTAMP WHERE id=? AND status=?`, id, protocol.RunStatusWaiting)
|
||||
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',cancel_requested=1,
|
||||
diagnostic=CASE WHEN status='waiting' THEN 'Canceled while waiting for a decision' ELSE 'Canceled while awaiting session export; computed visit retained' END,
|
||||
finished_at=CURRENT_TIMESTAMP WHERE id=? AND (status=? OR (status IN ('failed','interrupted') AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id)))`, id, protocol.RunStatusWaiting)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -323,7 +346,7 @@ func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
|
||||
func (s *Store) Cancel(ctx context.Context, id int64) error {
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET cancel_requested=1,
|
||||
diagnostic=CASE WHEN cleanup_unconfirmed=1 THEN diagnostic ELSE 'Stop requested; waiting for execution and cleanup to stop' END
|
||||
WHERE id=? AND status IN ('queued','running','waiting')`, id))
|
||||
WHERE id=? AND (status IN ('queued','running','waiting') OR (status IN ('failed','interrupted') AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id)))`, id))
|
||||
}
|
||||
|
||||
func (s *Store) FinishRun(ctx context.Context, id int64, status, outcome, diagnostic string, output map[string]any) error {
|
||||
@@ -347,6 +370,10 @@ AND NOT EXISTS(SELECT 1 FROM run_steps WHERE run_id=pipeline_runs.id AND agent_a
|
||||
if err := enqueueRunTransitionNotification(ctx, tx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
// Terminal targets have no successor BeginStep to consume their gate.
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM session_export_gates WHERE run_id=? AND EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=session_export_gates.run_id AND j.sequence=session_export_gates.sequence AND j.status='complete')`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package pipeline
|
||||
|
||||
import "context"
|
||||
|
||||
func (s *Store) SessionExportBlocked(ctx context.Context, id int64) (bool, error) {
|
||||
var blocked bool
|
||||
err := s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates WHERE run_id=?)`, id).Scan(&blocked)
|
||||
return blocked, err
|
||||
}
|
||||
|
||||
// PendingSessionExportGate authorizes only export repair of the exact completed
|
||||
// visit, never a model call or a human decision. No live executor is implied by
|
||||
// the workspace reservation maintained for this checkpoint.
|
||||
func (s *Store) PendingSessionExportGate(ctx context.Context, id int64, sequence int) (bool, error) {
|
||||
var pending bool
|
||||
err := s.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates g
|
||||
JOIN pipeline_runs r ON r.id=g.run_id JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
|
||||
WHERE g.run_id=? AND g.sequence=? AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
|
||||
AND s.status NOT IN ('running','waiting')
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE (a.run_id=g.run_id OR a.session_id=s.session_id) AND a.agent_active<>0))`, id, sequence).Scan(&pending)
|
||||
return pending, err
|
||||
}
|
||||
|
||||
// SessionExportReservations survive process restarts and cannot be confused
|
||||
// with agent-active/executing flags in the UI.
|
||||
func (s *Store) SessionExportReservations(ctx context.Context) (map[int64]string, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT r.id,r.workspace FROM session_export_gates g JOIN pipeline_runs r ON r.id=g.run_id WHERE r.cancel_requested=0 AND r.status<>'canceled'`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
reservations := map[int64]string{}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var workspace string
|
||||
if err := rows.Scan(&id, &workspace); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reservations[id] = workspace
|
||||
}
|
||||
return reservations, rows.Err()
|
||||
}
|
||||
|
||||
// ReadySessionExportGates includes interrupted continuations. The gate remains
|
||||
// durable until a successor is begun (or a terminal target is committed).
|
||||
func (s *Store) ReadySessionExportGates(ctx context.Context) (map[int64]int, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT g.run_id,g.sequence FROM session_export_gates g
|
||||
JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence
|
||||
JOIN pipeline_runs r ON r.id=g.run_id
|
||||
WHERE j.status='complete' AND r.status IN ('failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+exactExportJobOwner+` LIMIT 64`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
ready := map[int64]int{}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var sequence int
|
||||
if err := rows.Scan(&id, &sequence); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ready[id] = sequence
|
||||
}
|
||||
return ready, rows.Err()
|
||||
}
|
||||
|
||||
// ResumeSessionExportGate is a CAS, not a retry of the model invocation.
|
||||
func (s *Store) ResumeSessionExportGate(ctx context.Context, id int64, sequence int) error {
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET status='queued',outcome='',diagnostic='',finished_at=NULL
|
||||
WHERE id=? AND status IN ('failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0
|
||||
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND j.status='complete' AND `+exactExportJobOwner+`)
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence))
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) {
|
||||
s, id, _ := exportJobFixture(t)
|
||||
ctx := context.Background()
|
||||
step := protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"markdown": "actual result"}, SessionExport: &protocol.SessionExport{Status: "failed", SessionID: "ses_original", Backend: "v2", Diagnostic: "offline"}}
|
||||
if err := s.FinishStepExportBlocked(ctx, id, step); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.BeginStep(ctx, id, 2, "two", nil); err == nil {
|
||||
t.Fatal("successor started while export was incomplete")
|
||||
}
|
||||
if err := s.FinishRun(ctx, id, "failed", "error", "export gate", step.Output); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
|
||||
if err != nil || claimed == nil {
|
||||
t.Fatalf("missing export-only retry: %v", err)
|
||||
}
|
||||
if err := s.RecordSessionExportRepair(ctx, id, 1, &protocol.SessionExport{Status: "complete", SessionID: claimed.SessionID, Backend: claimed.Backend, Directory: claimed.Directory, Messages: 2, HTML: "session-repair-1-test.html", HTMLBytes: 100}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.FinishSessionExportJob(ctx, claimed, time.Now(), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.ResumeSessionExportGate(ctx, id, 1); err == nil {
|
||||
t.Fatal("duplicate continuation admitted")
|
||||
}
|
||||
// A crash after reserving the continuation but before starting its next
|
||||
// executor must retain the checkpoint for startup's interrupted run.
|
||||
if _, err := s.db.Exec(`UPDATE pipeline_runs SET status='interrupted' WHERE id=?`, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ready, err := s.ReadySessionExportGates(ctx)
|
||||
if err != nil || ready[id] != 1 {
|
||||
t.Fatalf("lost restart checkpoint: %v %v", ready, err)
|
||||
}
|
||||
if err := s.ResumeSessionExportGate(ctx, id, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.BeginStep(ctx, id, 2, "two", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if blocked, err := s.SessionExportBlocked(ctx, id); err != nil || blocked {
|
||||
t.Fatalf("successor did not consume checkpoint: %v %v", blocked, err)
|
||||
}
|
||||
if err := s.BeginStep(ctx, id, 2, "two", nil); err == nil {
|
||||
t.Fatal("successor started twice")
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,11 @@ status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','running','com
|
||||
next_at INTEGER NOT NULL, lease_until INTEGER NOT NULL DEFAULT 0, token TEXT NOT NULL DEFAULT '', diagnostic TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE);
|
||||
CREATE INDEX IF NOT EXISTS session_export_jobs_due ON session_export_jobs(status,next_at,lease_until);`)
|
||||
if err == nil {
|
||||
_, err = s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_gates (
|
||||
run_id INTEGER PRIMARY KEY, sequence INTEGER NOT NULL,
|
||||
FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE)`)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -45,7 +50,7 @@ SELECT s.run_id,s.sequence,s.session_id,o.backend,r.workspace,o.reference,?
|
||||
FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id
|
||||
JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence
|
||||
LEFT JOIN step_session_exports x ON x.run_id=s.run_id AND x.sequence=s.sequence
|
||||
WHERE s.status NOT IN ('running','waiting') AND s.session_id<>'' AND o.backend IN ('v1','v2')
|
||||
WHERE s.status NOT IN ('running','waiting') AND s.session_id<>'' AND (o.backend IN ('v1','v2') OR (o.backend='pi' AND o.reference<>''))
|
||||
AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=s.run_id AND j.sequence=s.sequence)
|
||||
AND NOT (COALESCE(json_extract(x.export_json,'$.status'),'complete')='complete'
|
||||
AND COALESCE(json_extract(x.export_json,'$.session_id'),'')=s.session_id
|
||||
|
||||
@@ -259,7 +259,7 @@ func TestSessionExportJobMigrationReconcilesMissingJobsWithoutReset(t *testing.T
|
||||
}
|
||||
|
||||
func TestSessionExportJobOnlyEnqueuesIncompleteNativeEvidence(t *testing.T) {
|
||||
for _, status := range []string{"complete", "failed", "empty", "partial", "missing", "unsupported", "wrong-session"} {
|
||||
for _, status := range []string{"complete", "failed", "empty", "partial", "missing", "unsupported", "pi-native", "wrong-session"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
s, id, _ := exportJobFixture(t)
|
||||
ctx := context.Background()
|
||||
@@ -272,10 +272,16 @@ func TestSessionExportJobOnlyEnqueuesIncompleteNativeEvidence(t *testing.T) {
|
||||
evidence.SessionID = "ses_wrong"
|
||||
}
|
||||
if status == "unsupported" {
|
||||
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi'"); err != nil {
|
||||
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi',reference=''"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if status == "pi-native" {
|
||||
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi',reference='/original/session.jsonl'"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
evidence.Backend, evidence.Status = "pi", "failed"
|
||||
}
|
||||
if err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", SessionExport: evidence, Output: map[string]any{}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -11,7 +11,8 @@ import (
|
||||
// legacy display lookup it never guesses a backend for a historical session.
|
||||
func (s *Store) RepairSessionOwner(ctx context.Context, id int64, sequence int) (SessionOwner, error) {
|
||||
var owner SessionOwner
|
||||
err := s.db.QueryRowContext(ctx, `SELECT s.session_id,o.backend,r.workspace,o.reference FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence WHERE s.run_id=? AND s.sequence=? AND s.session_id<>'' AND s.agent_active=0`, id, sequence).Scan(&owner.ID, &owner.Backend, &owner.Directory, &owner.Reference)
|
||||
err := s.db.QueryRowContext(ctx, `SELECT s.session_id,o.backend,r.workspace,o.reference FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence WHERE s.run_id=? AND s.sequence=? AND s.session_id<>'' AND s.agent_active=0
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.session_id=s.session_id AND a.agent_active<>0)`, id, sequence).Scan(&owner.ID, &owner.Backend, &owner.Directory, &owner.Reference)
|
||||
return owner, err
|
||||
}
|
||||
|
||||
|
||||
@@ -417,6 +417,7 @@ func waitTerminal(t *testing.T, r *runner, id int64) *protocol.RunDetail {
|
||||
// deterministically against B's active session.
|
||||
func TestOverlappingRunsDoNotFailQuiescence(t *testing.T) {
|
||||
_, r, pipelineID := agentStore(t)
|
||||
r.exportSession = validGateExport // Quiescence, not export transport, is under test.
|
||||
a, b := t.TempDir(), t.TempDir()
|
||||
|
||||
type agentRun struct {
|
||||
|
||||
@@ -95,6 +95,7 @@ esac
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cancel(); r.wg.Wait() }()
|
||||
r.exportSession = validGateExport // Export is a separate mandatory handoff gate.
|
||||
workspace, err := filepath.EvalSymlinks(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -455,6 +456,7 @@ esac
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store)
|
||||
handler.(*runHandler).runner.exportSession = validGateExport
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
items, err := store.List(ctx)
|
||||
if err != nil {
|
||||
@@ -566,6 +568,7 @@ esac
|
||||
toolsBlock := func(tools string) string {
|
||||
return "# stage worker\n\n```output\nResult.\n```\n\n```prompt\nDo the work.\n```\n\n```tools\n" + tools + "\n```\n\n```edges\nsuccess -> #success\nerror -> #failure\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=1\ntimeout=30s\n```\n"
|
||||
}
|
||||
r.exportSession = validGateExport
|
||||
runStage := func(name, tools string) (status, sessions, diagnostic string) {
|
||||
t.Helper()
|
||||
workspace := t.TempDir()
|
||||
|
||||
+88
-12
@@ -384,11 +384,26 @@ func (r *runner) launch(id int64, workspace string, fn func(context.Context)) {
|
||||
// checkWorkspaceFree refuses a run when an active or cleanup-unconfirmed run
|
||||
// reserves an overlapping resolved workspace. Callers must hold r.mu.
|
||||
func (r *runner) checkWorkspaceFree(workspace string) error {
|
||||
return r.checkWorkspaceFreeExcept(workspace, 0)
|
||||
}
|
||||
|
||||
func (r *runner) checkWorkspaceFreeExcept(workspace string, ownGate int64) error {
|
||||
for id, entry := range r.active {
|
||||
if workspacesOverlap(entry.workspace, workspace) {
|
||||
return fmt.Errorf("run %d is already active in workspace %s; wait for it to finish or start this run in a different workspace (for example another checkout of the repository)", id, workspace)
|
||||
}
|
||||
}
|
||||
check, cancel := context.WithTimeout(context.WithoutCancel(r.ctx), 5*time.Second)
|
||||
defer cancel()
|
||||
reservations, err := r.store.SessionExportReservations(check)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read pending export workspace reservations: %w", err)
|
||||
}
|
||||
for id, reserved := range reservations {
|
||||
if id != ownGate && workspacesOverlap(reserved, workspace) {
|
||||
return fmt.Errorf("run %d is awaiting its session export; workspace %s remains reserved", id, reserved)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -694,6 +709,11 @@ func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
} else if existing > 0 {
|
||||
return r.store.GetRun(r.ctx, existing)
|
||||
}
|
||||
if blocked, err := r.store.SessionExportBlocked(r.ctx, original.ID); err != nil {
|
||||
return nil, err
|
||||
} else if blocked {
|
||||
return nil, fmt.Errorf("run %d has a retained computation awaiting export-only recovery; model retry refused", original.ID)
|
||||
}
|
||||
if request.RetryVisit < 0 || (request.RetryVisit > 0 && request.RetryStage == "") {
|
||||
return nil, fmt.Errorf("retry visit requires a positive --stage visit selection")
|
||||
}
|
||||
@@ -947,6 +967,7 @@ type runState struct {
|
||||
// retryOverride is applied once, after the selected stage's memory bindings
|
||||
// have been resolved. It is intentionally not part of run input or memory.
|
||||
retryOverride stage.Values
|
||||
priorBudgets map[string]stage.BudgetSnapshot
|
||||
}
|
||||
|
||||
func newRunState(key string, input stage.Values) *runState {
|
||||
@@ -963,7 +984,18 @@ func rebuildRunState(detail *protocol.RunDetail, decided protocol.RunStep) *runS
|
||||
state.memory["input"] = memoryValue{output: original}
|
||||
}
|
||||
publicSeen := map[string]bool{}
|
||||
state.priorBudgets = map[string]stage.BudgetSnapshot{}
|
||||
for _, step := range detail.Steps {
|
||||
// The runtime stamps this cumulative snapshot on every executed model
|
||||
// visit. Reusing it preserves per-stage limits across export recovery.
|
||||
var audit struct {
|
||||
Limits struct {
|
||||
Budget stage.BudgetSnapshot `json:"budget"`
|
||||
} `json:"stage_limits"`
|
||||
}
|
||||
if json.Unmarshal([]byte(step.Transcript), &audit) == nil {
|
||||
state.priorBudgets[step.Stage] = audit.Limits.Budget
|
||||
}
|
||||
state.visits[step.Stage]++
|
||||
if step.Sequence >= state.nextSequence {
|
||||
state.nextSequence = step.Sequence + 1
|
||||
@@ -1715,6 +1747,9 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
modelConfig = spec.Config
|
||||
if budgets[key] == nil {
|
||||
budgets[key] = stage.NewBudget(modelConfig.BudgetUSD, modelConfig.BudgetTokens)
|
||||
if used, ok := state.priorBudgets[key]; ok {
|
||||
budgets[key].RestoreUsage(used)
|
||||
}
|
||||
}
|
||||
// The budget is shared across every visit to this stage key, so capture
|
||||
// its cumulative usage before the visit and record only this visit's
|
||||
@@ -1808,15 +1843,32 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
// Export the stage's session into its own artifacts directory before the
|
||||
// bounded scan, so the existing listing and serving paths find the files
|
||||
// with no special casing. This is bounded, best-effort evidence: it never
|
||||
// waits for session idleness and a failure never changes the stage's
|
||||
// status, outcome, target or result validation.
|
||||
// Only settled, exact recorded ownership can satisfy the handoff gate.
|
||||
// A canceled execution must not cancel this ownership read silently.
|
||||
var sessionExport *protocol.SessionExport
|
||||
exportRequired := artifactsDir != "" && r.execute == nil
|
||||
if artifactsDir != "" {
|
||||
if owner, ownerErr := r.store.RunSessionOwner(ctx, id, sequence); ownerErr == nil && owner.ID != "" {
|
||||
sessionExport = r.exportStageSession(ctx, owner, artifactsDir, id, sequence, key, modelConfig)
|
||||
ownerCtx, ownerCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
owner, ownerErr := r.store.RepairSessionOwner(ownerCtx, id, sequence)
|
||||
if ownerErr != nil && r.execute != nil {
|
||||
// Only genuinely unowned fake executors are exempt. A recorded
|
||||
// session with absent ownership metadata must still fail closed.
|
||||
if recorded, readErr := r.store.RunSessionOwner(ownerCtx, id, sequence); readErr != nil || recorded.ID != "" {
|
||||
exportRequired = true
|
||||
if readErr == nil {
|
||||
stageSession = recorded.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
ownerCancel()
|
||||
if ownerErr == nil && owner.ID != "" {
|
||||
exportRequired = true // Owned test executors exercise the real gate too.
|
||||
stageSession = owner.ID
|
||||
if !errors.Is(err, stage.ErrStageCleanup) {
|
||||
sessionExport = r.exportStageSession(ctx, owner, artifactsDir, id, sequence, key, modelConfig)
|
||||
}
|
||||
} else if exportRequired {
|
||||
sessionExport = &protocol.SessionExport{Status: "failed", Source: "native_export", Diagnostic: "exact settled session ownership unavailable; handoff refused"}
|
||||
}
|
||||
}
|
||||
artifacts, artifactsSkipped, artifactsTruncated := r.scanArtifacts(id, sequence, artifactsDir, visitTurn.PublicDir)
|
||||
@@ -1915,6 +1967,28 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
if panicked {
|
||||
step.Target = protocol.FailureEnd
|
||||
}
|
||||
_, _, _, failedTerminal := terminalFinish(step.Target, step.Stage, step.Outcome, step.Diagnostic)
|
||||
if exportRequired && !completeSessionExport(sessionExport) && !(err != nil && failedTerminal && step.Target != protocol.SuccessEnd) {
|
||||
// Preserve the real computation, files, outcome and target. In
|
||||
// particular an export failure is not an agent failure/retry request.
|
||||
save, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
saveErr := r.store.FinishStepExportBlocked(save, id, step)
|
||||
cancel()
|
||||
if saveErr != nil {
|
||||
finish("failed", "error", saveErr.Error())
|
||||
return
|
||||
}
|
||||
diagnostic := "Session export gate blocked; computed visit retained; export-only recovery pending"
|
||||
if sessionExport != nil && sessionExport.Diagnostic != "" {
|
||||
diagnostic += ": " + sessionExport.Diagnostic
|
||||
}
|
||||
if err != nil {
|
||||
diagnostic = err.Error() + "; " + diagnostic
|
||||
}
|
||||
// No finishStage, memory publication, recordTurn or next executor.
|
||||
r.finishRunState(id, name, workspace, "failed", "error", diagnostic, state.completed, sequence, key, stage.Values(step.Output))
|
||||
return
|
||||
}
|
||||
// Generate the one-line verdict reason once, at completion. It is
|
||||
// best-effort: a summariser failure or timeout returns NoSummary and must
|
||||
// never fail the stage or the run, so every error is swallowed. A failed
|
||||
@@ -1985,14 +2059,12 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
}
|
||||
|
||||
// exportStageSession writes the finished stage's session into its own artifacts
|
||||
// directory and returns the evidence for the step. A non-OpenCode harness (pi)
|
||||
// has no native export path, so it records that limitation and writes no files.
|
||||
// Any export error is recorded as a sanitized diagnostic and never returned to
|
||||
// the stage: the export is read-only, bounded and best-effort.
|
||||
// directory and returns verified evidence. The caller enforces the handoff gate;
|
||||
// an export failure never replaces the computation's result.
|
||||
func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionOwner, artifactsDir string, runID int64, sequence int, key string, modelConfig stage.ModelConfig) *protocol.SessionExport {
|
||||
evidence := &protocol.SessionExport{SessionID: owner.ID, Backend: owner.Backend, Directory: owner.Directory, RecordedAt: time.Now().UTC().Format(time.RFC3339)}
|
||||
defer classifySessionExport(evidence)
|
||||
if owner.Backend != "v1" && owner.Backend != "v2" {
|
||||
if owner.Backend != "v1" && owner.Backend != "v2" && !(owner.Backend == "pi" && owner.Reference != "") {
|
||||
evidence.Source = "unavailable"
|
||||
evidence.Diagnostic = "session export unavailable: harness has no native export path"
|
||||
return evidence
|
||||
@@ -2008,6 +2080,7 @@ func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionO
|
||||
result, err := export(exportCtx, stage.SessionExportRequest{
|
||||
SessionID: owner.ID,
|
||||
Backend: owner.Backend,
|
||||
Reference: owner.Reference,
|
||||
Directory: owner.Directory,
|
||||
ArtifactsDir: artifactsDir,
|
||||
RunID: runID,
|
||||
@@ -2029,6 +2102,9 @@ func (r *runner) exportStageSession(ctx context.Context, owner pipeline.SessionO
|
||||
evidence.Messages = result.Messages
|
||||
evidence.Truncated = result.Truncated
|
||||
evidence.Diagnostic = result.Diagnostic
|
||||
if result.Status != "" && result.Status != "complete" && evidence.Diagnostic == "" {
|
||||
evidence.Diagnostic = "native session export reported " + result.Status
|
||||
}
|
||||
verifySessionExportFile(evidence, artifactsDir)
|
||||
return evidence
|
||||
}
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func completeSessionExport(e *protocol.SessionExport) bool {
|
||||
return e != nil && e.Status == "complete" && e.SessionID != "" && e.Messages > 0 && e.HTML != "" && e.HTMLBytes > 0 && !e.Truncated && e.Diagnostic == ""
|
||||
}
|
||||
|
||||
// resumeExportGates is also run when no job is due: a manual export repair or
|
||||
// a crash after persisting repair evidence may have made a handoff eligible.
|
||||
func (r *runner) resumeExportGates(ctx context.Context) error {
|
||||
ready, err := r.store.ReadySessionExportGates(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for id, sequence := range ready {
|
||||
r.mu.Lock()
|
||||
if _, active := r.active[id]; active {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
d, err := r.store.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
r.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
if err := r.checkWorkspaceFreeExcept(d.Workspace, id); err != nil {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
owner, err := r.store.RepairSessionOwner(ctx, id, sequence)
|
||||
if err != nil {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
job, err := r.store.SessionExportJob(ctx, id, sequence)
|
||||
if err != nil || job == nil || job.SessionID != owner.ID || job.Backend != owner.Backend || job.Directory != owner.Directory || job.Reference != owner.Reference {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
var saved *protocol.RunStep
|
||||
for i := range d.Steps {
|
||||
if d.Steps[i].Sequence == sequence {
|
||||
saved = &d.Steps[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
verified := false
|
||||
if saved != nil {
|
||||
for _, e := range saved.SessionExportRepairs {
|
||||
if !completeSessionExport(e) || e.SessionID != owner.ID || e.Backend != owner.Backend || e.Directory != owner.Directory {
|
||||
continue
|
||||
}
|
||||
// New repairs stay private in the original visit. Historical
|
||||
// public repairs remain valid, but are never rewritten there.
|
||||
directory := r.turnPaths(id, sequence, saved.Stage).TurnDir
|
||||
if strings.HasPrefix(e.HTML, "public/") {
|
||||
directory = filepath.Join(r.artifactsRoot, strconv.FormatInt(id, 10))
|
||||
} else if filepath.Base(e.HTML) != e.HTML || !strings.HasPrefix(e.HTML, "session-repair-") {
|
||||
continue
|
||||
}
|
||||
if filepath.Clean(e.HTML) != e.HTML {
|
||||
continue
|
||||
}
|
||||
root, openErr := os.OpenRoot(directory)
|
||||
if openErr != nil {
|
||||
continue
|
||||
}
|
||||
info, statErr := root.Lstat(e.HTML)
|
||||
root.Close()
|
||||
if statErr == nil && info.Mode().IsRegular() && info.Size() == e.HTMLBytes && info.Size() > 0 && info.Size() <= stage.SessionExportFileLimit {
|
||||
verified = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !verified {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
if err := r.store.ResumeSessionExportGate(ctx, id, sequence); err != nil {
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
step := *saved
|
||||
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
|
||||
r.mu.Unlock()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
type gateTestOptions struct {
|
||||
missingOwner bool
|
||||
waitForCancel bool
|
||||
}
|
||||
|
||||
func gateRunner(t *testing.T, export func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error), active bool, options ...gateTestOptions) (*runHandler, *pipeline.Store, int64, *atomic.Int32, *atomic.Int32) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s, err := pipeline.Open(filepath.Join(t.TempDir(), "gate.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := Handler(ctx, s).(*runHandler)
|
||||
t.Cleanup(func() { cancel(); WaitRuns(h); s.Close() })
|
||||
h.runner.artifactsRoot = t.TempDir()
|
||||
h.runner.exportSession = export
|
||||
if _, err := s.Submit(ctx, "gate", pipeline.Template([]string{"one", "two"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := s.List(ctx)
|
||||
ready := make(chan struct{})
|
||||
var id int64
|
||||
first, second := &atomic.Int32{}, &atomic.Int32{}
|
||||
h.runner.execute = func(ctx context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
<-ready
|
||||
if def.Key == "one" {
|
||||
first.Add(1)
|
||||
metadata := []string{"v2", "exact-ref"}
|
||||
if len(options) > 0 && options[0].missingOwner {
|
||||
metadata = nil
|
||||
}
|
||||
if err := s.RunSession(ctx, id, 1, "ses_exact", active, metadata...); err != nil {
|
||||
return stage.Result{}, err
|
||||
}
|
||||
if len(options) > 0 && options[0].waitForCancel {
|
||||
<-ctx.Done()
|
||||
return stage.Result{}, ctx.Err()
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "computed output"}}, nil
|
||||
}
|
||||
second.Add(1)
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
|
||||
}
|
||||
d, err := h.runner.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "gate", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id = d.ID
|
||||
close(ready)
|
||||
return h, s, id, first, second
|
||||
}
|
||||
|
||||
func validGateExport(_ context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
if err := os.MkdirAll(req.ArtifactsDir, 0700); err != nil {
|
||||
return stage.SessionExportResult{}, err
|
||||
}
|
||||
name := req.FileName
|
||||
if name == "" {
|
||||
name = stage.SessionHTMLFile
|
||||
}
|
||||
body := []byte("<!doctype html><html>exact session</html>")
|
||||
if err := os.WriteFile(filepath.Join(req.ArtifactsDir, name), body, 0600); err != nil {
|
||||
return stage.SessionExportResult{}, err
|
||||
}
|
||||
return stage.SessionExportResult{Source: "native_export", Status: "complete", HTML: name, HTMLBytes: int64(len(body)), Messages: 2}, nil
|
||||
}
|
||||
|
||||
func TestSessionExportGateOrdersExecutors(t *testing.T) {
|
||||
entered, release := make(chan struct{}), make(chan struct{})
|
||||
h, s, id, first, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
if req.SessionID != "ses_exact" || req.Backend != "v2" || req.Reference != "exact-ref" {
|
||||
t.Errorf("wrong owner: %+v", req)
|
||||
}
|
||||
close(entered)
|
||||
<-release
|
||||
return validGateExport(ctx, req)
|
||||
}, false)
|
||||
<-entered
|
||||
if second.Load() != 0 {
|
||||
t.Fatal("next executor ran before verified export")
|
||||
}
|
||||
close(release)
|
||||
h.runner.wg.Wait()
|
||||
d, _ := s.GetRun(context.Background(), id)
|
||||
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" {
|
||||
t.Fatalf("handoff failed: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportGateRetryOnlyAndExactlyOnce(t *testing.T) {
|
||||
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
return stage.SessionExportResult{}, errors.New("export offline")
|
||||
}, false)
|
||||
h.runner.wg.Wait()
|
||||
ctx := context.Background()
|
||||
d, _ := s.GetRun(ctx, id)
|
||||
if d.Status != "failed" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
|
||||
t.Fatalf("lost computation/advanced: %+v", d)
|
||||
}
|
||||
h.runner.exportSession = validGateExport
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
if job == nil {
|
||||
t.Fatal("missing durable recovery job")
|
||||
}
|
||||
if _, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := h.runner.resumeExportGates(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.runner.wg.Wait()
|
||||
if err := h.runner.resumeExportGates(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.runner.wg.Wait()
|
||||
d, _ = s.GetRun(ctx, id)
|
||||
if first.Load() != 1 || second.Load() != 1 || len(d.Steps) != 2 || d.Status != "succeeded" {
|
||||
t.Fatalf("export retry replayed execution: %+v (%d/%d)", d, first.Load(), second.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportGateCleanupUnconfirmedDoesNotExportOrAdvance(t *testing.T) {
|
||||
var exports atomic.Int32
|
||||
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
exports.Add(1)
|
||||
return validGateExport(ctx, req)
|
||||
}, true)
|
||||
h.runner.wg.Wait()
|
||||
d, _ := s.GetRun(context.Background(), id)
|
||||
if !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 {
|
||||
t.Fatalf("unconfirmed cleanup advanced/exported: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportGateCanceledCleanupUnconfirmedDoesNotAdvance(t *testing.T) {
|
||||
var exports atomic.Int32
|
||||
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
exports.Add(1)
|
||||
return validGateExport(ctx, req)
|
||||
}, true, gateTestOptions{waitForCancel: true})
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
d, _ := s.GetRun(context.Background(), id)
|
||||
if len(d.Steps) > 0 && d.Steps[0].AgentActive {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("executor did not acquire session")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if err := h.runner.cancel(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.runner.wg.Wait()
|
||||
d, _ := s.GetRun(context.Background(), id)
|
||||
if !d.CancelRequested || !d.CleanupUnconfirmed || d.Status == "succeeded" || second.Load() != 0 || exports.Load() != 0 {
|
||||
t.Fatalf("canceled unconfirmed owner advanced: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportGateCanceledExecutionStillReadsExactOwnerAndExports(t *testing.T) {
|
||||
var exports atomic.Int32
|
||||
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
if ctx.Err() != nil || req.SessionID != "ses_exact" || req.Reference != "exact-ref" {
|
||||
t.Errorf("lost bounded uncanceled exact owner: %+v %v", req, ctx.Err())
|
||||
}
|
||||
exports.Add(1)
|
||||
return validGateExport(ctx, req)
|
||||
}, false, gateTestOptions{waitForCancel: true})
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
d, _ := s.GetRun(context.Background(), id)
|
||||
if len(d.Steps) > 0 && d.Steps[0].SessionID != "" {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("executor did not record session")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if err := h.runner.cancel(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.runner.wg.Wait()
|
||||
d, _ := s.GetRun(context.Background(), id)
|
||||
if d.Status != "canceled" || second.Load() != 0 || exports.Load() != 1 || d.Steps[0].SessionExport == nil {
|
||||
t.Fatalf("canceled ownership/export was silently lost: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportGateMissingOwnerFailsClosed(t *testing.T) {
|
||||
var exports atomic.Int32
|
||||
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
exports.Add(1)
|
||||
return validGateExport(ctx, req)
|
||||
}, false, gateTestOptions{missingOwner: true})
|
||||
h.runner.wg.Wait()
|
||||
d, _ := s.GetRun(context.Background(), id)
|
||||
if d.Status != "failed" || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
|
||||
t.Fatalf("missing owner did not fail closed: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportGateReservesWorkspaceWithoutExecutingAndCanCancel(t *testing.T) {
|
||||
h, s, id, _, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
return stage.SessionExportResult{}, errors.New("offline")
|
||||
}, false)
|
||||
h.runner.wg.Wait()
|
||||
ctx := context.Background()
|
||||
d, err := h.runner.getRun(id)
|
||||
if err != nil || d.Executing {
|
||||
t.Fatalf("parked export mislabeled executing: %+v %v", d, err)
|
||||
}
|
||||
pending, err := s.PendingSessionExportGate(ctx, id, 1)
|
||||
if err != nil || !pending {
|
||||
t.Fatalf("quiescent pending visit not exposed: %v %v", pending, err)
|
||||
}
|
||||
request := protocol.Message{PipelineID: d.PipelineID, Version: 1, RequestKey: "overlap", Workspace: d.Workspace, Input: map[string]any{"prompt": "another task"}}
|
||||
if _, err := h.runner.start(request); err == nil {
|
||||
t.Fatal("parked export released workspace")
|
||||
}
|
||||
if err := h.runner.cancel(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err = s.PendingSessionExportGate(ctx, id, 1)
|
||||
if err != nil || pending {
|
||||
t.Fatalf("canceled gate remains resumable: %v %v", pending, err)
|
||||
}
|
||||
h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
||||
}
|
||||
request.RequestKey = "after-cancel"
|
||||
if _, err := h.runner.start(request); err != nil {
|
||||
t.Fatalf("cancel did not release workspace: %v", err)
|
||||
}
|
||||
h.runner.wg.Wait()
|
||||
if second.Load() != 0 {
|
||||
t.Fatal("canceled gate advanced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportGateRejectsIncompleteAndInvalidFiles(t *testing.T) {
|
||||
for _, mode := range []string{"missing", "empty", "partial", "wrong-size", "symlink"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
h, s, id, _, second := gateRunner(t, func(ctx context.Context, req stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
result, err := validGateExport(ctx, req)
|
||||
path := filepath.Join(req.ArtifactsDir, result.HTML)
|
||||
switch mode {
|
||||
case "missing":
|
||||
err = os.Remove(path)
|
||||
case "empty":
|
||||
result.Status = "empty"
|
||||
case "partial":
|
||||
result.Status = "partial"
|
||||
case "wrong-size":
|
||||
result.HTMLBytes++
|
||||
case "symlink":
|
||||
if err = os.Remove(path); err == nil {
|
||||
err = os.Symlink("input.md", path)
|
||||
}
|
||||
}
|
||||
return result, err
|
||||
}, false)
|
||||
h.runner.wg.Wait()
|
||||
d, _ := s.GetRun(context.Background(), id)
|
||||
if d.Status != "failed" || second.Load() != 0 || d.Output["markdown"] != "computed output" {
|
||||
t.Fatalf("invalid export advanced: %+v", d)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportGateRestartResumesSuccessorNotOwner(t *testing.T) {
|
||||
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
return stage.SessionExportResult{}, errors.New("offline")
|
||||
}, false)
|
||||
h.runner.wg.Wait()
|
||||
path, root := filepath.Join(s.Directory(), "gate.db"), h.runner.artifactsRoot
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, err := pipeline.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
r, err := newRunner(ctx, reopened)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cancel(); r.wg.Wait(); r.schedulerWG.Wait(); reopened.Close() }()
|
||||
r.artifactsRoot = root
|
||||
r.exportSession = validGateExport
|
||||
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
||||
if def.Key != "two" {
|
||||
t.Errorf("replayed owner: %s", def.Key)
|
||||
}
|
||||
second.Add(1)
|
||||
body, err := os.ReadFile(def.Turn.Input)
|
||||
if err != nil || !strings.Contains(string(body), "computed output") {
|
||||
t.Errorf("lost computed handoff: %q %v", body, err)
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"markdown": "next output"}}, nil
|
||||
}
|
||||
job, _ := reopened.SessionExportJob(ctx, id, 1)
|
||||
if _, err := r.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.resumeExportGates(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
if err := r.resumeExportGates(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
d, _ := reopened.GetRun(ctx, id)
|
||||
if first.Load() != 1 || second.Load() != 1 || d.Status != "succeeded" {
|
||||
t.Fatalf("restart replayed or lost successor: %+v", d)
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,9 @@ func (r *runner) startSessionExportJobs() error {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := r.resumeExportGates(ctx); err != nil && ctx.Err() == nil {
|
||||
log.Printf("session export handoff continuation failed: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
|
||||
@@ -190,7 +190,8 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job != nil && job.Status == "complete" {
|
||||
settled, _ := s.GetRun(ctx, id)
|
||||
if job != nil && job.Status == "complete" && settled.Status == "succeeded" {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
@@ -203,7 +204,7 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, _ := json.Marshal(detail.Steps[0].SessionExport)
|
||||
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != original.Status {
|
||||
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "failed" {
|
||||
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/opencode"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
@@ -25,8 +26,8 @@ const sessionExportBody = `{"info":{"id":"ses_run"},"messages":[` +
|
||||
`{"type":"text","text":"done"},` +
|
||||
`{"type":"tool","name":"shell","state":{"status":"completed","input":{"cmd":"ls"},"content":[{"type":"text","text":"output here"}]}}]}]}`
|
||||
|
||||
// sessionExportCLI points PIPELINES_OPENCODE at a fake `opencode export` that
|
||||
// returns sessionExportBody, so the runner's real export path runs.
|
||||
// sessionExportCLI isolates both the authenticated V2 API and legacy CLI.
|
||||
// Neither export transport may fall through to the operator's real service.
|
||||
func sessionExportCLI(t *testing.T) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "opencode")
|
||||
@@ -38,6 +39,28 @@ func sessionExportCLI(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PIPELINES_OPENCODE", path)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
user, password, ok := r.BasicAuth()
|
||||
if !ok || user != "opencode" || password != "fixture-password" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
if r.Method != "GET" || r.URL.Path != "/api/experimental/session/ses_run/export" || r.URL.Query().Get("sanitize") != "false" {
|
||||
t.Errorf("unexpected export operation: %s %s", r.Method, r.URL.Path)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, `{"data":`+sessionExportBody+`}`)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
passwordPath := filepath.Join(t.TempDir(), "password")
|
||||
if err := os.WriteFile(passwordPath, []byte("fixture-password"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(opencode.EndpointEnvKey, srv.URL)
|
||||
t.Setenv(opencode.EndpointPasswordFileEnvKey, passwordPath)
|
||||
t.Setenv(opencode.EndpointUsernameEnvKey, "")
|
||||
}
|
||||
|
||||
// sessionExportRunner runs one stage that records a session for the run, so the
|
||||
@@ -145,7 +168,7 @@ func TestRunnerExportsTheHTMLSessionIntoArtifactsAndServesIt(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestRunnerSessionExportFailureTruncationAndUnavailableHarness proves the three
|
||||
// non-success export outcomes are recorded as evidence and never fail the run.
|
||||
// non-success exports block success without replacing the completed computation.
|
||||
func TestRunnerSessionExportFailureTruncationAndUnavailableHarness(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -188,8 +211,8 @@ func TestRunnerSessionExportFailureTruncationAndUnavailableHarness(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != "succeeded" {
|
||||
t.Fatalf("export outcome failed the run: %+v", detail.Status)
|
||||
if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "Session export gate blocked") || detail.Output["summary"] != "done" {
|
||||
t.Fatalf("export gate did not preserve and block computation: %+v", detail)
|
||||
}
|
||||
step := detail.Steps[0]
|
||||
if step.Status != "completed" || step.Outcome != "success" {
|
||||
|
||||
@@ -61,6 +61,7 @@ esac
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := Handler(ctx, store)
|
||||
handler.(*runHandler).runner.exportSession = validGateExport
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
|
||||
@@ -116,6 +116,7 @@ func TestRunnerRecordsPerStageUsageAcrossMixedModels(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.exportSession = validGateExport
|
||||
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "mixed", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -99,6 +99,15 @@ type Budget struct {
|
||||
func NewBudget(usd float64, tokens int64) *Budget {
|
||||
return &Budget{state: BudgetSnapshot{LimitUSD: usd, LimitTokens: tokens}, records: map[string]BudgetSnapshot{}, pollInterval: time.Second}
|
||||
}
|
||||
|
||||
// RestoreUsage seeds persisted earlier visits when a durable handoff resumes.
|
||||
// New sessions have new accounting keys; prior records must not be polled again.
|
||||
func (b *Budget) RestoreUsage(used BudgetSnapshot) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
used.LimitUSD, used.LimitTokens = b.state.LimitUSD, b.state.LimitTokens
|
||||
b.state = used
|
||||
}
|
||||
func (b *Budget) Enabled() bool { return b != nil && (b.state.LimitUSD > 0 || b.state.LimitTokens > 0) }
|
||||
func (b *Budget) Snapshot() BudgetSnapshot { b.mu.Lock(); defer b.mu.Unlock(); return b.state }
|
||||
|
||||
|
||||
Reference in New Issue
Block a user