runner: keep export handoffs nonterminal through recovery
This commit is contained in:
@@ -7,8 +7,10 @@ reads and final export attempts have bounded contexts independent of execution
|
||||
cancellation. Unconfirmed cleanup never exports or advances.
|
||||
|
||||
An export failure retains the computed step, outcome, target, output and file
|
||||
evidence. The run fails closed with an export-gate diagnostic, not an invented
|
||||
model failure. A separate `session_export_gates` checkpoint is committed with the
|
||||
evidence. The run parks nonterminally in `waiting`, with
|
||||
`export_pending_sequence` identifying the computed visit and no `finished_at`.
|
||||
This is not a human decision or a model failure. A separate
|
||||
`session_export_gates` checkpoint is committed with the
|
||||
step and its existing durable export job. Model retry is refused for that
|
||||
checkpoint: recovery reads only the original session. Missing exact ownership
|
||||
cannot be reconstructed or guessed and remains blocked.
|
||||
@@ -18,6 +20,15 @@ The checkpoint reserves the workspace durably even though no agent is executing.
|
||||
`PendingSessionExportGate(runID, sequence)` identifies the exact quiescent visit
|
||||
eligible for export-only repair. Cancellation releases the reservation without
|
||||
discarding or replaying the computation, and prevents automatic continuation.
|
||||
The repair handler must permit `waiting` only when that predicate is true for
|
||||
the requested sequence; unrelated human/resource waits remain ineligible.
|
||||
|
||||
Both CLI wait paths keep waiting for export recovery; `run --wait` still pauses
|
||||
for genuine human decisions. Lifecycle events carry the same pending sequence,
|
||||
and a lifecycle frame with zero/absent sequence clears it. Status-less repair
|
||||
freshness frames leave it unchanged. Continuation is `waiting -> running`, never
|
||||
terminal -> running. Startup normalizes historical failed/interrupted export
|
||||
checkpoints back to waiting before clients attach.
|
||||
|
||||
After a successful automatic or manual repair, the worker re-verifies ownership
|
||||
(including backend/reference) and the actual file, then rebuilds memory, turn
|
||||
|
||||
+5
-5
@@ -1461,16 +1461,16 @@ func (a App) wait(ctx context.Context, pos []string, o *options, call callFunc)
|
||||
return nil
|
||||
}
|
||||
|
||||
// awaitRuns polls every run that active reports as unfinished until each is
|
||||
// terminal, using the same GetRun reattachment the run --wait path uses, and
|
||||
// calls onTerminal for each as it finishes (including runs already terminal at
|
||||
// awaitRuns polls unfinished runs using GetRun reattachment. Export waits remain
|
||||
// unfinished even for run --wait, whose status-only predicate intentionally stops
|
||||
// at human decisions. It calls onTerminal for each as it finishes (including runs already terminal at
|
||||
// entry, in the order supplied). A deadline that passes first returns a
|
||||
// *waitLimitError naming the runs still active; ctx ending first returns
|
||||
// ctx.Err(). It never cancels, retries or otherwise changes a run.
|
||||
func (a App) awaitRuns(ctx context.Context, runs []protocol.Message, deadline <-chan time.Time, call callFunc, active func(string) bool, onTerminal func(protocol.Message) error) error {
|
||||
remaining := make([]protocol.Message, 0, len(runs))
|
||||
for _, r := range runs {
|
||||
if active(r.RunDetail.Status) {
|
||||
if active(r.RunDetail.Status) || (r.RunDetail.Status == protocol.RunStatusWaiting && r.RunDetail.ExportPendingSequence > 0) {
|
||||
remaining = append(remaining, r)
|
||||
continue
|
||||
}
|
||||
@@ -1495,7 +1495,7 @@ func (a App) awaitRuns(ctx context.Context, runs []protocol.Message, deadline <-
|
||||
if refreshed.RunDetail == nil {
|
||||
return fmt.Errorf("server did not return run detail for %d", r.RunDetail.ID)
|
||||
}
|
||||
if active(refreshed.RunDetail.Status) {
|
||||
if active(refreshed.RunDetail.Status) || (refreshed.RunDetail.Status == protocol.RunStatusWaiting && refreshed.RunDetail.ExportPendingSequence > 0) {
|
||||
next = append(next, refreshed)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func TestRunAndExplicitWaitDoNotReturnDuringExportRecovery(t *testing.T) {
|
||||
for _, mode := range []string{"run --wait", "wait"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
active := runInProgress
|
||||
if mode == "wait" {
|
||||
active = runUnfinished
|
||||
}
|
||||
initial := protocol.Message{RunDetail: &protocol.RunDetail{RunSummary: protocol.RunSummary{ID: 7, Status: protocol.RunStatusWaiting, ExportPendingSequence: 2}}}
|
||||
polls, finished := 0, false
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
err := (App{}).awaitRuns(ctx, []protocol.Message{initial}, nil, func(request protocol.Message) (protocol.Message, error) {
|
||||
polls++
|
||||
if request.Type != protocol.GetRun || request.RunID != 7 {
|
||||
t.Errorf("unexpected request: %+v", request)
|
||||
}
|
||||
return protocol.Message{RunDetail: &protocol.RunDetail{RunSummary: protocol.RunSummary{ID: 7, Status: "succeeded"}}}, nil
|
||||
}, active, func(message protocol.Message) error {
|
||||
if message.RunDetail.Status != "succeeded" {
|
||||
t.Errorf("wait returned before actual terminal: %+v", message.RunDetail)
|
||||
}
|
||||
finished = true
|
||||
return nil
|
||||
})
|
||||
if err != nil || polls != 1 || !finished {
|
||||
t.Fatalf("export wait ended early: polls=%d finished=%v error=%v", polls, finished, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -22,12 +22,13 @@ const runStamps = `strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),COALESCE(strftime
|
||||
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
|
||||
` + runStamps + `,r.diagnostic,
|
||||
CASE WHEN r.status='succeeded' THEN r.output_json ELSE COALESCE((SELECT output_json FROM run_steps WHERE run_id=r.id AND status='completed' ORDER BY sequence DESC LIMIT 1),'{}') END,
|
||||
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode,r.cancel_requested,r.cleanup_unconfirmed,r.output_format
|
||||
(SELECT COUNT(*) FROM run_steps WHERE run_id=r.id AND status='completed'),r.workspace,r.title,r.model,r.stage_models,r.notify_mode,r.cancel_requested,r.cleanup_unconfirmed,r.output_format,
|
||||
CASE WHEN r.status='waiting' THEN COALESCE((SELECT sequence FROM session_export_gates WHERE run_id=r.id),0) ELSE 0 END
|
||||
` + runFrom + ` JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version`
|
||||
|
||||
func scanRun(row interface{ Scan(...any) error }, r *protocol.RunSummary) error {
|
||||
var output, stageModels string
|
||||
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat); err != nil {
|
||||
if err := row.Scan(&r.ID, &r.PipelineID, &r.Name, &r.Version, &r.ContentHash, &r.Status, &r.Outcome, &r.CurrentStage, &r.CreatedAt, &r.FinishedAt, &r.Diagnostic, &output, &r.CompletedSteps, &r.Workspace, &r.Title, &r.Model, &stageModels, &r.NotifyMode, &r.CancelRequested, &r.CleanupUnconfirmed, &r.OutputFormat, &r.ExportPendingSequence); err != nil {
|
||||
return err
|
||||
}
|
||||
if stageModels != "" && stageModels != "{}" {
|
||||
|
||||
@@ -168,6 +168,9 @@ WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome,
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO session_export_gates(run_id,sequence) VALUES(?,?)`, id, step.Sequence); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := affected(tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='waiting',outcome='',diagnostic='Session export pending; computed visit retained',output_json=?,finished_at=NULL WHERE id=? AND status='running'`, string(output), id)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// The reported usage the stage already measured for budget accounting is
|
||||
// persisted as step-scoped evidence in the same transaction, mirroring the
|
||||
@@ -226,6 +229,13 @@ func (s *Store) ResolveDecision(ctx context.Context, id int64, stageKey, option,
|
||||
return protocol.RunStep{}, false, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var exportPending bool
|
||||
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM session_export_gates WHERE run_id=?)`, id).Scan(&exportPending); err != nil {
|
||||
return protocol.RunStep{}, false, err
|
||||
}
|
||||
if exportPending {
|
||||
return protocol.RunStep{}, false, fmt.Errorf("run %d is awaiting session export, not a human decision", id)
|
||||
}
|
||||
var status, currentStage, graphJSON string
|
||||
if err := tx.QueryRowContext(ctx, `SELECT r.status,r.current_stage,v.graph_json FROM pipeline_runs r
|
||||
JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version WHERE r.id=?`, id).Scan(&status, ¤tStage, &graphJSON); err != nil {
|
||||
@@ -253,6 +263,9 @@ JOIN pipeline_versions v ON v.pipeline_id=r.pipeline_id AND v.version=r.version
|
||||
found := false
|
||||
for _, declared := range graph.Stages {
|
||||
if declared.Key == stageKey {
|
||||
if stage.EffectiveType(declared.Type) != stage.HumanType {
|
||||
return protocol.RunStep{}, false, fmt.Errorf("stage %s is not a human decision stage", stageKey)
|
||||
}
|
||||
edges = declared.Edges
|
||||
found = true
|
||||
break
|
||||
@@ -324,7 +337,7 @@ func (s *Store) CancelWaiting(ctx context.Context, id int64) error {
|
||||
}
|
||||
defer tx.Rollback()
|
||||
res, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='canceled',cancel_requested=1,
|
||||
diagnostic=CASE WHEN status='waiting' THEN 'Canceled while waiting for a decision' ELSE 'Canceled while awaiting session export; computed visit retained' END,
|
||||
diagnostic=CASE WHEN EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id) THEN 'Canceled while awaiting session export; computed visit retained' ELSE 'Canceled while waiting for a decision' END,
|
||||
finished_at=CURRENT_TIMESTAMP WHERE id=? AND (status=? OR (status IN ('failed','interrupted') AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=pipeline_runs.id)))`, id, protocol.RunStatusWaiting)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -204,5 +204,14 @@ func (s *Store) RecoverRuns(ctx context.Context) error {
|
||||
if _, err := tx.ExecContext(ctx, "UPDATE pipeline_runs SET status=CASE WHEN cancel_requested=1 THEN 'canceled' ELSE 'interrupted' END,diagnostic='Server stopped; owned sessions stopped; run was not resumed',finished_at=CURRENT_TIMESTAMP WHERE status IN ('queued','running') AND status<>? AND cleanup_unconfirmed=0", protocol.RunStatusWaiting); err != nil {
|
||||
return err
|
||||
}
|
||||
// An export checkpoint is a paused handoff, not an interrupted model call.
|
||||
// Also migrate the earlier failed-gate representation before clients attach.
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE pipeline_runs SET status='waiting',outcome='',finished_at=NULL,diagnostic='Session export pending; computed visit retained'
|
||||
WHERE status IN ('queued','running','failed','interrupted','waiting') AND cancel_requested=0 AND cleanup_unconfirmed=0
|
||||
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN run_steps s ON s.run_id=g.run_id AND s.sequence=g.sequence
|
||||
WHERE g.run_id=pipeline_runs.id AND s.status IN ('completed','failed') AND s.agent_active=0
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.sequence>g.sequence OR a.agent_active<>0)))`); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ func (s *Store) ReadySessionExportGates(ctx context.Context) (map[int64]int, err
|
||||
rows, err := s.db.QueryContext(ctx, `SELECT g.run_id,g.sequence FROM session_export_gates g
|
||||
JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence
|
||||
JOIN pipeline_runs r ON r.id=g.run_id
|
||||
WHERE j.status='complete' AND r.status IN ('failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
|
||||
WHERE j.status='complete' AND r.status IN ('waiting','failed','interrupted') AND r.cancel_requested=0 AND r.cleanup_unconfirmed=0
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_active<>0 OR a.sequence>g.sequence)) AND `+exactExportJobOwner+` LIMIT 64`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -67,8 +67,8 @@ AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=g.run_id AND (a.agent_ac
|
||||
|
||||
// ResumeSessionExportGate is a CAS, not a retry of the model invocation.
|
||||
func (s *Store) ResumeSessionExportGate(ctx context.Context, id int64, sequence int) error {
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET status='queued',outcome='',diagnostic='',finished_at=NULL
|
||||
WHERE id=? AND status IN ('failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE pipeline_runs SET status='running',outcome='',diagnostic='',finished_at=NULL
|
||||
WHERE id=? AND status IN ('waiting','failed','interrupted') AND cancel_requested=0 AND cleanup_unconfirmed=0
|
||||
AND EXISTS(SELECT 1 FROM session_export_gates g JOIN session_export_jobs j ON j.run_id=g.run_id AND j.sequence=g.sequence WHERE g.run_id=pipeline_runs.id AND g.sequence=? AND j.status='complete' AND `+exactExportJobOwner+`)
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE a.run_id=pipeline_runs.id AND (a.agent_active<>0 OR a.sequence>?))`, id, sequence, sequence))
|
||||
}
|
||||
|
||||
@@ -18,8 +18,12 @@ func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) {
|
||||
if err := s.BeginStep(ctx, id, 2, "two", nil); err == nil {
|
||||
t.Fatal("successor started while export was incomplete")
|
||||
}
|
||||
if err := s.FinishRun(ctx, id, "failed", "error", "export gate", step.Output); err != nil {
|
||||
t.Fatal(err)
|
||||
paused, err := s.GetRun(ctx, id)
|
||||
if err != nil || paused.Status != "waiting" || paused.ExportPendingSequence != 1 || paused.FinishedAt != "" {
|
||||
t.Fatalf("gate became terminal: %+v %v", paused, err)
|
||||
}
|
||||
if _, _, err := s.ResolveDecision(ctx, id, "one", "success", ""); err == nil {
|
||||
t.Fatal("model export wait accepted human decision")
|
||||
}
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
|
||||
@@ -40,8 +44,17 @@ func TestSessionExportGateCheckpointAndSuccessorAreAtomic(t *testing.T) {
|
||||
}
|
||||
// A crash after reserving the continuation but before starting its next
|
||||
// executor must retain the checkpoint for startup's interrupted run.
|
||||
if _, err := s.db.Exec(`UPDATE pipeline_runs SET status='interrupted' WHERE id=?`, id); err != nil {
|
||||
t.Fatal(err)
|
||||
for _, prior := range []string{"failed", "interrupted", "running", "queued"} {
|
||||
if _, err := s.db.Exec(`UPDATE pipeline_runs SET status=?,finished_at=CURRENT_TIMESTAMP WHERE id=?`, prior, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.RecoverRuns(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recovered, err := s.GetRun(ctx, id)
|
||||
if err != nil || recovered.Status != "waiting" || recovered.ExportPendingSequence != 1 || recovered.FinishedAt != "" {
|
||||
t.Fatalf("restart exposed a terminal export wait from %s: %+v %v", prior, recovered, err)
|
||||
}
|
||||
}
|
||||
ready, err := s.ReadySessionExportGates(ctx)
|
||||
if err != nil || ready[id] != 1 {
|
||||
|
||||
@@ -139,7 +139,7 @@ func (s *Store) ClaimSessionExportJob(ctx context.Context, now time.Time, lease
|
||||
err = tx.QueryRowContext(ctx, `SELECT j.run_id,j.sequence,j.session_id,j.backend,j.directory,j.reference,j.status,j.attempts,j.next_at,j.diagnostic
|
||||
FROM session_export_jobs j JOIN pipeline_runs r ON r.id=j.run_id JOIN run_steps s ON s.run_id=j.run_id AND s.sequence=j.sequence
|
||||
WHERE ((j.status='pending' AND j.next_at<=?) OR (j.status='running' AND j.lease_until<=?)) AND j.attempts<?
|
||||
AND r.status IN ('succeeded','failed','canceled','interrupted') AND s.status NOT IN ('running','waiting')`+guard+`
|
||||
AND (r.status IN ('succeeded','failed','canceled','interrupted') OR (r.status='waiting' AND EXISTS(SELECT 1 FROM session_export_gates g WHERE g.run_id=j.run_id AND g.sequence=j.sequence))) AND s.status NOT IN ('running','waiting')`+guard+`
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE (a.run_id=j.run_id OR a.session_id=j.session_id) AND a.agent_active<>0)
|
||||
ORDER BY j.next_at,j.run_id,j.sequence LIMIT 1`, now.UnixMilli(), now.UnixMilli(), SessionExportJobAttempts).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic)
|
||||
if err == sql.ErrNoRows {
|
||||
|
||||
@@ -37,18 +37,20 @@ type RunEventFrame struct {
|
||||
Workspace string `json:"workspace,omitempty"`
|
||||
// PipelineID, Name, Title and Version identify the run's pipeline and its
|
||||
// operator-facing title, so a new run's row can be built without a fetch.
|
||||
PipelineID int64 `json:"pipeline_id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Version int `json:"version,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
CancelRequested bool `json:"cancel_requested,omitempty"`
|
||||
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
CurrentStage string `json:"current_stage,omitempty"`
|
||||
CompletedSteps int `json:"completed_steps,omitempty"`
|
||||
CreatedAt string `json:"created_at,omitempty"`
|
||||
FinishedAt string `json:"finished_at,omitempty"`
|
||||
PipelineID int64 `json:"pipeline_id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Version int `json:"version,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
// On a lifecycle frame, zero/absence clears a prior export wait.
|
||||
ExportPendingSequence int `json:"export_pending_sequence,omitempty"`
|
||||
CancelRequested bool `json:"cancel_requested,omitempty"`
|
||||
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
CurrentStage string `json:"current_stage,omitempty"`
|
||||
CompletedSteps int `json:"completed_steps,omitempty"`
|
||||
CreatedAt string `json:"created_at,omitempty"`
|
||||
FinishedAt string `json:"finished_at,omitempty"`
|
||||
// Artifacts is the number of files recorded for the referenced step, so a
|
||||
// client can surface artifact capture without reading the step.
|
||||
Artifacts int `json:"artifacts,omitempty"`
|
||||
|
||||
+18
-15
@@ -142,21 +142,24 @@ type RunSummary struct {
|
||||
// Title is the operator-facing, bounded single-line description of what the
|
||||
// run is for, generated from its prompt. It is never empty for a new run;
|
||||
// historical rows predating the title column may omit it.
|
||||
Title string `json:"title,omitempty"`
|
||||
Version int `json:"version"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
Status string `json:"status"`
|
||||
CancelRequested bool `json:"cancel_requested,omitempty"`
|
||||
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
|
||||
Executing bool `json:"executing"`
|
||||
Outcome string `json:"outcome"`
|
||||
CurrentStage string `json:"current_stage"`
|
||||
Workspace string `json:"workspace,omitempty"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
FinishedAt string `json:"finished_at"`
|
||||
Diagnostic string `json:"diagnostic"`
|
||||
ResultPreview string `json:"result_preview,omitempty"`
|
||||
CompletedSteps int `json:"completed_steps"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Version int `json:"version"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
Status string `json:"status"`
|
||||
// ExportPendingSequence distinguishes a quiescent export handoff from a
|
||||
// human decision while Status is waiting. It never implies agent execution.
|
||||
ExportPendingSequence int `json:"export_pending_sequence,omitempty"`
|
||||
CancelRequested bool `json:"cancel_requested,omitempty"`
|
||||
CleanupUnconfirmed bool `json:"cleanup_unconfirmed,omitempty"`
|
||||
Executing bool `json:"executing"`
|
||||
Outcome string `json:"outcome"`
|
||||
CurrentStage string `json:"current_stage"`
|
||||
Workspace string `json:"workspace,omitempty"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
FinishedAt string `json:"finished_at"`
|
||||
Diagnostic string `json:"diagnostic"`
|
||||
ResultPreview string `json:"result_preview,omitempty"`
|
||||
CompletedSteps int `json:"completed_steps"`
|
||||
// Model and StageModels are the launch-time overrides this run records so it
|
||||
// stays reproducible: Model covers every model stage and StageModels overrides
|
||||
// individual stages by key. They are empty for a run that used the pinned
|
||||
|
||||
@@ -451,6 +451,7 @@ func mergeRunEventFrame(into, in *protocol.RunEventFrame) {
|
||||
}
|
||||
if in.Status != "" {
|
||||
into.Status = in.Status
|
||||
into.ExportPendingSequence = in.ExportPendingSequence
|
||||
}
|
||||
if in.Outcome != "" {
|
||||
into.Outcome = in.Outcome
|
||||
@@ -509,22 +510,23 @@ func handleRunEvents(request protocol.Message, events *runEventHub, runs *runner
|
||||
continue
|
||||
}
|
||||
frame := protocol.RunEventFrame{
|
||||
GraphEvidence: summary.GraphEvidence,
|
||||
NotifyMode: summary.NotifyMode,
|
||||
RunID: summary.ID,
|
||||
Workspace: summary.Workspace,
|
||||
PipelineID: summary.PipelineID,
|
||||
Name: summary.Name,
|
||||
Title: summary.Title,
|
||||
Version: summary.Version,
|
||||
Status: summary.Status,
|
||||
Outcome: summary.Outcome,
|
||||
CurrentStage: summary.CurrentStage,
|
||||
CompletedSteps: summary.CompletedSteps,
|
||||
CreatedAt: summary.CreatedAt,
|
||||
FinishedAt: summary.FinishedAt,
|
||||
CancelRequested: summary.CancelRequested,
|
||||
CleanupUnconfirmed: summary.CleanupUnconfirmed,
|
||||
GraphEvidence: summary.GraphEvidence,
|
||||
NotifyMode: summary.NotifyMode,
|
||||
RunID: summary.ID,
|
||||
Workspace: summary.Workspace,
|
||||
PipelineID: summary.PipelineID,
|
||||
Name: summary.Name,
|
||||
Title: summary.Title,
|
||||
Version: summary.Version,
|
||||
Status: summary.Status,
|
||||
Outcome: summary.Outcome,
|
||||
CurrentStage: summary.CurrentStage,
|
||||
CompletedSteps: summary.CompletedSteps,
|
||||
CreatedAt: summary.CreatedAt,
|
||||
FinishedAt: summary.FinishedAt,
|
||||
CancelRequested: summary.CancelRequested,
|
||||
CleanupUnconfirmed: summary.CleanupUnconfirmed,
|
||||
ExportPendingSequence: summary.ExportPendingSequence,
|
||||
}
|
||||
if !sub.writeSnapshot(protocol.Message{Type: protocol.RunEvent, RunEvent: &frame}) {
|
||||
// The connection write failed; the subscription is closed and the
|
||||
|
||||
@@ -335,6 +335,9 @@ func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.Ru
|
||||
}
|
||||
if detail != nil {
|
||||
_, detail.Executing = r.active[id]
|
||||
if detail.ExportPendingSequence > 0 {
|
||||
detail.Executing = false
|
||||
}
|
||||
r.overlayEffectiveModels(detail)
|
||||
}
|
||||
if readErr != nil {
|
||||
@@ -421,6 +424,9 @@ func (r *runner) getRun(id int64) (*protocol.RunDetail, error) {
|
||||
d, err := r.store.GetRun(r.ctx, id)
|
||||
if d != nil {
|
||||
_, d.Executing = r.active[id]
|
||||
if d.ExportPendingSequence > 0 {
|
||||
d.Executing = false
|
||||
}
|
||||
for _, s := range d.Steps {
|
||||
if s.AgentActive {
|
||||
d.Executing = true
|
||||
@@ -467,6 +473,9 @@ func (r *runner) listRuns() ([]protocol.RunSummary, error) {
|
||||
items, err := r.store.ListRuns(r.ctx)
|
||||
for i := range items {
|
||||
_, items[i].Executing = r.active[items[i].ID]
|
||||
if items[i].ExportPendingSequence > 0 {
|
||||
items[i].Executing = false
|
||||
}
|
||||
if items[i].Status == "failed" && !items[i].Executing {
|
||||
d, err := r.store.GetRun(r.ctx, items[i].ID)
|
||||
if err != nil {
|
||||
@@ -797,6 +806,10 @@ func (r *runner) decide(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
r.mu.Unlock()
|
||||
return nil, fmt.Errorf("run %d is already active; wait for it to pause before deciding", detail.ID)
|
||||
}
|
||||
if detail.ExportPendingSequence > 0 {
|
||||
r.mu.Unlock()
|
||||
return nil, fmt.Errorf("run %d is awaiting session export, not a human decision", detail.ID)
|
||||
}
|
||||
if detail.Status == protocol.RunStatusWaiting {
|
||||
if err := r.checkWorkspaceFree(detail.Workspace); err != nil {
|
||||
r.mu.Unlock()
|
||||
@@ -1978,15 +1991,19 @@ func (r *runner) runLoop(ctx context.Context, id int64, graph protocol.Graph, na
|
||||
finish("failed", "error", saveErr.Error())
|
||||
return
|
||||
}
|
||||
diagnostic := "Session export gate blocked; computed visit retained; export-only recovery pending"
|
||||
if sessionExport != nil && sessionExport.Diagnostic != "" {
|
||||
diagnostic += ": " + sessionExport.Diagnostic
|
||||
}
|
||||
if err != nil {
|
||||
diagnostic = err.Error() + "; " + diagnostic
|
||||
}
|
||||
// No finishStage, memory publication, recordTurn or next executor.
|
||||
r.finishRunState(id, name, workspace, "failed", "error", diagnostic, state.completed, sequence, key, stage.Values(step.Output))
|
||||
check, done := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
detail, readErr := r.store.GetRun(check, id)
|
||||
done()
|
||||
if readErr != nil {
|
||||
log.Printf("read parked export gate for run %d: %v", id, readErr)
|
||||
return // The checkpoint is durable; never fabricate a terminal.
|
||||
}
|
||||
if detail.CancelRequested && detail.ExportPendingSequence == 0 {
|
||||
finish("canceled", "", "Canceled by operator; computed visit retained")
|
||||
return
|
||||
}
|
||||
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: workspace, Status: protocol.RunStatusWaiting, ExportPendingSequence: sequence, CurrentStage: key, CompletedSteps: completedSteps(detail.Steps), Step: &protocol.RunStepDelta{Sequence: sequence, Stage: key, Status: step.Status, Outcome: step.Outcome, Target: step.Target, Diagnostic: step.Diagnostic, Artifacts: len(step.Artifacts)}}, true)
|
||||
return
|
||||
}
|
||||
// Generate the one-line verdict reason once, at completion. It is
|
||||
|
||||
@@ -92,6 +92,7 @@ func (r *runner) resumeExportGates(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
step := *saved
|
||||
r.emitRun(protocol.RunEventFrame{RunID: id, Workspace: d.Workspace, Status: protocol.RunStatusRunning, CurrentStage: step.Stage, CompletedSteps: completedSteps(d.Steps)}, true)
|
||||
r.launch(id, d.Workspace, func(runCtx context.Context) { r.resume(runCtx, id, d, d.Workspace, step) })
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
@@ -111,7 +111,7 @@ func TestSessionExportGateRetryOnlyAndExactlyOnce(t *testing.T) {
|
||||
h.runner.wg.Wait()
|
||||
ctx := context.Background()
|
||||
d, _ := s.GetRun(ctx, id)
|
||||
if d.Status != "failed" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
|
||||
if d.Status != "waiting" || d.ExportPendingSequence != 1 || d.FinishedAt != "" || len(d.Steps) != 1 || d.Steps[0].Target != "two" || d.Output["markdown"] != "computed output" || second.Load() != 0 {
|
||||
t.Fatalf("lost computation/advanced: %+v", d)
|
||||
}
|
||||
h.runner.exportSession = validGateExport
|
||||
@@ -214,7 +214,7 @@ func TestSessionExportGateMissingOwnerFailsClosed(t *testing.T) {
|
||||
}, false, gateTestOptions{missingOwner: true})
|
||||
h.runner.wg.Wait()
|
||||
d, _ := s.GetRun(context.Background(), id)
|
||||
if d.Status != "failed" || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
|
||||
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || exports.Load() != 0 || d.Output["markdown"] != "computed output" || d.Steps[0].SessionID != "ses_exact" {
|
||||
t.Fatalf("missing owner did not fail closed: %+v", d)
|
||||
}
|
||||
}
|
||||
@@ -281,7 +281,7 @@ func TestSessionExportGateRejectsIncompleteAndInvalidFiles(t *testing.T) {
|
||||
}, false)
|
||||
h.runner.wg.Wait()
|
||||
d, _ := s.GetRun(context.Background(), id)
|
||||
if d.Status != "failed" || second.Load() != 0 || d.Output["markdown"] != "computed output" {
|
||||
if d.Status != "waiting" || d.ExportPendingSequence != 1 || second.Load() != 0 || d.Output["markdown"] != "computed output" {
|
||||
t.Fatalf("invalid export advanced: %+v", d)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -204,7 +204,7 @@ func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, _ := json.Marshal(detail.Steps[0].SessionExport)
|
||||
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "failed" {
|
||||
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != "succeeded" || original.Status != "waiting" {
|
||||
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -211,7 +211,7 @@ func TestRunnerSessionExportFailureTruncationAndUnavailableHarness(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if detail.Status != "failed" || !strings.Contains(detail.Diagnostic, "Session export gate blocked") || detail.Output["summary"] != "done" {
|
||||
if detail.Status != "waiting" || detail.ExportPendingSequence != 1 || detail.FinishedAt != "" || detail.Output["summary"] != "done" {
|
||||
t.Fatalf("export gate did not preserve and block computation: %+v", detail)
|
||||
}
|
||||
step := detail.Steps[0]
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func TestExportWaitEventCoalescingClearsOnlyOnLifecycleTransition(t *testing.T) {
|
||||
frame := protocol.RunEventFrame{RunID: 7, Status: "waiting", ExportPendingSequence: 2}
|
||||
mergeRunEventFrame(&frame, &protocol.RunEventFrame{RunID: 7, Artifacts: 1})
|
||||
if frame.ExportPendingSequence != 2 {
|
||||
t.Fatal("repair-only frame cleared export pending")
|
||||
}
|
||||
mergeRunEventFrame(&frame, &protocol.RunEventFrame{RunID: 7, Status: "running"})
|
||||
if frame.ExportPendingSequence != 0 || frame.Status != "running" {
|
||||
t.Fatalf("continuation did not clear export wait: %+v", frame)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportWaitPrivateRepairResumesWithoutTerminalRegression(t *testing.T) {
|
||||
h, s, id, first, second := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
return stage.SessionExportResult{}, errors.New("offline")
|
||||
}, false)
|
||||
h.runner.wg.Wait()
|
||||
ctx := context.Background()
|
||||
job, err := s.SessionExportJob(ctx, id, 1)
|
||||
if err != nil || job == nil {
|
||||
t.Fatalf("missing recovery checkpoint: %v", err)
|
||||
}
|
||||
name := "session-repair-1-private-test.html"
|
||||
result, err := validGateExport(ctx, stage.SessionExportRequest{ArtifactsDir: h.runner.turnPaths(id, 1, "one").TurnDir, FileName: name})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
evidence := &protocol.SessionExport{Status: "complete", SessionID: job.SessionID, Backend: job.Backend, Directory: job.Directory, Source: "native_export", HTML: name, HTMLBytes: result.HTMLBytes, Messages: result.Messages}
|
||||
if err := s.RecordDeliverable(ctx, id, 1, protocol.StageArtifact{Name: name, SizeBytes: result.HTMLBytes, ContentType: "text/html"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Reconciliation recognizes a durable manual repair without another export.
|
||||
if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || claimed != nil {
|
||||
t.Fatalf("repair triggered a redundant export: %+v %v", claimed, err)
|
||||
}
|
||||
if err := h.runner.resumeExportGates(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.runner.wg.Wait()
|
||||
if err := h.runner.resumeExportGates(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := h.runner.getRun(id)
|
||||
if err != nil || d.Status != "succeeded" || d.ExportPendingSequence != 0 || first.Load() != 1 || second.Load() != 1 {
|
||||
t.Fatalf("private repair replayed or lost continuation: %+v %v", d, err)
|
||||
}
|
||||
input, err := os.ReadFile(h.runner.turnPaths(id, 2, "two").InputPath)
|
||||
if err != nil || !strings.Contains(string(input), "computed output") || strings.Contains(string(input), name) {
|
||||
t.Fatalf("private export leaked or computed output lost: %q %v", input, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportWaitSubscriptionSnapshotCarriesPendingSequence(t *testing.T) {
|
||||
h, _, id, _, _ := gateRunner(t, func(context.Context, stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
return stage.SessionExportResult{}, errors.New("offline")
|
||||
}, false)
|
||||
h.runner.wg.Wait()
|
||||
hub := newRunEventHub()
|
||||
defer hub.unsubscribeClient("export-wait-test")
|
||||
var snapshot *protocol.RunEventFrame
|
||||
if !handleRunEvents(protocol.Message{Type: protocol.Subscribe, RunIDs: []int64{id}}, hub, h.runner, "export-wait-test", func(message protocol.Message) error {
|
||||
if message.RunEvent != nil {
|
||||
snapshot = message.RunEvent
|
||||
}
|
||||
return nil
|
||||
}) {
|
||||
t.Fatal("subscribe not handled")
|
||||
}
|
||||
if snapshot == nil || snapshot.Status != "waiting" || snapshot.ExportPendingSequence != 1 || snapshot.FinishedAt != "" {
|
||||
t.Fatalf("snapshot lost nonterminal export wait: %+v", snapshot)
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,9 @@ func lastWaitingStep(r *protocol.RunDetail) (protocol.RunStep, bool) {
|
||||
// waiting step and falls back to the pinned graph's edges when a step recorded
|
||||
// no options.
|
||||
func (m Model) waitingStage() (string, []string, string) {
|
||||
if m.runDetail != nil && m.runDetail.ExportPendingSequence > 0 {
|
||||
return m.runDetail.CurrentStage, nil, ""
|
||||
}
|
||||
if m.runDetail == nil {
|
||||
return "", nil, ""
|
||||
}
|
||||
@@ -43,7 +46,7 @@ func (m Model) waitingStage() (string, []string, string) {
|
||||
}
|
||||
|
||||
func (m Model) openDecision() (tea.Model, tea.Cmd) {
|
||||
if m.runDetail == nil || m.runDetail.Status != protocol.RunStatusWaiting {
|
||||
if m.runDetail == nil || m.runDetail.Status != protocol.RunStatusWaiting || m.runDetail.ExportPendingSequence > 0 {
|
||||
m.runError = "This run is not waiting for a decision."
|
||||
return m, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package tui
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func TestExportWaitRendersRecoveryNotHumanDecision(t *testing.T) {
|
||||
run := &protocol.RunDetail{RunSummary: protocol.RunSummary{ID: 7, Status: "waiting", ExportPendingSequence: 2, CurrentStage: "worker"}, Graph: protocol.Graph{Stages: []protocol.Stage{{Key: "worker", Type: "model", Edges: map[string]string{"success": "next", "error": "#failure"}}}}}
|
||||
frame := strings.Join(waitingLines(run), "\n")
|
||||
if !strings.Contains(frame, "Waiting for session export") || strings.Contains(frame, "person") || strings.Contains(frame, "Decide:") {
|
||||
t.Fatalf("export wait offered a human choice:\n%s", frame)
|
||||
}
|
||||
m := Model{runDetail: run}
|
||||
_, options, _ := m.waitingStage()
|
||||
if len(options) != 0 {
|
||||
t.Fatal("model-stage edges exposed as human options")
|
||||
}
|
||||
updated, _ := m.openDecision()
|
||||
if updated.(Model).decisionOpen {
|
||||
t.Fatal("opened decision for export wait")
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,9 @@ func resultLabel(r protocol.RunSummary) string {
|
||||
}
|
||||
return "Finished · no result recorded"
|
||||
}
|
||||
if r.Status == protocol.RunStatusWaiting && r.ExportPendingSequence > 0 {
|
||||
return "Waiting for session export · stage " + r.CurrentStage
|
||||
}
|
||||
if r.Diagnostic != "" {
|
||||
return "Stopped: " + ansi.Strip(r.Diagnostic)
|
||||
}
|
||||
@@ -111,6 +114,9 @@ func waitingLines(r *protocol.RunDetail) []string {
|
||||
if r.Status != protocol.RunStatusWaiting {
|
||||
return nil
|
||||
}
|
||||
if r.ExportPendingSequence > 0 {
|
||||
return []string{fmt.Sprintf("⏸ Waiting for session export · stage %s · visit %d", r.CurrentStage, r.ExportPendingSequence), "Computed output retained; automatic export-only recovery pending."}
|
||||
}
|
||||
lines := []string{"⏸ Waiting for a person · stage " + r.CurrentStage}
|
||||
if step, ok := lastWaitingStep(r); ok {
|
||||
if prompt := strings.TrimSpace(protocol.WaitingPrompt(step.Output)); prompt != "" {
|
||||
|
||||
@@ -657,7 +657,7 @@ func (m Model) runActions() string {
|
||||
if m.runDetail != nil {
|
||||
text += " · a evidence · A artifacts"
|
||||
}
|
||||
if m.runDetail != nil && m.runDetail.Status == protocol.RunStatusWaiting {
|
||||
if m.runDetail != nil && m.runDetail.Status == protocol.RunStatusWaiting && m.runDetail.ExportPendingSequence == 0 {
|
||||
text += " · c decide"
|
||||
}
|
||||
if m.selectedFailedRun() > 0 {
|
||||
@@ -740,7 +740,7 @@ func (m Model) runsDetailsView(height int) string {
|
||||
page := max(0, innerHeight-5)
|
||||
top := max(0, min(m.runTop, max(0, len(lines)-page)))
|
||||
footerHint := "o result · d diff · s progress · i input · t evidence · m details · A artifacts"
|
||||
if r.Status == protocol.RunStatusWaiting {
|
||||
if r.Status == protocol.RunStatusWaiting && r.ExportPendingSequence == 0 {
|
||||
footerHint += " · c decide"
|
||||
}
|
||||
footer := bar(footerHint, fmt.Sprintf("%d–%d / %d", min(top+1, len(lines)), min(top+page, len(lines)), len(lines)), max(1, m.width-4), false)
|
||||
|
||||
+3
-1
@@ -96,6 +96,7 @@ export function decisionTargetLabel(option, target) {
|
||||
*/
|
||||
export function pendingDecision(run) {
|
||||
if (run == null || run.status !== RUN_STATUS_WAITING) return null;
|
||||
if (Number(run.export_pending_sequence) > 0) return null;
|
||||
const step = lastWaitingStep(run.steps ?? []);
|
||||
const stepStage = str(step?.stage).trim() !== "" ? String(step.stage) : "";
|
||||
const runStage = str(run.current_stage).trim() !== "" ? String(run.current_stage) : "";
|
||||
@@ -103,6 +104,7 @@ export function pendingDecision(run) {
|
||||
if (stageKey === "") return null;
|
||||
|
||||
const declared = (run.graph?.stages ?? []).find((candidate) => candidate?.key === stageKey) ?? null;
|
||||
if (declared?.type && declared.type !== "human") return null;
|
||||
const recorded = waitingOptions(step?.output);
|
||||
const keys = recorded.length > 0 ? recorded : humanOptions(declared?.edges);
|
||||
if (keys.length === 0) return null;
|
||||
@@ -154,7 +156,7 @@ export class DecisionDraft {
|
||||
|
||||
/** Cancel is offered only while the run is executing. */
|
||||
export function canCancel(run) {
|
||||
return Boolean(run?.executing) && !run.cancel_requested && !run.cleanup_unconfirmed;
|
||||
return (Boolean(run?.executing) || (run?.status === RUN_STATUS_WAITING && Number(run.export_pending_sequence) > 0)) && !run.cancel_requested && !run.cleanup_unconfirmed;
|
||||
}
|
||||
|
||||
/** Retry is offered only for a loaded, non-executing, `failed` run. */
|
||||
|
||||
@@ -33,7 +33,8 @@ export const AttentionKind = Object.freeze({
|
||||
* so queued, running, succeeded, canceled, interrupted and any unknown status
|
||||
* can never invent an entry.
|
||||
*/
|
||||
export function attentionKindOf(status) {
|
||||
export function attentionKindOf(status, exportPendingSequence = 0) {
|
||||
if (Number(exportPendingSequence) > 0 && String(status ?? "").trim().toLowerCase() === "waiting") return null;
|
||||
switch (String(status ?? "").trim().toLowerCase()) {
|
||||
case "waiting":
|
||||
return AttentionKind.Decision;
|
||||
@@ -99,7 +100,7 @@ export function attentionEntries(runs, dismissed, { since = 0 } = {}) {
|
||||
dismissed instanceof Set ? dismissed : new Set(Array.isArray(dismissed) ? dismissed : []);
|
||||
return (runs ?? [])
|
||||
.map((run) => {
|
||||
const kind = attentionKindOf(run?.status);
|
||||
const kind = attentionKindOf(run?.status, run?.export_pending_sequence);
|
||||
if (!kind) return null;
|
||||
const key = attentionKey(run.id, kind);
|
||||
if (dismissedKeys.has(key)) return null;
|
||||
|
||||
+1
-1
@@ -122,7 +122,7 @@ export function createAlertLedger() {
|
||||
if (frame?.notify_mode != null) modes.set(runId, normalizeStatus(frame.notify_mode));
|
||||
const mode = modes.get(runId) || "auto";
|
||||
|
||||
const kind = status === "succeeded" && mode === "on" ? RunAlertKind.Succeeded : attentionKindOf(status);
|
||||
const kind = status === "succeeded" && mode === "on" ? RunAlertKind.Succeeded : attentionKindOf(status, frame?.export_pending_sequence);
|
||||
if (!kind) {
|
||||
forget(runId, AttentionKind.Decision);
|
||||
forget(runId, AttentionKind.Failed);
|
||||
|
||||
@@ -183,6 +183,7 @@ function runStateFields(current) {
|
||||
executing: Boolean(current.executing),
|
||||
cancel_requested: Boolean(current.cancel_requested),
|
||||
cleanup_unconfirmed: Boolean(current.cleanup_unconfirmed),
|
||||
export_pending_sequence: Number(current.export_pending_sequence) || 0,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -206,6 +207,9 @@ function applyRunStateFields(current, frame) {
|
||||
? (frame.resource_wait ?? null)
|
||||
: before.resource_wait,
|
||||
status,
|
||||
...(before.export_pending_sequence || Number(frame.export_pending_sequence) > 0 || Object.hasOwn(current, "export_pending_sequence")
|
||||
? { export_pending_sequence: blank(frame.status) ? before.export_pending_sequence : status === "waiting" ? Number(frame.export_pending_sequence) || 0 : 0 }
|
||||
: {}),
|
||||
outcome: blank(frame.outcome) ? before.outcome : frame.outcome,
|
||||
current_stage: blank(frame.current_stage) ? before.current_stage : frame.current_stage,
|
||||
completed_steps: Math.max(before.completed_steps, Number(frame.completed_steps) || 0),
|
||||
@@ -230,7 +234,8 @@ function applyRunStateFields(current, frame) {
|
||||
merged.executing !== before.executing ||
|
||||
merged.cancel_requested !== before.cancel_requested ||
|
||||
merged.cleanup_unconfirmed !== before.cleanup_unconfirmed;
|
||||
return changed ? { ...current, ...merged, ...(graphEvidence ? { graph_evidence: graphEvidence } : {}) } : current;
|
||||
const exportWaitChanged = (Number(merged.export_pending_sequence) || 0) !== before.export_pending_sequence;
|
||||
return changed || exportWaitChanged ? { ...current, ...merged, ...(graphEvidence ? { graph_evidence: graphEvidence } : {}) } : current;
|
||||
}
|
||||
|
||||
// Reuse the detail's monotonic visit merge, then compact by last stage visit and
|
||||
@@ -280,6 +285,7 @@ export function newRunSummaryFromFrame(frame) {
|
||||
title: blank(frame.title) ? null : frame.title,
|
||||
version: Number(frame.version) || 0,
|
||||
status: frame.status ?? "",
|
||||
...(Number(frame.export_pending_sequence) > 0 ? { export_pending_sequence: Number(frame.export_pending_sequence) } : {}),
|
||||
executing: executingFor(frame.status, true),
|
||||
cancel_requested: frame.cancel_requested === true,
|
||||
cleanup_unconfirmed: frame.cleanup_unconfirmed === true,
|
||||
|
||||
@@ -41,6 +41,7 @@ export function runStatusLabel(run) {
|
||||
if (["queued", "running", "waiting"].includes(run?.status)) {
|
||||
if (run.cleanup_unconfirmed) return "Cleanup unconfirmed — workspace reserved";
|
||||
if (run.cancel_requested) return "Stopping — waiting for execution to stop";
|
||||
if (run.status === "waiting" && Number(run.export_pending_sequence) > 0) return "Waiting for session export";
|
||||
}
|
||||
return run?.status || "unknown";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { foldRunEventDetail, foldRunEvent, isTerminalRunStatus } from "../src/runEvents.js";
|
||||
import { canCancel, canRetry, pendingDecision } from "../src/actions.js";
|
||||
import { runStatusLabel } from "../src/runs.js";
|
||||
import { attentionEntries, attentionKindOf } from "../src/attention.js";
|
||||
import { createAlertLedger } from "../src/notify.js";
|
||||
|
||||
test("export waiting remains nonterminal and folds forward without reopening its computed visit", () => {
|
||||
let run = { id: 7, status: "running", executing: true, steps: [{ sequence: 1, stage: "one", status: "running" }], graph: { stages: [{ key: "one", type: "model", edges: { success: "two", error: "#failure" } }] } };
|
||||
const waiting = { run_id: 7, status: "waiting", export_pending_sequence: 1, current_stage: "one", step: { sequence: 1, stage: "one", status: "completed", outcome: "success", target: "two" } };
|
||||
run = foldRunEventDetail(run, waiting);
|
||||
assert.equal(run.status, "waiting");
|
||||
assert.equal(run.export_pending_sequence, 1);
|
||||
assert.equal(run.executing, false);
|
||||
assert.equal(isTerminalRunStatus(run.status), false);
|
||||
assert.equal(pendingDecision(run), null);
|
||||
assert.equal(canCancel(run), true);
|
||||
assert.equal(canRetry(run), false);
|
||||
assert.equal(runStatusLabel(run), "Waiting for session export");
|
||||
assert.equal(attentionKindOf(run.status, run.export_pending_sequence), null);
|
||||
assert.deepEqual(attentionEntries([run], new Set()), []);
|
||||
const ledger = createAlertLedger();
|
||||
ledger.seed([{ id: 7, status: "running" }]);
|
||||
assert.equal(ledger.observe(waiting), null);
|
||||
const snapshot = foldRunEvent([], { ...waiting, pipeline_id: 3, name: "gate" }).runs[0];
|
||||
assert.equal(snapshot.export_pending_sequence, 1);
|
||||
// Repair-only frames must not clear the pending sequence.
|
||||
run = foldRunEventDetail(run, { run_id: 7, session_export_updated: true });
|
||||
assert.equal(run.export_pending_sequence, 1);
|
||||
run = foldRunEventDetail(run, { run_id: 7, status: "running" });
|
||||
assert.equal(run.export_pending_sequence, 0);
|
||||
assert.equal(run.steps[0].status, "completed");
|
||||
run = foldRunEventDetail(run, { run_id: 7, status: "succeeded", finished_at: "2026-10-20T20:00:00Z" });
|
||||
assert.equal(run.status, "succeeded");
|
||||
assert.equal(run.export_pending_sequence, 0);
|
||||
assert.equal(run.executing, false);
|
||||
assert.equal(foldRunEventDetail(run, waiting), run);
|
||||
});
|
||||
|
||||
test("model stages never become human decisions via waiting fallback", () => {
|
||||
assert.equal(pendingDecision({ id: 8, status: "waiting", current_stage: "one", steps: [], graph: { stages: [{ key: "one", type: "model", edges: { success: "two" } }] } }), null);
|
||||
});
|
||||
Reference in New Issue
Block a user