Recover incomplete session exports through durable runtime jobs
This commit is contained in:
@@ -898,3 +898,9 @@ command is `repair_session_export` with `run_id` and `export_sequence`; the repl
|
||||
is a run detail containing each visit's `session_export_repairs`. CLI JSON emits
|
||||
the repair record with explicit completeness `status`, and incomplete repair
|
||||
exits nonzero. Recovery still depends on upstream session availability.
|
||||
|
||||
Incomplete native stage-session exports also recover automatically after the
|
||||
run settles: up to three durable runtime attempts, with backoff and restart-safe
|
||||
leases. `inspect RUN_ID --json` includes each visit's `session_export_recovery`
|
||||
status/attempt count/diagnostic and preserved `session_export_repairs`. Exhausted
|
||||
automatic recovery remains a warning; manual versioned repair is still available.
|
||||
|
||||
@@ -43,6 +43,42 @@ There are at most 20 manual repair attempts per visit. Repairs cannot guarantee
|
||||
recovery if the upstream session or original directory is gone. The web pane
|
||||
explains original failure and lists repair attempts/links separately.
|
||||
|
||||
## Durable automatic recovery
|
||||
|
||||
Incomplete native exports now enqueue a `session_export_jobs` row in the same
|
||||
transaction as `FinishStep`. The snapshot records the exact original session ID,
|
||||
backend, directory and harness reference. Runtime startup also reconciles missing
|
||||
jobs from persisted ownership/steps, including interrupted initial capture; it
|
||||
never guesses missing ownership or edits original evidence. Reconciliation is
|
||||
bounded to 256 newly discovered jobs per sweep (startup, then every 30 seconds).
|
||||
|
||||
A runtime-owned worker claims jobs only for settled runs/steps, with no active
|
||||
agent in that run or using that session, and with `cleanup_unconfirmed=0` on
|
||||
current main. The runner's in-memory cleanup gate also defers execution without
|
||||
spending a retry. Each automatic attempt uses the existing versioned, read-only
|
||||
repair path and verifies ownership against the durable snapshot before readback.
|
||||
No stage prompt, pipeline definition, stage execution or run outcome changes.
|
||||
|
||||
The first attempt is due after 5 seconds; further attempts wait 20 and 60 seconds.
|
||||
There are at most 3 automatic claims, each with the existing 10-second readback
|
||||
budget. A 45-second persisted lease recovers interrupted workers after restart;
|
||||
attempt counts/backoff are not reset, including process kills. Saved complete
|
||||
repair evidence can acknowledge a job after a crash without exporting again.
|
||||
Expired final-attempt leases become exhausted, not an infinite retry loop.
|
||||
Wrong-session readbacks never qualify as complete evidence. Failed/empty/partial
|
||||
repairs and exhausted-job diagnostics remain visible in run inspection and the
|
||||
web visit pane (`session_export_recovery`). Originals remain unchanged.
|
||||
|
||||
Worker shutdown follows runner context cancellation and current main's
|
||||
`Store.Done()` (including cancellation of in-flight readback). The small startup
|
||||
helper detects that lifecycle interface without modifying this branch's older
|
||||
store/server-lock implementation. Its compatibility fallback exits on a closed
|
||||
database; current main's lifecycle/cleanup implementations must be retained when
|
||||
integrating. Upstream loss, missing original ownership/directory, inventory
|
||||
limits or sessions exceeding export caps can still leave exhausted evidence;
|
||||
manual versioned repair remains available. Browser captures for recovery status
|
||||
are not captured under the no-live-runs audit constraint.
|
||||
|
||||
No production service, live run or historical data was modified for this audit.
|
||||
Unit/fixture tests cover reports, failure gates, retry inheritance, idempotency,
|
||||
migration, transient/permanent export readback, empty-export diagnostics and
|
||||
|
||||
@@ -186,6 +186,13 @@ strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ'
|
||||
return nil, err
|
||||
}
|
||||
for i := range r.Steps {
|
||||
var recovery protocol.SessionExportRecovery
|
||||
jobErr := tx.QueryRowContext(ctx, "SELECT status,attempts,next_at,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?", id, r.Steps[i].Sequence).Scan(&recovery.Status, &recovery.Attempts, &recovery.NextAt, &recovery.Diagnostic)
|
||||
if jobErr == nil {
|
||||
r.Steps[i].SessionExportRecovery = &recovery
|
||||
} else if jobErr != sql.ErrNoRows {
|
||||
return nil, jobErr
|
||||
}
|
||||
repairs, err := tx.QueryContext(ctx, "SELECT export_json FROM session_export_repairs WHERE run_id=? AND sequence=? ORDER BY id", id, r.Steps[i].Sequence)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -145,6 +145,9 @@ WHERE run_id=? AND sequence=? AND status='running'`, step.Status, step.Outcome,
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := enqueueStepExportJob(ctx, tx, id, step.Sequence); err != nil {
|
||||
return err
|
||||
}
|
||||
// The reported usage the stage already measured for budget accounting is
|
||||
// persisted as step-scoped evidence in the same transaction, mirroring the
|
||||
// other step evidence. It is written only when the stage actually reported
|
||||
|
||||
@@ -79,7 +79,10 @@ CREATE TABLE IF NOT EXISTS retry_checkpoints (
|
||||
if err := s.dropResourceSchema(); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.migrateRunModels()
|
||||
if err := s.migrateRunModels(); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.initSessionExportJobs()
|
||||
}
|
||||
|
||||
// dropResourceSchema removes the retired resource pool, ownership, wait queue
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
const SessionExportJobAttempts = 3
|
||||
|
||||
// SessionExportJob is an immutable ownership snapshot plus durable recovery
|
||||
// state. Attempts count recovery operations, not the export CLI's inner reads.
|
||||
type SessionExportJob struct {
|
||||
RunID int64 `json:"run_id"`
|
||||
Sequence int `json:"sequence"`
|
||||
SessionID string `json:"session_id"`
|
||||
Backend string `json:"backend"`
|
||||
Directory string `json:"directory"`
|
||||
Reference string `json:"-"`
|
||||
Status string `json:"status"`
|
||||
Attempts int `json:"attempts"`
|
||||
NextAt int64 `json:"next_attempt_at_ms"`
|
||||
Diagnostic string `json:"diagnostic,omitempty"`
|
||||
Token string `json:"-"`
|
||||
}
|
||||
|
||||
func (s *Store) initSessionExportJobs() error {
|
||||
_, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS session_export_jobs (
|
||||
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL,
|
||||
session_id TEXT NOT NULL, backend TEXT NOT NULL, directory TEXT NOT NULL, reference TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','running','complete','exhausted')), attempts INTEGER NOT NULL DEFAULT 0 CHECK(attempts BETWEEN 0 AND 3),
|
||||
next_at INTEGER NOT NULL, lease_until INTEGER NOT NULL DEFAULT 0, token TEXT NOT NULL DEFAULT '', diagnostic TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence) ON DELETE CASCADE);
|
||||
CREATE INDEX IF NOT EXISTS session_export_jobs_due ON session_export_jobs(status,next_at,lease_until);`)
|
||||
return err
|
||||
}
|
||||
|
||||
// Missing or incomplete original evidence is recoverable only with an exact
|
||||
// native ownership record. No backend guesses, prompt parsing or stage reruns.
|
||||
const enqueueExportJobs = `INSERT INTO session_export_jobs(run_id,sequence,session_id,backend,directory,reference,next_at)
|
||||
SELECT s.run_id,s.sequence,s.session_id,o.backend,r.workspace,o.reference,?
|
||||
FROM run_steps s JOIN pipeline_runs r ON r.id=s.run_id
|
||||
JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence
|
||||
LEFT JOIN step_session_exports x ON x.run_id=s.run_id AND x.sequence=s.sequence
|
||||
WHERE s.status NOT IN ('running','waiting') AND s.session_id<>'' AND o.backend IN ('v1','v2')
|
||||
AND NOT EXISTS(SELECT 1 FROM session_export_jobs j WHERE j.run_id=s.run_id AND j.sequence=s.sequence)
|
||||
AND NOT (COALESCE(json_extract(x.export_json,'$.status'),'complete')='complete'
|
||||
AND COALESCE(json_extract(x.export_json,'$.session_id'),'')=s.session_id
|
||||
AND COALESCE(json_extract(x.export_json,'$.backend'),'')=o.backend
|
||||
AND COALESCE(json_extract(x.export_json,'$.messages'),0)>0
|
||||
AND COALESCE(json_extract(x.export_json,'$.html'),'')<>''
|
||||
AND COALESCE(json_extract(x.export_json,'$.html_bytes'),0)>0
|
||||
AND COALESCE(json_extract(x.export_json,'$.truncated'),0)=0
|
||||
AND COALESCE(json_extract(x.export_json,'$.diagnostic'),'')='')`
|
||||
|
||||
func enqueueStepExportJob(ctx context.Context, tx *sql.Tx, id int64, sequence int) error {
|
||||
_, err := tx.ExecContext(ctx, enqueueExportJobs+` AND s.run_id=? AND s.sequence=? ON CONFLICT(run_id,sequence) DO NOTHING`, time.Now().Add(5*time.Second).UnixMilli(), id, sequence)
|
||||
return err
|
||||
}
|
||||
|
||||
// ReconcileSessionExportJobs recovers interrupted initial capture and historical
|
||||
// missing jobs from source-of-truth step ownership. Existing schedules, exhausted
|
||||
// budgets and completed jobs are never reset, even across repeated restarts.
|
||||
func (s *Store) ReconcileSessionExportJobs(ctx context.Context, now time.Time) error {
|
||||
_, err := s.db.ExecContext(ctx, enqueueExportJobs+` LIMIT 256 ON CONFLICT(run_id,sequence) DO NOTHING`, now.Add(5*time.Second).UnixMilli())
|
||||
return err
|
||||
}
|
||||
|
||||
const exactExportJobOwner = `EXISTS(SELECT 1 FROM run_steps s JOIN run_session_owners o ON o.run_id=s.run_id AND o.sequence=s.sequence JOIN pipeline_runs r ON r.id=s.run_id WHERE s.run_id=j.run_id AND s.sequence=j.sequence AND s.session_id=j.session_id AND o.backend=j.backend AND o.reference=j.reference AND r.workspace=j.directory)`
|
||||
|
||||
// A complete repair must match the job's identity and have nonempty verified
|
||||
// file evidence. This also closes the crash window after repair persistence but
|
||||
// before acknowledgement of its job lease, without writing another export.
|
||||
const completeExportJobEvidence = `EXISTS(SELECT 1 FROM session_export_repairs p WHERE p.run_id=j.run_id AND p.sequence=j.sequence
|
||||
AND json_extract(p.export_json,'$.status')='complete'
|
||||
AND json_extract(p.export_json,'$.session_id')=j.session_id
|
||||
AND json_extract(p.export_json,'$.backend')=j.backend
|
||||
AND json_extract(p.export_json,'$.directory')=j.directory
|
||||
AND json_extract(p.export_json,'$.messages')>0 AND json_extract(p.export_json,'$.html_bytes')>0
|
||||
AND COALESCE(json_extract(p.export_json,'$.html'),'')<>''
|
||||
AND COALESCE(json_extract(p.export_json,'$.truncated'),0)=0
|
||||
AND COALESCE(json_extract(p.export_json,'$.diagnostic'),'')='')`
|
||||
|
||||
// cleanupGuard supports the older audit branch without editing its store or
|
||||
// cancellation schema. On current main the persisted flag is mandatory in the
|
||||
// claim predicate. The legacy schema has no cleanup-unconfirmed state.
|
||||
func (s *Store) exportJobCleanupGuard(ctx context.Context) (string, error) {
|
||||
rows, err := s.db.QueryContext(ctx, "PRAGMA table_info(pipeline_runs)")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var cid, notnull, pk int
|
||||
var name, kind string
|
||||
var def sql.NullString
|
||||
if err := rows.Scan(&cid, &name, &kind, ¬null, &def, &pk); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if name == "cleanup_unconfirmed" {
|
||||
return " AND r.cleanup_unconfirmed=0", nil
|
||||
}
|
||||
}
|
||||
return "", rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) ClaimSessionExportJob(ctx context.Context, now time.Time, lease time.Duration) (*SessionExportJob, error) {
|
||||
if lease <= 0 {
|
||||
return nil, fmt.Errorf("session export recovery lease must be positive")
|
||||
}
|
||||
guard, err := s.exportJobCleanupGuard(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
// A persisted successful repair wins, including after a process restart.
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='complete',token='',lease_until=0,next_at=0,diagnostic='' WHERE status<>'complete' AND `+completeExportJobEvidence); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='exhausted',token='',lease_until=0,next_at=0,diagnostic='Original session ownership changed or is unavailable; automatic recovery refused' WHERE status IN ('pending','running') AND NOT `+exactExportJobOwner); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs SET status='exhausted',token='',lease_until=0,next_at=0,diagnostic='Recovery budget exhausted; last worker stopped before acknowledgement' WHERE (status='pending' OR (status='running' AND lease_until<=?)) AND attempts>=?`, now.UnixMilli(), SessionExportJobAttempts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var job SessionExportJob
|
||||
err = tx.QueryRowContext(ctx, `SELECT j.run_id,j.sequence,j.session_id,j.backend,j.directory,j.reference,j.status,j.attempts,j.next_at,j.diagnostic
|
||||
FROM session_export_jobs j JOIN pipeline_runs r ON r.id=j.run_id JOIN run_steps s ON s.run_id=j.run_id AND s.sequence=j.sequence
|
||||
WHERE ((j.status='pending' AND j.next_at<=?) OR (j.status='running' AND j.lease_until<=?)) AND j.attempts<?
|
||||
AND r.status IN ('succeeded','failed','canceled','interrupted') AND s.status NOT IN ('running','waiting')`+guard+`
|
||||
AND NOT EXISTS(SELECT 1 FROM run_steps a WHERE (a.run_id=j.run_id OR a.session_id=j.session_id) AND a.agent_active<>0)
|
||||
ORDER BY j.next_at,j.run_id,j.sequence LIMIT 1`, now.UnixMilli(), now.UnixMilli(), SessionExportJobAttempts).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, tx.Commit()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var nonce [16]byte
|
||||
if _, err := rand.Read(nonce[:]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
job.Token = hex.EncodeToString(nonce[:])
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE session_export_jobs SET status='running',attempts=attempts+1,token=?,lease_until=? WHERE run_id=? AND sequence=?`, job.Token, now.Add(lease).UnixMilli(), job.RunID, job.Sequence); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
job.Attempts++
|
||||
job.Status = "running"
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &job, nil
|
||||
}
|
||||
|
||||
// DeferSessionExportJob returns a lease without spending a retry when the
|
||||
// runner's in-memory cleanup gate is still held. No native operation ran.
|
||||
func (s *Store) DeferSessionExportJob(ctx context.Context, job *SessionExportJob, now time.Time) error {
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE session_export_jobs SET status='pending',attempts=attempts-1,token='',lease_until=0,next_at=? WHERE run_id=? AND sequence=? AND status='running' AND token=?`, now.Add(5*time.Second).UnixMilli(), job.RunID, job.Sequence, job.Token))
|
||||
}
|
||||
|
||||
func (s *Store) FinishSessionExportJob(ctx context.Context, job *SessionExportJob, now time.Time, diagnostic string) error {
|
||||
// Never trust an optimistic worker return as proof of a full export.
|
||||
result, err := s.db.ExecContext(ctx, `UPDATE session_export_jobs AS j SET status='complete',token='',lease_until=0,next_at=0,diagnostic='' WHERE run_id=? AND sequence=? AND status='running' AND token=? AND `+completeExportJobEvidence, job.RunID, job.Sequence, job.Token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n, _ := result.RowsAffected(); n == 1 {
|
||||
return nil
|
||||
}
|
||||
status := "pending"
|
||||
delay := 20 * time.Second
|
||||
if job.Attempts == 2 {
|
||||
delay = 60 * time.Second
|
||||
}
|
||||
if job.Attempts >= SessionExportJobAttempts {
|
||||
status = "exhausted"
|
||||
}
|
||||
nextAt := now.Add(delay).UnixMilli()
|
||||
if status == "exhausted" {
|
||||
nextAt = 0
|
||||
}
|
||||
if diagnostic == "" {
|
||||
diagnostic = "Native session evidence remains incomplete; upstream recovery is not guaranteed"
|
||||
}
|
||||
if len(diagnostic) > 2048 {
|
||||
diagnostic = diagnostic[:2048]
|
||||
}
|
||||
return affected(s.db.ExecContext(ctx, `UPDATE session_export_jobs SET status=?,token='',lease_until=0,next_at=?,diagnostic=? WHERE run_id=? AND sequence=? AND status='running' AND token=?`, status, nextAt, diagnostic, job.RunID, job.Sequence, job.Token))
|
||||
}
|
||||
|
||||
func (s *Store) SessionExportJob(ctx context.Context, id int64, sequence int) (*SessionExportJob, error) {
|
||||
var job SessionExportJob
|
||||
err := s.db.QueryRowContext(ctx, `SELECT run_id,sequence,session_id,backend,directory,reference,status,attempts,next_at,diagnostic FROM session_export_jobs WHERE run_id=? AND sequence=?`, id, sequence).Scan(&job.RunID, &job.Sequence, &job.SessionID, &job.Backend, &job.Directory, &job.Reference, &job.Status, &job.Attempts, &job.NextAt, &job.Diagnostic)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("session export recovery: %w", err)
|
||||
}
|
||||
return &job, nil
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func exportJobFixture(t *testing.T) (*Store, int64, string) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "jobs.db")
|
||||
s, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { s.Close() })
|
||||
if _, err := s.Submit(ctx, "fixture", Template([]string{"one"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := s.List(ctx)
|
||||
definition, _ := s.Get(ctx, items[0].ID, 1)
|
||||
id, _, err := s.CreateRun(ctx, definition, "jobs", "/original/directory", false, nil, map[string]any{"prompt": "fixture"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.BeginStep(ctx, id, 1, "one", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.RunSession(ctx, id, 1, "ses_original", false, "v2", "original-reference"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s, id, path
|
||||
}
|
||||
|
||||
func finishExportJobStep(t *testing.T, s *Store, id int64) {
|
||||
t.Helper()
|
||||
if err := s.FinishStep(context.Background(), id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Output: map[string]any{}, SessionExport: &protocol.SessionExport{Status: "empty", Messages: 0, SessionID: "ses_original", Backend: "v2"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportJobEnqueueIsAtomicAndWaitsForSettlement(t *testing.T) {
|
||||
s, id, _ := exportJobFixture(t)
|
||||
ctx := context.Background()
|
||||
if _, err := s.db.Exec(`CREATE TRIGGER fail_usage BEFORE INSERT ON step_usage BEGIN SELECT RAISE(ABORT,'fixture usage failure'); END`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Output: map[string]any{}, Usage: &protocol.StageUsage{}})
|
||||
if err == nil {
|
||||
t.Fatal("fixture did not fail")
|
||||
}
|
||||
job, err := s.SessionExportJob(ctx, id, 1)
|
||||
if err != nil || job != nil {
|
||||
t.Fatalf("job escaped rolled-back step: %+v %v", job, err)
|
||||
}
|
||||
if _, err := s.db.Exec("DROP TRIGGER fail_usage"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
finishExportJobStep(t, s, id)
|
||||
job, err = s.SessionExportJob(ctx, id, 1)
|
||||
if err != nil || job == nil || job.SessionID != "ses_original" || job.Directory != "/original/directory" || job.Reference != "original-reference" {
|
||||
t.Fatalf("snapshot missing: %+v %v", job, err)
|
||||
}
|
||||
if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || claimed != nil {
|
||||
t.Fatalf("unsettled run claimed: %+v %v", claimed, err)
|
||||
}
|
||||
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt-1), time.Minute); err != nil || claimed != nil {
|
||||
t.Fatalf("backoff ignored: %+v %v", claimed, err)
|
||||
}
|
||||
claimed, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
|
||||
if err != nil || claimed == nil || claimed.Attempts != 1 {
|
||||
t.Fatalf("settled due job not claimed: %+v %v", claimed, err)
|
||||
}
|
||||
now := time.UnixMilli(job.NextAt)
|
||||
if err := s.FinishSessionExportJob(ctx, claimed, now, "empty"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, _ = s.SessionExportJob(ctx, id, 1)
|
||||
if job.Attempts != 1 || job.Status != "pending" || job.NextAt != now.Add(20*time.Second).UnixMilli() {
|
||||
t.Fatalf("first backoff wrong: %+v", job)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportJobBackoffRestartAndExhaustion(t *testing.T) {
|
||||
s, id, path := exportJobFixture(t)
|
||||
ctx := context.Background()
|
||||
finishExportJobStep(t, s, id)
|
||||
if err := s.FinishRun(ctx, id, "failed", "error", "original failure", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
now := time.UnixMilli(job.NextAt)
|
||||
claimed, err := s.ClaimSessionExportJob(ctx, now, time.Minute)
|
||||
if err != nil || claimed == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err = Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.ReconcileSessionExportJobs(ctx, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if other, err := s.ClaimSessionExportJob(ctx, now.Add(59*time.Second), time.Minute); err != nil || other != nil {
|
||||
t.Fatalf("live lease stolen: %+v %v", other, err)
|
||||
}
|
||||
recovered, err := s.ClaimSessionExportJob(ctx, now.Add(time.Minute), time.Minute)
|
||||
if err != nil || recovered == nil || recovered.Attempts != 2 || recovered.Token == claimed.Token {
|
||||
t.Fatalf("restart did not recover lease: %+v %v", recovered, err)
|
||||
}
|
||||
if err := s.FinishSessionExportJob(ctx, claimed, now, "stale worker"); err == nil {
|
||||
t.Fatal("stale lease overwrote recovery")
|
||||
}
|
||||
if err := s.FinishSessionExportJob(ctx, recovered, now, "upstream unavailable"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, _ = s.SessionExportJob(ctx, id, 1)
|
||||
if job.Status != "pending" || job.NextAt != now.Add(60*time.Second).UnixMilli() {
|
||||
t.Fatalf("second backoff wrong: %+v", job)
|
||||
}
|
||||
last, err := s.ClaimSessionExportJob(ctx, time.UnixMilli(job.NextAt), time.Minute)
|
||||
if err != nil || last == nil || last.Attempts != 3 {
|
||||
t.Fatalf("third attempt missing: %+v %v", last, err)
|
||||
}
|
||||
if err := s.FinishSessionExportJob(ctx, last, now.Add(time.Minute), "session gone"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.ReconcileSessionExportJobs(ctx, now.Add(time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, _ = s.SessionExportJob(ctx, id, 1)
|
||||
if job.Status != "exhausted" || job.Attempts != 3 || job.Diagnostic != "session gone" {
|
||||
t.Fatalf("budget reset: %+v", job)
|
||||
}
|
||||
if next, err := s.ClaimSessionExportJob(ctx, now.Add(24*time.Hour), time.Minute); err != nil || next != nil {
|
||||
t.Fatalf("infinite retry: %+v %v", next, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportJobGuardsCleanupAgentsAndExactOwnership(t *testing.T) {
|
||||
s, id, _ := exportJobFixture(t)
|
||||
ctx := context.Background()
|
||||
finishExportJobStep(t, s, id)
|
||||
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.db.Exec("ALTER TABLE pipeline_runs ADD COLUMN cleanup_unconfirmed INTEGER NOT NULL DEFAULT 0"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, queries := range [][2]string{
|
||||
{"UPDATE pipeline_runs SET cleanup_unconfirmed=1", "UPDATE pipeline_runs SET cleanup_unconfirmed=0"},
|
||||
{"UPDATE run_steps SET agent_active=1", "UPDATE run_steps SET agent_active=0"},
|
||||
} {
|
||||
if _, err := s.db.Exec(queries[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil {
|
||||
t.Fatalf("unsafe job claimed: %+v %v", job, err)
|
||||
}
|
||||
if _, err := s.db.Exec(queries[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='v1'"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil {
|
||||
t.Fatalf("ownership drift exported: %+v %v", job, err)
|
||||
}
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
if job.Status != "exhausted" || job.Attempts != 0 || job.Backend != "v2" {
|
||||
t.Fatalf("snapshot overwritten: %+v", job)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportJobRecoversPersistedSuccessWithoutAnotherRead(t *testing.T) {
|
||||
s, id, _ := exportJobFixture(t)
|
||||
ctx := context.Background()
|
||||
finishExportJobStep(t, s, id)
|
||||
if err := s.FinishRun(ctx, id, "succeeded", "success", "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute)
|
||||
if err != nil || job == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
evidence := &protocol.SessionExport{Status: "complete", SessionID: "ses_wrong", Backend: job.Backend, Directory: job.Directory, HTML: "public/repaired.html", HTMLBytes: 100, Messages: 2}
|
||||
if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.FinishSessionExportJob(ctx, job, time.Now(), "wrong identity"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, _ := s.SessionExportJob(ctx, id, 1)
|
||||
if state.Status == "complete" {
|
||||
t.Fatal("wrong-session evidence marked clean")
|
||||
}
|
||||
evidence.SessionID = job.SessionID
|
||||
if err := s.RecordSessionExportRepair(ctx, id, 1, evidence); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if claimed, err := s.ClaimSessionExportJob(ctx, time.Now().Add(24*time.Hour), time.Minute); err != nil || claimed != nil {
|
||||
t.Fatalf("successful persisted repair repeated: %+v %v", claimed, err)
|
||||
}
|
||||
state, _ = s.SessionExportJob(ctx, id, 1)
|
||||
if state.Status != "complete" || state.Attempts != 1 {
|
||||
t.Fatalf("acknowledgement not recovered: %+v", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportJobMigrationReconcilesMissingJobsWithoutReset(t *testing.T) {
|
||||
s, id, path := exportJobFixture(t)
|
||||
ctx := context.Background()
|
||||
finishExportJobStep(t, s, id)
|
||||
original, err := s.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.db.Exec("DROP TABLE session_export_jobs"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err = Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.ReconcileSessionExportJobs(ctx, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, err := s.SessionExportJob(ctx, id, 1)
|
||||
if err != nil || job == nil || job.SessionID != "ses_original" {
|
||||
t.Fatalf("historical incomplete export not reconciled: %+v %v", job, err)
|
||||
}
|
||||
if err := s.ReconcileSessionExportJobs(ctx, time.Now().Add(time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, _ := s.SessionExportJob(ctx, id, 1)
|
||||
if again.NextAt != job.NextAt || again.Attempts != job.Attempts {
|
||||
t.Fatal("restart reset retry schedule")
|
||||
}
|
||||
current, err := s.GetRun(ctx, id)
|
||||
if err != nil || current.Steps[0].SessionExport.Status != original.Steps[0].SessionExport.Status || current.Steps[0].SessionExport.Messages != 0 {
|
||||
t.Fatal("migration changed original evidence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportJobOnlyEnqueuesIncompleteNativeEvidence(t *testing.T) {
|
||||
for _, status := range []string{"complete", "failed", "empty", "partial", "missing", "unsupported", "wrong-session"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
s, id, _ := exportJobFixture(t)
|
||||
ctx := context.Background()
|
||||
evidence := &protocol.SessionExport{Status: status, SessionID: "ses_original", Backend: "v2", Messages: 2, HTML: "session.html", HTMLBytes: 100}
|
||||
if status == "missing" {
|
||||
evidence = nil
|
||||
}
|
||||
if status == "wrong-session" {
|
||||
evidence.Status = "complete"
|
||||
evidence.SessionID = "ses_wrong"
|
||||
}
|
||||
if status == "unsupported" {
|
||||
if _, err := s.db.Exec("UPDATE run_session_owners SET backend='pi'"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := s.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", SessionExport: evidence, Output: map[string]any{}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, err := s.SessionExportJob(ctx, id, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := status != "complete" && status != "unsupported"
|
||||
if (job != nil) != want {
|
||||
t.Fatalf("%s: %+v", status, job)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportJobFinalCrashedLeaseDoesNotResetBudget(t *testing.T) {
|
||||
s, id, _ := exportJobFixture(t)
|
||||
ctx := context.Background()
|
||||
finishExportJobStep(t, s, id)
|
||||
if err := s.FinishRun(ctx, id, "failed", "error", "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.db.Exec("UPDATE session_export_jobs SET status='running',attempts=3,lease_until=0,token='lost'"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job, err := s.ClaimSessionExportJob(ctx, time.Now().Add(time.Hour), time.Minute); err != nil || job != nil {
|
||||
t.Fatalf("final crash retried: %+v %v", job, err)
|
||||
}
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
if job.Status != "exhausted" || job.Attempts != 3 || job.NextAt != 0 {
|
||||
t.Fatalf("final lease not exhausted: %+v", job)
|
||||
}
|
||||
}
|
||||
+28
-19
@@ -125,6 +125,14 @@ type SessionExport struct {
|
||||
Diagnostic string `json:"diagnostic,omitempty"`
|
||||
}
|
||||
|
||||
// SessionExportRecovery describes the runtime's bounded durable repair job.
|
||||
type SessionExportRecovery struct {
|
||||
Status string `json:"status"`
|
||||
Attempts int `json:"attempts"`
|
||||
NextAt int64 `json:"next_attempt_at_ms"`
|
||||
Diagnostic string `json:"diagnostic,omitempty"`
|
||||
}
|
||||
|
||||
type RunSummary struct {
|
||||
NotifyMode string `json:"notify_mode,omitempty"`
|
||||
ID int64 `json:"id"`
|
||||
@@ -168,25 +176,26 @@ func (r RunSummary) Label() string {
|
||||
}
|
||||
|
||||
type RunStep struct {
|
||||
Control *ControlEvidence `json:"control,omitempty"`
|
||||
FileChanges *StageFileChanges `json:"file_changes,omitempty"`
|
||||
Verdict *StageVerdict `json:"verdict,omitempty"`
|
||||
SessionExport *SessionExport `json:"session_export,omitempty"`
|
||||
SessionExportRepairs []*SessionExport `json:"session_export_repairs,omitempty"`
|
||||
Usage *StageUsage `json:"usage,omitempty"`
|
||||
Notifications []NotificationReceipt `json:"notifications,omitempty"`
|
||||
Artifacts []StageArtifact `json:"artifacts,omitempty"`
|
||||
Sequence int `json:"sequence"`
|
||||
Stage string `json:"stage"`
|
||||
Status string `json:"status"`
|
||||
Outcome string `json:"outcome"`
|
||||
Target string `json:"target"`
|
||||
Input map[string]any `json:"input"`
|
||||
Output map[string]any `json:"output"`
|
||||
Diagnostic string `json:"diagnostic"`
|
||||
SessionID string `json:"session_id"`
|
||||
AgentActive bool `json:"agent_active"`
|
||||
Transcript string `json:"transcript"`
|
||||
Control *ControlEvidence `json:"control,omitempty"`
|
||||
FileChanges *StageFileChanges `json:"file_changes,omitempty"`
|
||||
Verdict *StageVerdict `json:"verdict,omitempty"`
|
||||
SessionExport *SessionExport `json:"session_export,omitempty"`
|
||||
SessionExportRepairs []*SessionExport `json:"session_export_repairs,omitempty"`
|
||||
SessionExportRecovery *SessionExportRecovery `json:"session_export_recovery,omitempty"`
|
||||
Usage *StageUsage `json:"usage,omitempty"`
|
||||
Notifications []NotificationReceipt `json:"notifications,omitempty"`
|
||||
Artifacts []StageArtifact `json:"artifacts,omitempty"`
|
||||
Sequence int `json:"sequence"`
|
||||
Stage string `json:"stage"`
|
||||
Status string `json:"status"`
|
||||
Outcome string `json:"outcome"`
|
||||
Target string `json:"target"`
|
||||
Input map[string]any `json:"input"`
|
||||
Output map[string]any `json:"output"`
|
||||
Diagnostic string `json:"diagnostic"`
|
||||
SessionID string `json:"session_id"`
|
||||
AgentActive bool `json:"agent_active"`
|
||||
Transcript string `json:"transcript"`
|
||||
// Provider and Model are the effective provider/model this visit actually ran
|
||||
// with, recorded for every model stage (including failures). They are empty
|
||||
// for non-model stages and historical steps.
|
||||
|
||||
@@ -118,6 +118,9 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
||||
}
|
||||
}
|
||||
r := &runner{ctx: ctx, store: s, artifactsRoot: root, active: active, idents: map[int64]runIdentity{}, attentionSent: map[int64]map[string]bool{}, notifyModes: map[int64]string{}, exportSession: stage.ExportSession}
|
||||
if err := r.startSessionExportJobs(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.schedulerWG.Add(1)
|
||||
go func() { defer r.schedulerWG.Done(); r.resumeRunNotifications() }()
|
||||
return r, nil
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
// startSessionExportJobs is the only runner startup integration. Current main's
|
||||
// Store.Done stops the worker and in-flight readback even with Background callers.
|
||||
// The interface avoids duplicating that newer store implementation/server lock.
|
||||
func (r *runner) startSessionExportJobs() error {
|
||||
if err := r.store.ReconcileSessionExportJobs(r.ctx, time.Now()); err != nil {
|
||||
return err
|
||||
}
|
||||
var done <-chan struct{}
|
||||
if lifecycle, ok := any(r.store).(interface{ Done() <-chan struct{} }); ok {
|
||||
done = lifecycle.Done()
|
||||
}
|
||||
ctx, cancel := sessionExportWorkerContext(r.ctx, done)
|
||||
r.schedulerWG.Add(1)
|
||||
go func() {
|
||||
defer r.schedulerWG.Done()
|
||||
defer cancel()
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
nextReconcile := time.Now().Add(30 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-done:
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
if time.Now().After(nextReconcile) {
|
||||
nextReconcile = time.Now().Add(30 * time.Second)
|
||||
if err := r.store.ReconcileSessionExportJobs(ctx, time.Now()); err != nil {
|
||||
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
|
||||
return
|
||||
}
|
||||
log.Printf("session export recovery reconciliation failed: %v", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Serial bounded batches; each readback has a 10-second budget.
|
||||
for range 8 {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
worked, err := r.recoverSessionExportJob(ctx, time.Now())
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || strings.Contains(err.Error(), "database is closed") {
|
||||
return
|
||||
}
|
||||
log.Printf("session export recovery failed: %v", err)
|
||||
break
|
||||
}
|
||||
if !worked {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func sessionExportWorkerContext(parent context.Context, done <-chan struct{}) (context.Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-done:
|
||||
cancel()
|
||||
}
|
||||
}()
|
||||
return ctx, cancel
|
||||
}
|
||||
|
||||
func (r *runner) recoverSessionExportJob(ctx context.Context, now time.Time) (bool, error) {
|
||||
job, err := r.store.ClaimSessionExportJob(ctx, now, 45*time.Second)
|
||||
if err != nil || job == nil {
|
||||
return false, err
|
||||
}
|
||||
r.mu.Lock()
|
||||
_, executing := r.active[job.RunID]
|
||||
r.mu.Unlock()
|
||||
if executing {
|
||||
return true, r.store.DeferSessionExportJob(ctx, job, now)
|
||||
}
|
||||
owner := &pipeline.SessionOwner{ID: job.SessionID, Backend: job.Backend, Directory: job.Directory, Reference: job.Reference, RunID: job.RunID}
|
||||
detail, repairErr := r.repairSessionExportOwned(ctx, job.RunID, job.Sequence, owner)
|
||||
diagnostic := ""
|
||||
if repairErr != nil {
|
||||
diagnostic = stage.RedactEvidence(repairErr.Error())
|
||||
} else if detail != nil {
|
||||
for _, step := range detail.Steps {
|
||||
if step.Sequence == job.Sequence && len(step.SessionExportRepairs) > 0 {
|
||||
last := step.SessionExportRepairs[len(step.SessionExportRepairs)-1]
|
||||
diagnostic = last.Diagnostic
|
||||
if diagnostic == "" && last.Status != "complete" {
|
||||
diagnostic = "Automatic native session export remains " + last.Status
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Save acknowledgement even on shutdown after an attempt. A process kill
|
||||
// before this point leaves a recoverable, budgeted lease.
|
||||
save, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer cancel()
|
||||
return true, r.store.FinishSessionExportJob(save, job, time.Now(), diagnostic)
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func recoveryRunner(t *testing.T) (*runHandler, *pipeline.Store, int64) {
|
||||
t.Helper()
|
||||
h, s, id := sessionExportRunner(t, "v2", func(stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
return stage.SessionExportResult{}, errors.New("initial readback unavailable")
|
||||
})
|
||||
// Synthetic executor has no agent process; model confirmed runtime cleanup.
|
||||
if err := s.ConfirmRecoveredSession(context.Background(), pipeline.SessionOwner{RunID: id, ID: "ses_run"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.runner.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
t.Error("export recovery reran a stage")
|
||||
return stage.Result{}, errors.New("forbidden rerun")
|
||||
}
|
||||
return h, s, id
|
||||
}
|
||||
|
||||
func TestAutomaticSessionExportRecoveryTransientThenComplete(t *testing.T) {
|
||||
h, s, id := recoveryRunner(t)
|
||||
ctx := context.Background()
|
||||
original, _ := s.GetRun(ctx, id)
|
||||
before, _ := json.Marshal(original.Steps[0].SessionExport)
|
||||
owner, _ := s.RepairSessionOwner(ctx, id, 1)
|
||||
var calls atomic.Int32
|
||||
h.runner.exportSession = func(ctx context.Context, request stage.SessionExportRequest) (stage.SessionExportResult, error) {
|
||||
if request.SessionID != owner.ID || request.Backend != owner.Backend || request.Directory != owner.Directory {
|
||||
t.Errorf("ownership changed: %+v", request)
|
||||
}
|
||||
if calls.Add(1) == 1 {
|
||||
return stage.SessionExportResult{}, errors.New("temporary upstream read failure")
|
||||
}
|
||||
body := []byte("<!doctype html><p>recorded messages</p>")
|
||||
if err := os.WriteFile(filepath.Join(request.ArtifactsDir, request.FileName), body, 0600); err != nil {
|
||||
return stage.SessionExportResult{}, err
|
||||
}
|
||||
return stage.SessionExportResult{Source: "native_export", Messages: 2, HTML: request.FileName, HTMLBytes: int64(len(body)), Attempts: 1}, nil
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt))
|
||||
if err != nil || !worked {
|
||||
t.Fatalf("attempt %d: %v %v", i, worked, err)
|
||||
}
|
||||
}
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
if job.Status != "complete" || job.Attempts != 2 || calls.Load() != 2 {
|
||||
t.Fatalf("recovery not complete: %+v calls=%d", job, calls.Load())
|
||||
}
|
||||
detail, _ := s.GetRun(ctx, id)
|
||||
after, _ := json.Marshal(detail.Steps[0].SessionExport)
|
||||
if string(before) != string(after) || detail.Status != original.Status || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 2 {
|
||||
t.Fatalf("original evidence/execution changed: %+v", detail)
|
||||
}
|
||||
if detail.Steps[0].SessionExportRecovery == nil || detail.Steps[0].SessionExportRecovery.Status != "complete" {
|
||||
t.Fatal("recovery state not visible")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutomaticSessionExportRecoveryWrongSessionExhaustsNeverClean(t *testing.T) {
|
||||
h, s, id := recoveryRunner(t)
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
calls := filepath.Join(dir, "calls")
|
||||
bin := filepath.Join(dir, "opencode")
|
||||
script := fmt.Sprintf("#!/bin/sh\necho read >> %q\necho '{\"info\":{\"id\":\"ses_wrong\"},\"messages\":[{\"info\":{\"role\":\"assistant\"},\"parts\":[{\"type\":\"text\",\"text\":\"wrong\"}]}]}'\n", calls)
|
||||
if err := os.WriteFile(bin, []byte(script), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PIPELINES_OPENCODE", bin)
|
||||
h.runner.exportSession = stage.ExportSession
|
||||
for range 3 {
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
|
||||
t.Fatalf("job failed: %v %v", worked, err)
|
||||
}
|
||||
}
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
if job.Status != "exhausted" || job.Attempts != 3 || !strings.Contains(job.Diagnostic, "identity mismatch") {
|
||||
t.Fatalf("wrong session looks clean: %+v", job)
|
||||
}
|
||||
if worked, err := h.runner.recoverSessionExportJob(ctx, time.Now().Add(24*time.Hour)); err != nil || worked {
|
||||
t.Fatalf("unbounded recovery: %v %v", worked, err)
|
||||
}
|
||||
body, _ := os.ReadFile(calls)
|
||||
if strings.Count(string(body), "read") != 3 {
|
||||
t.Fatalf("wrong-session read was retried internally: %s", body)
|
||||
}
|
||||
detail, _ := s.GetRun(ctx, id)
|
||||
if len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 3 {
|
||||
t.Fatalf("lost failed attempts: %+v", detail.Steps)
|
||||
}
|
||||
for _, repair := range detail.Steps[0].SessionExportRepairs {
|
||||
if repair.Status != "failed" || repair.HTML != "" {
|
||||
t.Fatalf("wrong session marked clean: %+v", repair)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutomaticSessionExportRecoveryDefersRuntimeCleanup(t *testing.T) {
|
||||
h, s, id := recoveryRunner(t)
|
||||
ctx := context.Background()
|
||||
h.runner.mu.Lock()
|
||||
h.runner.active[id] = activeRun{}
|
||||
h.runner.mu.Unlock()
|
||||
defer func() { h.runner.mu.Lock(); delete(h.runner.active, id); h.runner.mu.Unlock() }()
|
||||
job, _ := s.SessionExportJob(ctx, id, 1)
|
||||
if worked, err := h.runner.recoverSessionExportJob(ctx, time.UnixMilli(job.NextAt)); err != nil || !worked {
|
||||
t.Fatalf("deferral failed: %v %v", worked, err)
|
||||
}
|
||||
job, _ = s.SessionExportJob(ctx, id, 1)
|
||||
if job.Status != "pending" || job.Attempts != 0 {
|
||||
t.Fatalf("cleanup spent attempt: %+v", job)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportRecoveryWorkerStopsWithContextAndClosedStore(t *testing.T) {
|
||||
for _, closeStore := range []bool{false, true} {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s, err := pipeline.Open(filepath.Join(t.TempDir(), "worker.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := newRunner(ctx, s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if closeStore {
|
||||
s.Close()
|
||||
} else {
|
||||
cancel()
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() { r.schedulerWG.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("export worker did not stop")
|
||||
}
|
||||
cancel()
|
||||
s.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportRecoveryReopensAndRepairsAutomaticallyWithoutStageRerun(t *testing.T) {
|
||||
h, s, id := recoveryRunner(t)
|
||||
root := h.runner.artifactsRoot
|
||||
path := filepath.Join(s.Directory(), "session.db")
|
||||
original, _ := s.GetRun(context.Background(), id)
|
||||
before, _ := json.Marshal(original.Steps[0].SessionExport)
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := pipeline.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
sessionExportCLI(t) // read-only native CLI fixture, never a live model/run
|
||||
r, err := newRunner(ctx, s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { cancel(); r.schedulerWG.Wait(); s.Close() }()
|
||||
r.artifactsRoot = root
|
||||
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
||||
t.Error("restart recovery reran a stage")
|
||||
return stage.Result{}, errors.New("forbidden rerun")
|
||||
}
|
||||
deadline := time.Now().Add(9 * time.Second)
|
||||
for {
|
||||
job, err := s.SessionExportJob(ctx, id, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job != nil && job.Status == "complete" {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("persisted job did not auto-recover: %+v", job)
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
detail, err := s.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, _ := json.Marshal(detail.Steps[0].SessionExport)
|
||||
if string(before) != string(after) || len(detail.Steps) != 1 || len(detail.Steps[0].SessionExportRepairs) != 1 || detail.Status != original.Status {
|
||||
t.Fatalf("recovery overwrote history or reran stage: %+v", detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExportWorkerStoreDoneCancelsInflightReadbackContext(t *testing.T) {
|
||||
done := make(chan struct{})
|
||||
ctx, cancel := sessionExportWorkerContext(context.Background(), done)
|
||||
defer cancel()
|
||||
exportCtx, stop := context.WithTimeout(ctx, stage.SessionExportTimeout)
|
||||
defer stop()
|
||||
close(done)
|
||||
select {
|
||||
case <-exportCtx.Done():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Store.Done did not cancel in-flight readback context")
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
@@ -17,6 +18,10 @@ import (
|
||||
// repairSessionExport is runtime-owned and read-only to OpenCode. It never
|
||||
// resumes a session, runs a stage, changes a definition, or overwrites evidence.
|
||||
func (r *runner) repairSessionExport(ctx context.Context, id int64, sequence int) (*protocol.RunDetail, error) {
|
||||
return r.repairSessionExportOwned(ctx, id, sequence, nil)
|
||||
}
|
||||
|
||||
func (r *runner) repairSessionExportOwned(ctx context.Context, id int64, sequence int, expected *pipeline.SessionOwner) (*protocol.RunDetail, error) {
|
||||
r.exportRepairMu.Lock()
|
||||
defer r.exportRepairMu.Unlock()
|
||||
if id <= 0 || sequence <= 0 {
|
||||
@@ -52,6 +57,9 @@ func (r *runner) repairSessionExport(ctx context.Context, id int64, sequence int
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("exact original session ownership metadata unavailable: %w", err)
|
||||
}
|
||||
if expected != nil && (owner.ID != expected.ID || owner.Backend != expected.Backend || owner.Directory != expected.Directory || owner.Reference != expected.Reference) {
|
||||
return nil, fmt.Errorf("original session ownership differs from durable recovery snapshot; export refused")
|
||||
}
|
||||
active, err := r.store.ActiveSessionOwners(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -31,6 +31,11 @@ Audit additions (2026-10-20): **partial** — explicit run Report format selecto
|
||||
empty/failed/partial explanations. Unit-tested; browser captures missing under
|
||||
the audit's no-live-runs constraint. Existing captured states below are unchanged.
|
||||
|
||||
Automatic session export recovery additionally shows pending/running/complete/
|
||||
exhausted job status and its bounded attempt count in the visit pane. Backend
|
||||
recovery is fixture-tested; this new status line remains **partial**, without a
|
||||
browser capture (no live runs were authorized).
|
||||
|
||||
## Transport and connection
|
||||
| # | Feature (Android counterpart) | Protocol | Status |
|
||||
| --- | --- | --- | --- |
|
||||
|
||||
@@ -1294,6 +1294,8 @@ function artifactAction(kind, label, disabled, onclick) {
|
||||
function renderSessionExport(step) {
|
||||
const file = sessionExportArtifact(step);
|
||||
const wrap = h("div", { class: "session-export" });
|
||||
const recovery = step?.session_export_recovery;
|
||||
if (recovery) wrap.append(h("div", { class: "muted", text: `Automatic export recovery: ${recovery.status} · ${recovery.attempts}/3 attempts. ${recovery.diagnostic || ""}` }));
|
||||
for (const repair of step?.session_export_repairs || []) {
|
||||
const row = h("div", { class: "session-export-repair" });
|
||||
row.append(h("span", { text: `Repair: ${repair.status || "partial"} · ${repair.messages || 0} messages. ${repair.diagnostic || ""}` }));
|
||||
|
||||
Reference in New Issue
Block a user