Record run input directory and retry failed runs with original inputs
This commit is contained in:
@@ -124,5 +124,8 @@ CLI-created ones. **e** reopens a draft, **v** validates it, and **s** opens exp
|
||||
submission confirmation. Drafts remain retained after submission.
|
||||
|
||||
From a run, press **p** / **:pipeline** to open the pipeline version it used.
|
||||
Initial run input also records `pwd`. Press **r** / **:retry** on a failed run
|
||||
(or `pipelines retry <run-id>`) to start a new attempt with the same original
|
||||
input, directory, and pipeline version. Completed side effects are not undone.
|
||||
|
||||
More: [CLI](docs/cli.md) · [installation](docs/install.md) · [pipeline format](internal/author/skill/references/format.md).
|
||||
|
||||
+14
-1
@@ -94,7 +94,7 @@ printf '%s' '{"type":"get_pipeline","pipeline_id":1,"version":1}' | pipelines rp
|
||||
|
||||
Existing protocol commands: `ping`, `list_pipelines`, `get_pipeline`,
|
||||
`list_templates`, `validate`, `submit`, `run_pipeline`, `list_runs`, `get_run`,
|
||||
`cancel_run`. See `internal/protocol/message.go` for JSON envelopes. Mutating RPCs
|
||||
`cancel_run`, `retry_run`. See `internal/protocol/message.go` for JSON envelopes. Mutating RPCs
|
||||
have real effects; they are not previews. TUI-only animation, scrolling, and
|
||||
transient client logs are presentation, not separate server capabilities.
|
||||
|
||||
@@ -103,6 +103,19 @@ transient client logs are presentation, not separate server capabilities.
|
||||
On either the runs list or run inspector, press `p` (or use `:pipeline`) to open
|
||||
the selected run's exact pipeline version. Navigation does not cancel the run.
|
||||
|
||||
Initial input includes `pwd`, the resolved run directory (normally the directory
|
||||
you invoked Pipelines from), alongside the task and any custom input fields.
|
||||
It is recorded by the server and shown in the run's Input section.
|
||||
|
||||
Press `r` or use `:retry` on a failed run to restart it from the first stage.
|
||||
CLI equivalent: `pipelines retry <run-id>` (optional `--json`, `--wait`, and
|
||||
`--request-key`). The new attempt retains the initial input, directory, pinned
|
||||
pipeline version, and edit scope, and records `retry_of`. Original history is
|
||||
unchanged. Legacy runs gain `pwd` from their recorded workspace on retry.
|
||||
Retry replays the entire workflow; earlier edits or notifications are not undone
|
||||
and may happen again. A missing original directory blocks retry rather than
|
||||
silently using your current directory.
|
||||
|
||||
```sh
|
||||
cd ~/repos/project
|
||||
pipelines run "bug fix" "Fix the layout" --json
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
Definitions are Markdown made only of `# stage KEY` headings and fenced blocks.
|
||||
An optional `# settings` section precedes all stages. No unrelated prose outside
|
||||
blocks. The first stage receives `input.prompt` from the runtime user's task;
|
||||
blocks. The first stage receives `input.prompt` from the runtime user's task
|
||||
and `input.pwd`, the resolved run directory recorded by the server;
|
||||
later stages receive exactly the previous executed stage's output map. There are
|
||||
no stage `input` blocks. Preserve needed context explicitly in `output` handoffs.
|
||||
|
||||
|
||||
+21
-2
@@ -73,6 +73,9 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
|
||||
f.BoolVar(&o.wait, "wait", false, "In JSON mode, wait for a terminal run state")
|
||||
case "runs":
|
||||
f.StringVar(&o.name, "pipeline", "", "Filter by exact pipeline name or ID")
|
||||
case "retry":
|
||||
f.StringVar(&o.key, "request-key", "", "Idempotency key (default: generated unique key)")
|
||||
f.BoolVar(&o.wait, "wait", false, "Wait for the new run to finish")
|
||||
}
|
||||
f.Usage = func() {
|
||||
fmt.Fprintf(a.Out, "Usage: pipelines %s\n", usage[command])
|
||||
@@ -404,7 +407,7 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("template %q not found", key)
|
||||
case "inspect", "cancel":
|
||||
case "inspect", "cancel", "retry":
|
||||
value, err := one()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -417,10 +420,21 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
if command == "cancel" {
|
||||
typeName = protocol.CancelRun
|
||||
}
|
||||
r, err := call(protocol.Message{Type: typeName, RunID: id})
|
||||
key := ""
|
||||
if command == "retry" {
|
||||
typeName = protocol.RetryRun
|
||||
key = o.key
|
||||
if key == "" {
|
||||
key = fmt.Sprintf("retry-%d-%d", os.Getpid(), time.Now().UnixNano())
|
||||
}
|
||||
}
|
||||
r, err := call(protocol.Message{Type: typeName, RunID: id, RequestKey: key})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if command == "retry" {
|
||||
return a.finishRun(ctx, r, o, call)
|
||||
}
|
||||
return a.output(r.RunDetail)
|
||||
case "rpc":
|
||||
path, err := one()
|
||||
@@ -647,6 +661,11 @@ func (a App) run(ctx context.Context, pos []string, o *options, call callFunc, r
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return a.finishRun(ctx, r, o, call)
|
||||
}
|
||||
|
||||
func (a App) finishRun(ctx context.Context, r protocol.Message, o *options, call callFunc) error {
|
||||
var err error
|
||||
if o.wait {
|
||||
if r.RunDetail == nil {
|
||||
return fmt.Errorf("server did not return run detail")
|
||||
|
||||
@@ -17,6 +17,7 @@ var usage = map[string]string{
|
||||
"runs": "runs [--pipeline NAME] [--json]",
|
||||
"inspect": "inspect <run-id> [--json]",
|
||||
"cancel": "cancel <run-id> [--json]",
|
||||
"retry": "retry <run-id> [--json] [--wait]",
|
||||
"connection": "connection [--json]",
|
||||
"ping": "ping [--json]",
|
||||
"rpc": "rpc <request.json|->",
|
||||
@@ -29,6 +30,7 @@ var notes = map[string]string{
|
||||
"validate": "Uses the same server parser as submission. Does not persist or activate schedules. Structural validity does not prove runtime/tool availability.",
|
||||
"submit": "This command is explicit approval to publish; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.",
|
||||
"run": "Runs in your current directory. Supplied name/prompt start immediately; otherwise select a pipeline and send a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or RUN confirmation. Cancellation does not undo edits.",
|
||||
"retry": "Start a new attempt of a failed run with the same initial input, directory, pipeline version, and edit scope. Original history is retained. Already-completed side effects are not undone.",
|
||||
"rpc": "Accepts a JSON protocol envelope; '-' reads stdin. Every TUI server operation is available here too. Mutating commands are not dry runs.",
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ EXECUTE
|
||||
runs List audited runs; --pipeline filters results
|
||||
inspect <run-id> Inspect stages, outputs, errors and status
|
||||
cancel <run-id> Cancel a run; changes are NOT rolled back
|
||||
retry <run-id> Restart a failed run with its original input and directory
|
||||
connection Server and public phone connection (no secrets)
|
||||
ping Check the server connection
|
||||
rpc <request.json|-> Send any protocol command (automation escape hatch)
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRetryStorageMigrationPersistenceAndKeyConflicts(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "legacy.db")
|
||||
store, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.Submit(ctx, "workflow", Template([]string{"one"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := store.List(ctx)
|
||||
d, _ := store.Get(ctx, items[0].ID, 1)
|
||||
input := map[string]any{"prompt": "legacy input"}
|
||||
id, _, err := store.CreateRun(ctx, d, "original", "/original", false, nil, input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishRun(ctx, id, "failed", "error", "failed", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
original, err := store.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate the schema before retry support, preserving its run history.
|
||||
if _, err := store.db.ExecContext(ctx, "DROP TABLE run_retries"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err = Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, err := store.GetRun(ctx, id)
|
||||
if err != nil || !reflect.DeepEqual(original, after) {
|
||||
t.Fatal("migration modified legacy history", after, err)
|
||||
}
|
||||
retryID, created, err := store.CreateRun(ctx, d, "retry", "/original", false, nil, input, id)
|
||||
if err != nil || !created {
|
||||
t.Fatal("retry creation failed", err)
|
||||
}
|
||||
if _, _, err := store.CreateRun(ctx, d, "retry", "/original", false, nil, input); err == nil {
|
||||
t.Fatal("request key accepted with different provenance")
|
||||
}
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err = Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
retry, err := store.GetRun(ctx, retryID)
|
||||
if err != nil || retry.RetryOf != id || !reflect.DeepEqual(retry.Input, input) {
|
||||
t.Fatal("retry metadata did not survive reopen", retry, err)
|
||||
}
|
||||
existing, created, err := store.CreateRun(ctx, d, "retry", "/original", false, nil, input, id)
|
||||
if err != nil || created || existing != retryID {
|
||||
t.Fatal("persisted retry lost idempotency", existing, created, err)
|
||||
}
|
||||
}
|
||||
@@ -30,11 +30,20 @@ run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, receipts_json TEXT NOT NULL,
|
||||
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
||||
CREATE TABLE IF NOT EXISTS run_session_owners (
|
||||
run_id INTEGER NOT NULL, sequence INTEGER NOT NULL, backend TEXT NOT NULL, reference TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));`)
|
||||
PRIMARY KEY(run_id,sequence), FOREIGN KEY(run_id,sequence) REFERENCES run_steps(run_id,sequence));
|
||||
CREATE TABLE IF NOT EXISTS run_retries (
|
||||
run_id INTEGER PRIMARY KEY REFERENCES pipeline_runs(id), original_run_id INTEGER NOT NULL REFERENCES pipeline_runs(id));`)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) CreateRun(ctx context.Context, d *protocol.PipelineDetail, key, workspace string, tools bool, paths []string, input map[string]any) (int64, bool, error) {
|
||||
func (s *Store) CreateRun(ctx context.Context, d *protocol.PipelineDetail, key, workspace string, tools bool, paths []string, input map[string]any, retryOf ...int64) (int64, bool, error) {
|
||||
var parent int64
|
||||
if len(retryOf) > 0 {
|
||||
parent = retryOf[0]
|
||||
}
|
||||
if parent < 0 {
|
||||
return 0, false, fmt.Errorf("invalid retry parent")
|
||||
}
|
||||
encoded, err := json.Marshal(input)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
@@ -51,7 +60,11 @@ func (s *Store) CreateRun(ctx context.Context, d *protocol.PipelineDetail, key,
|
||||
var oldTools bool
|
||||
err = s.db.QueryRowContext(ctx, "SELECT id,pipeline_id,version,input_json,workspace,write_paths,allow_tools FROM pipeline_runs WHERE request_key=?", key).Scan(&id, &pid, &version, &oldInput, &oldWorkspace, &oldPaths, &oldTools)
|
||||
if err == nil {
|
||||
if pid != d.ID || version != d.Version || oldInput != string(encoded) || oldWorkspace != workspace || oldPaths != string(writes) || oldTools != tools {
|
||||
var oldParent int64
|
||||
if err := s.db.QueryRowContext(ctx, "SELECT COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=?),0)", id).Scan(&oldParent); err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
if pid != d.ID || version != d.Version || oldInput != string(encoded) || oldWorkspace != workspace || oldPaths != string(writes) || oldTools != tools || oldParent != parent {
|
||||
return 0, false, fmt.Errorf("request key already used for a different run")
|
||||
}
|
||||
return id, false, nil
|
||||
@@ -59,12 +72,28 @@ func (s *Store) CreateRun(ctx context.Context, d *protocol.PipelineDetail, key,
|
||||
if err != sql.ErrNoRows {
|
||||
return 0, false, err
|
||||
}
|
||||
result, err := s.db.ExecContext(ctx, "INSERT INTO pipeline_runs(pipeline_id,version,request_key,workspace,allow_tools,write_paths,input_json) VALUES (?,?,?,?,?,?,?)", d.ID, d.Version, key, workspace, tools, string(writes), string(encoded))
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
result, err := tx.ExecContext(ctx, "INSERT INTO pipeline_runs(pipeline_id,version,request_key,workspace,allow_tools,write_paths,input_json) VALUES (?,?,?,?,?,?,?)", d.ID, d.Version, key, workspace, tools, string(writes), string(encoded))
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
id, err = result.LastInsertId()
|
||||
return id, true, err
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
if parent > 0 {
|
||||
if _, err := tx.ExecContext(ctx, "INSERT INTO run_retries(run_id,original_run_id) VALUES(?,?)", id, parent); err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
return id, true, nil
|
||||
}
|
||||
|
||||
const runSelect = `SELECT r.id,r.pipeline_id,p.name,r.version,v.content_hash,r.status,r.outcome,r.current_stage,
|
||||
@@ -121,7 +150,7 @@ func (s *Store) GetRun(ctx context.Context, id int64) (*protocol.RunDetail, erro
|
||||
return nil, err
|
||||
}
|
||||
var input, output, paths string
|
||||
if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output); err != nil {
|
||||
if err := tx.QueryRowContext(ctx, "SELECT workspace,allow_tools,write_paths,input_json,output_json,COALESCE((SELECT original_run_id FROM run_retries WHERE run_id=pipeline_runs.id),0) FROM pipeline_runs WHERE id=?", id).Scan(&r.Workspace, &r.AllowTools, &paths, &input, &output, &r.RetryOf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := decodeValues(input, &r.Input); err != nil {
|
||||
|
||||
@@ -8,6 +8,7 @@ const (
|
||||
GetRun = "get_run"
|
||||
Run = "run"
|
||||
CancelRun = "cancel_run"
|
||||
RetryRun = "retry_run"
|
||||
)
|
||||
|
||||
type RunSummary struct {
|
||||
@@ -52,6 +53,7 @@ type NotificationReceipt struct {
|
||||
|
||||
type RunDetail struct {
|
||||
RunSummary
|
||||
RetryOf int64 `json:"retry_of,omitempty"`
|
||||
Workspace string `json:"workspace"`
|
||||
AllowTools bool `json:"allow_tools"`
|
||||
WritePaths []string `json:"write_paths"`
|
||||
|
||||
@@ -58,7 +58,7 @@ func TestCLIHeadlessRunInspectionCancellationAndRPC(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.RunDetail.Workspace != cwd {
|
||||
if response.RunDetail.Workspace != cwd || response.RunDetail.Input["pwd"] != cwd {
|
||||
t.Fatal("CLI did not use invocation directory", response.RunDetail.Workspace, cwd)
|
||||
}
|
||||
output, err = run("run", "workflow", "exact task", "--yes", "--json", "--request-key", "same-key")
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/cli"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func TestRetryReplaysInitialInputAndPinnedVersionThroughCLI(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
if _, err := store.Submit(ctx, "workflow", pipeline.Template([]string{"plan", "implement"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := store.List(ctx)
|
||||
handler := Handler(ctx, store)
|
||||
r := handler.(*runHandler).runner
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
workspace, err := filepath.EvalSymlinks(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
initial := map[string]any{"prompt": "exact original task\nwith two lines", "pwd": "spoofed", "custom": map[string]any{"value": "keep me"}}
|
||||
r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
if def.Key == "plan" {
|
||||
if input["pwd"] != workspace || input["prompt"] != initial["prompt"] {
|
||||
t.Error("wrong initial input", input)
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "intermediate, not the task"}}, nil
|
||||
}
|
||||
return stage.Result{}, fmt.Errorf("deliberate failure")
|
||||
}
|
||||
original, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "original", Workspace: workspace, WritePaths: []string{"README.md"}, Input: initial})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
original, err = store.GetRun(ctx, original.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if original.Status != "failed" || original.Input["pwd"] != workspace || initial["pwd"] != "spoofed" {
|
||||
t.Fatal("initial input was not recorded independently", original)
|
||||
}
|
||||
if _, err := store.Submit(ctx, "workflow", pipeline.Template([]string{"different"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
||||
r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
if def.Key == "plan" {
|
||||
if !reflect.DeepEqual(map[string]any(input), original.Input) {
|
||||
t.Error("retry used changed input", input, original.Input)
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "new handoff"}}, nil
|
||||
}
|
||||
if def.Key != "implement" || input["summary"] != "new handoff" {
|
||||
t.Error("retry did not restart at entry", def.Key, input)
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
|
||||
}
|
||||
runCLI := func(args ...string) (protocol.Message, error) {
|
||||
var out bytes.Buffer
|
||||
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
|
||||
err := app.Run(ctx, append([]string{"--url", url}, args...))
|
||||
var result protocol.Message
|
||||
if err == nil {
|
||||
err = json.Unmarshal(out.Bytes(), &result)
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
result, err := runCLI("retry", fmt.Sprint(original.ID), "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retried := result.RunDetail
|
||||
if retried == nil || retried.ID == original.ID || retried.Status != "succeeded" || retried.Version != 1 || retried.RetryOf != original.ID || retried.Workspace != workspace || !reflect.DeepEqual(retried.Input, original.Input) || !reflect.DeepEqual(retried.WritePaths, original.WritePaths) || retried.AllowTools != original.AllowTools {
|
||||
t.Fatal("retry did not preserve recorded run settings", retried)
|
||||
}
|
||||
r.wg.Wait()
|
||||
again, err := runCLI("retry", fmt.Sprint(original.ID), "--json", "--request-key", "retry-key")
|
||||
if err != nil || again.RunDetail.ID != retried.ID {
|
||||
t.Fatal("retry lost idempotency", again, err)
|
||||
}
|
||||
unchanged, err := store.GetRun(ctx, original.ID)
|
||||
if err != nil || !reflect.DeepEqual(original, unchanged) {
|
||||
t.Fatal("retry modified failed history", unchanged, err)
|
||||
}
|
||||
if _, err := runCLI("retry", fmt.Sprint(retried.ID), "--json"); err == nil {
|
||||
t.Fatal("succeeded run accepted for retry")
|
||||
}
|
||||
if _, err := r.retry(protocol.Message{RunID: original.ID}); err == nil {
|
||||
t.Fatal("retry accepted without idempotency key")
|
||||
}
|
||||
if _, err := r.start(protocol.Message{PipelineID: original.PipelineID, Version: 1, RequestKey: "retry-key", Workspace: workspace, Input: original.Input, WritePaths: original.WritePaths}); err == nil {
|
||||
t.Fatal("ordinary run reused a retry's key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
store.Submit(ctx, "workflow", pipeline.Template([]string{"one"}, true))
|
||||
items, _ := store.List(ctx)
|
||||
d, _ := store.Get(ctx, items[0].ID, 1)
|
||||
r, err := newRunner(ctx, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, status := range []string{"queued", "running", "succeeded", "canceled", "failed"} {
|
||||
id, _, err := store.CreateRun(ctx, d, status, filepath.Join(t.TempDir(), "missing"), false, nil, map[string]any{"prompt": "old task"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != "queued" {
|
||||
if err := store.FinishRun(ctx, id, status, "error", "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status})
|
||||
if err == nil {
|
||||
t.Fatal("invalid retry accepted", status)
|
||||
}
|
||||
if status == "failed" && !strings.Contains(err.Error(), "workspace is unavailable") {
|
||||
t.Fatal("retry did not validate original directory", err)
|
||||
}
|
||||
}
|
||||
if _, err := r.retry(protocol.Message{RunID: 999, RequestKey: "missing"}); err == nil {
|
||||
t.Fatal("nonexistent run accepted")
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -44,7 +45,7 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
||||
return &runner{ctx: ctx, store: s, active: map[int64]context.CancelFunc{}}, nil
|
||||
}
|
||||
|
||||
func (r *runner) start(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
|
||||
if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 {
|
||||
return nil, fmt.Errorf("an explicit version and request key are required")
|
||||
}
|
||||
@@ -97,9 +98,11 @@ func (r *runner) start(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
return nil, fmt.Errorf("write path must stay within the run directory")
|
||||
}
|
||||
}
|
||||
request.Input = maps.Clone(request.Input)
|
||||
if request.Input == nil {
|
||||
request.Input = map[string]any{}
|
||||
}
|
||||
request.Input["pwd"] = workspace
|
||||
if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" {
|
||||
return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt")
|
||||
}
|
||||
@@ -149,7 +152,7 @@ func (r *runner) start(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
return nil, fmt.Errorf("another run is active; inspect it before starting more work")
|
||||
}
|
||||
}
|
||||
id, created, err := r.store.CreateRun(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input)
|
||||
id, created, err := r.store.CreateRun(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, retryOf...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -167,6 +170,20 @@ func (r *runner) start(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
return r.store.GetRun(r.ctx, id)
|
||||
}
|
||||
|
||||
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
if request.RunID <= 0 {
|
||||
return nil, fmt.Errorf("retry requires a positive run ID")
|
||||
}
|
||||
original, err := r.store.GetRun(r.ctx, request.RunID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if original.Status != "failed" {
|
||||
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
|
||||
}
|
||||
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths}, original.ID)
|
||||
}
|
||||
|
||||
func (r *runner) cancel(id int64) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -32,7 +32,7 @@ func TestRunnerRoutingPersistenceAndIdempotency(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
r.execute = func(ctx context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
if def.Key == "one" {
|
||||
if len(input) != 1 || input["prompt"] != "task" {
|
||||
if len(input) != 2 || input["prompt"] != "task" || input["pwd"] == "" {
|
||||
t.Error("entry input must be the user prompt unchanged")
|
||||
}
|
||||
close(entered)
|
||||
|
||||
@@ -121,12 +121,15 @@ func Handler(ctx context.Context, stores ...*pipeline.Store) http.Handler {
|
||||
} else {
|
||||
response.Pipelines = items
|
||||
}
|
||||
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun) {
|
||||
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun) {
|
||||
var err error
|
||||
switch request.Type {
|
||||
case protocol.RunPipeline:
|
||||
response.Type = protocol.RunStarted
|
||||
response.RunDetail, err = runs.start(request)
|
||||
case protocol.RetryRun:
|
||||
response.Type = protocol.RunStarted
|
||||
response.RunDetail, err = runs.retry(request)
|
||||
case protocol.ListRuns:
|
||||
response.Type = protocol.Runs
|
||||
response.Runs, err = runs.store.ListRuns(ctx)
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
var commands = []string{":logs", ":new", ":drafts", ":templates", ":refresh", ":ping", ":edit", ":submit", ":versions", ":run", ":runs", ":pipeline", ":home", ":quit"}
|
||||
var commands = []string{":logs", ":new", ":drafts", ":templates", ":refresh", ":ping", ":edit", ":submit", ":versions", ":run", ":runs", ":retry", ":pipeline", ":home", ":quit"}
|
||||
|
||||
func (m Model) footerContent() string {
|
||||
label := fmt.Sprintf(" LOGS %d · :logs ", len(m.lines))
|
||||
@@ -82,6 +82,10 @@ func (m Model) pageCommands() []string {
|
||||
detailPage := m.keys.detail && !m.runOpen
|
||||
for _, command := range commands {
|
||||
switch command {
|
||||
case ":retry":
|
||||
if m.selectedFailedRun() <= 0 {
|
||||
continue
|
||||
}
|
||||
case ":pipeline":
|
||||
if id, version := m.selectedRunPipeline(); id <= 0 || version <= 0 {
|
||||
continue
|
||||
@@ -145,6 +149,8 @@ func (m Model) updatePalette(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
m.paletteOpen = false
|
||||
m.keys.palette = false
|
||||
switch command {
|
||||
case ":retry":
|
||||
return m.retrySelectedRun()
|
||||
case ":pipeline":
|
||||
return m.openRunPipeline()
|
||||
case ":drafts":
|
||||
|
||||
@@ -75,7 +75,7 @@ func (k keyMap) ShortHelp() []key.Binding {
|
||||
return []key.Binding{key.NewBinding(key.WithKeys("n", "p"), key.WithHelp("n/p", "template")), key.NewBinding(key.WithKeys("v"), key.WithHelp("v", "render/source")), key.NewBinding(key.WithKeys("enter"), key.WithHelp("Enter", "use template")), k.cancel}
|
||||
}
|
||||
if k.runs {
|
||||
return []key.Binding{k.commands, key.NewBinding(key.WithKeys("p"), key.WithHelp("p", "pipeline")), key.NewBinding(key.WithKeys("tab"), key.WithHelp("Tab", "pane")), key.NewBinding(key.WithKeys("j", "k"), key.WithHelp("j/k", "select/scroll")), key.NewBinding(key.WithKeys("pgup", "pgdown"), key.WithHelp("PgUp/Dn", "page")), key.NewBinding(key.WithKeys("esc"), key.WithHelp("Esc", "back"))}
|
||||
return []key.Binding{k.commands, key.NewBinding(key.WithKeys("p"), key.WithHelp("p", "pipeline")), key.NewBinding(key.WithKeys("r"), key.WithHelp("r", "retry failed")), key.NewBinding(key.WithKeys("tab"), key.WithHelp("Tab", "pane")), key.NewBinding(key.WithKeys("j", "k"), key.WithHelp("j/k", "select/scroll")), key.NewBinding(key.WithKeys("pgup", "pgdown"), key.WithHelp("PgUp/Dn", "page")), key.NewBinding(key.WithKeys("esc"), key.WithHelp("Esc", "back"))}
|
||||
}
|
||||
if k.detail {
|
||||
return []key.Binding{k.commands, k.search, key.NewBinding(key.WithKeys("n", "p"), key.WithHelp("n/p", "next/prev stage")), key.NewBinding(key.WithKeys("tab"), key.WithHelp("Tab", "pane")), key.NewBinding(key.WithKeys("r"), key.WithHelp("r", "runs")), key.NewBinding(key.WithKeys("j", "k"), key.WithHelp("j/k", "scroll")), key.NewBinding(key.WithKeys("esc"), key.WithHelp("esc", "home"))}
|
||||
@@ -289,6 +289,9 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
} else if msg.Type == protocol.Run || msg.Type == protocol.RunStarted {
|
||||
accepted := m.runOpen && msg.RunDetail != nil && ((msg.Type == protocol.Run && msg.RunDetail.ID == m.requestedRunID) || (msg.Type == protocol.RunStarted && msg.ID == m.runStartRequest))
|
||||
if accepted {
|
||||
if msg.Type == protocol.RunStarted {
|
||||
m.runStartRequest = ""
|
||||
}
|
||||
m.keys.runs = true
|
||||
m.runDetail = msg.RunDetail
|
||||
m.requestedRunID = msg.RunDetail.ID
|
||||
@@ -304,6 +307,9 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
}
|
||||
if strings.HasPrefix(msg.ID, "run-") && m.runOpen {
|
||||
m.runError = msg.Error
|
||||
if msg.ID == m.runStartRequest {
|
||||
m.runStartRequest = ""
|
||||
}
|
||||
}
|
||||
if msg.ID != "" && msg.ID == m.detailRequest {
|
||||
m.detailLoading = false
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
package tui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/server"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
)
|
||||
|
||||
func TestRetryShortcutAndPaletteSendServerRetryWithoutReplacingDirectory(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
store.Submit(ctx, "workflow", pipeline.Template([]string{"one"}, true))
|
||||
items, _ := store.List(ctx)
|
||||
d, _ := store.Get(ctx, items[0].ID, 1)
|
||||
srv := httptest.NewServer(server.Handler(ctx, store))
|
||||
defer srv.Close()
|
||||
id, _, err := store.CreateRun(ctx, d, "failed", filepath.Join(t.TempDir(), "missing"), false, nil, map[string]any{"prompt": "original task"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.FinishRun(ctx, id, "failed", "error", "original failure", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run, err := store.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := New(ctx, "ws"+strings.TrimPrefix(srv.URL, "http")+"/connect")
|
||||
connected := m.connect()().(connectedMsg)
|
||||
defer connected.conn.CloseNow()
|
||||
updated, read := m.Update(connected)
|
||||
m = updated.(Model)
|
||||
updated, read = m.Update(read())
|
||||
m = updated.(Model)
|
||||
for _, palette := range []bool{false, true} {
|
||||
m.runOpen = true
|
||||
m.keys.runs = true
|
||||
m.runDetail = run
|
||||
m.requestedRunID = id
|
||||
m.runs = []protocol.RunSummary{run.RunSummary}
|
||||
if !slices.Contains(m.pageCommands(), ":retry") {
|
||||
t.Fatal("failed run missing retry action")
|
||||
}
|
||||
var send tea.Cmd
|
||||
if palette {
|
||||
m.runDetail = nil
|
||||
m.requestedRunID = 0
|
||||
m.commandInput = newInput()
|
||||
m.commandInput.SetValue(":retry")
|
||||
updated, send = m.updatePalette(tea.KeyMsg{Type: tea.KeyEnter})
|
||||
} else {
|
||||
updated, send = m.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'r'}})
|
||||
}
|
||||
m = updated.(Model)
|
||||
if send == nil || m.runStartRequest == "" || m.runDetail != nil || m.requestedRunID != 0 {
|
||||
t.Fatal("retry did not start cleanly")
|
||||
}
|
||||
updated, _ = m.Update(receivedMsg(protocol.Message{Type: protocol.Run, RunDetail: run}))
|
||||
m = updated.(Model)
|
||||
if m.runDetail != nil {
|
||||
t.Fatal("stale poll restored original run during retry")
|
||||
}
|
||||
if _, ok := send().(sentMsg); !ok {
|
||||
t.Fatal("retry request failed")
|
||||
}
|
||||
updated, read = m.Update(read())
|
||||
m = updated.(Model)
|
||||
if !strings.Contains(m.runError, "workspace is unavailable") || m.runStartRequest != "" {
|
||||
t.Fatal("retry replaced original directory or stuck loading after server error", m.runError)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryResponseClearsPendingRequestAndAllowsAnotherFailedAttempt(t *testing.T) {
|
||||
m := New(context.Background(), "")
|
||||
m.runOpen = true
|
||||
m.keys.runs = true
|
||||
m.runStartRequest = "run-2"
|
||||
run := &protocol.RunDetail{RunSummary: protocol.RunSummary{ID: 3, Status: "failed"}, RetryOf: 1, Input: map[string]any{"prompt": "task", "pwd": "/project"}}
|
||||
updated, _ := m.Update(receivedMsg(protocol.Message{Type: protocol.RunStarted, ID: "run-2", RunDetail: run}))
|
||||
m = updated.(Model)
|
||||
if m.runDetail != run || m.requestedRunID != 3 || m.runStartRequest != "" || m.selectedFailedRun() != 3 {
|
||||
t.Fatal("retry response was not displayed or another retry is blocked")
|
||||
}
|
||||
if !slices.Contains(m.pageCommands(), ":retry") {
|
||||
t.Fatal("failed retry lacks retry command")
|
||||
}
|
||||
for _, status := range []string{"queued", "running", "succeeded", "canceled"} {
|
||||
m.runDetail.Status = status
|
||||
if slices.Contains(m.pageCommands(), ":retry") {
|
||||
t.Fatal("retry action visible for nonfailed run", status)
|
||||
}
|
||||
updated, cmd := m.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'r'}})
|
||||
m = updated.(Model)
|
||||
if cmd != nil || m.runError != "Select a failed run to retry." {
|
||||
t.Fatal("nonfailed run accepted", status)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,38 @@ import (
|
||||
"github.com/charmbracelet/x/ansi"
|
||||
)
|
||||
|
||||
func (m Model) selectedFailedRun() int64 {
|
||||
if !m.runOpen || m.runStartRequest != "" {
|
||||
return 0
|
||||
}
|
||||
if m.runDetail != nil {
|
||||
if m.runDetail.Status == "failed" {
|
||||
return m.runDetail.ID
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if m.requestedRunID > 0 {
|
||||
return 0
|
||||
}
|
||||
runs := m.visibleRuns()
|
||||
if len(runs) > 0 {
|
||||
r := runs[min(m.runIndex, len(runs)-1)]
|
||||
if r.Status == "failed" {
|
||||
return r.ID
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (m Model) retrySelectedRun() (tea.Model, tea.Cmd) {
|
||||
id := m.selectedFailedRun()
|
||||
if id <= 0 {
|
||||
m.runError = "Select a failed run to retry."
|
||||
return m, nil
|
||||
}
|
||||
return m.runRequest(protocol.Message{Type: protocol.RetryRun, RunID: id, RequestKey: fmt.Sprintf("retry-%d", time.Now().UnixNano())})
|
||||
}
|
||||
|
||||
func (m Model) selectedRunPipeline() (int64, int) {
|
||||
if !m.runOpen {
|
||||
return 0, 0
|
||||
@@ -279,6 +311,9 @@ func (m Model) runSectionLines(width int) []string {
|
||||
editScope = strings.Join(r.WritePaths, ", ")
|
||||
}
|
||||
content := [][]string{summary, valueLines(r.Input), valueLines(r.Output), {"Select a stage to inspect its tool calls."}, {"Workspace: " + r.Workspace, "Source hash: " + r.ContentHash, fmt.Sprintf("Shell/edit enabled: %t", r.AllowTools), "Stage tools: defined by the pinned pipeline version", "Edit scope: " + editScope}}
|
||||
if r.RetryOf > 0 {
|
||||
content[4] = append(content[4], fmt.Sprintf("Retry of run #%d", r.RetryOf))
|
||||
}
|
||||
if m.runStepIndex > 0 && m.runStepIndex <= len(r.Steps) {
|
||||
step := r.Steps[m.runStepIndex-1]
|
||||
summary = []string{"Status: " + m.progress(m.stepProgressStatus(step)), "Route: " + outcomeLabel(step.Outcome, step.Outcome) + " → " + step.Target, "Started: " + dateLabel(step.StartedAt), "Finished: " + dateLabel(step.FinishedAt), "Duration: " + runDuration(step.StartedAt, step.FinishedAt)}
|
||||
@@ -353,12 +388,12 @@ func (m Model) runsView(height int) string {
|
||||
if m.runError != "" {
|
||||
rows = append(rows, m.runError)
|
||||
}
|
||||
return panel(m.width, height, []string{bar(title, fmt.Sprint(len(runs)), max(1, m.width-4), true)}, rows, "j/k select · Enter inspect · p pipeline", true)
|
||||
return panel(m.width, height, []string{bar(title, fmt.Sprint(len(runs)), max(1, m.width-4), true)}, rows, "j/k select · Enter inspect · p pipeline · r retry failed", true)
|
||||
}
|
||||
r := m.runDetail
|
||||
header := bar(fmt.Sprintf("#%d %s · v%d", r.ID, r.Name, r.Version), m.progress(r.Status)+" · "+runDuration(r.CreatedAt, r.FinishedAt), m.width, false)
|
||||
if height <= 6 {
|
||||
return fitBlock([]string{header, m.runSelectionLabel(), "p pipeline · Tab pane · Esc back"}, m.width, height)
|
||||
return fitBlock([]string{header, m.runSelectionLabel(), "p pipeline · r retry failed · Tab pane · Esc back"}, m.width, height)
|
||||
}
|
||||
innerHeight := height - 2
|
||||
sidebar := m.runSidebarWidth()
|
||||
@@ -398,7 +433,7 @@ func (m Model) runsView(height int) string {
|
||||
lines := m.runSectionLines(max(1, rightWidth-4))
|
||||
page := max(0, innerHeight-4)
|
||||
top := max(0, min(m.runTop, max(0, len(lines)-page)))
|
||||
footer := fmt.Sprintf("%d–%d / %d · s/i/o/t/m sections · p pipeline · PgUp/Dn · Esc back", min(top+1, len(lines)), min(top+page, len(lines)), len(lines))
|
||||
footer := fmt.Sprintf("%d–%d / %d · s/i/o/t/m sections · p pipeline · r retry failed · PgUp/Dn · Esc back", min(top+1, len(lines)), min(top+page, len(lines)), len(lines))
|
||||
label := m.runSelectionLabel()
|
||||
if m.runStepIndex > 0 && m.runStepIndex <= len(r.Steps) {
|
||||
label = fmt.Sprintf("Details · %s · step %d", r.Steps[m.runStepIndex-1].Stage, r.Steps[m.runStepIndex-1].Sequence)
|
||||
|
||||
+12
-2
@@ -31,8 +31,16 @@ func (m Model) runRequest(request protocol.Message) (tea.Model, tea.Cmd) {
|
||||
m.runCollapsed = nil
|
||||
m.runError = ""
|
||||
}
|
||||
if request.Type == protocol.RunPipeline {
|
||||
if request.Workspace == "" {
|
||||
if request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun {
|
||||
if request.Type == protocol.RetryRun {
|
||||
m.runOpen = true
|
||||
m.requestedRunID = 0
|
||||
m.runDetail = nil
|
||||
m.runStepIndex = 0
|
||||
m.runTop = 0
|
||||
m.runCollapsed = nil
|
||||
}
|
||||
if request.Type == protocol.RunPipeline && request.Workspace == "" {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
m.appendLine("cannot determine current directory: " + err.Error())
|
||||
@@ -158,6 +166,8 @@ func (m Model) updateRuns(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
switch msg.String() {
|
||||
case "p":
|
||||
return m.openRunPipeline()
|
||||
case "r":
|
||||
return m.retrySelectedRun()
|
||||
case "esc":
|
||||
if m.runDetail != nil || m.requestedRunID > 0 || m.runStartRequest != "" {
|
||||
m.runDetail = nil
|
||||
|
||||
Reference in New Issue
Block a user