Record run input directory and retry failed runs with original inputs
This commit is contained in:
@@ -58,7 +58,7 @@ func TestCLIHeadlessRunInspectionCancellationAndRPC(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.RunDetail.Workspace != cwd {
|
||||
if response.RunDetail.Workspace != cwd || response.RunDetail.Input["pwd"] != cwd {
|
||||
t.Fatal("CLI did not use invocation directory", response.RunDetail.Workspace, cwd)
|
||||
}
|
||||
output, err = run("run", "workflow", "exact task", "--yes", "--json", "--request-key", "same-key")
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/cli"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func TestRetryReplaysInitialInputAndPinnedVersionThroughCLI(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
if _, err := store.Submit(ctx, "workflow", pipeline.Template([]string{"plan", "implement"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := store.List(ctx)
|
||||
handler := Handler(ctx, store)
|
||||
r := handler.(*runHandler).runner
|
||||
defer func() { cancel(); WaitRuns(handler) }()
|
||||
workspace, err := filepath.EvalSymlinks(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
initial := map[string]any{"prompt": "exact original task\nwith two lines", "pwd": "spoofed", "custom": map[string]any{"value": "keep me"}}
|
||||
r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
if def.Key == "plan" {
|
||||
if input["pwd"] != workspace || input["prompt"] != initial["prompt"] {
|
||||
t.Error("wrong initial input", input)
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "intermediate, not the task"}}, nil
|
||||
}
|
||||
return stage.Result{}, fmt.Errorf("deliberate failure")
|
||||
}
|
||||
original, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "original", Workspace: workspace, WritePaths: []string{"README.md"}, Input: initial})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
original, err = store.GetRun(ctx, original.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if original.Status != "failed" || original.Input["pwd"] != workspace || initial["pwd"] != "spoofed" {
|
||||
t.Fatal("initial input was not recorded independently", original)
|
||||
}
|
||||
if _, err := store.Submit(ctx, "workflow", pipeline.Template([]string{"different"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect"
|
||||
r.execute = func(_ context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
if def.Key == "plan" {
|
||||
if !reflect.DeepEqual(map[string]any(input), original.Input) {
|
||||
t.Error("retry used changed input", input, original.Input)
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "new handoff"}}, nil
|
||||
}
|
||||
if def.Key != "implement" || input["summary"] != "new handoff" {
|
||||
t.Error("retry did not restart at entry", def.Key, input)
|
||||
}
|
||||
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil
|
||||
}
|
||||
runCLI := func(args ...string) (protocol.Message, error) {
|
||||
var out bytes.Buffer
|
||||
app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")}
|
||||
err := app.Run(ctx, append([]string{"--url", url}, args...))
|
||||
var result protocol.Message
|
||||
if err == nil {
|
||||
err = json.Unmarshal(out.Bytes(), &result)
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
result, err := runCLI("retry", fmt.Sprint(original.ID), "--json", "--wait", "--timeout", "5s", "--request-key", "retry-key")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retried := result.RunDetail
|
||||
if retried == nil || retried.ID == original.ID || retried.Status != "succeeded" || retried.Version != 1 || retried.RetryOf != original.ID || retried.Workspace != workspace || !reflect.DeepEqual(retried.Input, original.Input) || !reflect.DeepEqual(retried.WritePaths, original.WritePaths) || retried.AllowTools != original.AllowTools {
|
||||
t.Fatal("retry did not preserve recorded run settings", retried)
|
||||
}
|
||||
r.wg.Wait()
|
||||
again, err := runCLI("retry", fmt.Sprint(original.ID), "--json", "--request-key", "retry-key")
|
||||
if err != nil || again.RunDetail.ID != retried.ID {
|
||||
t.Fatal("retry lost idempotency", again, err)
|
||||
}
|
||||
unchanged, err := store.GetRun(ctx, original.ID)
|
||||
if err != nil || !reflect.DeepEqual(original, unchanged) {
|
||||
t.Fatal("retry modified failed history", unchanged, err)
|
||||
}
|
||||
if _, err := runCLI("retry", fmt.Sprint(retried.ID), "--json"); err == nil {
|
||||
t.Fatal("succeeded run accepted for retry")
|
||||
}
|
||||
if _, err := r.retry(protocol.Message{RunID: original.ID}); err == nil {
|
||||
t.Fatal("retry accepted without idempotency key")
|
||||
}
|
||||
if _, err := r.start(protocol.Message{PipelineID: original.PipelineID, Version: 1, RequestKey: "retry-key", Workspace: workspace, Input: original.Input, WritePaths: original.WritePaths}); err == nil {
|
||||
t.Fatal("ordinary run reused a retry's key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryRejectsUnavailableOriginalDirectoryAndNonfailedRuns(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
store.Submit(ctx, "workflow", pipeline.Template([]string{"one"}, true))
|
||||
items, _ := store.List(ctx)
|
||||
d, _ := store.Get(ctx, items[0].ID, 1)
|
||||
r, err := newRunner(ctx, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, status := range []string{"queued", "running", "succeeded", "canceled", "failed"} {
|
||||
id, _, err := store.CreateRun(ctx, d, status, filepath.Join(t.TempDir(), "missing"), false, nil, map[string]any{"prompt": "old task"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != "queued" {
|
||||
if err := store.FinishRun(ctx, id, status, "error", "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_, err = r.retry(protocol.Message{RunID: id, RequestKey: "retry-" + status})
|
||||
if err == nil {
|
||||
t.Fatal("invalid retry accepted", status)
|
||||
}
|
||||
if status == "failed" && !strings.Contains(err.Error(), "workspace is unavailable") {
|
||||
t.Fatal("retry did not validate original directory", err)
|
||||
}
|
||||
}
|
||||
if _, err := r.retry(protocol.Message{RunID: 999, RequestKey: "missing"}); err == nil {
|
||||
t.Fatal("nonexistent run accepted")
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -44,7 +45,7 @@ func newRunner(ctx context.Context, s *pipeline.Store) (*runner, error) {
|
||||
return &runner{ctx: ctx, store: s, active: map[int64]context.CancelFunc{}}, nil
|
||||
}
|
||||
|
||||
func (r *runner) start(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
func (r *runner) start(request protocol.Message, retryOf ...int64) (*protocol.RunDetail, error) {
|
||||
if request.Version <= 0 || request.RequestKey == "" || len(request.RequestKey) > 128 {
|
||||
return nil, fmt.Errorf("an explicit version and request key are required")
|
||||
}
|
||||
@@ -97,9 +98,11 @@ func (r *runner) start(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
return nil, fmt.Errorf("write path must stay within the run directory")
|
||||
}
|
||||
}
|
||||
request.Input = maps.Clone(request.Input)
|
||||
if request.Input == nil {
|
||||
request.Input = map[string]any{}
|
||||
}
|
||||
request.Input["pwd"] = workspace
|
||||
if prompt, ok := request.Input["prompt"].(string); !ok || strings.TrimSpace(prompt) == "" {
|
||||
return nil, fmt.Errorf("a nonempty user prompt is required in input.prompt")
|
||||
}
|
||||
@@ -149,7 +152,7 @@ func (r *runner) start(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
return nil, fmt.Errorf("another run is active; inspect it before starting more work")
|
||||
}
|
||||
}
|
||||
id, created, err := r.store.CreateRun(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input)
|
||||
id, created, err := r.store.CreateRun(r.ctx, d, request.RequestKey, workspace, request.AllowTools, request.WritePaths, request.Input, retryOf...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -167,6 +170,20 @@ func (r *runner) start(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
return r.store.GetRun(r.ctx, id)
|
||||
}
|
||||
|
||||
func (r *runner) retry(request protocol.Message) (*protocol.RunDetail, error) {
|
||||
if request.RunID <= 0 {
|
||||
return nil, fmt.Errorf("retry requires a positive run ID")
|
||||
}
|
||||
original, err := r.store.GetRun(r.ctx, request.RunID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if original.Status != "failed" {
|
||||
return nil, fmt.Errorf("only failed runs can be retried (run %d is %s)", original.ID, original.Status)
|
||||
}
|
||||
return r.start(protocol.Message{PipelineID: original.PipelineID, Version: original.Version, RequestKey: request.RequestKey, Workspace: original.Workspace, Input: original.Input, AllowTools: original.AllowTools, WritePaths: original.WritePaths}, original.ID)
|
||||
}
|
||||
|
||||
func (r *runner) cancel(id int64) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -32,7 +32,7 @@ func TestRunnerRoutingPersistenceAndIdempotency(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
r.execute = func(ctx context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
|
||||
if def.Key == "one" {
|
||||
if len(input) != 1 || input["prompt"] != "task" {
|
||||
if len(input) != 2 || input["prompt"] != "task" || input["pwd"] == "" {
|
||||
t.Error("entry input must be the user prompt unchanged")
|
||||
}
|
||||
close(entered)
|
||||
|
||||
@@ -121,12 +121,15 @@ func Handler(ctx context.Context, stores ...*pipeline.Store) http.Handler {
|
||||
} else {
|
||||
response.Pipelines = items
|
||||
}
|
||||
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun) {
|
||||
} else if runs != nil && (request.Type == protocol.RunPipeline || request.Type == protocol.RetryRun || request.Type == protocol.ListRuns || request.Type == protocol.GetRun || request.Type == protocol.CancelRun) {
|
||||
var err error
|
||||
switch request.Type {
|
||||
case protocol.RunPipeline:
|
||||
response.Type = protocol.RunStarted
|
||||
response.RunDetail, err = runs.start(request)
|
||||
case protocol.RetryRun:
|
||||
response.Type = protocol.RunStarted
|
||||
response.RunDetail, err = runs.retry(request)
|
||||
case protocol.ListRuns:
|
||||
response.Type = protocol.Runs
|
||||
response.Runs, err = runs.store.ListRuns(ctx)
|
||||
|
||||
Reference in New Issue
Block a user