Validate providers and retain complete run debugging context

This commit is contained in:
2026-10-09 23:21:01 +01:00
parent f3d829bea5
commit 08b8647d5b
51 changed files with 1036 additions and 159 deletions
+75 -5
View File
@@ -19,6 +19,7 @@ import (
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/runinspect"
"git.a-wan.net/alan/pipelines/internal/stage"
"git.a-wan.net/alan/pipelines/internal/statusview"
"git.a-wan.net/alan/pipelines/internal/tui"
tea "github.com/charmbracelet/bubbletea"
@@ -29,10 +30,10 @@ type App struct {
In io.Reader
}
type options struct {
url, name, prompt, template, workspace, paths, key, opencode string
json, source, noOpen, yes, allow, createOnly, force, wait bool
version, expected int
timeout time.Duration
url, name, prompt, template, workspace, paths, key, opencode, model string
json, source, noOpen, yes, allow, createOnly, force, wait bool
version, expected int
timeout time.Duration
}
func (a App) flags(command string) (*flag.FlagSet, *options) {
@@ -43,7 +44,11 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
f.BoolVar(&o.json, "json", false, "Machine-readable JSON; no interactive picker/editor")
f.DurationVar(&o.timeout, "timeout", 30*time.Second, "Timeout per server request")
switch command {
case "provider-check":
f.StringVar(&o.model, "model", envOr("PIPELINES_DEFAULT_MODEL", ""), "Provider/model to validate with one tool-free generation")
f.BoolVar(&o.yes, "yes", false, "Approve the small validation generation without prompting")
case "new", "edit":
f.BoolVar(&o.yes, "yes", false, "Approve draft creation/editor launch without prompting")
f.StringVar(&o.name, "name", "", "Pipeline name (or edit ID)")
f.StringVar(&o.prompt, "prompt", "", "Request to give OpenCode; omitted asks what to change")
f.BoolVar(&o.noOpen, "no-open", false, "Create the draft without opening OpenCode")
@@ -83,6 +88,7 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
case "cancel", "draft-delete":
f.BoolVar(&o.yes, "yes", false, "Confirm this action without an interactive prompt")
case "debug":
f.BoolVar(&o.yes, "yes", false, "Approve opening a debugging session without prompting")
f.StringVar(&o.opencode, "opencode", envOr("PIPELINES_OPENCODE", "opencode"), "Host OpenCode executable")
f.BoolVar(&o.noOpen, "no-open", false, "Print the investigation prompt without launching OpenCode")
}
@@ -202,6 +208,39 @@ func (a App) Run(ctx context.Context, args []string) error {
return fmt.Errorf("timeout must be positive and versions nonnegative")
}
pos := f.Args()
if command == "provider-check" {
if len(pos) != 0 {
return fmt.Errorf("provider-check takes no positional arguments")
}
provider, model, ok := strings.Cut(o.model, "/")
if !ok || !stage.ValidModel(provider, model) {
return fmt.Errorf("--model requires provider/model")
}
if err := a.confirmAction("Validate "+o.model+" with one small tool-free generation (provider charges may apply)?", o.yes); err != nil {
return err
}
check, cancel := context.WithTimeout(ctx, o.timeout)
defer cancel()
operations := &stage.OperationLog{}
session := ""
c := stage.OpenCodeClient{Operations: operations, OnGenerationSession: func(id, backend, reference string, active bool) error { session = id; return nil }}
text, err := c.Generate(check, stage.ModelConfig{Provider: provider, Model: model}, "Provider connection validation only. Do not use tools. Reply with exactly PIPELINES_PROVIDER_OK.")
if err == nil && strings.TrimSpace(text) != "PIPELINES_PROVIDER_OK" {
err = fmt.Errorf("model returned a response but did not complete the validation instruction")
}
result := map[string]any{"healthy": err == nil, "model": o.model, "session_id": session, "failures": operations.Failures}
if err != nil {
result["diagnostic"] = err.Error()
}
if o.json {
if outputErr := a.output(result); outputErr != nil {
return outputErr
}
} else if err == nil {
fmt.Fprintln(a.Out, "Validated", o.model, "· retained session", session)
}
return err
}
if command == "draft-delete" {
if len(pos) != 1 {
return fmt.Errorf("draft-delete requires one retained draft path")
@@ -356,6 +395,20 @@ func (a App) Run(ctx context.Context, args []string) error {
return err
}
debug := runinspect.New(response.RunDetail, executable, o.url)
definition, definitionErr := call(protocol.Message{Type: protocol.GetPipeline, PipelineID: response.RunDetail.PipelineID, Version: response.RunDetail.Version})
if definitionErr != nil {
debug.Prompt += "\nPinned pipeline definition unavailable: " + definitionErr.Error() + "\n"
} else if definition.Detail != nil {
data, _ := json.Marshal(definition.Detail)
debug.Prompt += "\nExact immutable pipeline definition and routing graph:\n" + string(data) + "\n"
}
live, evidenceErr := call(protocol.Message{Type: protocol.GetRunEvidence, RunID: id})
if evidenceErr != nil {
debug.Prompt += "\nServer-side live session evidence unavailable: " + evidenceErr.Error() + "\n"
} else {
data, _ := json.Marshal(live.Evidence)
debug.Prompt += "\nRead-only retained OpenCode evidence retrieved on the execution computer:\n" + string(data) + "\n"
}
if o.json {
return a.output(debug)
}
@@ -363,6 +416,16 @@ func (a App) Run(ctx context.Context, args []string) error {
_, err := fmt.Fprintln(a.Out, debug.Prompt)
return err
}
if err := a.confirmAction(fmt.Sprintf("Open a fresh debugging session for run #%d?", id), o.yes); err != nil {
return err
}
debug, err = debug.Prepare()
if err != nil {
return fmt.Errorf("retain debugging context: %w", err)
}
if debug.ContextFile != "" {
fmt.Fprintln(a.Out, "Retained debugging context:", debug.ContextFile)
}
cmd := debug.Command(ctx, o.opencode)
cmd.Stdin = a.In
cmd.Stdout = a.Out
@@ -668,7 +731,7 @@ func (a App) author(ctx context.Context, command string, pos []string, o *option
return fmt.Errorf("pipeline already exists; use edit or choose another name")
}
}
source = pipeline.Template([]string{"worker"}, true)
source = pipeline.WithDefaultModel(pipeline.Template([]string{"worker"}, true), r.DefaultModel)
if o.template != "" {
r, err := call(protocol.Message{Type: protocol.ListTemplates})
if err != nil {
@@ -687,6 +750,13 @@ func (a App) author(ctx context.Context, command string, pos []string, o *option
}
}
}
confirmation := "Create a retained draft?"
if !o.noOpen && !o.json {
confirmation = "Create a retained draft and open OpenCode?"
}
if err := a.confirmAction(confirmation, o.yes); err != nil {
return err
}
d, err := draft.Create(o.name, source, o.url, command == "new", base)
if err != nil {
return err
+2 -2
View File
@@ -32,7 +32,7 @@ func setup(t *testing.T) (func(...string) (string, error), *pipeline.Store, stri
url := "ws" + strings.TrimPrefix(http.URL, "http") + "/connect"
t.Setenv("PIPELINES_DRAFT_DIR", t.TempDir())
return func(args ...string) (string, error) {
if len(args) > 0 && args[0] == "submit" {
if len(args) > 0 && (args[0] == "submit" || args[0] == "new" || args[0] == "edit") {
args = append(args, "--yes")
}
var out, stderr bytes.Buffer
@@ -63,7 +63,7 @@ func TestCLIStatusAndStatistics(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, text := range []string{"Pipelines server", "ntfy", "not configured", "Server computer IPs", "Port:"} {
for _, text := range []string{"Pipelines server", "ntfy", "not configured", "Port:"} {
if !strings.Contains(output, text) {
t.Fatal("missing CLI status", text, output)
}
+25 -24
View File
@@ -2,30 +2,31 @@ package cli
// Help is also the public command inventory for agents and parity checks.
var usage = map[string]string{
"new": "new [name] [prompt] [--template KEY] [--no-open | --json]",
"edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]",
"drafts": "drafts [--json]",
"draft-delete": "draft-delete <draft.md> [--json]",
"validate": "validate <draft.md> [--json]",
"submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]",
"list": "list [--json]",
"show": "show <name|id> [--version N] [--json]",
"versions": "versions <name|id> [--json]",
"export": "export <name|id> [--version N]",
"templates": "templates [--json]",
"template": "template <key> [--source | --json]",
"run": "run [name|id] [prompt] [--json [--wait]]",
"runs": "runs [--pipeline NAME] [--json]",
"inspect": "inspect <run-id> [--json]",
"cancel": "cancel <run-id> [--json]",
"retry": "retry <run-id> [--json] [--wait]",
"debug": "debug <run-id> [--json | --no-open]",
"connection": "connection [--json]",
"ping": "ping [--json]",
"status": "status [--json]",
"stats": "stats [--json]",
"rpc": "rpc <request.json|->",
"tui": "tui",
"provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]",
"new": "new [name] [prompt] [--template KEY] [--no-open | --json]",
"edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]",
"drafts": "drafts [--json]",
"draft-delete": "draft-delete <draft.md> [--json]",
"validate": "validate <draft.md> [--json]",
"submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]",
"list": "list [--json]",
"show": "show <name|id> [--version N] [--json]",
"versions": "versions <name|id> [--json]",
"export": "export <name|id> [--version N]",
"templates": "templates [--json]",
"template": "template <key> [--source | --json]",
"run": "run [name|id] [prompt] [--json [--wait]]",
"runs": "runs [--pipeline NAME] [--json]",
"inspect": "inspect <run-id> [--json]",
"cancel": "cancel <run-id> [--json]",
"retry": "retry <run-id> [--json] [--wait]",
"debug": "debug <run-id> [--json | --no-open]",
"connection": "connection [--json]",
"ping": "ping [--json]",
"status": "status [--json]",
"stats": "stats [--json]",
"rpc": "rpc <request.json|->",
"tui": "tui",
}
var notes = map[string]string{
+55
View File
@@ -0,0 +1,55 @@
package cli_test
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/runinspect"
)
func TestDebugIncludesPinnedDefinitionAndRecordedFailure(t *testing.T) {
run, store, _ := setup(t)
ctx := context.Background()
source := pipeline.Template([]string{"original-worker"}, true)
if _, err := store.Submit(ctx, "workflow", source); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
definition, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
id, _, err := store.CreateRun(ctx, definition, "debug-check", t.TempDir(), false, nil, map[string]any{"prompt": "original task for diagnosis"})
if err != nil {
t.Fatal(err)
}
if err = store.FinishRun(ctx, id, "failed", "error", "recorded provider failure", nil); err != nil {
t.Fatal(err)
}
if _, err = store.Submit(ctx, "workflow", pipeline.Template([]string{"new-worker"}, true)); err != nil {
t.Fatal(err)
}
output, err := run("debug", fmt.Sprint(id), "--json")
if err != nil {
t.Fatal(err)
}
var debug runinspect.Debug
if err = json.Unmarshal([]byte(output), &debug); err != nil {
t.Fatal(err)
}
for _, expected := range []string{"original task for diagnosis", "recorded provider failure", "original-worker", "Exact immutable pipeline definition", "Read-only retained OpenCode evidence retrieved on the execution computer"} {
if !strings.Contains(debug.Prompt, expected) {
t.Fatal("missing debug context", expected, debug.Prompt)
}
}
if strings.Contains(debug.Prompt, "new-worker") {
t.Fatal("debug used latest definition instead of pinned version")
}
}
+2 -1
View File
@@ -20,6 +20,7 @@ BROWSE
template <key> Inspect a template; --source prints Markdown
EXECUTE
provider-check Validate a provider/model with a tool-free response
run [name] [prompt] Fuzzy selection, multiline prompt, live inspector
runs List audited runs; --pipeline filters results
inspect <run-id> Inspect stages, outputs, errors and status
@@ -50,7 +51,7 @@ Use pipelines help <command> for its flags. Names with spaces must be quoted.
Examples:
pipelines edit "bug fix" "Add a review step"
pipelines new "release notes" "Summarize changes and notify my phone"
pipelines new "weekly digest" --template scheduled-summary --no-open --json
pipelines new "weekly digest" --template scheduled-summary --no-open --json --yes
pipelines validate /path/to/draft.md --json
pipelines submit /path/to/draft.md
pipelines run "bug fix" "Fix the layout" --json
+48
View File
@@ -0,0 +1,48 @@
package cli_test
import (
"bytes"
"context"
"git.a-wan.net/alan/pipelines/internal/cli"
"os"
"path/filepath"
"strings"
"testing"
)
func TestProviderCheckExecutesGenerationAndReportsFailures(t *testing.T) {
for _, fail := range []bool{false, true} {
bin := filepath.Join(t.TempDir(), "opencode")
script := `#!/bin/sh
case "$*" in
*--help*) echo 'api';;
*'/api/session/ses_check/generate'*) ` + map[bool]string{true: `echo '{"_tag":"UnauthorizedError","message":"expired provider credentials","token":"private-token"}'; exit 7`, false: `echo '{"data":{"text":"PIPELINES_PROVIDER_OK"}}'`}[fail] + `;;
*'/api/session/ses_check/interrupt'*) echo '{}';;
*'/api/session'*) echo '{"data":{"id":"ses_check"}}';;
*) exit 9;;
esac
`
if err := os.WriteFile(bin, []byte(script), 0700); err != nil {
t.Fatal(err)
}
t.Setenv("PIPELINES_OPENCODE", bin)
var out, stderr bytes.Buffer
app := cli.App{Out: &out, Err: &stderr, In: strings.NewReader("")}
err := app.Run(context.Background(), []string{"provider-check", "--model", "fixture/test", "--json", "--yes"})
if (err != nil) != fail || !strings.Contains(out.String(), "ses_check") {
t.Fatal(err, out.String())
}
if fail && (!strings.Contains(out.String(), "expired provider credentials") || strings.Contains(out.String(), "private-token")) {
t.Fatal(out.String())
}
}
}
func TestProviderCheckDeclineDoesNotExecute(t *testing.T) {
t.Setenv("PIPELINES_OPENCODE", "/does-not-exist")
var out bytes.Buffer
err := (cli.App{Out: &out, Err: &out, In: strings.NewReader("n\n")}).Run(context.Background(), []string{"provider-check", "--model", "fixture/test"})
if err == nil || strings.Contains(err.Error(), "OpenCode") {
t.Fatal(err)
}
}