Validate providers and retain complete run debugging context
This commit is contained in:
+75
-5
@@ -19,6 +19,7 @@ import (
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/runinspect"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
"git.a-wan.net/alan/pipelines/internal/statusview"
|
||||
"git.a-wan.net/alan/pipelines/internal/tui"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
@@ -29,10 +30,10 @@ type App struct {
|
||||
In io.Reader
|
||||
}
|
||||
type options struct {
|
||||
url, name, prompt, template, workspace, paths, key, opencode string
|
||||
json, source, noOpen, yes, allow, createOnly, force, wait bool
|
||||
version, expected int
|
||||
timeout time.Duration
|
||||
url, name, prompt, template, workspace, paths, key, opencode, model string
|
||||
json, source, noOpen, yes, allow, createOnly, force, wait bool
|
||||
version, expected int
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func (a App) flags(command string) (*flag.FlagSet, *options) {
|
||||
@@ -43,7 +44,11 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
|
||||
f.BoolVar(&o.json, "json", false, "Machine-readable JSON; no interactive picker/editor")
|
||||
f.DurationVar(&o.timeout, "timeout", 30*time.Second, "Timeout per server request")
|
||||
switch command {
|
||||
case "provider-check":
|
||||
f.StringVar(&o.model, "model", envOr("PIPELINES_DEFAULT_MODEL", ""), "Provider/model to validate with one tool-free generation")
|
||||
f.BoolVar(&o.yes, "yes", false, "Approve the small validation generation without prompting")
|
||||
case "new", "edit":
|
||||
f.BoolVar(&o.yes, "yes", false, "Approve draft creation/editor launch without prompting")
|
||||
f.StringVar(&o.name, "name", "", "Pipeline name (or edit ID)")
|
||||
f.StringVar(&o.prompt, "prompt", "", "Request to give OpenCode; omitted asks what to change")
|
||||
f.BoolVar(&o.noOpen, "no-open", false, "Create the draft without opening OpenCode")
|
||||
@@ -83,6 +88,7 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
|
||||
case "cancel", "draft-delete":
|
||||
f.BoolVar(&o.yes, "yes", false, "Confirm this action without an interactive prompt")
|
||||
case "debug":
|
||||
f.BoolVar(&o.yes, "yes", false, "Approve opening a debugging session without prompting")
|
||||
f.StringVar(&o.opencode, "opencode", envOr("PIPELINES_OPENCODE", "opencode"), "Host OpenCode executable")
|
||||
f.BoolVar(&o.noOpen, "no-open", false, "Print the investigation prompt without launching OpenCode")
|
||||
}
|
||||
@@ -202,6 +208,39 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
return fmt.Errorf("timeout must be positive and versions nonnegative")
|
||||
}
|
||||
pos := f.Args()
|
||||
if command == "provider-check" {
|
||||
if len(pos) != 0 {
|
||||
return fmt.Errorf("provider-check takes no positional arguments")
|
||||
}
|
||||
provider, model, ok := strings.Cut(o.model, "/")
|
||||
if !ok || !stage.ValidModel(provider, model) {
|
||||
return fmt.Errorf("--model requires provider/model")
|
||||
}
|
||||
if err := a.confirmAction("Validate "+o.model+" with one small tool-free generation (provider charges may apply)?", o.yes); err != nil {
|
||||
return err
|
||||
}
|
||||
check, cancel := context.WithTimeout(ctx, o.timeout)
|
||||
defer cancel()
|
||||
operations := &stage.OperationLog{}
|
||||
session := ""
|
||||
c := stage.OpenCodeClient{Operations: operations, OnGenerationSession: func(id, backend, reference string, active bool) error { session = id; return nil }}
|
||||
text, err := c.Generate(check, stage.ModelConfig{Provider: provider, Model: model}, "Provider connection validation only. Do not use tools. Reply with exactly PIPELINES_PROVIDER_OK.")
|
||||
if err == nil && strings.TrimSpace(text) != "PIPELINES_PROVIDER_OK" {
|
||||
err = fmt.Errorf("model returned a response but did not complete the validation instruction")
|
||||
}
|
||||
result := map[string]any{"healthy": err == nil, "model": o.model, "session_id": session, "failures": operations.Failures}
|
||||
if err != nil {
|
||||
result["diagnostic"] = err.Error()
|
||||
}
|
||||
if o.json {
|
||||
if outputErr := a.output(result); outputErr != nil {
|
||||
return outputErr
|
||||
}
|
||||
} else if err == nil {
|
||||
fmt.Fprintln(a.Out, "Validated", o.model, "· retained session", session)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if command == "draft-delete" {
|
||||
if len(pos) != 1 {
|
||||
return fmt.Errorf("draft-delete requires one retained draft path")
|
||||
@@ -356,6 +395,20 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
debug := runinspect.New(response.RunDetail, executable, o.url)
|
||||
definition, definitionErr := call(protocol.Message{Type: protocol.GetPipeline, PipelineID: response.RunDetail.PipelineID, Version: response.RunDetail.Version})
|
||||
if definitionErr != nil {
|
||||
debug.Prompt += "\nPinned pipeline definition unavailable: " + definitionErr.Error() + "\n"
|
||||
} else if definition.Detail != nil {
|
||||
data, _ := json.Marshal(definition.Detail)
|
||||
debug.Prompt += "\nExact immutable pipeline definition and routing graph:\n" + string(data) + "\n"
|
||||
}
|
||||
live, evidenceErr := call(protocol.Message{Type: protocol.GetRunEvidence, RunID: id})
|
||||
if evidenceErr != nil {
|
||||
debug.Prompt += "\nServer-side live session evidence unavailable: " + evidenceErr.Error() + "\n"
|
||||
} else {
|
||||
data, _ := json.Marshal(live.Evidence)
|
||||
debug.Prompt += "\nRead-only retained OpenCode evidence retrieved on the execution computer:\n" + string(data) + "\n"
|
||||
}
|
||||
if o.json {
|
||||
return a.output(debug)
|
||||
}
|
||||
@@ -363,6 +416,16 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
_, err := fmt.Fprintln(a.Out, debug.Prompt)
|
||||
return err
|
||||
}
|
||||
if err := a.confirmAction(fmt.Sprintf("Open a fresh debugging session for run #%d?", id), o.yes); err != nil {
|
||||
return err
|
||||
}
|
||||
debug, err = debug.Prepare()
|
||||
if err != nil {
|
||||
return fmt.Errorf("retain debugging context: %w", err)
|
||||
}
|
||||
if debug.ContextFile != "" {
|
||||
fmt.Fprintln(a.Out, "Retained debugging context:", debug.ContextFile)
|
||||
}
|
||||
cmd := debug.Command(ctx, o.opencode)
|
||||
cmd.Stdin = a.In
|
||||
cmd.Stdout = a.Out
|
||||
@@ -668,7 +731,7 @@ func (a App) author(ctx context.Context, command string, pos []string, o *option
|
||||
return fmt.Errorf("pipeline already exists; use edit or choose another name")
|
||||
}
|
||||
}
|
||||
source = pipeline.Template([]string{"worker"}, true)
|
||||
source = pipeline.WithDefaultModel(pipeline.Template([]string{"worker"}, true), r.DefaultModel)
|
||||
if o.template != "" {
|
||||
r, err := call(protocol.Message{Type: protocol.ListTemplates})
|
||||
if err != nil {
|
||||
@@ -687,6 +750,13 @@ func (a App) author(ctx context.Context, command string, pos []string, o *option
|
||||
}
|
||||
}
|
||||
}
|
||||
confirmation := "Create a retained draft?"
|
||||
if !o.noOpen && !o.json {
|
||||
confirmation = "Create a retained draft and open OpenCode?"
|
||||
}
|
||||
if err := a.confirmAction(confirmation, o.yes); err != nil {
|
||||
return err
|
||||
}
|
||||
d, err := draft.Create(o.name, source, o.url, command == "new", base)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -32,7 +32,7 @@ func setup(t *testing.T) (func(...string) (string, error), *pipeline.Store, stri
|
||||
url := "ws" + strings.TrimPrefix(http.URL, "http") + "/connect"
|
||||
t.Setenv("PIPELINES_DRAFT_DIR", t.TempDir())
|
||||
return func(args ...string) (string, error) {
|
||||
if len(args) > 0 && args[0] == "submit" {
|
||||
if len(args) > 0 && (args[0] == "submit" || args[0] == "new" || args[0] == "edit") {
|
||||
args = append(args, "--yes")
|
||||
}
|
||||
var out, stderr bytes.Buffer
|
||||
@@ -63,7 +63,7 @@ func TestCLIStatusAndStatistics(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, text := range []string{"Pipelines server", "ntfy", "not configured", "Server computer IPs", "Port:"} {
|
||||
for _, text := range []string{"Pipelines server", "ntfy", "not configured", "Port:"} {
|
||||
if !strings.Contains(output, text) {
|
||||
t.Fatal("missing CLI status", text, output)
|
||||
}
|
||||
|
||||
+25
-24
@@ -2,30 +2,31 @@ package cli
|
||||
|
||||
// Help is also the public command inventory for agents and parity checks.
|
||||
var usage = map[string]string{
|
||||
"new": "new [name] [prompt] [--template KEY] [--no-open | --json]",
|
||||
"edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]",
|
||||
"drafts": "drafts [--json]",
|
||||
"draft-delete": "draft-delete <draft.md> [--json]",
|
||||
"validate": "validate <draft.md> [--json]",
|
||||
"submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]",
|
||||
"list": "list [--json]",
|
||||
"show": "show <name|id> [--version N] [--json]",
|
||||
"versions": "versions <name|id> [--json]",
|
||||
"export": "export <name|id> [--version N]",
|
||||
"templates": "templates [--json]",
|
||||
"template": "template <key> [--source | --json]",
|
||||
"run": "run [name|id] [prompt] [--json [--wait]]",
|
||||
"runs": "runs [--pipeline NAME] [--json]",
|
||||
"inspect": "inspect <run-id> [--json]",
|
||||
"cancel": "cancel <run-id> [--json]",
|
||||
"retry": "retry <run-id> [--json] [--wait]",
|
||||
"debug": "debug <run-id> [--json | --no-open]",
|
||||
"connection": "connection [--json]",
|
||||
"ping": "ping [--json]",
|
||||
"status": "status [--json]",
|
||||
"stats": "stats [--json]",
|
||||
"rpc": "rpc <request.json|->",
|
||||
"tui": "tui",
|
||||
"provider-check": "provider-check --model PROVIDER/MODEL [--yes] [--json]",
|
||||
"new": "new [name] [prompt] [--template KEY] [--no-open | --json]",
|
||||
"edit": "edit [name|id] [prompt] [--version N] [--no-open | --json]",
|
||||
"drafts": "drafts [--json]",
|
||||
"draft-delete": "draft-delete <draft.md> [--json]",
|
||||
"validate": "validate <draft.md> [--json]",
|
||||
"submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]",
|
||||
"list": "list [--json]",
|
||||
"show": "show <name|id> [--version N] [--json]",
|
||||
"versions": "versions <name|id> [--json]",
|
||||
"export": "export <name|id> [--version N]",
|
||||
"templates": "templates [--json]",
|
||||
"template": "template <key> [--source | --json]",
|
||||
"run": "run [name|id] [prompt] [--json [--wait]]",
|
||||
"runs": "runs [--pipeline NAME] [--json]",
|
||||
"inspect": "inspect <run-id> [--json]",
|
||||
"cancel": "cancel <run-id> [--json]",
|
||||
"retry": "retry <run-id> [--json] [--wait]",
|
||||
"debug": "debug <run-id> [--json | --no-open]",
|
||||
"connection": "connection [--json]",
|
||||
"ping": "ping [--json]",
|
||||
"status": "status [--json]",
|
||||
"stats": "stats [--json]",
|
||||
"rpc": "rpc <request.json|->",
|
||||
"tui": "tui",
|
||||
}
|
||||
|
||||
var notes = map[string]string{
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package cli_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
"git.a-wan.net/alan/pipelines/internal/runinspect"
|
||||
)
|
||||
|
||||
func TestDebugIncludesPinnedDefinitionAndRecordedFailure(t *testing.T) {
|
||||
run, store, _ := setup(t)
|
||||
ctx := context.Background()
|
||||
source := pipeline.Template([]string{"original-worker"}, true)
|
||||
if _, err := store.Submit(ctx, "workflow", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := store.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, err := store.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, _, err := store.CreateRun(ctx, definition, "debug-check", t.TempDir(), false, nil, map[string]any{"prompt": "original task for diagnosis"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = store.FinishRun(ctx, id, "failed", "error", "recorded provider failure", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = store.Submit(ctx, "workflow", pipeline.Template([]string{"new-worker"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
output, err := run("debug", fmt.Sprint(id), "--json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var debug runinspect.Debug
|
||||
if err = json.Unmarshal([]byte(output), &debug); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, expected := range []string{"original task for diagnosis", "recorded provider failure", "original-worker", "Exact immutable pipeline definition", "Read-only retained OpenCode evidence retrieved on the execution computer"} {
|
||||
if !strings.Contains(debug.Prompt, expected) {
|
||||
t.Fatal("missing debug context", expected, debug.Prompt)
|
||||
}
|
||||
}
|
||||
if strings.Contains(debug.Prompt, "new-worker") {
|
||||
t.Fatal("debug used latest definition instead of pinned version")
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ BROWSE
|
||||
template <key> Inspect a template; --source prints Markdown
|
||||
|
||||
EXECUTE
|
||||
provider-check Validate a provider/model with a tool-free response
|
||||
run [name] [prompt] Fuzzy selection, multiline prompt, live inspector
|
||||
runs List audited runs; --pipeline filters results
|
||||
inspect <run-id> Inspect stages, outputs, errors and status
|
||||
@@ -50,7 +51,7 @@ Use pipelines help <command> for its flags. Names with spaces must be quoted.
|
||||
Examples:
|
||||
pipelines edit "bug fix" "Add a review step"
|
||||
pipelines new "release notes" "Summarize changes and notify my phone"
|
||||
pipelines new "weekly digest" --template scheduled-summary --no-open --json
|
||||
pipelines new "weekly digest" --template scheduled-summary --no-open --json --yes
|
||||
pipelines validate /path/to/draft.md --json
|
||||
pipelines submit /path/to/draft.md
|
||||
pipelines run "bug fix" "Fix the layout" --json
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package cli_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"git.a-wan.net/alan/pipelines/internal/cli"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProviderCheckExecutesGenerationAndReportsFailures(t *testing.T) {
|
||||
for _, fail := range []bool{false, true} {
|
||||
bin := filepath.Join(t.TempDir(), "opencode")
|
||||
script := `#!/bin/sh
|
||||
case "$*" in
|
||||
*--help*) echo 'api';;
|
||||
*'/api/session/ses_check/generate'*) ` + map[bool]string{true: `echo '{"_tag":"UnauthorizedError","message":"expired provider credentials","token":"private-token"}'; exit 7`, false: `echo '{"data":{"text":"PIPELINES_PROVIDER_OK"}}'`}[fail] + `;;
|
||||
*'/api/session/ses_check/interrupt'*) echo '{}';;
|
||||
*'/api/session'*) echo '{"data":{"id":"ses_check"}}';;
|
||||
*) exit 9;;
|
||||
esac
|
||||
`
|
||||
if err := os.WriteFile(bin, []byte(script), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PIPELINES_OPENCODE", bin)
|
||||
var out, stderr bytes.Buffer
|
||||
app := cli.App{Out: &out, Err: &stderr, In: strings.NewReader("")}
|
||||
err := app.Run(context.Background(), []string{"provider-check", "--model", "fixture/test", "--json", "--yes"})
|
||||
if (err != nil) != fail || !strings.Contains(out.String(), "ses_check") {
|
||||
t.Fatal(err, out.String())
|
||||
}
|
||||
if fail && (!strings.Contains(out.String(), "expired provider credentials") || strings.Contains(out.String(), "private-token")) {
|
||||
t.Fatal(out.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderCheckDeclineDoesNotExecute(t *testing.T) {
|
||||
t.Setenv("PIPELINES_OPENCODE", "/does-not-exist")
|
||||
var out bytes.Buffer
|
||||
err := (cli.App{Out: &out, Err: &out, In: strings.NewReader("n\n")}).Run(context.Background(), []string{"provider-check", "--model", "fixture/test"})
|
||||
if err == nil || strings.Contains(err.Error(), "OpenCode") {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user