From 88dfc051e54a262d494a3b29176c1903cd3da1b6 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Wed, 4 Nov 2026 17:20:58 +0000 Subject: [PATCH 1/3] Model stages: outcome-first output shape and one in-session outcome repair The prompt now says output.md must start with # outcome and the outcome word alone on the next line, then # handoff, then detail. Runs 241/243 followed the old "start with # handoff" wording and dropped the outcome. When a completed turn lacks a valid outcome, the OpenCode/agent and Pi clients send one short follow-up in the same session asking only for the outcome line. An outcome-only reply goes at the top of the original report; a still-invalid reply fails with the same validation error as before. The follow-up's tokens accrue to the session usage. No outcome is inferred. Missing tools/evidence now name blocked when declared, else error. The parser tolerates a trailing period on the outcome word. --- docs/model-results.md | 14 +- internal/stage/agent.go | 6 + internal/stage/model_diagnostic_test.go | 15 ++- internal/stage/model_prompt.go | 27 +++- internal/stage/model_repair.go | 88 ++++++++++++ internal/stage/model_repair_test.go | 171 ++++++++++++++++++++++++ internal/stage/model_run.go | 4 +- internal/stage/model_test.go | 2 +- internal/stage/opencode.go | 7 + internal/stage/pi.go | 25 +++- internal/stage/pi_test.go | 23 +++- internal/turn/output.go | 59 +++++++- internal/turn/output_test.go | 32 +++++ 13 files changed, 457 insertions(+), 16 deletions(-) create mode 100644 internal/stage/model_repair.go create mode 100644 internal/stage/model_repair_test.go diff --git a/docs/model-results.md b/docs/model-results.md index 65300f4a..a0a635f6 100644 --- a/docs/model-results.md +++ b/docs/model-results.md @@ -59,12 +59,21 @@ success The rest of the output is free-form markdown. ``` -Matching ignores case and surrounding whitespace and returns the canonical edge +Matching ignores case, surrounding whitespace and one trailing period, and returns the canonical edge spelling. Headings and words inside fenced code blocks are ignored, so a code sample cannot add a false outcome. Missing, duplicate, empty or undeclared outcomes fail the stage. The parser never guesses, repairs the text, or looks for any other section. +When a turn completes but its final message has no valid outcome, the harness +client (OpenCode or Pi) sends one short follow-up in the same session asking +only for the `# outcome` line and listing the allowed outcomes, without redoing +work. An outcome-only reply is placed at the top of the original output (any +invalid outcome section is replaced); a full reply replaces it. The outcome is +always the model's explicit answer, never inferred from prose. Its tokens count +toward the stage usage. If the reply is still invalid, the stage fails with the +same validation error as before. + Everything after the outcome value line is retained as the turn body under the single `markdown` key of the internal output map. No JSON envelope, output schema, field selection, `output_contract`, or format-repair pass exists for @@ -81,7 +90,8 @@ facts: - the paths to `memory.md`, `input.md` and `output.md` (or their contents for a text-only stage); - the stage prompt verbatim; -- the worked outcome example and the complete allowed outcome list; +- the required shape, outcome first (`# outcome` and one of the allowed + outcomes, then `# handoff`, then any detail), with a worked example; - the execution time and timezone; - a granted resource target, when one exists; - the writable `private/` and shared `public/` artifact directories, when they diff --git a/internal/stage/agent.go b/internal/stage/agent.go index e71b7e6f..007df8cb 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -241,6 +241,12 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s if len(data) > 0 { c.Transcript = string(data) } + if err == nil { + text, data, err = c.repairOutput(ctx, config, c.SessionID, text) + if len(data) > 0 { + c.Transcript = string(data) + } + } // Budgeted stages already settle through the single watch.finish read. For an // unbudgeted V2 stage the watch was never started, so record best-effort final // session usage from exactly one bounded aggregate read; missing/invalid usage diff --git a/internal/stage/model_diagnostic_test.go b/internal/stage/model_diagnostic_test.go index af4128ed..88cf42a6 100644 --- a/internal/stage/model_diagnostic_test.go +++ b/internal/stage/model_diagnostic_test.go @@ -46,13 +46,14 @@ func TestModelFailureCategoriesAreSanitizedAndDistinct(t *testing.T) { // TestAgentPostEditValidationFailureRetainsWorkWithoutReplay reproduces a // tool-using stage that edits once and then returns a nested-duplicate final -// result. The validation failure must not re-prompt the agent, must retain the -// admitted session, transcript and evidence, and must carry the precise category. +// result. The only extra prompt is the single outcome repair follow-up, which +// does not redo work; when the reply is still invalid the stage fails as +// before, retaining the admitted session, transcript, evidence and category. func TestAgentPostEditValidationFailureRetainsWorkWithoutReplay(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "README.md") review := "# outcome\nsuccess\n\n# evidence\n\nprivate-evidence\n\n# outcome\nerror\n" - creates, edits, owned := 0, 0, 0 + creates, edits, repairs, owned := 0, 0, 0, 0 agent := &AgentClient{OpenCodeClient: OpenCodeClient{Directory: dir}, OnSession: func(string, bool) error { owned++; return nil }} agent.command = func(_ context.Context, _ string, args ...string) ([]byte, error) { if args[1] == "delete" { @@ -63,6 +64,10 @@ func TestAgentPostEditValidationFailureRetainsWorkWithoutReplay(t *testing.T) { creates++ return []byte(fmt.Sprintf(`{"data":{"id":"ses_%d"}}`, creates)), nil case "/api/session/ses_1/prompt": + if strings.Contains(args[len(args)-1], "no valid `# outcome`") { + repairs++ + return []byte(`{}`), nil + } edits++ if err := os.WriteFile(path, []byte("edited once"), 0600); err != nil { t.Fatal(err) @@ -85,8 +90,8 @@ func TestAgentPostEditValidationFailureRetainsWorkWithoutReplay(t *testing.T) { if strings.Contains(err.Error(), "HARNESS TRANSPORT") || strings.Contains(err.Error(), "private-evidence") { t.Fatalf("validation diagnostic leaked content or a false category: %v", err) } - if creates != 1 || edits != 1 || owned != 2 || agent.SessionID != "ses_1" || result.Output != nil { - t.Fatal("validation failure replayed work or dropped retained ownership", creates, edits, owned, agent.SessionID, result) + if creates != 1 || edits != 1 || repairs != 1 || owned != 2 || agent.SessionID != "ses_1" || result.Output != nil { + t.Fatal("validation failure replayed work or dropped retained ownership", creates, edits, repairs, owned, agent.SessionID, result) } if !json.Valid([]byte(agent.Transcript)) || !strings.Contains(agent.Transcript, "private-evidence") { t.Fatal("retained readback evidence missing", agent.Transcript) diff --git a/internal/stage/model_prompt.go b/internal/stage/model_prompt.go index 456bb5fd..1b7ee346 100644 --- a/internal/stage/model_prompt.go +++ b/internal/stage/model_prompt.go @@ -105,19 +105,40 @@ func modelPrompt(def Definition, spec ModelSpec, input Values) (string, error) { } b.WriteString("\n\nProduce ") b.WriteString(def.Turn.Output) - b.WriteString(" as markdown; return its complete content as your final message (the server writes that file). Start it with a short # handoff section (at most about 1 KB): what you did, files changed, and open issues or findings the next stage must act on. The next stage receives only that section as essential memory; the rest of your output stays in history. The only required section is the outcome, for example:\n\n# handoff\n...\n\n# outcome\n") + b.WriteString(" as markdown and return its complete content as your final message (the server writes that file; no JSON envelope). ") + b.WriteString(outputShape(outcomes)) + b.WriteString(" The next stage receives only the handoff as essential memory. For example:\n\n# outcome\n") b.WriteString(preferredOutcome(outcomes)) - fmt.Fprintf(&b, "\n\nAllowed outcomes: %s.\n\nEverything else is free-form markdown for the next stage and the operator. The server does not parse or validate any other section. Return the complete output.md content as your final message; do not wrap it in a JSON envelope.\n", strings.Join(outcomes, ", ")) + b.WriteString("\n\n# handoff\nWhat you did, files changed, open issues for the next stage.\n\n# details\n...\n") fmt.Fprintf(&b, "\nExecution time: %s. Timezone: %s.\n", time.Now().Format(time.RFC3339), time.Local.String()) if def.Turn.Public != "" { fmt.Fprintf(&b, "\nShared artifacts go to %s; files there are visible to every later stage. That absolute path is the shared folder: a relative public/ inside the workspace is a different, unshared directory. Turn-local work goes to %s and is not handed to later stages.\n", def.Turn.Public, def.Turn.Private) } b.WriteString("\n") b.WriteString(toolsLine) - b.WriteString(" Do not invent file edits, command results, screenshots, or observations. If the task requires unavailable tools or evidence, choose the stage's error outcome and explain the limitation. Never claim unperformed work succeeded.") + b.WriteString(" Do not invent file edits, command results, screenshots, or observations. If the task requires unavailable tools or evidence, ") + b.WriteString(limitationOutcome(outcomes)) + b.WriteString(" and explain the limitation. Never claim unperformed work succeeded.") return b.String(), nil } +// outputShape is the one sentence stating the required output.md layout, +// outcome first. The repair follow-up reuses it so both say the same thing. +func outputShape(outcomes []string) string { + return "output.md must start with `# outcome` and the outcome word alone on the next line, one of: " + strings.Join(outcomes, ", ") + "; then `# handoff` (a few lines); then any detail." +} + +// limitationOutcome names the outcome for missing tools or evidence: blocked +// when the stage declares it, otherwise error. +func limitationOutcome(outcomes []string) string { + for _, want := range []string{"blocked", "error"} { + if slices.Contains(outcomes, want) { + return "choose `" + want + "`" + } + } + return "choose the stage's error outcome" +} + // serverChangedFiles reads the runner's reserved changed-path list from stage // input, accepting the []string the runner supplies and the []any JSON decoding // produces for a replayed handoff. Other values are ignored. diff --git a/internal/stage/model_repair.go b/internal/stage/model_repair.go new file mode 100644 index 00000000..db75e888 --- /dev/null +++ b/internal/stage/model_repair.go @@ -0,0 +1,88 @@ +package stage + +import ( + "context" + "errors" + "strings" + + "git.a-wan.net/alan/pipelines/internal/turn" +) + +// outputRepair is the single outcome repair follow-up. When a turn completes +// but its final message lacks a valid outcome, the client sends one short +// message in the same session asking only for the outcome, instead of failing +// a stage whose admitted work a full retry would repeat. The outcome is always +// stated by the model; the server never infers one from prose. +type outputRepair struct{ outcomes []string } + +type outputRepairKey struct{} + +// withOutputRepair lets a model client repair a missing or invalid outcome +// for the given declared outcomes while its session is still open. +func withOutputRepair(ctx context.Context, outcomes []string) context.Context { + return context.WithValue(ctx, outputRepairKey{}, &outputRepair{outcomes: outcomes}) +} + +// outputRepairFrom returns the attempt's repair, or nil when none applies. +func outputRepairFrom(ctx context.Context) *outputRepair { + repair, _ := ctx.Value(outputRepairKey{}).(*outputRepair) + return repair +} + +// followUp reports whether text needs the outcome follow-up and returns its +// message. Valid and empty outputs need none: an empty output has no report +// to keep, so it fails as before. +func (r *outputRepair) followUp(text string) (string, bool) { + if r == nil || len(r.outcomes) == 0 || strings.TrimSpace(text) == "" { + return "", false + } + _, _, err := turn.ParseOutput(text, r.outcomes) + if !errors.Is(err, turn.ErrMissingOutcome) && !errors.Is(err, turn.ErrOutcomeNotAllowed) && !errors.Is(err, turn.ErrDuplicateOutcome) { + return "", false + } + allowed := strings.Join(r.outcomes, ", ") + return "Your final message has no valid `# outcome` section. Do not redo any work or repeat your report; the server keeps it. " + + outputShape(r.outcomes) + " Reply with only these two lines:\n\n# outcome\n\n", true +} + +// merge combines the original output with the follow-up reply. An +// outcome-only reply is placed at the top of the original report, replacing +// any invalid outcome section. A reply carrying a full document replaces the +// original. An invalid reply keeps the original, so validation fails as before. +func (r *outputRepair) merge(original, reply string) string { + outcome, body, err := turn.ParseOutput(reply, r.outcomes) + if err != nil { + return original + } + if strings.TrimSpace(body) == "" { + return turn.WithOutcome(original, outcome) + } + return reply +} + +// repairOutput sends the outcome follow-up in an open OpenCode session when +// text needs it, waits for that turn and returns the merged output with the +// raw final readback. Tokens it uses accrue to the same session, so the +// existing budget watch and session usage read count them. +func (c OpenCodeClient) repairOutput(ctx context.Context, config ModelConfig, sessionID, text string) (string, []byte, error) { + repair := outputRepairFrom(ctx) + message, ok := repair.followUp(text) + if !ok { + return text, nil, nil + } + if _, err := c.call(ctx, "post", "/api/session/"+sessionID+"/prompt", generationBody("text", message, config, c.Backend)); err != nil { + return "", nil, err + } + if c.Backend == "v1" { + if _, err := c.call(ctx, "post", "/api/experimental/session/"+sessionID+"/wait", nil); err != nil { + return "", nil, err + } + } else if err := c.waitForSessionIdle(ctx, sessionID); err != nil { + return "", nil, err + } + data, reply, err := c.readFinalAgentResult(ctx, sessionID) + if err != nil { + return "", data, err + } + return repair.merge(text, reply), data, nil +} diff --git a/internal/stage/model_repair_test.go b/internal/stage/model_repair_test.go new file mode 100644 index 00000000..4ba6d178 --- /dev/null +++ b/internal/stage/model_repair_test.go @@ -0,0 +1,171 @@ +package stage + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/turn" +) + +// repairAgent serves one V2 agent session whose first final message is +// handoffOnly and whose follow-up reply is reply. It records the call order. +func repairAgent(handoffOnly, reply string, calls *[]string) *AgentClient { + prompts := 0 + return &AgentClient{OpenCodeClient: OpenCodeClient{Backend: "v2", sessionPollInterval: time.Millisecond, command: func(_ context.Context, _ string, args ...string) ([]byte, error) { + path := args[2] + switch { + case path == "/api/session" && args[1] == "post": + return []byte(`{"data":{"id":"ses_1"}}`), nil + case path == "/api/session/ses_1/prompt": + prompts++ + body := args[len(args)-1] + if prompts == 2 && !strings.Contains(body, "no valid `# outcome`") { + return nil, fmt.Errorf("second prompt is not the repair follow-up: %s", body) + } + *calls = append(*calls, fmt.Sprintf("prompt%d", prompts)) + return []byte(`{}`), nil + case path == "/api/session/active": + return []byte(`{"data":{}}`), nil + case strings.HasSuffix(path, finalAssistantReadback): + text := handoffOnly + if prompts == 2 { + text = reply + } + return json.Marshal(map[string]any{"data": []any{map[string]any{"finish": "stop", "content": []any{map[string]string{"type": "text", "text": text}}}}}) + case path == "/api/session/ses_1": + *calls = append(*calls, "usage") + return usageResponse("ses_1", .2, 42), nil + case strings.HasSuffix(path, "/interrupt"): + return []byte(`{}`), nil + } + return nil, fmt.Errorf("unexpected path %s", path) + }}} +} + +// TestHandoffOnlyOutputIsRepairedInSession reproduces runs 241/243: the model +// starts with # handoff and omits the outcome. One follow-up in the same +// session asks only for the outcome; its reply is used and the report kept. +func TestHandoffOnlyOutputIsRepairedInSession(t *testing.T) { + report := "# handoff\n\nImplemented cache.go; tests pass.\n\n# details\n\nLong report.\n" + var calls []string + agent := repairAgent(report, "# outcome\nsuccess\n", &calls) + def := turnDefinition(t) + def.Blocks["tools"] = "edit" + budget := NewBudget(0, 0) + result, err := (ModelStage{Client: agent}).Run(WithBudget(context.Background(), budget), def, nil) + if err != nil || result.Outcome != "success" { + t.Fatalf("repair did not recover the outcome: %v %v", result, err) + } + written, _ := os.ReadFile(def.Turn.Output) + if string(written) != "# outcome\nsuccess\n\n"+report { + t.Fatalf("output.md = %q", written) + } + if body := result.Output["markdown"].(string); !strings.Contains(body, "Implemented cache.go") || !strings.Contains(body, "Long report.") { + t.Fatalf("report dropped: %q", body) + } + // The usage read follows the follow-up, so its tokens are counted. + if strings.Join(calls, ",") != "prompt1,prompt2,usage" || budget.Snapshot().UsedTokens != 42 { + t.Fatalf("calls=%v usage=%+v", calls, budget.Snapshot()) + } +} + +// TestFailedRepairFailsAsBefore proves a follow-up that still lacks a valid +// outcome fails with today's missing-outcome error and keeps the report. +func TestFailedRepairFailsAsBefore(t *testing.T) { + report := "# handoff\n\nDid the work.\n" + for _, reply := range []string{"I think it went well.\n", "# outcome\nmaybe\n"} { + var calls []string + def := turnDefinition(t) + def.Blocks["tools"] = "edit" + _, err := (ModelStage{Client: repairAgent(report, reply, &calls)}).Run(context.Background(), def, nil) + if !errors.Is(err, ErrInvalidModelResult) || !strings.Contains(err.Error(), turn.ErrMissingOutcome.Error()) || !strings.Contains(err.Error(), "FINAL RESULT VALIDATION") { + t.Fatalf("reply %q: err = %v", reply, err) + } + if strings.Count(strings.Join(calls, ","), "prompt") != 2 { + t.Fatalf("expected exactly one follow-up: %v", calls) + } + if written, _ := os.ReadFile(def.Turn.Output); string(written) != report { + t.Fatalf("output.md = %q", written) + } + } +} + +// TestValidOutputSendsNoFollowUp keeps a valid first answer to one prompt. +func TestValidOutputSendsNoFollowUp(t *testing.T) { + var calls []string + def := turnDefinition(t) + def.Blocks["tools"] = "edit" + result, err := (ModelStage{Client: repairAgent("# outcome\nerror\n\n# handoff\n\nblocked\n", "", &calls)}).Run(context.Background(), def, nil) + if err != nil || result.Outcome != "error" || strings.Count(strings.Join(calls, ","), "prompt") != 1 { + t.Fatalf("%v %v %v", result, err, calls) + } +} + +// TestModelPromptStatesOutcomeFirst checks the required shape: outcome first, +// then handoff, then detail, with the allowed outcomes listed. +func TestModelPromptStatesOutcomeFirst(t *testing.T) { + def := modelDefinition() + spec, err := (ModelStage{}).Spec(def) + if err != nil { + t.Fatal(err) + } + prompt, err := modelPrompt(def, spec, nil) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(prompt, "output.md must start with `# outcome` and the outcome word alone on the next line, one of: error, success; then `# handoff` (a few lines); then any detail.") { + t.Fatalf("shape sentence missing: %s", prompt) + } + outcome, handoff := strings.Index(prompt, "\n# outcome\n"), strings.Index(prompt, "\n# handoff\n") + if outcome < 0 || handoff < 0 || outcome > handoff { + t.Fatalf("example does not put the outcome before the handoff: %s", prompt) + } + if strings.Contains(prompt, "Start it with a short # handoff") { + t.Fatal("prompt still asks to start with the handoff") + } + if !strings.Contains(prompt, "unavailable tools or evidence, choose `error` and explain") { + t.Fatalf("limitation outcome not error: %s", prompt) + } +} + +func TestLimitationOutcomePrefersBlocked(t *testing.T) { + for outcomes, want := range map[string]string{ + "blocked,error,success": "choose `blocked`", + "error,success": "choose `error`", + "done,retry": "choose the stage's error outcome", + } { + if got := limitationOutcome(strings.Split(outcomes, ",")); got != want { + t.Fatalf("%s: %q, want %q", outcomes, got, want) + } + } +} + +func TestRepairMerge(t *testing.T) { + r := &outputRepair{outcomes: []string{"error", "success"}} + if _, ok := r.followUp("# outcome\nsuccess\n"); ok { + t.Fatal("valid output needs no follow-up") + } + if _, ok := r.followUp(" \n"); ok { + t.Fatal("empty output is not repaired") + } + message, ok := r.followUp("# handoff\n\nx\n") + if !ok || !strings.Contains(message, "error, success") || !strings.Contains(message, "Do not redo any work") { + t.Fatalf("follow-up = %q", message) + } + if got := r.merge("# handoff\n\nx\n", "# outcome\nSuccess\n"); got != "# outcome\nsuccess\n\n# handoff\n\nx\n" { + t.Fatalf("outcome-only merge = %q", got) + } + full := "# outcome\nerror\n\n# handoff\n\ny\n" + if got := r.merge("# handoff\n\nx\n", full); got != full { + t.Fatalf("full reply merge = %q", got) + } + if got := r.merge("# handoff\n\nx\n", "no outcome here"); got != "# handoff\n\nx\n" { + t.Fatalf("invalid reply merge = %q", got) + } +} diff --git a/internal/stage/model_run.go b/internal/stage/model_run.go index 774e39e9..8839b4c7 100644 --- a/internal/stage/model_run.go +++ b/internal/stage/model_run.go @@ -61,7 +61,9 @@ func (m ModelStage) Run(ctx context.Context, def Definition, input Values) (Resu } spec.Config.Budget.Attempt() attemptCtx, cancel := context.WithTimeoutCause(ctx, spec.Config.Timeout, &stageTimeoutCause{timeout: spec.Config.Timeout}) - text, generateErr := client.Generate(attemptCtx, spec.Config, prompt) + // A completed turn without a valid outcome gets one in-session follow-up + // asking only for the outcome, rather than a full retry of its work. + text, generateErr := client.Generate(withOutputRepair(attemptCtx, outcomesFor(def.Edges)), spec.Config, prompt) if generateErr == nil && attemptCtx.Err() != nil { generateErr = attemptCtx.Err() } diff --git a/internal/stage/model_test.go b/internal/stage/model_test.go index 4d246b90..8d85b22c 100644 --- a/internal/stage/model_test.go +++ b/internal/stage/model_test.go @@ -78,7 +78,7 @@ func TestModelPromptUsesTurnFilesAndNoJSONEnvelope(t *testing.T) { "Do the task.", "# outcome", "success", - "free-form markdown", + "must start with `# outcome`", } { if !strings.Contains(prompt, want) { t.Fatalf("prompt missing %q:\n%s", want, prompt) diff --git a/internal/stage/opencode.go b/internal/stage/opencode.go index f47643e2..3aa964d0 100644 --- a/internal/stage/opencode.go +++ b/internal/stage/opencode.go @@ -431,6 +431,13 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp if err != nil { return "", err } + text, data, err = c.repairOutput(ctx, config, session.Data.ID, text) + if len(data) > 0 { + c.FormatAudit = string(data) + } + if err != nil { + return "", err + } if !config.Budget.Enabled() { // As with AgentClient, missing usage is not a generation failure when // no limit is enabled (and a nil ledger needs no aggregate read). diff --git a/internal/stage/pi.go b/internal/stage/pi.go index 6228509f..c13b4353 100644 --- a/internal/stage/pi.go +++ b/internal/stage/pi.go @@ -329,8 +329,31 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri if _, err = c.rpc(ctx, "prompt", map[string]any{"message": prompt}); err != nil { return "", err } + if text, err = c.settleTurn(ctx); err != nil { + return "", err + } + // One outcome repair follow-up in the same Pi session; get_session_stats + // reports session totals, so its tokens are counted by the re-read. + repair := outputRepairFrom(ctx) + if message, ok := repair.followUp(text); ok { + c.settled = false + if _, err = c.rpc(ctx, "prompt", map[string]any{"message": message}); err != nil { + return "", err + } + reply, err := c.settleTurn(ctx) + if err != nil { + return "", err + } + text = repair.merge(text, reply) + } + return text, nil +} + +// settleTurn waits for the prompted Pi turn to settle, records its session +// usage and returns the final assistant text. +func (c *PiClient) settleTurn(ctx context.Context) (string, error) { for !c.settled { - if _, err = c.receive(ctx); err != nil { + if _, err := c.receive(ctx); err != nil { return "", err } } diff --git a/internal/stage/pi_test.go b/internal/stage/pi_test.go index 4f83c75e..c4a9ae3d 100644 --- a/internal/stage/pi_test.go +++ b/internal/stage/pi_test.go @@ -30,6 +30,12 @@ done if os.Getenv("PI_FIXTURE_CASE") == "missing" { delete(message, "usage") } + // repair: the first turn drops its outcome; the follow-up states only it. + repair := os.Getenv("PI_FIXTURE_CASE") == "repair" + prompts, total := 0, 0 + if repair { + message["content"] = []map[string]string{{"type": "text", "text": "# handoff\n\nReport kept.\n"}} + } emit := func(v any) { data, _ := json.Marshal(v); fmt.Println(string(data)) } scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { @@ -45,12 +51,17 @@ done emit(map[string]any{"type": "response", "id": request["id"], "success": true}) continue } + prompts++ + total += 10 + if repair && prompts == 2 { + message = map[string]any{"role": "assistant", "stopReason": "stop", "timestamp": 2, "content": []map[string]string{{"type": "text", "text": "# outcome\nsuccess\n"}}, "usage": map[string]any{"totalTokens": 10, "cost": map[string]any{"total": 0.1}}} + } emit(map[string]any{"type": "message_end", "message": message}) emit(map[string]string{"type": "agent_settled"}) case "abort": emit(map[string]string{"type": "agent_settled"}) case "get_session_stats": - data = map[string]any{"sessionId": sessionID, "assistantMessages": 1, "tokens": map[string]int{"total": 10}, "cost": 0.1} + data = map[string]any{"sessionId": sessionID, "assistantMessages": max(prompts, 1), "tokens": map[string]int{"total": max(total, 10)}, "cost": 0.1 * float64(max(prompts, 1))} case "get_messages": data = map[string]any{"messages": []any{message}} } @@ -60,7 +71,7 @@ done } func TestPiAuditedRPCAndBudgets(t *testing.T) { - for _, which := range []string{"success", "budget", "missing", "hang"} { + for _, which := range []string{"success", "budget", "missing", "hang", "repair"} { t.Run(which, func(t *testing.T) { root := t.TempDir() t.Setenv("XDG_CACHE_HOME", root) @@ -87,12 +98,20 @@ func TestPiAuditedRPCAndBudgets(t *testing.T) { ctx, cancel = context.WithTimeout(context.Background(), 200*time.Millisecond) } defer cancel() + if which == "repair" { + ctx = withOutputRepair(ctx, []string{"error", "success"}) + } text, err := client.Generate(ctx, ModelConfig{Provider: "fixture", Model: "model", Budget: budget}, "task") switch which { case "success": if err != nil || text == "" || budget.Snapshot().UsedTokens != 10 { t.Fatal(text, err, budget.Snapshot()) } + case "repair": + // The follow-up's tokens count: two turns of 10 each. + if err != nil || text != "# outcome\nsuccess\n\n# handoff\n\nReport kept.\n" || budget.Snapshot().UsedTokens != 20 { + t.Fatal(text, err, budget.Snapshot()) + } case "budget": if !errors.Is(err, ErrBudgetExceeded) { t.Fatal(err) diff --git a/internal/turn/output.go b/internal/turn/output.go index 3dfc0ebb..4e176706 100644 --- a/internal/turn/output.go +++ b/internal/turn/output.go @@ -64,7 +64,8 @@ func ParseOutput(markdown string, allowedOutcomes []string) (Outcome string, Bod if value == len(lines) { return "", "", fmt.Errorf("%w: outcome heading has no value", ErrMissingOutcome) } - word := strings.TrimSpace(lines[value].text) + // The value line is the word alone; a trailing period is tolerated. + word := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(lines[value].text), ".")) canonical, ok := allowedOutcome(word, allowedOutcomes) if !ok { return "", "", fmt.Errorf("%w: %q", ErrOutcomeNotAllowed, word) @@ -89,6 +90,62 @@ func allowedOutcome(word string, outcomes []string) (string, bool) { return "", false } +// WithOutcome returns markdown with a leading "# outcome" section holding the +// given word, replacing any outcome headings (and their value lines) already in +// it. The word is never derived from the text: callers pass an outcome the +// model stated explicitly, such as its reply to the outcome repair follow-up. +// Every other byte of the original document is kept in order. +func WithOutcome(markdown, outcome string) string { + lines := markdownLines(markdown) + drop := make(map[int]bool) + var fence fenceState + for i := 0; i < len(lines); i++ { + text := lines[i].text + if fence.open() { + if fence.closes(text) { + fence = fenceState{} + } + continue + } + if marker, ok := openingFence(text); ok { + fence = marker + continue + } + if !isOutcomeHeading(text) { + continue + } + drop[i] = true + value := i + 1 + for value < len(lines) && strings.TrimSpace(lines[value].text) == "" { + value++ + } + if value == len(lines) { + continue + } + _, _, heading := atxHeading(lines[value].text) + _, fenced := openingFence(lines[value].text) + if !heading && !fenced { + drop[value] = true + i = value + } + } + var b strings.Builder + b.WriteString("# outcome\n") + b.WriteString(outcome) + b.WriteString("\n\n") + for i, line := range lines { + if drop[i] { + continue + } + start := 0 + if i > 0 { + start = lines[i-1].next + } + b.WriteString(markdown[start:line.next]) + } + return b.String() +} + type markdownLine struct { text string next int diff --git a/internal/turn/output_test.go b/internal/turn/output_test.go index a42dc92b..a6c7cfc9 100644 --- a/internal/turn/output_test.go +++ b/internal/turn/output_test.go @@ -143,6 +143,38 @@ func TestParseOutputReturnsCanonicalOutcome(t *testing.T) { } } +func TestParseOutputToleratesTrailingPeriod(t *testing.T) { + for _, input := range []string{"# outcome\nsuccess.\n", "# outcome\n Success . \n"} { + if outcome, _, err := ParseOutput(input, []string{"success"}); err != nil || outcome != "success" { + t.Fatalf("%q: %q %v", input, outcome, err) + } + } + if _, _, err := ParseOutput("# outcome\nsuccess, mostly\n", []string{"success"}); !errors.Is(err, ErrOutcomeNotAllowed) { + t.Fatalf("prose accepted as outcome: %v", err) + } +} + +func TestWithOutcome(t *testing.T) { + tests := []struct{ name, input, want string }{ + {"no outcome", "# handoff\n\nx\n", "# outcome\nsuccess\n\n# handoff\n\nx\n"}, + {"invalid value replaced", "# handoff\n\nx\n\n# outcome\nmaybe\n\n# log\n", "# outcome\nsuccess\n\n# handoff\n\nx\n\n\n# log\n"}, + {"duplicates removed", "## outcome\nerror\n# body\n## outcome\nsuccess\n", "# outcome\nsuccess\n\n# body\n"}, + {"empty heading keeps next heading", "# outcome\n\n# handoff\nx\n", "# outcome\nsuccess\n\n\n# handoff\nx\n"}, + {"fenced kept", "```\n# outcome\nerror\n```\n", "# outcome\nsuccess\n\n```\n# outcome\nerror\n```\n"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := WithOutcome(tt.input, "success") + if got != tt.want { + t.Fatalf("WithOutcome = %q, want %q", got, tt.want) + } + if outcome, _, err := ParseOutput(got, []string{"success", "error"}); err != nil || outcome != "success" { + t.Fatalf("result does not parse: %q %v", outcome, err) + } + }) + } +} + func TestParseOutputRequiresWhitespaceAfterHeadingMarker(t *testing.T) { if _, _, err := ParseOutput("#outcome\nsuccess\n", []string{"success"}); !errors.Is(err, ErrMissingOutcome) { t.Fatalf("error = %v, want missing outcome", err) From d3a2369d1f04d012312d570ddc74b7c6f8453565 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Wed, 4 Nov 2026 17:26:31 +0000 Subject: [PATCH 2/3] Stage rows show cached share and cost; Changes explain no_patch_reason MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Timeline stage rows add the cached share (when cache_known) and a positive cost: 20.1M tokens · 97% cached · $1.42. The file viewer words each server no_patch_reason, keeping the old truncated/binary text as the fallback. --- .../changes/domain/change_wording.dart | 23 +++++ .../features/changes/domain/file_change.dart | 11 +++ .../changes/widgets/file_diff_viewer.dart | 6 +- app/lib/features/runs/domain/run_tokens.dart | 15 ++++ .../features/runs/ui/run_detail_content.dart | 2 +- app/notes/no-patch-reason.md | 9 ++ app/notes/run-tokens.md | 7 +- .../changes/no_patch_reason_test.dart | 87 +++++++++++++++++++ .../runs/ui/run_stage_tokens_test.dart | 47 +++++++++- 9 files changed, 202 insertions(+), 5 deletions(-) create mode 100644 app/notes/no-patch-reason.md create mode 100644 app/test/features/changes/no_patch_reason_test.dart diff --git a/app/lib/features/changes/domain/change_wording.dart b/app/lib/features/changes/domain/change_wording.dart index cf81814d..e1896e60 100644 --- a/app/lib/features/changes/domain/change_wording.dart +++ b/app/lib/features/changes/domain/change_wording.dart @@ -100,8 +100,31 @@ String noChangeDetail(RunChanges changes) { return emptyCaptureDetail; } +/// The message for the server's `no_patch_reason`, or null when it sent none +/// or one this client does not know (the generic wording then applies). +String? noPatchReasonMessage(String? reason) => switch (reason) { + 'binary' => 'Binary file; only its metadata was recorded.', + 'file_size' => 'This file is over 1 MiB, so no diff text was recorded.', + 'budget' => + 'The workspace snapshot ran out of text budget, so no diff text was ' + 'recorded.', + 'lines' => 'The change is over 5,000 lines, so no diff text was recorded.', + 'evidence_size' => + "This visit's recorded changes reached their size limit, so this patch " + 'was dropped.', + 'git_status' => + "This change was seen only in Git status, so the file's contents weren't " + 'compared.', + 'baseline' => + "The file's content at stage start couldn't be recovered from Git, so no " + 'diff was recorded.', + _ => null, +}; + /// Why a file has no patch to render, in the entry's own recorded terms. String patchUnavailableReason(FileChange file) { + final reason = noPatchReasonMessage(file.noPatchReason); + if (reason != null) return reason; if (file.truncated) { return "The server truncated this file's patch at its capture limit; no " 'diff text was recorded.'; diff --git a/app/lib/features/changes/domain/file_change.dart b/app/lib/features/changes/domain/file_change.dart index 0f8ccdcf..6068d6f2 100644 --- a/app/lib/features/changes/domain/file_change.dart +++ b/app/lib/features/changes/domain/file_change.dart @@ -12,12 +12,14 @@ class FileChange { String? patch, this.binary = false, this.truncated = false, + String? noPatchReason, String? beforeMode, String? afterMode, this.steps = const [], ParsedPatch? parsedPatch, }) : patch = _nonBlank(patch), beforeMode = _nonBlank(beforeMode), + noPatchReason = _nonBlank(noPatchReason), afterMode = _nonBlank(afterMode), // Aggregate copies pass their already-parsed patch through instead of // re-parsing the same recorded bytes for every stage that touched a path. @@ -32,6 +34,11 @@ class FileChange { final String? patch; final bool binary; final bool truncated; + + /// Why the server recorded no (complete) patch: `binary`, `file_size`, + /// `budget`, `lines`, `evidence_size`, `git_status` or `baseline`. Null from + /// an older server or when a patch was recorded. + final String? noPatchReason; final String? beforeMode; final String? afterMode; @@ -68,6 +75,7 @@ class FileChange { patch: patch, binary: binary, truncated: truncated, + noPatchReason: noPatchReason, beforeMode: beforeMode, afterMode: afterMode, steps: List.unmodifiable(value), @@ -88,6 +96,9 @@ class FileChange { patch: json['patch'] is String ? json['patch'] as String : null, binary: json['binary'] == true, truncated: json['truncated'] == true, + noPatchReason: json['no_patch_reason'] is String + ? json['no_patch_reason'] as String + : null, beforeMode: _textOrNull(json['before_mode']), afterMode: _textOrNull(json['after_mode']), ); diff --git a/app/lib/features/changes/widgets/file_diff_viewer.dart b/app/lib/features/changes/widgets/file_diff_viewer.dart index 8d5db5bd..2f049ea8 100644 --- a/app/lib/features/changes/widgets/file_diff_viewer.dart +++ b/app/lib/features/changes/widgets/file_diff_viewer.dart @@ -211,7 +211,11 @@ class _FileDiffViewerState extends State { Widget _body(BuildContext context, TextStyle mono) { final file = widget.file; - if (file.binary) return _message(context, binaryDiffMessage); + // A recorded reason is more specific than the generic binary message. + final reason = noPatchReasonMessage(file.noPatchReason); + if (file.binary && reason == null) { + return _message(context, binaryDiffMessage); + } if (!file.hasPatch) { return _message(context, patchUnavailableReason(file)); } diff --git a/app/lib/features/runs/domain/run_tokens.dart b/app/lib/features/runs/domain/run_tokens.dart index 4189c7d7..a683fc22 100644 --- a/app/lib/features/runs/domain/run_tokens.dart +++ b/app/lib/features/runs/domain/run_tokens.dart @@ -122,6 +122,21 @@ String formatTokenCount(int tokens) { String tokenUsageLabel(TokenUsage usage) => '${usage.partial ? '≥' : ''}${formatTokenCount(usage.total)} tokens'; +/// A stage row's compact label: the total, the cached share when the stage +/// recorded its cache split, and the cost when a positive one was reported. +/// `20.1M tokens · 95% cached · $0.42`. A missing figure is omitted, never +/// shown as zero. +String stageTokenRowLabel(TokenUsage usage) { + final cacheRead = usage.cacheRead; + final cost = usage.cost; + return [ + tokenUsageLabel(usage), + if (cacheRead != null && usage.total > 0) + '${(cacheRead * 100 / usage.total).round()}% cached', + if (cost != null) cost < 0.005 ? '<\$0.01' : '\$${cost.toStringAsFixed(2)}', + ].join(' · '); +} + /// The detail's token value: the compact total, then fresh versus cached /// when every stage reported its cache components, the in/out split when /// reported, and how many stages reported when the total is partial. diff --git a/app/lib/features/runs/ui/run_detail_content.dart b/app/lib/features/runs/ui/run_detail_content.dart index bb296eb4..236fb0ff 100644 --- a/app/lib/features/runs/ui/run_detail_content.dart +++ b/app/lib/features/runs/ui/run_detail_content.dart @@ -153,7 +153,7 @@ List detailTimeline(dynamic steps) { target: '${step['target'] ?? ''}'.trim(), duration: _visitDuration(step), tokens: switch (stageTokenUsage(JsonMap.from(step))) { - final usage? => tokenUsageLabel(usage), + final usage? => stageTokenRowLabel(usage), null => null, }, ); diff --git a/app/notes/no-patch-reason.md b/app/notes/no-patch-reason.md new file mode 100644 index 00000000..4e06852b --- /dev/null +++ b/app/notes/no-patch-reason.md @@ -0,0 +1,9 @@ +# Changes: why a file has no patch + +The server's `FileChange.no_patch_reason` (`binary`, `file_size`, `budget`, +`lines`, `evidence_size`, `git_status`, `baseline`) is parsed into +`FileChange.noPatchReason` and worded by `noPatchReasonMessage` in +`lib/features/changes/domain/change_wording.dart`. The file viewer shows it in +place of the generic truncated/binary message. A missing, blank or unknown +reason (older server) keeps the old wording. Tests: +`test/features/changes/no_patch_reason_test.dart`. diff --git a/app/notes/run-tokens.md b/app/notes/run-tokens.md index 96041afb..8c37944a 100644 --- a/app/notes/run-tokens.md +++ b/app/notes/run-tokens.md @@ -31,8 +31,11 @@ Runs that spend 20M+ tokens now show it the way they show disk size. - Summary: `Size on disk`, `Tokens` (`22.4M · 2.1M fresh, 20.3M cached · 1.8M in, 240K out`; fresh is total minus cache reads, shown only when `cache_known`) and `Cost`. -- Stage rows: the Timeline row subtitle adds `· 20.1M tokens`. The visit facts - add `Tokens` and `Cost`. +- Stage rows: the Timeline row subtitle adds `· 20.1M tokens · 97% cached · + $1.42` (`stageTokenRowLabel`). The cached share (cache reads over the total, + rounded) appears only when the stage recorded `cache_known`; the cost only + when positive, to two places, `<$0.01` below half a cent. The visit facts + add `Tokens` and `Cost` (`Not reported` stays there). - Cost is shown only when a positive `cost_usd` was reported. Subscription models report `cost_known` with no amount, which shows as `Not reported`. The stats page's `Reported cost` uses the same rule. diff --git a/app/test/features/changes/no_patch_reason_test.dart b/app/test/features/changes/no_patch_reason_test.dart new file mode 100644 index 00000000..d7e737a9 --- /dev/null +++ b/app/test/features/changes/no_patch_reason_test.dart @@ -0,0 +1,87 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/changes/changes.dart'; + +import 'widget_hosts.dart'; + +const _legacy = + "The server truncated this file's patch at its capture limit; no diff " + 'text was recorded.'; + +const _expected = { + 'file_size': 'This file is over 1 MiB, so no diff text was recorded.', + 'budget': + 'The workspace snapshot ran out of text budget, so no diff text was ' + 'recorded.', + 'lines': 'The change is over 5,000 lines, so no diff text was recorded.', + 'evidence_size': + "This visit's recorded changes reached their size limit, so this patch " + 'was dropped.', + 'git_status': + "This change was seen only in Git status, so the file's contents weren't " + 'compared.', + 'baseline': + "The file's content at stage start couldn't be recovered from Git, so no " + 'diff was recorded.', + 'binary': 'Binary file; only its metadata was recorded.', +}; + +FileChange _noPatch(Object? reason, {bool binary = false}) => + FileChange.fromJson({ + 'path': 'big.txt', + 'kind': 'modified', + 'truncated': true, + 'no_patch': true, + 'binary': binary, + 'no_patch_reason': ?reason, + }); + +void main() { + test('each recorded no-patch reason has its own message', () { + for (final entry in _expected.entries) { + final file = _noPatch(entry.key, binary: entry.key == 'binary'); + expect(file.noPatchReason, entry.key); + expect(patchUnavailableReason(file), entry.value, reason: entry.key); + } + }); + + test('no reason, a blank one or an unknown one keeps the old wording', () { + expect(patchUnavailableReason(_noPatch(null)), _legacy); + expect(patchUnavailableReason(_noPatch('')), _legacy); + expect(patchUnavailableReason(_noPatch('future_reason')), _legacy); + expect(patchUnavailableReason(_noPatch(42)), _legacy); + }); + + test('the reason survives aggregation across stages', () { + final file = _noPatch('budget').withSteps(const ['build']); + expect(file.noPatchReason, 'budget'); + }); + + testWidgets('the viewer shows the reason, binary included', (tester) async { + await tester.pumpWidget( + materialHost( + Column( + children: [ + Expanded(child: FileDiffViewer(file: _noPatch('lines'))), + Expanded( + child: FileDiffViewer(file: _noPatch('binary', binary: true)), + ), + Expanded( + child: FileDiffViewer( + file: FileChange( + path: 'logo.bin', + kind: ChangeKind.modified, + binary: true, + ), + ), + ), + ], + ), + ), + ); + expect(find.text(_expected['lines']!), findsOneWidget); + expect(find.text(_expected['binary']!), findsOneWidget); + expect(find.text(binaryDiffMessage), findsOneWidget); + expect(find.text(_legacy), findsNothing); + }); +} diff --git a/app/test/features/runs/ui/run_stage_tokens_test.dart b/app/test/features/runs/ui/run_stage_tokens_test.dart index ae6e12ac..f0706ef9 100644 --- a/app/test/features/runs/ui/run_stage_tokens_test.dart +++ b/app/test/features/runs/ui/run_stage_tokens_test.dart @@ -40,7 +40,52 @@ void main() { test('a stage row carries its compact token total', () { final visits = detailTimeline([_reported, _unreported]); - expect(visits[0].tokens, '1.5M tokens'); + expect(visits[0].tokens, '1.5M tokens · \$0.42'); expect(visits[1].tokens, isNull); }); + + String? rowLabel(Map usage) => detailTimeline([ + {..._unreported, 'usage': usage}, + ])[0].tokens; + + test('a stage row adds its cached share and cost when reported', () { + const base = {'tokens_known': true, 'tokens_total': 20100000}; + expect( + rowLabel({ + ...base, + 'cache_known': true, + 'cache_read': 19100000, + 'cost_known': true, + 'cost_usd': 0.4249, + }), + '20.1M tokens · 95% cached · \$0.42', + ); + // A known split with no reads is a real 0%, not an absence. + expect( + rowLabel({...base, 'cache_known': true}), + '20.1M tokens · 0% cached', + ); + // An older row without cache_known, or a cache read alone, shows no share. + expect(rowLabel({...base, 'cache_read': 19100000}), '20.1M tokens'); + // Tiny costs stay visible; zero or unknown costs are omitted on the row. + expect( + rowLabel({...base, 'cost_known': true, 'cost_usd': 0.0042}), + '20.1M tokens · <\$0.01', + ); + expect( + rowLabel({...base, 'cost_known': true, 'cost_usd': 0.005}), + '20.1M tokens · \$0.01', + ); + expect( + rowLabel({...base, 'cost_known': true, 'cost_usd': 0}), + '20.1M tokens', + ); + expect(rowLabel({...base, 'cost_usd': 1.5}), '20.1M tokens'); + // The stage detail still says the cost was not reported. + final facts = detailVisitFacts({ + ..._unreported, + 'usage': {...base, 'cost_known': true}, + }); + expect(facts, contains((label: 'Cost', value: 'Not reported'))); + }); } From 5090f1cf4e1a2f77e4f95d8d85bf535dc67cd0fb Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Wed, 4 Nov 2026 17:27:01 +0000 Subject: [PATCH 3/3] Pipelines skill: outcome-first output, aligned review and loop guidance, operator material moved to docs Rewrite format.md around turn files, the output.md order (# outcome, edge word, # handoff, detail), checks, configs, human stages and the land stage delivery gate (sh -c in a fresh detached worktree, clean checkouts incl. untracked, 64 KB output cap, opencode only). Align review guidance across practices, verification, orchestration and the feedback-loop example. Fix loop-limit semantics in control.md against runner.go, budget_tokens sizing, read for large inputs, blocked-else-error and one stage naming set. Move dated verification logs to docs/verification-history.md and brokered chat, TUI keys and harness internals to docs/operator-reference.md; label operator-only references in SKILL.md. --- docs/operator-reference.md | 291 ++++++++ docs/verification-history.md | 612 +++++++++++++++++ examples/feedback-loop.md | 8 +- examples/two-stages.md | 2 +- internal/author/skill/SKILL.md | 63 +- internal/author/skill/references/control.md | 83 +-- internal/author/skill/references/debugging.md | 2 +- internal/author/skill/references/format.md | 439 ++++-------- internal/author/skill/references/harnesses.md | 214 +----- .../author/skill/references/operations.md | 243 +------ .../author/skill/references/orchestration.md | 12 +- internal/author/skill/references/practices.md | 68 +- .../author/skill/references/verification.md | 644 ++---------------- 13 files changed, 1243 insertions(+), 1438 deletions(-) create mode 100644 docs/operator-reference.md create mode 100644 docs/verification-history.md diff --git a/docs/operator-reference.md b/docs/operator-reference.md new file mode 100644 index 00000000..3545527d --- /dev/null +++ b/docs/operator-reference.md @@ -0,0 +1,291 @@ +# Operator reference + +Operator and developer material moved out of the packaged Pipelines skill +(`internal/author/skill/references/operations.md`) because authoring agents +never need it. + +## TUI keys + +In the TUI, Home advertises `/` omni search and `:` commands. Home `/` opens one +popup over pipelines, versions, runs, steps/visits, artifact names and stage +input/output/evidence text; each row shows a short type badge and a text match +shows a bracketed snippet, and a bare status word or date selects runs. The same +query is available as `pipelines search QUERY [--type KIND]`, where `--type` +narrows the kind. Run Tab switches only +Details ↔ Graph. `n/p` select recorded visits, including repeated stages; `d` +focuses/toggles file changes, `A` opens the run's grouped artifacts view (name, +content type, size and URL per entry, `PgUp`/`PgDn` scroll so a trailing visit +that recorded none stays reachable, `Enter`/`o` to open the selected URL and +`y` to copy it), `a` selects the active/latest retained +configured-harness agent session's bounded, redacted tool evidence, `D`/`:debug` +opens fresh investigation, and `P` opens the pinned pipeline version. `:quit` +exits; `q` does not. Results are retained reports, not independently confirmed +achievements. Distinguish partial output from full success. + +## Brokered OpenCode chat + +The protocol declares the brokered `chat_*` surface (`chat_sessions`, +`chat_create`, `chat_directories`, `chat_prompt`, `chat_messages`, +`chat_interrupt`, `chat_permission`, `chat_reply`). The server dispatches all of +them, and pushes `chat_event` frames derived from OpenCode's global SSE firehose +(`/event` on the native serve, `/api/event` on the V2 API), with retained +per-session/per-message state so a reconnecting client re-syncs instead of +replaying deltas. + +At startup `cmd/pipelines-server` calls `opencode.OpenChat`, which detects the +installed backend and opens exactly one owned `opencode serve` endpoint for it: +the native wire for v1, the V2 API for the installed V2 CLI. Chat is enabled only +when that connection carries a URL-bearing, SSE-capable event endpoint; otherwise +chat stays disabled and every `chat_*` request is answered with a protocol error +naming the missing configuration rather than hanging. Stage generation still +reaches V2 through the `opencode api` CLI, so the owned serve process is chat's +single transport and starts no process for stages. The adapter skips firehose +events it does not surface before decoding their data, so an unrelated event +whose `delta` has a different shape (`session.instructions.updated` carries an +object) never fails the stream. + +The operator can instead broker chat from their own OpenCode service with the +opt-in `-opencode-endpoint` flag or `PIPELINES_OPENCODE_ENDPOINT`, with precedence +flag > environment > the endpoint recorded on a previous run and the spawned +instance as the default. `opencode.OpenEndpoint` validates the reference once +(the established http(s)/host/no-credentials/no-query/no-path idiom), detects the +backend from the service's own `/api/info` (V2) or `/global/health` (native) +contract and requires a streamable `text/event-stream` event path; it returns a +connection with **no owned process** and never calls `startServe`. A malformed +reference, an unreachable host, a reachable non-OpenCode service or a +health-ok-but-no-SSE service each disable chat with an error naming the endpoint +and never fall back to the spawned instance. An authenticating endpoint accepts an +optional password only through the private file +`PIPELINES_OPENCODE_ENDPOINT_PASSWORD_FILE` (regular, 0600, single-owner), and +`PIPELINES_OPENCODE_ENDPOINT_USERNAME` overrides the detected default user. The +validated choice is recorded in the additive `server_settings` table so a restart +with no flag/environment reuses it; an unreachable recorded endpoint disables chat +with a clear message and no fallback. `pipelines service enable` preserves the +endpoint and the password-file reference in autostart. + +`chat_sessions` lists the operator's own chat sessions, not the sessions pipeline +runs create. Operator sessions are recorded in the dedicated `operator_sessions` +table (`session_id` primary key, plus `directory`, `title`, `tag`, `created_at`, +`last_used_at`), separate from `run_steps`/`run_session_owners`, so an operator +chat is a first-class thing and a pipeline stage session — recorded as run +ownership, never in the operator registry — is never listed. The handler reads the +registry once (bounded to 128 rows, most-recently-used first), calls OpenCode's +session list once, and keeps only registry rows the endpoint still reports, so a +session with no recorded tag is never listed and a deleted upstream session does +not linger. It fills each session's `directory`, `tag` and `created_at`/ +`updated_at` from the registry and marks `busy` when another client currently +holds the session (an in-server claim, released when the stream ends or the client +disconnects); a second client's `chat_prompt` for a held session is rejected +before any OpenCode call. It is fail-closed: with no registry configured it +returns an empty list, never the endpoint's whole session list. The OpenCode +session list is read unscoped by directory (the V2 `GET /api/session` otherwise +defaults to the caller's directory), because an operator session may live in any +directory. + +`chat_create` creates one operator session and records it in the same registry +with the stored tag `operator` (the display label `Operator` is derived from it, +so there is no second tag vocabulary). It takes a `title` (default "New chat") and +a `directory` (default `~`): an omitted or empty value, `~`, and any `~/...` value +all resolve to `os.UserHomeDir`, never the server's working directory; a relative +path or a missing directory is rejected before any OpenCode call, and a session +created but not recordable is reported as an error naming its id. Create posts the +verified V2 shape `POST /api/session` `{"title":...,"location":{"directory":...}}`. +`chat_directories` returns the operator's home first, then the directories of +previous operator sessions most-recently-used first, deduplicated and bounded (8); +it is a dedicated request so the create dialog can load the choices without first +listing sessions and `chat_sessions` stays a pure, fail-closed session list. A +successful `chat_prompt` touches the session's `last_used_at` in one UPDATE, so it +moves to the front of the list; a non-operator session id touches no row. + +`chat_adopt` records a session that already exists on the configured endpoint in +the same operator registry, using the endpoint's own directory and title +verbatim (never a guess, the server's working directory or the home directory). +It reads the endpoint once through the shared `Chat.Session` (which reuses +`Sessions`/`decodeChatSessions`), records through the existing +`RecordOperatorSession` with the `operator` tag, and echoes the adopted session +plus its registry row like `chat_create`, so `chat_sessions` lists it and +`chat_prompt` continues it. A nil payload, a blank or foreign-shaped id, a nil +broker or a nil registry is refused before any endpoint call; an id the endpoint +does not report is refused naming the id and records nothing; an endpoint record +with an empty directory is refused rather than substituting a directory. The CLI +equivalent is `pipelines chat-adopt ` (`--json` prints the full +response, otherwise a short `Adopted · ` line). + +While a `chat_prompt` subscription is open the server also polls +`GET /api/permission/request` on a fixed small interval (first poll immediately) and +pushes one `chat_permission` frame per newly pending approval request, deduped by +request id so a repeated poll never re-pushes the same request. The app answers with +`chat_reply`, which the server forwards as +`POST /api/session/{id}/permission/{requestID}/reply` with +`{"decision":"once"|"always"|"reject"}`. A missing session/request id or an invalid +decision is rejected before any OpenCode call, and the poller stops when the stream +ends or the client disconnects. + +The permission list and reply are implemented against the V2 HTTP API only: their +field names match the service's `Permission.Request`/`Permission.Reply` schemas +(verified against the installed service's `/openapi.json`). The native v1 serve +translation rejects them explicitly instead of guessing an endpoint, so on v1 the +permission poller stops quietly rather than reporting the documented limitation as +a per-prompt error. + +### Verification evidence (2026-10-17) + +Verified against the installed OpenCode 2.0.20 on 2026-10-17: + +- `opencode --version` reports `2.0.20`; `opencode --help` advertises `api` and + `serve`. `GET /openapi.json` on an owned `opencode serve` lists `GET /api/event` + (SSE) and the session/prompt/interrupt/permission endpoints. +- A live turn streamed token-level `session.text.delta` and + `session.reasoning.delta` events (each with `assistantMessageID` and `ordinal`), + plus `session.step.*` and `session.usage.updated`; the V2 API server + authenticates the fixed `opencode` basic-auth user. +- The opt-in live end-to-end test (`PIPELINES_TEST_OPENCODE_CHAT_LIVE=1 go test + ./internal/server -run TestChatLiveEndToEnd -v`) opened an owned V2 endpoint, + created an operator session through `chat_create` in a chosen directory, listed + exactly that session through `chat_sessions` (with its directory and tag + `operator`), recorded a genuine pipeline stage session in + `run_steps`/`run_session_owners` and showed it never appeared in the list, and + streamed a completed turn as `[step:started,text,step:streamed,step:ended]` + (first delta `"pong"`). The operator-session registry, the filter, directory + defaulting and the recents bound are also covered by the `pipeline`, `opencode` + and `server` unit tests, which run without the live gate. +- Not verified live: a genuine pending-approval round trip and any backend other + than the installed 2.0.20. The permission wire shapes are unit-tested over an + httptest fixture, not against a live pending request. +- Operator-supplied endpoint selection and `chat_adopt` (added 2026-10-17): + `opencode.OpenEndpoint` and the flag/env/recorded precedence, the URL-shape + validation failures (scheme, empty host, userinfo, query, fragment, non-empty + path), the unreachable/non-OpenCode/no-SSE/auth-mismatch failures, the + no-owned-process property (`conn.process == nil`) and the password-file rule + are unit-tested over httptest fixtures in `internal/opencode/endpoint_test.go`. + `chat_adopt` (endpoint directory/title recorded with the `operator` tag, echo, + missing id refused naming the id and recording nothing, nil payload/blank id/nil + broker/nil registry refused before any endpoint call, empty endpoint directory + refused) is unit-tested over the server fake broker in + `internal/server/chat_test.go`; the recorded-endpoint round-trip across a store + Close/Open is in `internal/pipeline/settings_test.go`; the protocol round-trip + and the CLI dispatch (`pipelines chat-adopt`) are covered in + `internal/protocol/chat_test.go` and `internal/cli/chat_adopt_test.go`. The + live scratch-server check against the operator's real endpoint (adopt, list, + prompt) was **not** run in this change; only the unit fixtures above were + executed. + +Deployment state: not deployed. This is reviewed and committed locally only; +deploying it (and restarting the server) is the operator's step. Do not describe +brokered chat as already deployed. + +Decide a waiting human stage in the TUI: open the run, press `c`, pick an option, +optionally type feedback, Enter. + +## Docker release resolution + +Docker setup resolves the current supported OpenCode V2 release from the single +documented complete `GET .../git/matching-refs/tags/v2` response (a bounded +complete array with no pagination parameters; never the moving `latest` tag or +the registry's insertion order) and the current Pi release from npm, resolves the +selected tag's immutable GHCR `sha256` digest, builds `FROM` that digest, and +verifies the built image reports exactly those executable versions plus a +Node-compatible Pi engine. +It fails closed, keeping the last verified Dockerfile fallback pins, when discovery, +digest resolution, completeness, build or compatibility cannot be established: never +a hardcoded fallback, never a silent stale-pin substitution. A full install skips a +rebuild only when the release receipt **and** the actual built image's release and +source identity labels already match; a dirty source always invokes the Docker +build and relies on layer caching. A model/login-only reconfiguration compares only +the verified release image identity, so a changed source alone does not rebuild. +`pipelines harness setup opencode|pi` reconfigures inside the existing container and +refuses an image that does not match; it does not rebuild or restart services (an +explicit ordinary `./install` does when the user asks). + +## Harness internals + +CLI `new/edit/debug` and TUI author/debug build one shared, Docker-aware CLI +prefix. A Docker-linked profile embeds the in-container `/usr/local/bin/pipelines` +(never the host binary: Linux can see but cannot execute a mounted macOS host +executable). Loopback is recognized only via `net.ParseIP(...).IsLoopback()` plus +an exact case-insensitive `localhost`, so a remote DNS name such as +`127.remote.example` stays remote and runs no probe. For a loopback logical +`--url`, a single bounded `docker compose port` probe parses each returned line +as a complete `host:port` binding (IPv4/IPv6/wildcard) and the prefix maps only +when a binding covers the requested loopback address and port; malformed or +non-matching bindings fail closed. A loopback `wss` endpoint is rejected as +having no supported local TLS mapping rather than downgraded to plaintext. On a +match it embeds scoped `PIPELINES_TRANSPORT_SOURCE`/`PIPELINES_TRANSPORT_TARGET` +aliases in the prefix string (not `cmd.Env`, so the pre-existing harness +server's tools inherit them). The client rewrites an endpoint only on an exact +source match; remote endpoints stay remote. Draft metadata (server, base +version, create-only guard) is preserved unchanged. +`provider-check` with a Docker-linked profile runs inside that profile's existing +Compose service with `-T`, the original cwd/model/timeout and exact stdout/exit +propagation; container profiles have no `DockerDirectory` and cannot recurse, and +explicit native profiles keep the host path. The picker uses a true `x/term` TTY +check, so `/dev/null` and pipes fail fast instead of hanging. + +On an existing Docker setup, `pipelines harness setup opencode|pi` configures inside its container and link the host +launcher without restarting/deploying services. They resolve and verify the current +OpenCode V2 release from the single documented complete +`GET .../git/matching-refs/tags/v2` response (a bounded complete array with no +pagination parameters; never the moving `latest` tag or the registry's insertion +order) and the current Pi release from npm, resolve the selected tag's immutable +GHCR `sha256` digest, and require a rebuild only when the verified release image +identity no longer matches. A full install always invokes the Docker build for a +dirty source (relying on layer caching); a model/login-only reconfiguration does +not rebuild. Setup writes its generated release receipt to the machine-local root +file `.pipelines-releases.json` (with atomic `.pipelines-releases-*` temporary +files); both are git-ignored, so a fresh clean checkout stays clean and reinstall +still reuses the verified release/image identity. The actual built image's labels +stay authoritative and a dirty source still rebuilds. If discovery, digest +resolution or completeness cannot be established they stop with an error (keeping +the last verified Dockerfile fallback pins); they never guess, fall back to a host +runtime or install a stale pin. +(`./install --opencode`/`--pi` were removed and only point to that command.) The native installation reuses the host OpenCode +configuration, writes **host** (non-Docker) profiles for the harnesses it detects +into XDG config home's `pipelines/harnesses.json`, and clears a stale Docker link +so a Docker-era profile is not left Docker-linked. It never installs, upgrades or +downloads a harness. Container and host credentials and accounts remain independent and +are reused on reinstall/reconfigure. `./install` installs natively by default +(`--native` is equivalent); explicit `./install --docker` selects the Docker +server + ntfy installation. +Fresh setup accepts `./install --docker --harness pi` to skip OpenCode +login. Existing setup can configure either harness independently. Pi setup does +not run a generation check; `provider-check --harness pi --model PROVIDER/MODEL` +is a separate confirmed, potentially chargeable check. +Profiles live in XDG config home's `pipelines/harnesses.json`, store preferences +only, and preserve unrelated configuration. A profile launches a native host +harness unless it records a Docker checkout directory, in which case it launches +through that installation's Compose service. `harness setup --host` +configures or converts a host profile and clears any Docker link; without +`--host`, reconfiguring a Docker-linked profile re-verifies that the recorded +directory still has `compose.yaml` and fails naming it rather than saving a link +that cannot launch. Setup's summary and `harness list` report the resulting mode +(and, for Docker, the directory). Harness CLI setup/default commands +always configure the local execution user, not a remote `--url` server. +OpenCode interactive launch uses +`mini --model`: verified V2 full CLI lacks a model flag. Pi uses its CLI with the +packaged skill explicitly loaded. Pi has no standalone `auth login` command +(verified `pi auth --help`: only credential check/print): setup prints a +title-plus-three-lines Pi sign-in handoff and opens a fresh, no-context, +extension-free, no-prompt-templates, no-tools session with Pi's documented +`--tui-mode regular` flag, so the header and handoff stay visible above Pi's +interface at 80x30 instead of the default fullscreen clear. The user types +`/login`, chooses an authentication method ("Sign in with an API key" for +provider API credentials; account and Radius remain the user's choice), finishes +provider sign-in, then exits with `ctrl+c`/`ctrl+d` to return to Pipelines model +configuration. Existing sign-ins are reused by exiting when no new sign-in is +needed; completing a sign-in can update the credentials stored in the Docker +volume. No coding/task prompt or scripted keystrokes are injected. +OpenCode V2 model stages settle long sessions by polling `GET /api/session/active` +about once per second under the stage context, instead of the `.../wait` request +whose CLI-side request timeout failed long stages after roughly six minutes +(HTTP 499, dated 2026-10-10). V1 native settlement is unchanged. Final results +and debug session evidence use the same single bounded latest-assistant readback +(`order=desc&limit=1&type=assistant`), so a small final result is visible behind +large prior tool content; retained transcripts over 32 KiB are stored as valid +bounded preview envelopes that preserve stage limits and diagnostics, and exact +outputs remain authoritative. Pi evidence is JSONL and is not parsed as one JSON +object. Short-lived context-controlled CLI calls (`--help` detection, API calls +and the native export fallback) share one finite inherited-pipe drain grace +(`os/exec.Cmd.WaitDelay` ~250ms) so a canceled context or a descendant that still +holds stdout/stderr cannot block for the child's natural lifetime; the +`context.Canceled`/`DeadlineExceeded` and `exec.ErrWaitDelay` identities are +preserved and partial stdout after the grace is never accepted as success. diff --git a/docs/verification-history.md b/docs/verification-history.md new file mode 100644 index 00000000..8d72b9d4 --- /dev/null +++ b/docs/verification-history.md @@ -0,0 +1,612 @@ +# Verification history + +Dated verification logs and backend caveats moved out of the packaged +Pipelines skill (`internal/author/skill/references/verification.md` and +`harnesses.md`) so authoring agents do not load them. They are a historical +record: the JSON/line-format result envelope and the format-repair ladder they +describe are retired. Model stages now write `output.md` and only its +`# outcome` section is parsed (`internal/author/skill/references/format.md`, +`docs/model-results.md`). + +## From the skill verification reference + +### Budgets and backend caveats + +Defaults remain two retries and one hour per stage attempt; a run-level wall-clock +deadline is not added. Budgets are optional, positive and independent USD/token +reported-usage limits shared across attempts and repeat visits. +Accounting failures stop budget-limited stages. In-flight calls can overshoot; +these are **not provider-side hard caps or a guaranteed bill ceiling**. Reported +USD may validly be zero; do not fabricate prices/accounting from output length. + +Per-run usage is recorded as typed step evidence from the usage the stage already +measured for its budget (no second harness read and no transcript parse), and +totalled for the run and over all history. A stage revisited within a run shares +one budget, so each step records only that visit's increment of it and the run +total is not the cumulative stage budget counted again; a revisit that reports +nothing new (a failure or timeout before any usage poll, or a Pi visit that never +reaches a usage stat) records no usage for that visit rather than a known zero. +`CLI_PREFIX inspect RUN_ID --json` +exposes each model step's `tokens_total`, `tokens_in`/`tokens_out` (only when +`split_known`), `cost_usd` (only when `cost_known`) and the model it ran with, +plus the totalled `usage`; `CLI_PREFIX stats --json` adds the history-wide totals, +which equal the sum of the per-run totals. A stage that reported no cost exposes +its tokens and marks the cost unavailable rather than showing zero; a harness that +reports only a total (Pi) exposes the total and marks the in/out split +unavailable rather than presenting the total as input tokens. An upgraded or +historical database gains an empty usage table with no backfill, so old runs omit +`usage` instead of showing zero. This is current source, not deployment evidence. + +Dated current-source evidence (2026-10-17, this machine). A throwaway read-only +reader (`internal` store API) over a scratch copy of +`~/.local/share/pipelines/data/pipelines.db`, taken with +`sqlite3 "file:$HOME/.local/share/pipelines/data/pipelines.db?mode=ro" ".backup "` +so the live DB is never mutated, opened 228 runs and found **no** `step_usage` +rows: every stored run predates this change, so per-run figures are unavailable +rather than zero. Run 314 (this item's own run) and run 239 (whose implementer +step ran `opencode-go/kimi-k2.7-code`, a stage model different from the pinned +`opencode-go/deepseek-v4.1-flash`) both report `usage=(none recorded)`, and +`stats` omits the history-wide totals. The recorded figures themselves are +covered by the passing current-source round trips — `internal/pipeline/usage_test.go`, +`internal/server/usage_test.go` (including `inspect --json` over a websocket) and +the pre-upgrade/read-bounded/stats-sum cases — not by a live run, because the +deployed server predates this change and was not restarted. No live model call, +deploy or rebuild produced this evidence. + +A live V2 trial found `/generate` returned text while its retained session had no +messages and zero counters. Current source uses tracked `/prompt` → bounded +active-session polling → message readback for **budgeted V2 text-only calls, with +all tools denied**. Completed token-budgeted responses reporting zero tokens fail +closed. No new definition flags or agent tools are required for this fix. +Unbudgeted tool-free V2 generation can still have empty backend message history; +pipeline inputs, results and command audit remain retained. An unbudgeted V2 +audited agent stage makes exactly one extra bounded final session-aggregate read +to record any reported USD/token usage best-effort: missing or invalid usage stays +unknown and never turns a valid run into a failure, cumulative sessions/retries +are keyed per session so they cannot double count, and totals are never inferred +from final-assistant-only or native-partial data. Budgeted settlement and native +V1 enforcement are unchanged. Native V1 uses its existing path. + +The V2 `.../experimental/session/{id}/wait` request is **no longer used**: a dated +Docker log shows it timing out at the CLI's own request timeout after about six +minutes (run started 2026-10-10T23:07:16.547Z, `TimeoutError` logged at +2026-10-10T23:13:17.262Z, server `http.status=499`), which aborted a long +implementation stage. Instead current source polls `GET /api/session/active` +about once per second with a context-aware delay and no busy loop, until the +admitted session is absent (inactive). Missing/null/non-object `data`, or a state +other than the documented `running` type, fails closed and is never treated as +inactive. This is **mocked regression coverage** in `internal/stage` (fake CLI), +not live-provider evidence; V1 native settlement is unchanged. + +Current source reads the V2 final result with a single bounded descending +final-assistant readback (`order=desc&limit=1&type=assistant`) after the settled +poll, for both audited agent stages and budgeted V2 text-only calls, so large +retained history (including prior tool-read contents) is no longer fetched just +to select the completed result. A truncated/incomplete readback, an empty data +list, or a latest message that is neither completed `stop` nor a completed +structured tool-call fails closed with a sanitized diagnostic that omits +provider content; retained audit previews are **not** executable final results. +Session/debug evidence now reuses the same single latest-assistant readback +instead of a recent-20 all-message fetch, and returns each payload as a labelled +bounded, credential-redacted envelope with explicit preview/truncation/missing +diagnostics; a small final assistant result is therefore selected even when large +prior tool content exists. Pi evidence remains documented JSONL and is never +parsed as one JSON object. This describes current source behavior, not a guarantee +that a user's installed server includes the fix. An updated skill/client is not +deployment evidence; never rebuild/restart services or log into providers without +specific authorization. + +Retained model-stage failures now carry a fixed sanitized category. A strictly +invalid final result (a duplicate nested field, or malformed/ambiguous JSON with a +byte offset) is labeled `FINAL RESULT VALIDATION`; request, V2 active-session poll +and final-readback failures are labeled `HARNESS TRANSPORT`. A pre-admission +request failure replays nothing and is retried for any stage; a post-admission +failure is retried only on an explicit read-only stage (see the 2026-10-17 section +below). A validation failure +preserves the admitted session, transcript and evidence and performs no second tool +execution. Confirmed ordinary-endpoint failures on 2026-10-11: an all-history +readback of exactly 262144 bytes of incomplete JSON (`invalid agent messages`), a +reviewer result with a duplicate nested `performance_review` field, and an +onboarding visual-review result with `invalid character 'z' after object key:value +pair at byte 3968` where unsafe lossless repair was rejected. These are **mocked +`internal/stage` fixtures**: executable source-level evidence, not live-provider +evidence and not a deployment. OpenCode V2 prose `output_contract` does not enforce +structured output; V2 results are validated locally and no unsupported V2 format +field is sent. A review-only fresh execution is a future recovery idea, **not +implemented**. Onboarding UI visual approval remains unresolved and must not be +reported complete. + +### Bounded malformed-result recovery: 2026-10-16 + +Source-level, mocked `internal/stage` regression coverage (not live-provider +evidence and not a deployment): malformed final-result recovery now tries a +deterministic local pass first (append missing closers, drop one trailing comma, +or insert the single comma/colon the parser expected), then at most one deny-all +formatter request whose deadline is derived from the payload size, the stage +timeout and the attempt's remaining budget rather than a fixed 30-second +constant. Every path +records `repaired`, `partial` or `unrecoverable` in `format_repairs`; a `partial` +or `unrecoverable` payload is never returned as a completed result, and an +ambiguous payload (an unescaped quote/colon inside a string value) is never +guessed at. A shell-enabled stage's malformed but deterministically recoverable +result now completes without replaying admitted tool work; `safeToReplay` is +unchanged. The run-55 `ui_reviewer` payload was retrieved read-only with +`pipelines inspect 55 --json` and reproduced its exact byte offset 7837; it is the +ambiguous error class, so it is reported unrecoverable rather than repaired. This +does not update or restart any running server. + +### Harness transport reach retry and repair deadline: 2026-10-17 + +Source-level, mocked `internal/stage`, `internal/opencode` and `internal/server` +regression coverage (not live-provider evidence and not a deployment): the +harness transport path now distinguishes a pre-admission reach failure +(connecting or creating the session) from a post-admission readback/poll/agent +failure under the one `HARNESS TRANSPORT` category. A pre-admission reach failure +replays nothing, so it is retried within `retries` for any stage with a fresh +session and no replayed prompt; a post-admission failure is retried only on an +explicit read-only stage, and an `edit`/`shell`/legacy stage still never replays +admitted work. A context that expires during a request keeps both the sanitized +transport detail (operation/status/exit) and +`errors.Is(err, context.DeadlineExceeded)`; a bare attempt-boundary deadline is +not treated as a reach failure and stays terminal for a mutating stage. An +unrecognized provider execution-error name is reported as a bounded, sanitized +identifier plus any HTTP status. The isolated formatter deadline is now +proportional to the stage timeout (floored, capped, and clamped to +`remaining - margin` so it can never extend the stage/attempt deadline). These are +mocked fixtures; a deployed server is not updated by this change. + +### Failure-handling source changes: 2026-10-15 + +Source-level, mocked `internal/stage` and `internal/server` regression coverage +(not live-provider evidence): an explicit read-only tools stage now actually +retries a post-admission stage deadline or harness transport failure, creating a +fresh bounded session per attempt, while a stage with `edit`/`shell` or no tools +block still never replays admitted work; a malformed final result is re-emitted +up to `retries` only on such a read-only stage with its `format_repairs` evidence +retained. A timed-out agent stage settles knowable V2 usage and retains one +bounded, redacted latest-assistant readback plus a `retained_session` pointer in +its stored transcript. A declared outcome routed to `#failure` with no step +diagnostic now yields a run diagnostic like `stage returned outcome +"blocked"`; an existing step diagnostic is still used verbatim. + +### Dated source-level coverage: 2026-10-11 + +Source-level/mocked regression coverage (not live-provider evidence): +`resolve-releases` now uses a single bounded +`GET .../git/matching-refs/tags/v2` response that the GitHub REST docs describe as +complete with no pagination; offline fixtures cover a 100+ ref array, +`MAX_REFS`/`MAX_BODY` caps, ignored prerelease/build tags, malformed refs and a +missing newest-tag digest failing closed with no older fallback, with no +`tags/list` cross-check. Picker tests cover blink/index preservation, +ANSI-readable selection without a colliding background, and +40x16/80x24/120x36 height budgets. The compiled picker PTY check (opt-in via +`PIPELINES_HARNESS_PTY_DIR`) waits through blink intervals and asserts exact +saved/arrow-scrolled models. `setup-docker.py` accepts the actual +`opencode v2.0.26` probe token and compares source identity only on a full +install; model/login-only reconfiguration compares the verified release image +identity. Python fixtures bind temporary HOME directories so the suite passes +under the container default `/home/pipelines` without weakening mount guards. +The Docker authoring transport is also covered at source level: the shared +prefix picks `/usr/local/bin/pipelines` for a Docker-linked profile. Loopback is +recognized only via `net.ParseIP(...).IsLoopback()` and an exact case-insensitive +`localhost`, so a remote DNS name such as `127.remote.example` stays remote with +no probe. A single bounded `docker compose port` probe parses complete +`host:port` bindings, and scoped source/target aliases are embedded only when a +binding covers the requested loopback address and port; malformed or non-matching +bindings and a loopback `wss` endpoint fail closed, and `client.Dial` rewrites +only an exact source match. `provider-check` dispatches through a Docker-linked +profile's existing service with `-T`, the original cwd/model/timeout, exact +stdout/exit propagation and `--json` passed only in JSON mode (fixture tests, no +real provider), while explicit native profiles keep the host generator. The +picker uses a true `x/term` TTY check so `/dev/null` and pipes fail fast, with a +short context deadline so a regression fails rather than hangs. Real Docker +rebuild/reconfigure against live registries, a real +40x16/80x24/120x36 terminal capture, and real model inference remain **not tested** +in this change. + +#### 2026-10-11 Pi sign-in handoff (source + Root PTY observation) + +Root separately exercised current-source `pipelines harness setup pi --login` +under a real Docker PTY with copied harness preferences. Root observed that +Pi 1.1.0 defaults to fullscreen and clears the Pipelines login instructions, so +the user sees a coding interface without the `/login` guidance, and that +`/login` first shows "Select authentication method" (account / API key / Radius) +rather than a provider picker. The same handler run with a temporary executable +wrapper passing `pi --tui-mode regular` retained the Pipelines header and +instructions above Pi's interface. Installed `pi --help` documents the supported +`--tui-mode regular` flag and the fullscreen default; `pi auth --help` confirms +there is no standalone login (only print/check, never run). These are source/root +observations with copied preferences: **no provider was selected, no key was +entered and no inference occurred**, and this source is **not deployed** and is +not live-provider evidence. A first reviewer capture at 80x30 found the longer +handoff plus Pi's own banner scrolled the Pipelines header off the top, so the +copy was shortened to a title plus three short lines (existing-credential reuse +folded into the `/login` line, with the account/API key/Radius method choice and +`ctrl+c`/`ctrl+d` exit keys kept) and the redundant trailing blank removed. +Independent PTY re-capture at 80x30/100x36 after this shortening is **not +tested** in this change. The matching exact login-argv and copy assertions are +covered by fixture tests in `internal/cli`/`internal/harness`. + +#### 2026-10-11 server CLI fixture isolation (source-level + Root-reported run) + +Root-reported runtime evidence supplied for this change (not re-executed by the +authoring stages): the deterministic seeded-profile regression +`go test ./tmp/preview-verification -run TestServerCLIFixtureDoesNotInheritUserDockerProfile -count=1` +seeds a synthetic Docker-linked harness profile in a child +`go test ./internal/server -run '^TestCLIHeadlessRunInspectionCancellationAndRPC$'`. +It previously failed because the child inherited the caller's real +`PIPELINES_HARNESS_CONFIG`, so the CLI `debug` prefix failed closed +("refusing to redirect") against the test's ephemeral httptest endpoint. Root's +four independent preview tests for the bounded transcript helpers pass; Root's +full host race run produced no race warning and only this fixture failure, and +`go vet ./...` passed. Current source therefore isolates the server CLI fixture +with the same missing-temp-file +`t.Setenv("PIPELINES_HARNESS_CONFIG", filepath.Join(t.TempDir(), "harnesses.json"))` +pattern already used by `internal/cli`'s setup, so the host executable stays +authoritative and no Docker probe runs for an ephemeral test endpoint. The +production fail-closed redirect guard in `internal/harness/prefix.go` and its +`refusing to redirect` / loopback-wss regressions are unchanged; the failing +verification was fixed rather than removed. + +This is a current-source and Root-reported observation. At the time of that +observation, independent re-execution of the full `go build`/`go vet`/`go test` +matrix (including Docker-only race results) and the final installed-host Docker +verification were **pending**; they are now recorded in the 2026-10-11 FINAL +INSTALLED section below, so this phase-specific wording is superseded by that dated +install. The packaged reference describes source behavior; a source fix or updated +skill is not deployment evidence. + +#### 2026-10-11 retained run11 reviewer re-capture (source-level, not a final install) + +Independently reading the retained run11 verification at +`tmp/maintenance-run11-verification.json` records the source-level review +sequence: an earlier reviewer PTY capture rejected the longer handoff because at +80x30 it scrolled the Pipelines header off the top; the implementer then +shortened `AuthHandoff` to a title plus three short lines; and the re-run +`reviewer` and `ui_reviewer` stages approved from real current-source Pipelines +PTY captures at 80x30 and 100x36 that showed the header and handoff visible above +installed Pi 1.1.0's regular interface. A labelled controlled fixture that +stripped `--tui-mode regular` reproduced the pre-fix fullscreen clear, and the +review stages report no provider selected, no key entered and no inference. This +is source-level reviewer evidence, read from retained output; it was not a +deployed/installed state at that time, and the final full Docker install +verification was then **pending** — it is now recorded in the 2026-10-11 FINAL +INSTALLED section below, so this phase-specific wording is superseded by that dated +install. + +Root-reported actual runs (source-level observations supplied by Root for this +change and not re-verified here): run7 showed both configured providers' exact +handoff markers; runs6/9/10 exercised live maintenance with final-assistant +polling/readback succeeding. These are Root observations, not mocked-regression +coverage and not a completed installation; no final deployed/installed state is +claimed. + +#### 2026-10-11 run13 interruption, Docker race limitation and state separation + +Four states are kept explicitly separate and must not be conflated: + +1. **Current source** (this checkout, not deployed): bounded 32 KiB + (`transcriptBound = 32<<10`) redacted transcript previews produce a valid JSON + envelope with redaction applied before truncation and a rune-safe 8 KiB preview; + session/debug evidence is scoped `latest_bounded_evidence` with the single + bounded latest-assistant readback and at most the 3 most recent session-bearing + steps; an unbudgeted V2 audited agent stage records reported usage best-effort + through exactly one 2s-bounded final session-aggregate read, leaving + missing/invalid usage unknown; and the isolated server CLI fixture uses the + missing-temp-file `PIPELINES_HARNESS_CONFIG` pattern while the production + fail-closed redirect guard in `internal/harness/prefix.go` is unchanged. Pi JSONL + is never parsed as one JSON object; native V1 keeps its existing path. +2. **Host verification evidence independently recorded by Root**: after the fixture + fix, Root ran the full `go test -race -count=1 ./...` and `go vet ./...`; after + three harness-neutral evidence labels and Step wording changes Root re-ran the + affected `tui`/`author`/`cli` race and vet checks, the Python suite (80) and the + opt-in compiled picker PTY check (4). These are Root-recorded host results, not + re-executed by the authoring stages. +3. **Deployed old image**: the running Docker image predates these source changes + and does not include them. +4. **Final install (completed 2026-10-11)**: the final install and installed + authoring/provider verification were **pending** when this separation was + written and are now recorded in the 2026-10-11 FINAL INSTALLED section below; an + updated skill/client or source checkout is still not deployment evidence on its + own. + +Docker race limitation (confirmed): the Docker runtime is built with +`CGO_ENABLED=0` and has **no C compiler** (the Alpine runtime stage installs no +GCC/musl-dev), so `go test -race` cannot run there. In-Docker reviewers must use +ordinary `go build`/`go test`/`go vet` and rely on independently executed host +`-race` runs. Race checks must not be attempted in Docker and no compiler may be +installed or searched for. + +Run13 interruption (dated evidence, not a fix): the previous reviewer ran `find /` +looking for GCC; the tool timeout left a process in disk sleep, Docker could not +stop the container, and an approved Desktop restart recovered run13 as interrupted +(`executing=false`, every step `agent_active=false`) with **no automatic replay**. +This is an environment/interruption observation only: no general cancellation fix +is inferred and process termination is unchanged. Docker startup recovery already +sets running steps to `interrupted` with `agent_active=0`. + +#### 2026-10-11 short-lived CLI inherited-pipe cancellation (current source) + +Current source (not deployed) adds one shared, dependency-free constructor in +`internal/opencode`: `CommandContext(ctx, executable, args...)` sets the single +finite `os/exec.Cmd.WaitDelay` constant `ShortLivedWaitDelay = 250ms` and is the +only place that value is defined. It is reused by the `internal/stage` +`OpenCodeClient.call()` API/agent path (including `Interrupt`), `opencode.Detect()` +`--help` and the native `Connection.Messages()` export fallback. It bounds +**draining of inherited output pipes only**: after the child exits os/exec waits at +most the grace for descendants holding the inherited stdout/stderr pipes, then +closes them and reports `exec.ErrWaitDelay`; it does not terminate or confirm +cleanup of descendants. + +`context.Canceled`/`context.DeadlineExceeded` identities, `exec.ErrWaitDelay` on +the sanitized errors, the ≤64 KiB request / ≤1 MiB response / 8 KiB stderr bounds, +`boundedBuffer`/`boundedOutput` drain-with-overflow behavior, redaction, strict +result validation and cleanup-unconfirmed active ownership (`ErrStageCleanup`, no +replay) are unchanged; no new module dependency is added and a parent that exits +successfully while a descendant still holds partial stdout is never treated as +success. + +Dated synthetic reproduction (`tmp/interrupt-verification`, synthetic +real-subprocess diagnostics, **not provider evidence**): with Root waiting for the +descendant itself to start before cancelling public +`OpenCodeClient{Backend:"v2"}.Interrupt`, two Docker/Linux returns took +3996/3995ms despite `context.Canceled` (`descendant-started.jsonl`); the stronger +host reproduction took 4003/4000ms (`reproduction.log`). Added source-level +regressions use real `os/exec` fixtures with start-marker barriers, bounded +watchdogs and naturally terminating children for cancel-after-descendant, +deadline identity, descendant-held pipes → `ErrWaitDelay`, ordinary +output/exit/redaction, failed-interrupt ownership/no replay, plus `Detect`/export +wait-delay and cancellation cases. This is **current source, not deployed**: +descendant termination, confirmed remote interruption and any run13 Docker disk +sleep fix are **not claimed**, and Docker race remains unavailable (no C +compiler). + +#### 2026-10-11 FINAL INSTALLED verification (friend-ready preview) + +This is the dated receipt for the installed runtime, recorded by Root from +`tmp/friend-final-verification/summary.json`; it supersedes the earlier +"pending final install" statements above, which stay as phase-specific history. +A normal interactive `./install --model opencode-go/deepseek-v4.1-flash +--harness-model opencode-go/deepseek-v4.1-flash` exited **0**, resolving the current +releases live and building fresh dirty source into actual image +`sha256:1810ab50f3ea3991e7f3548c79250e65a114bb1821eac52efd872b91cf7c3136` +(OpenCode 2.0.26 / Pi 1.1.0 / Node 24.18.1) with real tool-free provider +validation, ntfy authenticated publish, server + host health, home/workspace entry, +preserved authoritative endpoint and `ok` pre-install database/home backup +integrity. Post-install preservation all passed (original definitions 1–4, history +1–16, host/container default Pi and both models/profiles, logical endpoint, +service URLs/timezone/ports/models, Pi credential hash, OpenCode connected accounts, +ntfy token); neither original-repo commit/push nor credential/data cleanup ran. +Installed host-CLI authoring with **both** Docker harnesses (separate retained +drafts, each changing only the worker prompt) performed real `ping` → `pong` and +`validate` → `validated` through the generated Linux `/usr/local/bin/pipelines` and +scoped transport aliases preserving the logical `8081` endpoint and draft metadata, +independently inspected from native OpenCode tool outputs and retained Pi JSONL +`toolResults`, with no draft submit/run/publication. Docker-proxy submit on +verification-only definition 5 published v2, a second stale v1 draft was rejected as +conflict, and **Run17** succeeded in 7 seconds with the exact +`pipelines-docker-verified` markers in OpenCode then Pi, both inactive and exactly +one completed marker read verified from retained native history (not the +latest-assistant preview); best-effort unbudgeted usage was OpenCode 4011 +tokens/$0.00029769599999999996 and Pi 1071 tokens/$0.000142002, and +`/api/session/active` was empty after exit. Installed-UI captures +(`installed-pi-*`, `pi-signin-*`, `installed-pi-models-*`) cover the actual run17 +80x30/120x36 neutral-Pi/green-both-stage renders, the Pi auth handoff/method menu +viewed/canceled/exit0 with no sign-in, and the model wizard using copied +preferences only while production preferences stayed preserved. Root's independent +checks passed: full host `go test -race -count=1 ./...` and `go vet ./...`, +Python 80 tests (1 optional PTY skipped), Node policy, isolated developer lifecycle +fixture (no inference/deploy) and author package tests; earlier independent +reviewers ran ordinary Docker build/vet/tests and focused subprocess regressions +(no Docker race, no GCC search, no global `find`). **UNVERIFIED** here: fresh +OAuth/provider login completion, phone delivery, other real providers, the +D-disk-sleep fix and active lease expiry; authentication menus and +existing-credential inference are not fresh sign-in proof. The stronger synthetic +PUBLIC `Interrupt` child-start barrier (host 252/252 ms, Docker 250/251 ms, context +canceled true, bounded true vs ~4 s before) is not proof of descendant termination, +remote cleanup or a Docker VM disk fix. Archive extraction, build-identity and +compile verification remain pending until Root executes them. + +#### 2026-10-11 generated release receipt ignore (source-hygiene regression) + +Current source adds two root-anchored `.gitignore` rules for the installer's +machine-local generated artifacts: `/.pipelines-releases.json` (the receipt path +from `setup-docker.py` `release_receipt_path()`) and `/.pipelines-releases-*` (the +`tempfile.mkstemp` `prefix=".pipelines-releases-"` in `record_release_receipt()`); +no installer/runtime code changed and the pre-existing receipt and other files are +retained. A new `scripts/test_build_metadata.py` case is a controlled, isolated Git +fixture using the repository's actual `.gitignore`: a clean tracked source plus a +representative generated receipt/`.pipelines-releases-*` temporary file leave `git +status --porcelain` empty and `build_metadata.docker_args` at +`PIPELINES_DIRTY=false` with no `-dirty` version; editing the tracked source then +makes the status non-empty and `docker_args` report `PIPELINES_DIRTY=true` with a +`-dirty` version. This is **offline source-level regression coverage**, not the +recorded actual new-account install/provider login above and not live-provider or +deployment evidence; this docs-and-ignore change has no rendered UI. + +### Dated live reference validation: 2026-10-10 + +Environment: Docker Linux aarch64, OpenCode v2.0.20, +**opencode-go/deepseek-v4.1-flash**. Isolated databases/fixtures and local HTTP +notification endpoints; normal published pipelines/repositories were not changed. + +- Incident extraction → report: exact IDs/severities/counts, unchanged handoff, + retained session readback and one local success receipt passed (6.63s). +- File edit → verification: exact final release-file bytes, unchanged fixtures, + independently executed `cmp`, completed read/edit/shell evidence, retained edit + diff and no verifier edits passed (14.60s). +- Missing required input: blocked route ended failed; missing file was not invented; + retained read attempt and one local failure receipt passed (4.21s). +- Accounting/one-token budget initially failed its acceptance check because session + zeros let the run succeed. After the tracked-prompt fix, a focused live rerun + passed: **952 reported tokens, $0.00016440**, budget-exceeded failure, stopped + owned session (2.43s). This is not an independently reconciled provider bill. + +The earlier three workflows were rechecked from retained artifacts with current +assertions, without more inference. There was **no fresh full-suite live run of +all four workflows after the fix**. Do not generalize these results to all five +starter templates, other models/providers, or compatible native schema/tool usage. +Real ntfy authentication/storage, phone reception, provider outages/timeouts, +long-stream interruption and promptness of cleanup were **not tested** by this trial. +The trial ran rebuilt test binaries; it did not deploy the fix to the ordinary server. + +### Dated live reference validation: 2026-10-11 (run7) + +One existing real run7 (2026-10-11) is the retained live evidence dated for this +change: it executed both configured harnesses, OpenCode 2.0.26 and Pi 1.1.0, with +`opencode-go/deepseek-v4.1-flash` stages on the Docker server, and both steps +succeeded (6s; the retained Pi evidence panel reports 1054/10000 tokens). The +captures are retained at `tmp/friend-ui-verification/source-pi-80-*` and +`source-pi-120-*`, and a current-source TUI inspection of that run recorded actual +80x30 and 120x36 renderings, including the run graph and the bounded evidence +panel. + +This is a single real run, **not** a fresh full reference suite. Other providers, +real outages/long-stream interruption, fresh OAuth and phone delivery remain +**not tested**; the new-source installer and installed-authoring verification were +separately completed by the dated 2026-10-11 FINAL INSTALLED section above. The +[2026-10-10 OpenCode v2.0.20 trial](#dated-live-reference-validation-2026-10-10) +above remains dated history. + +### Reference suite when explicitly requested + +The source checkout's `TestReferenceWorkflowsLive` uses six model stage attempts +plus possible compatibility/format-repair calls. Retries are disabled and stages +have two-minute timeouts. It uses real existing credentials and retains private +`runs.db`, pinned definitions, fixture directories, run/session JSON and report. +The accounting case has a one-token/$0.25 budget; other cases have no spending +limit. Test notifications use a local endpoint, not the configured phone topic. + +Live inference requires explicit `PIPELINES_REFERENCE_LIVE=1`, a chosen +`PIPELINES_REFERENCE_MODEL=PROVIDER/MODEL`, and an existing private +`PIPELINES_REFERENCE_DIR`. Get consent for chargeable inference and confirm which +execution computer/account to use. Ordinary tests skip live inference. Do not +search the repository or run the suite during normal draft authoring. + +If the user explicitly asks to run the suite and provides a source checkout: + +```sh +PIPELINES_REFERENCE_LIVE=1 \ +PIPELINES_REFERENCE_MODEL=PROVIDER/MODEL \ +PIPELINES_REFERENCE_DIR=/private/approved/artifacts \ +go test ./internal/server -run '^TestReferenceWorkflowsLive$' -count=1 -v -timeout 15m +``` + +Docker credentials require running the correctly compiled test binary in that +container; do not copy host credentials or publish tests to the normal database. +`TestReferenceDefinitions` checks definitions without inference. +`TestReferenceArtifactsReview` rechecks copied run/fixture evidence without +inference using `PIPELINES_REFERENCE_REVIEW_DIR`; select only cases present in +that artifact directory. Missing cases skip and must not be reported as passed. + +### 2026-10-12 run22 success, run23 live CSV fixture and run24 interruption + +Source/structural and live-fixture evidence stays separate from runtime +deployment; nothing here is a deployment claim. + +- **run22 (structural/source checks, success):** the retained run22 verification + passed its structural/source checks of the scoped template and prompt changes. +- **run23 (live Pi CSV fixture):** a real exact-template Pi CSV run fetched the + official Python `csv` documentation through the conditional official-source + webfetch and implemented CSV parsing with the standard library's `csv.reader` + through an existing shared helper, adding no duplicate preview parser; both the + implementer and reviewer checks actually passed. Root independently verified the + full retained native tool results and protected-file hashes against its saved + baseline, rather than relying on model-written reports. +- **run24 (interrupted, not a pass):** the run24 implementation completed, but its + reviewer ignored the scope instruction and ran a prohibited filesystem-root scan + (`find /`), leaving a process in uninterruptible D-state and stalling Docker + mount/snapshot reads. Root canceled the run, made a consistent SQLite backup + (`integrity ok`), recovered Docker with the supported stop/start path, and + restored the existing ntfy/Pipelines services with the original definitions, run + history, credentials and settings verified unchanged. run24 is retained as + **interrupted**, not a success. + +- **run25 (live Pi source-only review):** the reviewer identified duplicated + implementation, a hardcoded special case, suppressed errors and test gaps. + Root inspected retained native calls: only local read/search tools ran, with no + shell/edit/network calls. Original fixture hashes were unchanged; runtime + behavior was explicitly marked untested. +- **run26 (final maintenance review, success):** the scoped plan/implement/review + workflow passed package tests, vet, build and formatting checks. Root inspected + the actual retained outputs and changes. No rendered UI change was claimed. + +The scoped prompt guidance asks stages to keep inspection/verification to relevant +repository paths and supported commands; it is a working instruction, **not** an OS +sandbox, and neither fixes Docker VM/disk bugs nor universally enforces tool +boundaries. At these recorded checkpoints the source changes had not been deployed. +Catalog deployment requires a separate check of the running server and its actual +template contents; these observations do not establish publication or deployment. +Other providers and phone delivery remain **not tested** by this record. + +### 2026-10-12 final template verification and Docker catalog deployment + +The final three-stage template ran as live Pi run27 (verification definition7, +version2) in a fresh disposable CSV fixture. Root inspected full retained native +results: the planner fetched official Python CSV documentation, the implementer +used the standard library through the existing shared parser, and implementer and +reviewer independently ran `python3 checks.py` successfully. Root reran those +checks, verified the protected fixture hashes and planner-owned review criteria, +and confirmed the preview reused the shared parser. + +The Docker server was then refreshed from clean source commit +`143f237268dbde2efd6c5ce5722a3e1fd7e08c9c`, retaining OpenCode 2.0.26, Pi 1.1.0 +and Node 24.18.1. The running server reported that exact commit and `dirty=false`; +all five served templates matched current source with the saved Pi/model defaults. +A consistent pre-deployment database backup passed integrity checks. Every stored +definition version/source, retained run ID, saved setting, endpoint and provider +credential/account checked by Root was preserved. This record is a documentation +follow-up to that deployed build; it changes no runtime code or template source. + +These bounded fixtures establish the observed behavior on this instance, not a +guarantee of every future model review, provider, task or platform. They do not +establish fresh account sign-in or phone notification reception. + + +## From the skill harness reference + +2026-10-10 verification: `scripts/check-pi-rpc.py` checked installed Pi 0.87.1 +readiness, metadata/config RPC and **idle** lease cleanup without inference. +Go RPC fixtures cover result/budget/missing-accounting/timeout/format-repair paths; +Node tests cover permission checks. `PIPELINES_PI_LOCAL=1 go test ./internal/stage +-run TestPiInstalledLocalIntegration` also exercised the **installed Pi** against +scripted loopback responses: actual allowed/denied writes, budget exhaustion, +and active-shell cancellation (including no delayed write after cleanup). +The installed-Pi loopback suite also covers isolated deny-all syntax repair with +retained ownership and cumulative usage. The compiled fuzzy picker passed PTY +selection/cancellation; installed Pi catalog discovery/configuration passed with +an unreachable local endpoint and no prompt. +This is integration evidence, not real-provider/model evidence. Active lease-expiry +recovery, web search delivery and fresh interactive Docker Pi setup remain +**not tested**. Pi catalogs exclude ambient provider extensions for parity with +audited stages; compatible custom endpoints can use Pi's private `models.json`. +An isolated built Docker image also passed non-root, network-disabled policy/RPC +readiness and idle lease cleanup, without credentials/mounts or inference. +Codex/Claude +adapters and AI-assisted server guidelines validation remain future work. + +2026-10-11 source-level coverage (source-level regressions; the separate dated +2026-10-11 final install is live evidence): the +shared Docker-aware CLI prefix, exact-match client transport rewriting, +Docker-linked `provider-check` dispatch and the `x/term` TTY check have +source-level regression tests (`internal/harness`, `internal/client`, +`internal/cli`). The prefix tests cover the remote-DNS negative +(`127.remote.example` stays unmapped with no probe), IPv4/IPv6/`localhost` +positives, complete `host:port` binding matching (a different service on the same +port, or a family mismatch, does not match), malformed/empty output failing +clearly, multiple bindings under a single probe, and loopback `wss` rejection. +Terminal Pick regression tests use a short deadline so a TTY regression fails +fast rather than hanging. Docker dispatch tests assert `--json` is passed only in +JSON mode while plain text stays readable. Python fixtures cover arbitrary-digit +OpenCode patches (a parsed `2.0.260` is still rejected against an expected +`2.0.26` by exact equality; 260/100 are illustrative valid/future examples, not +verified releases) and reject both raw and canonical reserved homes. Real +in-container `provider-check`/ping confirmation, a compiled PTY run and a real +Docker reconfigure remain **not tested** in this change. These source-level +regressions are not a claim about the deployed image; the separate dated 2026-10-11 +FINAL INSTALLED verification recorded that the runtime was actually installed and +used (OpenCode 2.0.26 / Pi 1.1.0 / Node 24.18.1, image +`sha256:1810ab50f3ea3991e7f3548c79250e65a114bb1821eac52efd872b91cf7c3136`, with +both real stages executed in run17 and passed in Root's full host race/vet runs). +Active lease-expiry recovery, web search delivery and fresh interactive Docker Pi +sign-in remain **not tested**. diff --git a/examples/feedback-loop.md b/examples/feedback-loop.md index 161d1e65..c1bc32ae 100644 --- a/examples/feedback-loop.md +++ b/examples/feedback-loop.md @@ -37,9 +37,11 @@ error -> #failure # stage review ```prompt -Independently review the actual change against the original request in -memory.md, not just the implementer's summary: `git diff --stat` first, then the -changed hunks and only the code around them. Run no test suites. Return +Review the actual change against the original request in memory.md, not just +the implementer's summary: start from the server-captured changed files and +`git diff --stat`, then read the per-file diffs and only the code around them. +Judge the checks the implementer reported and name missing coverage; run no +test suites. Return rejected only for a must-fix problem, giving its location, evidence and fix. Otherwise return approved, with any non-blocking notes in the body. Do not edit, commit or push. diff --git a/examples/two-stages.md b/examples/two-stages.md index 1b7048b7..38e99ac7 100644 --- a/examples/two-stages.md +++ b/examples/two-stages.md @@ -29,7 +29,7 @@ error -> #failure # stage check ```prompt -input.md is the previous stage's draft; memory.md holds the original request. +input.md holds the original request and the previous stage's draft. Check the draft against its sources and the request, correct mistakes and gaps, and return the final text under `# Final` with a short list of corrections. Change nothing. Return blocked if the draft cannot be checked. diff --git a/internal/author/skill/SKILL.md b/internal/author/skill/SKILL.md index dbe4182d..b20972fa 100644 --- a/internal/author/skill/SKILL.md +++ b/internal/author/skill/SKILL.md @@ -1,6 +1,6 @@ --- name: Pipelines -description: Use existing Pipelines workflows to run, inspect, wait, recover and deliver project work. Use only when the work belongs in an existing pipeline; authoring is a separate, explicit task. +description: Use existing Pipelines workflows to run, inspect, wait, recover and deliver work, and author definitions when the user explicitly asks. --- # Pipelines @@ -34,63 +34,44 @@ explain the gap; ask before authoring. Do not add a task-specific pipeline as a ## Authoring -A definition is Markdown: `# stage ` sections with `prompt`, `configs`, -`tools`, `edges`, and `limits`/`checks` where required: every stage that can -modify the workspace declares `checks` (commands on the `delivery=true` stage, -otherwise `none: `), and every loop has `limits`. Start from -`examples/` and read `references/practices.md` before writing prompts. Use stage -names that describe the work: - -`Plan` · `Code` · `Review` · `UIReview` · `Land` (`Prepare` and `Survey` are -useful variants). +Start from `pipelines templates` and `pipelines new NAME --template KEY` +(`one-stage`, `two-stages`, `feedback-loop`, `human-decision`); these fill in +the server's provider/model. Read `references/format.md` and +`references/practices.md` before writing prompts. Stage names: `plan` · +`implement` · `review` · `ui-review` · `decide` (human) · `land`. Keep prompts short and objective: what the stage should produce and any real -constraints. The runner enforces turns, tools and limits; prompts do not need to -repeat them. Only `# outcome` is parsed from `output.md`; everything else is -free markdown. Use `links` and `memory` only when they clarify a handoff. +constraints. The runner enforces turns, tools and limits and explains the output +format; prompts do not repeat them. Only `# outcome` is parsed from `output.md`; +everything else is free markdown. There are no memory, links, input or output +blocks; stages communicate through `output.md` (`# handoff`) and `public/`. -`pipelines validate DRAFT` checks a draft, including the authoring rules in -`references/practices.md` (errors block submit; `warning:` lines do not but should -be fixed); `pipelines submit DRAFT` publishes a new immutable version. +`pipelines validate DRAFT` checks a draft, including the authoring rules +(errors block submit; `warning:` lines do not but should be fixed); +`pipelines submit DRAFT` publishes a new immutable version. `pipelines help ` has flags. -Schedules are separate saved run configurations: pipeline/version, prompt, -workspace and other run options, plus timing. Never add `# settings` or schedule -blocks to a workflow to arrange recurring work. One generic pipeline can have -many independent topics and timers. Read `references/schedules.md` when the user -explicitly asks to manage a schedule; scheduling alone is not workflow authoring. - -## Turns and artifacts - -Each visit has `input.md` (the run prompt; later visits add the previous body), `memory.md` -(short essential memory inlined in the prompt: task, previous stage's `# handoff`, -outcome ledger, changed files; 4 KB cap), `history.md` (completed turns in full, -read on demand), `output.md`, and `private/`. Every run also has one shared -`public/` directory, addressed by its absolute path; files there are visible to -later stages and are the stage's real artifacts. `input.md`, `memory.md`, -`history.md`, `output.md` and `session.html` are system evidence for auditing, not stage -output. - ## References -Read only what the task needs: +Read only what the task needs. Rows marked operator only are for setting up or +debugging the server; skip them when authoring. | Need | Reference | | --- | --- | -| Definition syntax and turn files | `references/format.md` | +| Definition syntax, turn files, output.md, checks, landing | `references/format.md` | | Good practices (tokens, checks, prompts, loops) | `references/practices.md` | -| Limits, tools and permissions | `references/control.md` | +| Loop limits | `references/control.md` | | Stages, parallel work, recovery | `references/orchestration.md` | -| CLI, storage and evidence | `references/operations.md` | -| Verification and accounting | `references/verification.md` | +| Verification and evidence | `references/verification.md` | | Failures | `references/debugging.md` | | Delivery and installation | `references/delivery.md` | -| Harnesses and models | `references/harnesses.md` | | Notifications | `references/notifications.md` | -| Independent saved run schedules | `references/schedules.md` | +| Saved run schedules (never in a definition) | `references/schedules.md` | | Transfer and versions | `references/transfer.md` | -| Hosting, auth and deployment | `references/access.md` | | Queue and work items | `references/work.md`, `references/queue.md` | +| CLI, storage and run evidence (operator only; not needed for authoring) | `references/operations.md` | +| Harnesses and models (operator only) | `references/harnesses.md` | +| Hosting, auth and deployment (operator only) | `references/access.md` | Ask about real ambiguity, destructive actions or changes outside the request; routine steps of an authorized task do not need another confirmation. diff --git a/internal/author/skill/references/control.md b/internal/author/skill/references/control.md index 5a441e09..c13b7978 100644 --- a/internal/author/skill/references/control.md +++ b/internal/author/skill/references/control.md @@ -1,11 +1,16 @@ -# Runner-owned limits and stage memory +# Runner-owned limits and loops -Use these blocks for bounded feedback loops. Never ask a model to count visits, -increment revision counters, or enforce retry caps. Model verdicts concern current -work only. Counters are runner audit data, not model input. A fresh session alone -does not guarantee independent review: explicitly select its input fields. +Use `limits` for bounded feedback loops. Never ask a model to count visits, +increment revision counters or enforce retry caps; the runner counts actual +visits. Model verdicts concern the current work only. + +Put the cap on the reviewer, and leave the loop target without `limits` (it then +has the runner's default of five visits) or give it a cap at least as large as the +reviewer's: ````markdown +# stage review + ```limits {"max_visits":3,"on":["rejected"]} ``` @@ -18,48 +23,32 @@ error -> #failure ``` ```` -These blocks accompany a normal review stage's prompt/tools/configs. -Pipelines counts actual visits, ignoring model-generated counters. On visit three, -`approved` can succeed; `rejected` retains its verdict but routes to `#failure`. -`rejected` is for a must-fix finding with a concrete location, evidence and remedy; -`advisory` carries non-blocking notes and continues the run without another -implementer visit, so an advisory-only review never buys a full re-implementation. -An implementer may likewise declare `blocked_retryable -> implementer` for a -transient environment, tooling or device problem a fresh attempt could survive, -keeping `blocked -> #failure` for an acceptance criterion that genuinely cannot be -met. `max_visits` caps every visit of that stage, including re-entries from the -reviewer's `rejected` edge, so an implementer that is also the rejection target must -use a cap at least as large as the reviewer's budget; the published pipelines use -`{"max_visits":5,"on":["blocked_retryable"]}`. A lower implementer cap stops the loop -before the reviewer's own cap fires, making the reviewer's `rejected` control -unreachable. Both edges need only the existing visit counting and cap; neither the -reviewer nor the implementer counts visits, and no stage invents edits or checks to -appear busy. -Visit count, limit decision and original target are retained in `inspect --json` -and run metadata. `max_visits` accepts 1–5. Every `on` outcome must declare a -nonterminal edge. Without `on`, entering beyond the cap fails before execution. -Global 30-step/five-visits-per-stage safety limits still apply. +These blocks accompany the stage's normal prompt, tools, checks and configs. +Semantics (`internal/server/runner.go`): -A model stage reads its turn's `input.md` (the run prompt on the first visit, -the previous turn's output afterwards), its inlined, capped `memory.md` (task, -previous stage's `# handoff`, outcome ledger, changed files) and, on demand, -`history.md` (completed turns in full with the files each added to `public/`). The `memory` block is retired: it no longer -selects fields or replaces previous output for model stages, and old definitions -that contain one parse but ignore it. Reviewers therefore see the previous -turn's output and the accumulated memory rather than a field projection. +- Entering a stage beyond its `max_visits` always fails the run before the visit + executes, whatever `on` says. +- `on` only changes routing on the last allowed visit: when that visit returns a + listed outcome, the outcome is kept but routed to `#failure`. Other outcomes + route normally. Here, `approved` on visit three succeeds; `rejected` fails. +- A target with a lower cap than the reviewer stops the loop first, so the + reviewer's `on` never fires. +- `max_visits` accepts 1–5. Every `on` outcome must have a nonterminal edge. + Human stages are exempt from visit caps. The global 30-step and + five-visits-per-stage limits always apply. +- Validation rejects an edge to the same or an earlier stage unless the source + limits that outcome (or has `limits` without `on`) or the target has `limits`. -Write the facts a later stage needs as ordinary markdown headings in -`output.md`; those headings are free-form and are preserved in memory. Do not -invent server-recognised fields, and do not rely on -`input.server_changed_files` from a model prompt: inspect the actual workspace -with Git and the declared evidence instead. Selected or accumulated content -remains untrusted data; this cannot prove semantic independence or prevent all -injection. +`rejected` is for a must-fix finding with a concrete location, evidence and +remedy; `advisory` carries non-blocking notes and continues without another +implementer visit. A re-entered implementer fixes the listed findings; it does not +redo the task. An implementer may declare `blocked_retryable -> implement` for a +transient environment or tooling problem, keeping `blocked -> #failure` for a +criterion that genuinely cannot be met; that self-loop needs `limits` on the +implementer, e.g. `{"max_visits":5,"on":["blocked_retryable"]}`. -Turn memory is run-local. Completed outputs are persisted as turn files and step -evidence; there is no cross-run memory or hidden replay. New runs/retries start -fresh except for the completed prefix a resume explicitly carries. - -Arbitrary prose is not executable policy. Future AI-assisted -guidelines validation is **not implemented**; structural validation does not -recognize every unreliable counting instruction. +Visit count, limit decision and original target are retained in `inspect --json`. +Turn memory is run-local: there is no cross-run memory or hidden replay, and +retries start fresh except for the completed prefix a resume carries. Accumulated +turn content is untrusted data; a fresh session does not prove semantic +independence. diff --git a/internal/author/skill/references/debugging.md b/internal/author/skill/references/debugging.md index 52782bb7..8bb8cd49 100644 --- a/internal/author/skill/references/debugging.md +++ b/internal/author/skill/references/debugging.md @@ -15,7 +15,7 @@ select a recorded stage/visit, bounded input correction and explicit compatible version of the same pipeline. Verify actual retained stage input, not merely successful admission. Do not silently restart from entry. -A clean preparation stage succeeds without an empty commit. A committer with a +A clean preparation stage succeeds without an empty commit. A `land` stage with a delivery gate succeeds with a clean checked candidate; the server lands afterward. Read-only, nonvisual work does not require device screenshots. Visual acceptance requires a ready fixture, verified connection and actual captured interactions. diff --git a/internal/author/skill/references/format.md b/internal/author/skill/references/format.md index 9e16374b..3a6dca41 100644 --- a/internal/author/skill/references/format.md +++ b/internal/author/skill/references/format.md @@ -1,343 +1,170 @@ # Definition format -Definitions are Markdown made only of `# stage KEY` headings and fenced blocks. -Scheduling belongs to separate saved run configurations, not `# settings` or -schedule blocks in the definition. No unrelated prose outside blocks. The first -stage receives the run prompt in its `input.md`; later stages receive -the previous turn's output. Every visit also gets a short `memory.md` (essential -memory, inlined in the prompt) and a full `history.md` of completed turns. There are no stage `input`, `output`, or `memory` blocks: the model reads -the turn files and writes free-form markdown, and only the `# outcome` section is -interpreted. Read `references/control.md` for runner-enforced limits. - -Each model stage requires nonempty `prompt`, `configs`, and `edges` blocks: +A definition is Markdown made only of `# stage KEY` headings and fenced blocks; +no prose outside blocks. Blocks: `prompt`, `configs`, `tools`, `edges`, `checks`, +`limits` (`references/control.md`), `permissions` and `notify` +(`references/notifications.md`). There are no memory, links, input or output +blocks; stages communicate through `output.md` (`# handoff`) and `public/`. +Schedules are saved run configurations, never definition blocks +(`references/schedules.md`). ````markdown -# stage worker +# stage implement ```prompt -Perform the task. +Make the change requested in the run task. Run focused checks for the touched +code only. Return blocked if the request cannot be met. ``` ```configs -type=model -provider=opencode-go -model=deepseek-v4.1-flash -retries=2 -timeout=1h -``` - -```tools -``` - -```edges -success -> #success -blocked -> #failure -error -> #failure -``` -```` - -Stage/outcome keys begin with a letter and contain letters, digits, `_`, or `-`. -Each outcome has exactly one target: another stage, `#success`, or `#failure`. -An omitted error edge defaults to failure. Execution is sequential; forks choose -one outcome, never execute parallel branches. Cycles are allowed; every stage -must be reachable from entry and have a possible path to a terminal. Use bounded -runner-owned `limits` for feedback loops, never revision counters in model -prompts/outputs: publishing rejects an edge to the same or an earlier stage -(definition order) unless the source stage limits that outcome or the target -stage has `limits` (a human stage's choices are exempt). Read -`references/control.md` for the exact format. - -## Checks - -Every model stage that can modify the workspace (it declares `edit` or `shell`, -has no `tools` block, or sets `delivery=true`) must declare a `checks` block, so -the author decides where verification happens. Read-only stages need none. - -- A `delivery=true` stage lists 1–20 commands, one per line. The server runs them - on the committed candidate (and again on a merged tree) before landing; a - failing command blocks landing. These are the run's deterministic checks. -- Every other stage writes exactly one line, `none: `, naming where - verification happens instead, for example - `none: focused tests only; the Land delivery gate runs the suite` or - `none: read-only review; shell is used only for git diff`. Commands on a - non-delivery stage are rejected because nothing would run them. - -A check must fail when it cannot run. Validation rejects guards that pass -without running anything: `|| true`, `|| :`, `|| exit 0`, `|| echo`, -`[ ! -x tool ] || ...`, `test ! -f ... || ...`, `! command -v tool || ...`, -an `if ...; then ...; fi` with no failing else, and `set +e`. Cover every -language the run can change (a Go-only check list does not verify Flutter work). -`pipelines validate` reports these with the remaining authoring rules -(`references/practices.md`). - -## Result format - -Each visit gets `input.md`, `memory.md`, `history.md`, `output.md` and -`private/` in `/stages/-/`, plus the run-level shared -`public/`. What a stage receives: - -- `memory.md` (essential, hard cap 4 KB, always inlined in the prompt): the run - task, the previous stage's handoff with its outcome and public/ additions, a - one-line-per-stage outcome ledger (latest visit; a revisit is marked) and the - server-captured changed files. A stage must be able to work from this alone. -- `input.md` (on demand): the full run task and the previous turn's full output. -- `history.md` (on demand, never inlined): every completed turn in full, the - format `memory.md` had before essential memory existed. - -Stages with `read` are told the two on-demand paths; stages without it get -`input.md` inlined too (capped at 32 KB) but never `history.md`. - -The previous stage's handoff is its `# handoff` section (any level, up to the -next heading of the same or higher level); without one the server uses the head -of its output, marked as such. Either is truncated to fit with a pointer to -`input.md`, so nothing depends on the model following the convention. The -prompt asks every stage to start its output with a short handoff: what it did, -files changed, open issues or findings. The only required section is the -outcome heading, followed by one of the stage's declared edge words: - -````markdown -# handoff - -Fixed both review findings in cache.go; tests pass. No open issues. - -# outcome - -success - -# Result - -Free-form markdown for the next stage and the operator. -```` - -`internal/turn.ParseOutput` matches the heading case-insensitively at any ATX -level and ignores headings or words inside fenced code blocks. A missing, -duplicate, empty or undeclared outcome fails the stage with -`FINAL RESULT VALIDATION`. Every other heading is uninterpreted and preserved -verbatim in the body. There is no JSON envelope, output schema, field selection -or format-repair pass: do not ask for JSON fields or invent server sections. -Keep turn-local files in `private/`; put reports, logs and screenshots for later -stages in `public/`. Never fabricate a result. - -Model configs may set `harness=opencode` or `harness=pi`; omission preserves -historical OpenCode execution. Read `references/harnesses.md` for setup/limits. -Use the exact provider/model IDs validated during server setup. New drafts/templates -inherit that selection; do not assume the host's login matches the container's. -Other OpenCode provider/model IDs are accepted syntactically; actual availability, -authentication and inference require `pipelines provider-check` on the execution -computer/container. Optional `retries` defaults to 2 (three total attempts), -and accepts integers 0–100; explicit 0 disables execution retries. Optional -`timeout` defaults to `1h` per attempt and accepts positive durations such as -`30m` or `90s`. - -A stage that declares a `skipped` edge (for example a UI reviewer) is eligible -for the server skip fallback: when every earlier visit's file-change capture was -complete and recorded no changed files or screenshots, the server completes the -stage as `skipped` without a model call (`reason_source: fallback`). Any recorded -change or incomplete capture runs the model. Optional `auto_skip=false` (values -`true`/`false`, default `true`) always runs the model. Read -`references/operations.md` ("Server skip fallback") for the exact rule. - -Optional `reasoning_effort=high` selects a model's reasoning/thinking level for -this stage. Values are `default`, `none` (alias `off`), `minimal`, `low`, `medium`, -`high`, `xhigh`, or `max`. Omission or `default` preserves the harness/model -default; never add a level automatically to existing workflows. Availability is -model-specific: OpenCode V2 selects the corresponding named model variant and -rejects unavailable variants; Pi uses its native thinking level (`none` becomes -`off`) and verifies the effective level before prompting, rejecting unsupported -or silently clamped levels. Explicit effort on OpenCode V1 is unsupported. -Do not instruct the agent to “think harder” as a substitute for this native -setting. Effort is not a token budget or a guaranteed quality improvement; keep -usage budgets/provider spending limits separate. Both text-only and tool-using -model stages honor the setting, without changing their tools or permissions. - -Automatic retries cover retryable transport failures and -timeouts; an explicit read-only stage may additionally retry a post-admission -deadline or transport failure, while a stage with `edit`/`shell` (or no tools -block) never replays admitted work. Optional `budget_usd=1.50` and -`budget_tokens=50000` independently cap reported usage. -They accumulate across attempts and repeated visits -to this stage in the same run. Omit either budget to leave that resource unlimited. -Both budgets must be positive; tokens include input, output, reasoning and cache -reads/writes. Accounting failures stop budget-limited stages. In-flight requests -may exceed limits before usage is reported; these are not provider-side hard caps. -In current source, all OpenCode V2 text-only calls use tracked prompting with -all tools denied, rather than the usage-less `/generate` path. Completed -token-budgeted responses reporting zero tokens fail closed. This is an execution -implementation detail, not a new Markdown field: never add backend/format/API -settings to definitions. Read `references/verification.md` for the dated live -accounting result and its provider/deployment limitations when relevant. -The run has no additional wall-clock deadline; step/visit caps and cancellation -still apply. A native ntfy stage accepts `timeout` but never automatically retries -delivery and has no model budget. - -Explicit empty `tools` denies all; allowed tools are `read`, `glob`, -`grep`, `webfetch`, `websearch`, `edit`, `shell`, one per line. No wildcards, -delegation, or browser tools. Tools are per-stage and automatically enabled by -starting a run. `edit` supports any file type and creating files/directories in the -run directory; no file list is required. `shell` supports commands appropriate to -the repository's language and tooling, not a language-specific allowlist. A missing -tools block uses local read/glob/grep/edit/shell tools; prefer explicit stage roles. -Planning stages must not label a task blocked simply because they defer edits to -an implementer stage. The intended role split: an implementer may -read/glob/grep/edit/shell; a planner reads, plus conditional web search/fetch only -when external knowledge is needed; a reviewer reads and may run read-only checks -with shell but never edits; a source-only review stage has no edit/shell; a -research stage uses web search/fetch; a committer has read/glob/grep and shell but -no edit. The bundled examples (`one-stage`, `two-stages`, `feedback-loop`, -`human-decision`) follow it without web tools. Never force web access for -trivial, fully understood local edits, and return blocked only when required -reliable evidence cannot be obtained rather than whenever one search tool fails. -Keep examples and prompts language-neutral unless the task itself asks for a -specific language. - -Optional `permissions` is a JSON object with `edit` (run-directory-relative -patterns) and `shell` (exact commands) arrays. It narrows declared tools and any -run-level file scope; it never enables a missing tool. Empty arrays deny actions. -Nulls, duplicate/unknown keys, traversal and shell wildcards/chaining/redirects or -expansion are rejected. If edits are restricted and shell is enabled, provide an -explicit shell list too. Example: `{"edit":["README.md","plugin/*.qml"],"shell":["bash -n install.sh"]}`. -Use this block for intentional file/command restrictions; prose is not a runtime -permission policy. Keep ordinary workflows language-neutral and unrestricted -within their run directory unless the user asks for a narrower scope. - -## Human (decision) stages - -A `type=human` stage pauses its run until a person chooses one of its outgoing -options. It has no model, agent, tools or configs: it needs a nonempty `prompt` -block (the operator instructions) and at least one non-`error` `edges` outcome. -Its outcomes are exactly its non-error edges, so a draft can express choices such -as `deploy`, `revise` or `abort`; the mandatory error edge is a failure route, -not a choice. It validates, renders, exports and imports like every other type. - -````markdown -# stage gate - -```prompt -Review the screenshots and artifacts above, then choose where the run goes. -``` - -```edges -deploy -> publish -revise -> worker -abort -> #failure -error -> #failure -``` -```` - -When a run reaches a human stage its step becomes `waiting`, the run reports it is -waiting on a person, and nothing later runs. Decide from the TUI (open the run, -press `c`, pick an option, optionally type feedback, Enter) or the CLI: -`pipelines decide [stage] --option KEY [--feedback TEXT]`. The decision -and feedback are recorded as the step's evidence (the feedback stays in the step -output, never an error diagnostic) and the run resumes along that -edge. The wait is persisted, so it survives a server restart. An unknown option is -refused with the valid list; a decision to a stage that is not waiting is refused; -a repeated decision for an already-decided visit does not double-apply. Cancel and -retry keep working while waiting and after a decision, and a human stage is never -killed by a stage timeout or `max_visits` (the shared run budget still applies). - -## Committer stages - -A workflow that changes the repository ends in a committer: a normal model stage -whose prompt commits the run's change locally and never pushes, tags, deploys or -rewrites history. It is not a native type. Declare `shell` (and `read`/`glob`/`grep` -when it inspects content) and no `edit`, so a failed check is reported rather than -patched, and select the runner's captured change set alongside the producer's -declared list: - -````markdown -# stage committer - -```prompt -Commit this run's change locally; never push. +provider=default +model=default ``` ```tools read glob grep +edit shell ``` ```checks -none: commits locally; the manager lands and runs the suite +none: focused tests only; the land delivery gate runs the suite ``` ```edges -success -> #success +success -> review blocked -> #failure error -> #failure ``` ```` -A landing committer instead sets `delivery=true` in `configs` and lists the -repository's required check commands in `checks`; the server runs them, so its -prompt does not run the suite again. +Stage and outcome keys begin with a letter and contain letters, digits, `_` or +`-`. Name roles `plan`, `implement`, `review`, `ui-review`, `decide` (human) and +`land`; other jobs take a plain verb (`draft`, `check`, `act`). Each outcome has exactly one target: another +stage, `#success` or `#failure`. An omitted `error` edge defaults to `#failure` +for every stage type, human included. Execution is sequential; an outcome picks +one edge, never parallel branches. Every stage must be reachable from the entry +and have a path to a terminal; loops need runner-owned `limits`. -State this pre-commit checklist in the prompt: +## Turn files -1. Inspect `git status` and the full diff. Stage this run's change, including - inherited work, restricted to paths that still exist, and keep every - exclusion. Do not rely on a model-declared file list. -2. Inspect the staged diff before committing; never commit unrelated work, secrets - or generated artifacts. -3. Without a delivery gate, run the repository's required checks once (summary - output) and record the exact commands and observed results as evidence. Do not - commit when a required check fails; return `blocked` with the failing output - instead. With `delivery=true` the server runs the `checks` block; skip this step. -4. Create no empty commit: when nothing is staged, return `success` with a clear - note. -5. Commit with a concise message naming the run's change and report the commit hash; - never push, tag, rebase or amend. -6. Ensure the item's per-item documentation note (`docs/notes/.md`, written when - the change makes a shared document stale) is staged with the change. Never edit a - shared document (a README or topic doc, or a packaged skill reference) here: a single - documentation writer folds notes later (`references/orchestration.md`). An item that - staled no shared document has no note and needs no step 6. +Each visit gets `input.md`, `memory.md`, `history.md`, `output.md` and `private/` +in `/stages/-/`, plus the run-level shared `public/` addressed by +its absolute path. -This list is the canonical checklist. Repeat it in each committer prompt, because a -definition is published on its own; keep those copies in sync with this list. The -bundled examples leave changes uncommitted and have no committer. +- `memory.md` (capped at 4 KB, always inlined in the prompt): the run task, the + previous stage's `# handoff` with its outcome and `public/` additions, a + one-line-per-stage outcome ledger and the server-captured changed files. +- `input.md`: the run task, plus on later turns the previous turn's full output + (`internal/turn/render.go`). A stage with `read` reads it on demand; a stage + without `read` gets it inlined (up to 32 KB) on every call, so give `read` to + stages that follow a large output. +- `history.md`: every completed turn in full; on demand for stages with `read`, + never inlined. -The committer prepares the complete clean candidate and resolves integration conflicts. -When `delivery=true`, the server executes the declared checks and lands after the -stage returns success; do not run the post-landing ancestry check beforehand. -If canonical `main` moved meanwhile, the server merges it into the item (a normal -merge commit), re-runs the checks on the merged tree and lands that; a conflicting -catch-up is aborted with the workspace unchanged and the conflicting files named. -Without a delivery gate, the original workflow or manager completes the merge -and verifies ancestry. No separate merger pipeline is required (`references/queue.md`). +Keep turn-local files in `private/`; put reports, logs and screenshots for later +stages in `public/`. `input.md`, `memory.md`, `history.md`, `output.md` and +`session.html` are server-written evidence, not stage output. -A stage returns the markdown above, never the retired -`{ "outcome": ..., "output": ... }` envelope. Do not treat tool-call/DSML markup, -plain prose, or a missing outcome as success, and never rerun tool work merely to -fix a response's formatting. +## output.md -## Scheduling +The stage returns the complete `output.md` in this order: `# outcome`, the edge +word alone on the next non-blank line, then `# handoff` (a few lines: what it did, +files changed, open findings), then any detail: -Publishing a workflow does not schedule it. Create a separate saved schedule -with an explicit pipeline/version and complete run information (prompt, server -workspace, model overrides, notifications, output format and write restrictions -when used). Multiple saved schedules can reuse one workflow with different -research items or timers; updates and deletion do not rewrite the definition. -Use exactly one timing rule: `times` (24-hour clock values) or `every` (a duration -of at least one minute), with optional weekdays and timezone. +````markdown +# outcome -New definitions with embedded schedules are rejected. Historical immutable -versions remain readable; their latest legacy slots migrate into separate saved -schedules without changing source/history or replaying old occurrences. Never -reintroduce legacy scheduling blocks or make a new workflow just to set another -topic/timer. Warn explicitly before enabling unattended runs. See -`references/schedules.md` for the API and retry/concurrency guards. +success -## Notifications +# handoff -Stage hooks use the global phone connection, never store credentials. Optional -fenced `notify` block: `on=success,failure` (add `waiting` to fire when a human -stage parks the run), `artifacts=markdown`. -Read this skill's `references/notifications.md` for the minimal terminal-path -recipe and optional native publisher format. Prefer hooks for ordinary success/error -alerts; a request to notify is not a request for a new delivery stage. Do not scan -the user's repository for Pipelines docs. Bundled examples are generic and manual-only. -Notification attachment/report support and model availability need real runtime -tests; structural validation alone cannot confirm delivery. +Fixed both review findings in cache.go; focused tests pass. No open issues. + +Free-form detail for the operator and later stages. +```` + +The outcome heading matches case-insensitively at any ATX level, outside fenced +code. Never add a second heading named Outcome. If the outcome is missing, the +runner sends one repair follow-up asking only for it; a still-missing, duplicate, +empty or undeclared outcome fails with `FINAL RESULT VALIDATION`. Every other +heading is free-form and preserved. The handoff is the `# handoff` section (or, +without one, the head of the output, marked as such), truncated to fit memory. +The `{ "outcome": ..., "output": ... }` JSON envelope and its format-repair ladder +are retired: never ask for JSON fields or invent server sections. Missing tools or +evidence route to `blocked` when declared, else `error`; never fabricate a result. + +## Configs + +- `provider`/`model`: exact IDs validated during server setup. Templates + (`pipelines new NAME --template KEY`) fill them in; copy those IDs into a + hand-written draft. Availability needs `pipelines provider-check`. +- `harness=opencode|pi`; omission means OpenCode (`references/harnesses.md`). +- `retries` 0–100, default 2 (three attempts). Retries cover transport failures + and timeouts; a stage with `edit`/`shell` (or no tools block) never replays + admitted work. +- `timeout` per attempt, default `1h` (e.g. `30m`, `90s`). +- `reasoning_effort`: `default`, `none` (`off`), `minimal`, `low`, `medium`, + `high`, `xhigh`, `max`. Omit unless asked; unsupported levels are rejected. + Do not prompt "think harder" instead. +- `budget_usd`, `budget_tokens`: positive caps on reported usage, shared across + attempts and visits of this stage; totals include cache reads and writes. Set + only when the user asks or a measured visit exists (`references/practices.md`). +- `delivery=true`: the landing stage's gate (below). OpenCode harness only. +- `auto_skip=false`: disable the server skip fallback for a stage that declares a + `skipped` edge (`references/operations.md`, Server skip fallback). + +## Tools and permissions + +`tools` lists `read`, `glob`, `grep`, `webfetch`, `websearch`, `edit`, `shell`, +one per line; an empty block denies all and a missing block grants local +read/glob/grep/edit/shell (prefer explicit). Roles: `plan` reads (web only when +external knowledge is needed); `implement` reads, edits and runs shell; `review` +reads and may use shell for `git diff`, never edits; `land` has read/glob/grep and +shell, no edit. Planning stages do not return `blocked` merely because edits come +later. Keep prompts language-neutral unless the task names a language. + +Optional `permissions` is a JSON object with `edit` (run-directory-relative +patterns) and `shell` (exact commands) arrays. It narrows declared tools; it never +enables a missing one. Empty arrays deny; wildcards, chaining, redirects and +traversal are rejected. With restricted edits and shell enabled, list shell too: +`{"edit":["README.md","plugin/*.qml"],"shell":["bash -n install.sh"]}`. + +## Checks + +Every model stage that can modify the workspace (`edit` or `shell`, no `tools` +block, or `delivery=true`) declares `checks`. A `delivery=true` stage lists 1–20 +commands, one per line. Every other stage writes exactly one line, +`none: `, e.g. `none: read-only review; shell is used only for git diff`. +A check must fail when it cannot run: validation rejects `|| true`, `|| :`, +`|| exit 0`, `|| echo`, negated file tests, `! command -v tool ||`, an `if` with no +failing else, and `set +e`. Cover every language the run can change. + +## Human stages + +`type=human` pauses the run until a person picks one of its non-`error` edges +(e.g. `approve -> act`, `revise -> plan`, `abort -> #failure`; see the +`human-decision` template). It needs only `prompt` (operator instructions), +`configs` with `type=human` and `edges`; no model or tools. Decide with +`pipelines decide RUN_ID [STAGE] --option KEY [--feedback TEXT]` or the apps. The +wait survives restarts and is never killed by a stage timeout or `max_visits`. + +## Land stage and delivery gate + +A workflow that commits its change ends in `land`: a model stage with +`read`/`glob`/`grep`/`shell`, no `edit`, that commits locally. Its prompt states: + +1. Commit only this run's changes (server list + git status). +2. Check the staged diff for unrelated work and secrets. +3. No empty commit, push, amend or rebase. Report the hash. + +With `delivery=true` and `checks` commands, after `land` succeeds the server: +requires the item and main checkouts to be clean, untracked files included; merges +a moved `main` into the item (a conflict aborts with the workspace unchanged); +runs each check with `sh -c` from the root of a fresh detached worktree of the +commit, so include dependency bootstrap (`npm ci && npm test`); caps each check's +output at 64 KB; and fast-forwards `main`. A failing check blocks landing. Do not +check ancestry before the gate runs. Without a gate, the manager merges and +verifies ancestry; no separate merger pipeline is needed (`references/queue.md`). diff --git a/internal/author/skill/references/harnesses.md b/internal/author/skill/references/harnesses.md index ae425712..f679e232 100644 --- a/internal/author/skill/references/harnesses.md +++ b/internal/author/skill/references/harnesses.md @@ -1,191 +1,45 @@ -# Configured harnesses +# Configured harnesses (operator only) -Current source supports OpenCode and Pi interactive authoring, debugging and -model stages. nvim remains a file editor/read-only debug packet viewer, not a -stage backend. Only configured harnesses appear in TUI pickers, default first. -CLI `new/edit/debug --harness opencode|pi|nvim` overrides the default. +Authoring needs only the `harness=` config (`references/format.md`). The rest is +for an operator setting up or debugging the execution computer. + +OpenCode and Pi run interactive authoring, debugging and model stages; Claude Code +and Codex are chat backends. nvim is a file editor/read-only debug packet viewer, +not a stage backend. Only configured harnesses appear in pickers, default first. +CLI `new/edit/debug --harness opencode|pi|nvim` overrides the default, and `pipelines harness` launches the configured default in the invocation directory. -CLI `new/edit/debug` and TUI author/debug build one shared, Docker-aware CLI -prefix. A Docker-linked profile embeds the in-container `/usr/local/bin/pipelines` -(never the host binary: Linux can see but cannot execute a mounted macOS host -executable). Loopback is recognized only via `net.ParseIP(...).IsLoopback()` plus -an exact case-insensitive `localhost`, so a remote DNS name such as -`127.remote.example` stays remote and runs no probe. For a loopback logical -`--url`, a single bounded `docker compose port` probe parses each returned line -as a complete `host:port` binding (IPv4/IPv6/wildcard) and the prefix maps only -when a binding covers the requested loopback address and port; malformed or -non-matching bindings fail closed. A loopback `wss` endpoint is rejected as -having no supported local TLS mapping rather than downgraded to plaintext. On a -match it embeds scoped `PIPELINES_TRANSPORT_SOURCE`/`PIPELINES_TRANSPORT_TARGET` -aliases in the prefix string (not `cmd.Env`, so the pre-existing harness -server's tools inherit them). The client rewrites an endpoint only on an exact -source match; remote endpoints stay remote. Draft metadata (server, base -version, create-only guard) is preserved unchanged. -`provider-check` with a Docker-linked profile runs inside that profile's existing -Compose service with `-T`, the original cwd/model/timeout and exact stdout/exit -propagation; container profiles have no `DockerDirectory` and cannot recurse, and -explicit native profiles keep the host path. The picker uses a true `x/term` TTY -check, so `/dev/null` and pipes fail fast instead of hanging. - `pipelines harness setup ` is the single setup path on native and Docker installations: it checks the CLI is reachable from the server (printing the install command when missing), signs in only when needed (`--login` -forces it) and verifies with the server's own check. Claude Code and Codex have -no models to pick. For OpenCode/Pi it runs a styled onboarding -wizard (bold title, verified Docker harness versions) and selects separate -harness and new-pipeline models using type-to-filter fuzzy pickers, not numbered -prompts. Each picker itself shows its numbered step with the title and purpose -(no duplicate numbered prose is printed outside it); the interactive harness -model is `Step 1 of 2` and the new-pipeline model is `Step 2 of 2`, matching -fresh Docker setup. Typing filters, arrow keys or Tab navigate, Enter confirms, -and Esc cancels. The saved value is preselected and marked `(current)`; the window -follows the active row on narrow terminals and the heading, selected row and -no-match state stay within the real terminal height. -Automation: `--model PROVIDER/MODEL --pipeline-model PROVIDER/MODEL`. -`harness default [opencode|pi]` switches defaults; `harness list` shows installed / -signed-in / configured / default status for all four (`--json` adds profiles). -Setup reads catalogs/login state without inference. Changing defaults never -rewrites published definitions or retained drafts. Canceling or failing either -picker leaves saved preferences byte-identical. Non-terminal automation with a -missing model fails clearly instead of starting a picker. +forces it) and verifies with the server's own check. For OpenCode/Pi it picks the +interactive harness model and the new-pipeline model; automation passes +`--model PROVIDER/MODEL --pipeline-model PROVIDER/MODEL`, and a non-terminal run +with a missing model fails instead of opening a picker. `harness default +[opencode|pi]` switches defaults; `harness list` shows installed / signed-in / +configured / default status (`--json` adds profiles). Setup reads catalogs and +login state without inference. Changing defaults never rewrites published +definitions or drafts; canceling a picker leaves saved preferences unchanged. +Harness setup/default commands configure the local execution user, not a remote +`--url` server. Pi setup runs no generation check; `provider-check --harness pi +--model PROVIDER/MODEL` is a separate, potentially chargeable check. -On an existing Docker setup, `pipelines harness setup opencode|pi` configures inside its container and link the host -launcher without restarting/deploying services. They resolve and verify the current -OpenCode V2 release from the single documented complete -`GET .../git/matching-refs/tags/v2` response (a bounded complete array with no -pagination parameters; never the moving `latest` tag or the registry's insertion -order) and the current Pi release from npm, resolve the selected tag's immutable -GHCR `sha256` digest, and require a rebuild only when the verified release image -identity no longer matches. A full install always invokes the Docker build for a -dirty source (relying on layer caching); a model/login-only reconfiguration does -not rebuild. Setup writes its generated release receipt to the machine-local root -file `.pipelines-releases.json` (with atomic `.pipelines-releases-*` temporary -files); both are git-ignored, so a fresh clean checkout stays clean and reinstall -still reuses the verified release/image identity. The actual built image's labels -stay authoritative and a dirty source still rebuilds. If discovery, digest -resolution or completeness cannot be established they stop with an error (keeping -the last verified Dockerfile fallback pins); they never guess, fall back to a host -runtime or install a stale pin. -(`./install --opencode`/`--pi` were removed and only point to that command.) The native installation reuses the host OpenCode -configuration, writes **host** (non-Docker) profiles for the harnesses it detects -into XDG config home's `pipelines/harnesses.json`, and clears a stale Docker link -so a Docker-era profile is not left Docker-linked. It never installs, upgrades or -downloads a harness. Container and host credentials and accounts remain independent and -are reused on reinstall/reconfigure. `./install` installs natively by default -(`--native` is equivalent); explicit `./install --docker` selects the Docker -server + ntfy installation. -Fresh setup accepts `./install --docker --harness pi` to skip OpenCode -login. Existing setup can configure either harness independently. Pi setup does -not run a generation check; `provider-check --harness pi --model PROVIDER/MODEL` -is a separate confirmed, potentially chargeable check. -Profiles live in XDG config home's `pipelines/harnesses.json`, store preferences -only, and preserve unrelated configuration. A profile launches a native host -harness unless it records a Docker checkout directory, in which case it launches -through that installation's Compose service. `harness setup --host` -configures or converts a host profile and clears any Docker link; without -`--host`, reconfiguring a Docker-linked profile re-verifies that the recorded -directory still has `compose.yaml` and fails naming it rather than saving a link -that cannot launch. Setup's summary and `harness list` report the resulting mode -(and, for Docker, the directory). Harness CLI setup/default commands -always configure the local execution user, not a remote `--url` server. -OpenCode interactive launch uses -`mini --model`: verified V2 full CLI lacks a model flag. Pi uses its CLI with the -packaged skill explicitly loaded. Pi has no standalone `auth login` command -(verified `pi auth --help`: only credential check/print): setup prints a -title-plus-three-lines Pi sign-in handoff and opens a fresh, no-context, -extension-free, no-prompt-templates, no-tools session with Pi's documented -`--tui-mode regular` flag, so the header and handoff stay visible above Pi's -interface at 80x30 instead of the default fullscreen clear. The user types -`/login`, chooses an authentication method ("Sign in with an API key" for -provider API credentials; account and Radius remain the user's choice), finishes -provider sign-in, then exits with `ctrl+c`/`ctrl+d` to return to Pipelines model -configuration. Existing sign-ins are reused by exiting when no new sign-in is -needed; completing a sign-in can update the credentials stored in the Docker -volume. No coding/task prompt or scripted keystrokes are injected. -OpenCode V2 model stages settle long sessions by polling `GET /api/session/active` -about once per second under the stage context, instead of the `.../wait` request -whose CLI-side request timeout failed long stages after roughly six minutes -(HTTP 499, dated 2026-10-10). V1 native settlement is unchanged. Final results -and debug session evidence use the same single bounded latest-assistant readback -(`order=desc&limit=1&type=assistant`), so a small final result is visible behind -large prior tool content; retained transcripts over 32 KiB are stored as valid -bounded preview envelopes that preserve stage limits and diagnostics, and exact -outputs remain authoritative. Pi evidence is JSONL and is not parsed as one JSON -object. Short-lived context-controlled CLI calls (`--help` detection, API calls -and the native export fallback) share one finite inherited-pipe drain grace -(`os/exec.Cmd.WaitDelay` ~250ms) so a canceled context or a descendant that still -holds stdout/stderr cannot block for the child's natural lifetime; the -`context.Canceled`/`DeadlineExceeded` and `exec.ErrWaitDelay` identities are -preserved and partial stdout after the grace is never accepted as success. +Docker image resolution, profile storage, the Docker-aware CLI prefix, Pi sign-in +and OpenCode V2 settlement internals are in `docs/operator-reference.md` in the +source checkout; dated verification is in `docs/verification-history.md`. -Stages set `harness=pi` or `harness=opencode` in configs; omission preserves -historical OpenCode behavior. Pi must be configured on the execution server. A run may override the pinned model at launch: `pipelines run --model PROVIDER/MODEL` covers every model stage and repeatable `--stage-model -STAGE=PROVIDER/MODEL` overrides one stage, with precedence **per-stage > -whole-run > the model pinned in the pipeline version**. Overrides are validated -against the pinned version before the run starts (an unknown stage key fails -naming the valid keys; a non-model stage cannot be overridden), apply to that run -only, and are recorded on the run so `retry` reproduces them. The effective -provider/model is stamped on the step and exposed by the run's graph, and the -stage's repair/formatter path uses the same resolved model because both read the -stage's effective configs. -The RPC adapter starts new retained sessions per visit with explicit tools and -a Pipelines permission extension. Ambient extensions, skills, context files, -prompt templates, automatic retries/compaction are disabled. Tool gates are not -an OS sandbox: approved shell can have broad side effects. Usage accumulates -across attempts/visits; missing required accounting fails closed. Strict envelope -validation and bounded syntax-only repair never replay tools. A model-driven -repair, when needed, uses a separate deny-all session with retained ownership -and the same cumulative budget. Its output is checked locally for value changes. +STAGE=PROVIDER/MODEL` overrides one stage (per-stage > whole-run > pinned). +Overrides are validated against the pinned version before the run starts (an +unknown stage key fails naming the valid keys; a non-model stage cannot be +overridden), apply to that run only, and are recorded so `retry` reproduces them. +The effective provider/model is stamped on the step. -Session/RPC audit lives in the execution user's cache under `pipelines/pi-sessions`. -Ownership persists before prompts. Cleanup waits for settlement; expired leases -request abort through the extension. Unconfirmed cleanup blocks continuation/retry. - -2026-10-10 verification: `scripts/check-pi-rpc.py` checked installed Pi 0.87.1 -readiness, metadata/config RPC and **idle** lease cleanup without inference. -Go RPC fixtures cover result/budget/missing-accounting/timeout/format-repair paths; -Node tests cover permission checks. `PIPELINES_PI_LOCAL=1 go test ./internal/stage --run TestPiInstalledLocalIntegration` also exercised the **installed Pi** against -scripted loopback responses: actual allowed/denied writes, budget exhaustion, -and active-shell cancellation (including no delayed write after cleanup). -The installed-Pi loopback suite also covers isolated deny-all syntax repair with -retained ownership and cumulative usage. The compiled fuzzy picker passed PTY -selection/cancellation; installed Pi catalog discovery/configuration passed with -an unreachable local endpoint and no prompt. -This is integration evidence, not real-provider/model evidence. Active lease-expiry -recovery, web search delivery and fresh interactive Docker Pi setup remain -**not tested**. Pi catalogs exclude ambient provider extensions for parity with -audited stages; compatible custom endpoints can use Pi's private `models.json`. -An isolated built Docker image also passed non-root, network-disabled policy/RPC -readiness and idle lease cleanup, without credentials/mounts or inference. -Codex/Claude -adapters and AI-assisted server guidelines validation remain future work. - -2026-10-11 source-level coverage (source-level regressions; the separate dated -2026-10-11 final install is live evidence): the -shared Docker-aware CLI prefix, exact-match client transport rewriting, -Docker-linked `provider-check` dispatch and the `x/term` TTY check have -source-level regression tests (`internal/harness`, `internal/client`, -`internal/cli`). The prefix tests cover the remote-DNS negative -(`127.remote.example` stays unmapped with no probe), IPv4/IPv6/`localhost` -positives, complete `host:port` binding matching (a different service on the same -port, or a family mismatch, does not match), malformed/empty output failing -clearly, multiple bindings under a single probe, and loopback `wss` rejection. -Terminal Pick regression tests use a short deadline so a TTY regression fails -fast rather than hanging. Docker dispatch tests assert `--json` is passed only in -JSON mode while plain text stays readable. Python fixtures cover arbitrary-digit -OpenCode patches (a parsed `2.0.260` is still rejected against an expected -`2.0.26` by exact equality; 260/100 are illustrative valid/future examples, not -verified releases) and reject both raw and canonical reserved homes. Real -in-container `provider-check`/ping confirmation, a compiled PTY run and a real -Docker reconfigure remain **not tested** in this change. These source-level -regressions are not a claim about the deployed image; the separate dated 2026-10-11 -FINAL INSTALLED verification recorded that the runtime was actually installed and -used (OpenCode 2.0.26 / Pi 1.1.0 / Node 24.18.1, image -`sha256:1810ab50f3ea3991e7f3548c79250e65a114bb1821eac52efd872b91cf7c3136`, with -both real stages executed in run17 and passed in Root's full host race/vet runs). -Active lease-expiry recovery, web search delivery and fresh interactive Docker Pi -sign-in remain **not tested**. +Pi stages start a new retained session per visit with explicit tools and a +Pipelines permission extension; ambient extensions, skills, context files, prompt +templates, automatic retries and compaction are disabled. Tool gates are not an +OS sandbox: approved shell can have broad side effects. Usage accumulates across +attempts and visits; missing required accounting fails closed. Session/RPC audit +lives in the execution user's cache under `pipelines/pi-sessions`; unconfirmed +cleanup blocks continuation and retry. diff --git a/internal/author/skill/references/operations.md b/internal/author/skill/references/operations.md index 01d0614a..8e66ffeb 100644 --- a/internal/author/skill/references/operations.md +++ b/internal/author/skill/references/operations.md @@ -38,23 +38,6 @@ parallel/distributed task engine or a guarantee that model reports are correct. - Services are native or Docker-based. With an existing Docker installation do not start a competing native server. Server APIs are unauthenticated: trusted networks only; never recommend public exposure. Tool scope is not a universal OS sandbox. -- Docker setup resolves the current supported OpenCode V2 release from the single - documented complete `GET .../git/matching-refs/tags/v2` response (a bounded - complete array with no pagination parameters; never the moving `latest` tag or - the registry's insertion order) and the current Pi release from npm, resolves the - selected tag's immutable GHCR `sha256` digest, builds `FROM` that digest, and - verifies the built image reports exactly those executable versions plus a - Node-compatible Pi engine. - It fails closed, keeping the last verified Dockerfile fallback pins, when discovery, - digest resolution, completeness, build or compatibility cannot be established: never - a hardcoded fallback, never a silent stale-pin substitution. A full install skips a - rebuild only when the release receipt **and** the actual built image's release and - source identity labels already match; a dirty source always invokes the Docker - build and relies on layer caching. A model/login-only reconfiguration compares only - the verified release image identity, so a changed source alone does not rebuild. - `pipelines harness setup opencode|pi` reconfigures inside the existing container and - refuses an image that does not match; it does not rebuild or restart services (an - explicit ordinary `./install` does when the user asks). ## Where data lives @@ -72,7 +55,6 @@ delete databases, sessions, volumes or credentials to repair a workflow. Use the authoritative `CLI_PREFIX` from the prompt: -- `CLI_PREFIX runs --json`: run history (optional `--pipeline NAME` scope). - `CLI_PREFIX search QUERY [--type KIND] [--limit N] --json`: one read-only query across pipelines, versions, runs, steps, field text and artifacts. QUERY is free text, a bare status word (`queued`/`running`/`succeeded`/`failed`/`canceled`/ @@ -112,17 +94,7 @@ Use the authoritative `CLI_PREFIX` from the prompt: limits, failure/format diagnostics and the retained session pointer), not the complete tool history. - `CLI_PREFIX decide RUN_ID [STAGE] --option KEY [--feedback TEXT] [--json]`: answer - a run that is `waiting` on a `type=human` stage. The status and the stage's - instructions/options are visible in `runs` and `inspect --json` (and the TUI - picker); the stage is inferred from the run's current stage unless named. The - choice and feedback are recorded as the step's evidence (its output and derived - verdict; the feedback is never an error diagnostic) and the run resumes along - `edges[KEY]`. An unknown option is - refused with the valid list; a stage that is not waiting is refused; a repeated - decision for an already-decided visit does not double-apply. The wait is persisted, - so `decide` still works after a server restart. `cancel` and `retry` keep working - while waiting and after a decision; a human stage is never killed by a stage - timeout or `max_visits` (the shared 30-step run budget still applies). + a run `waiting` on a `type=human` stage (`references/format.md`). - `CLI_PREFIX run ... --json --wait --wait-timeout DURATION`: `--wait` polls until terminal and `--wait-timeout` bounds only the client wait after admission, independent of per-request `--timeout`. A reached wait limit leaves the admitted run @@ -160,22 +132,6 @@ runs natively through `./install` by default (`scripts/setup-native.py`, with `--native` equivalent); the Docker server + ntfy installation is explicit with `./install --docker`. -In the TUI, Home advertises `/` omni search and `:` commands. Home `/` opens one -popup over pipelines, versions, runs, steps/visits, artifact names and stage -input/output/evidence text; each row shows a short type badge and a text match -shows a bracketed snippet, and a bare status word or date selects runs. The same -query is available as `pipelines search QUERY [--type KIND]`, where `--type` -narrows the kind. Run Tab switches only -Details ↔ Graph. `n/p` select recorded visits, including repeated stages; `d` -focuses/toggles file changes, `A` opens the run's grouped artifacts view (name, -content type, size and URL per entry, `PgUp`/`PgDn` scroll so a trailing visit -that recorded none stays reachable, `Enter`/`o` to open the selected URL and -`y` to copy it), `a` selects the active/latest retained -configured-harness agent session's bounded, redacted tool evidence, `D`/`:debug` -opens fresh investigation, and `P` opens the pinned pipeline version. `:quit` -exits; `q` does not. Results are retained reports, not independently confirmed -achievements. Distinguish partial output from full success. - ## Retained stage file changes Each newly executed visit compares a bounded, read-only workspace snapshot just @@ -228,18 +184,11 @@ file diffs: current capture is server-owned. Do not promise full files/rollback. ## Retained stage artifacts -Every model visit gets a file-based turn under the data directory -(`/artifacts//stages/-/`) with `input.md`, `memory.md` -(capped essential memory, inlined), `history.md` (full earlier turns, read on -demand), `output.md` and `private/`; `public/` is run-level and shared by every stage. -The root is configurable and defaults next to the database. The writable -`private/` and shared `public/` directories are stated in the agent prompt for -every harness, and `PIPELINES_ARTIFACTS_DIR` points at the turn directory for a -locally spawned Pi process. OpenCode V2 tools run inside the shared serve -process, so there the prompt, not the environment, is the reliable carrier. -Turn-local files belong in `private/`; reports, logs and screenshots for later -stages belong in `public/`. The server owns `input.md`, `memory.md`, -`history.md` and `output.md` and never asks the model to write them directly. +Turn files (`input.md`, `memory.md`, `history.md`, `output.md`, `private/`) and +the run-level `public/` are described in `references/format.md`. They live under +`/artifacts//stages/-/`; the root is configurable and +defaults next to the database. The server owns `input.md`, `memory.md`, +`history.md` and `output.md`. After the visit the server makes one bounded inventory of the turn and public directories and records each file's name, size, content type (extension, sniff @@ -247,9 +196,8 @@ fallback) and a relative url; these appear under each step's `artifacts` in `inspect RUN_ID --json` and the TUI's `A` artifacts view, which groups them by stage and visit, marks images and shows the recorded count, capture-pending/not-recorded wording and partial-capture notes. Public entries -carry a `public/` prefix. `input.md`, `memory.md`, `history.md` and `output.md` -are the primary artifacts of every model visit. To see what a late stage was -given, read its `memory.md`; `history.md` is the full record it could consult. +carry a `public/` prefix. To see what a late stage was given, read its +`memory.md`; `history.md` is the full record it could consult. `GET /artifacts///` serves the bytes (correct Content-Type and Content-Length, HEAD, and Range/206/416 via `http.ServeContent`), refusing traversal, absolute paths, symlink escapes and unknown runs/steps. A pre-turn @@ -302,158 +250,6 @@ and one WAL checkpoint, reporting measured bytes. Pipeline definitions and their immutable versions are never touched. The commands are local, require an existing `--db`, and need no running server. -## Brokered OpenCode chat - -The protocol declares the brokered `chat_*` surface (`chat_sessions`, -`chat_create`, `chat_directories`, `chat_prompt`, `chat_messages`, -`chat_interrupt`, `chat_permission`, `chat_reply`). The server dispatches all of -them, and pushes `chat_event` frames derived from OpenCode's global SSE firehose -(`/event` on the native serve, `/api/event` on the V2 API), with retained -per-session/per-message state so a reconnecting client re-syncs instead of -replaying deltas. - -At startup `cmd/pipelines-server` calls `opencode.OpenChat`, which detects the -installed backend and opens exactly one owned `opencode serve` endpoint for it: -the native wire for v1, the V2 API for the installed V2 CLI. Chat is enabled only -when that connection carries a URL-bearing, SSE-capable event endpoint; otherwise -chat stays disabled and every `chat_*` request is answered with a protocol error -naming the missing configuration rather than hanging. Stage generation still -reaches V2 through the `opencode api` CLI, so the owned serve process is chat's -single transport and starts no process for stages. The adapter skips firehose -events it does not surface before decoding their data, so an unrelated event -whose `delta` has a different shape (`session.instructions.updated` carries an -object) never fails the stream. - -The operator can instead broker chat from their own OpenCode service with the -opt-in `-opencode-endpoint` flag or `PIPELINES_OPENCODE_ENDPOINT`, with precedence -flag > environment > the endpoint recorded on a previous run and the spawned -instance as the default. `opencode.OpenEndpoint` validates the reference once -(the established http(s)/host/no-credentials/no-query/no-path idiom), detects the -backend from the service's own `/api/info` (V2) or `/global/health` (native) -contract and requires a streamable `text/event-stream` event path; it returns a -connection with **no owned process** and never calls `startServe`. A malformed -reference, an unreachable host, a reachable non-OpenCode service or a -health-ok-but-no-SSE service each disable chat with an error naming the endpoint -and never fall back to the spawned instance. An authenticating endpoint accepts an -optional password only through the private file -`PIPELINES_OPENCODE_ENDPOINT_PASSWORD_FILE` (regular, 0600, single-owner), and -`PIPELINES_OPENCODE_ENDPOINT_USERNAME` overrides the detected default user. The -validated choice is recorded in the additive `server_settings` table so a restart -with no flag/environment reuses it; an unreachable recorded endpoint disables chat -with a clear message and no fallback. `pipelines service enable` preserves the -endpoint and the password-file reference in autostart. - -`chat_sessions` lists the operator's own chat sessions, not the sessions pipeline -runs create. Operator sessions are recorded in the dedicated `operator_sessions` -table (`session_id` primary key, plus `directory`, `title`, `tag`, `created_at`, -`last_used_at`), separate from `run_steps`/`run_session_owners`, so an operator -chat is a first-class thing and a pipeline stage session — recorded as run -ownership, never in the operator registry — is never listed. The handler reads the -registry once (bounded to 128 rows, most-recently-used first), calls OpenCode's -session list once, and keeps only registry rows the endpoint still reports, so a -session with no recorded tag is never listed and a deleted upstream session does -not linger. It fills each session's `directory`, `tag` and `created_at`/ -`updated_at` from the registry and marks `busy` when another client currently -holds the session (an in-server claim, released when the stream ends or the client -disconnects); a second client's `chat_prompt` for a held session is rejected -before any OpenCode call. It is fail-closed: with no registry configured it -returns an empty list, never the endpoint's whole session list. The OpenCode -session list is read unscoped by directory (the V2 `GET /api/session` otherwise -defaults to the caller's directory), because an operator session may live in any -directory. - -`chat_create` creates one operator session and records it in the same registry -with the stored tag `operator` (the display label `Operator` is derived from it, -so there is no second tag vocabulary). It takes a `title` (default "New chat") and -a `directory` (default `~`): an omitted or empty value, `~`, and any `~/...` value -all resolve to `os.UserHomeDir`, never the server's working directory; a relative -path or a missing directory is rejected before any OpenCode call, and a session -created but not recordable is reported as an error naming its id. Create posts the -verified V2 shape `POST /api/session` `{"title":...,"location":{"directory":...}}`. -`chat_directories` returns the operator's home first, then the directories of -previous operator sessions most-recently-used first, deduplicated and bounded (8); -it is a dedicated request so the create dialog can load the choices without first -listing sessions and `chat_sessions` stays a pure, fail-closed session list. A -successful `chat_prompt` touches the session's `last_used_at` in one UPDATE, so it -moves to the front of the list; a non-operator session id touches no row. - -`chat_adopt` records a session that already exists on the configured endpoint in -the same operator registry, using the endpoint's own directory and title -verbatim (never a guess, the server's working directory or the home directory). -It reads the endpoint once through the shared `Chat.Session` (which reuses -`Sessions`/`decodeChatSessions`), records through the existing -`RecordOperatorSession` with the `operator` tag, and echoes the adopted session -plus its registry row like `chat_create`, so `chat_sessions` lists it and -`chat_prompt` continues it. A nil payload, a blank or foreign-shaped id, a nil -broker or a nil registry is refused before any endpoint call; an id the endpoint -does not report is refused naming the id and records nothing; an endpoint record -with an empty directory is refused rather than substituting a directory. The CLI -equivalent is `pipelines chat-adopt ` (`--json` prints the full -response, otherwise a short `Adopted · ` line). - -While a `chat_prompt` subscription is open the server also polls -`GET /api/permission/request` on a fixed small interval (first poll immediately) and -pushes one `chat_permission` frame per newly pending approval request, deduped by -request id so a repeated poll never re-pushes the same request. The app answers with -`chat_reply`, which the server forwards as -`POST /api/session/{id}/permission/{requestID}/reply` with -`{"decision":"once"|"always"|"reject"}`. A missing session/request id or an invalid -decision is rejected before any OpenCode call, and the poller stops when the stream -ends or the client disconnects. - -The permission list and reply are implemented against the V2 HTTP API only: their -field names match the service's `Permission.Request`/`Permission.Reply` schemas -(verified against the installed service's `/openapi.json`). The native v1 serve -translation rejects them explicitly instead of guessing an endpoint, so on v1 the -permission poller stops quietly rather than reporting the documented limitation as -a per-prompt error. - -### Verification evidence (2026-10-17) - -Verified against the installed OpenCode 2.0.20 on 2026-10-17: - -- `opencode --version` reports `2.0.20`; `opencode --help` advertises `api` and - `serve`. `GET /openapi.json` on an owned `opencode serve` lists `GET /api/event` - (SSE) and the session/prompt/interrupt/permission endpoints. -- A live turn streamed token-level `session.text.delta` and - `session.reasoning.delta` events (each with `assistantMessageID` and `ordinal`), - plus `session.step.*` and `session.usage.updated`; the V2 API server - authenticates the fixed `opencode` basic-auth user. -- The opt-in live end-to-end test (`PIPELINES_TEST_OPENCODE_CHAT_LIVE=1 go test - ./internal/server -run TestChatLiveEndToEnd -v`) opened an owned V2 endpoint, - created an operator session through `chat_create` in a chosen directory, listed - exactly that session through `chat_sessions` (with its directory and tag - `operator`), recorded a genuine pipeline stage session in - `run_steps`/`run_session_owners` and showed it never appeared in the list, and - streamed a completed turn as `[step:started,text,step:streamed,step:ended]` - (first delta `"pong"`). The operator-session registry, the filter, directory - defaulting and the recents bound are also covered by the `pipeline`, `opencode` - and `server` unit tests, which run without the live gate. -- Not verified live: a genuine pending-approval round trip and any backend other - than the installed 2.0.20. The permission wire shapes are unit-tested over an - httptest fixture, not against a live pending request. -- Operator-supplied endpoint selection and `chat_adopt` (added 2026-10-17): - `opencode.OpenEndpoint` and the flag/env/recorded precedence, the URL-shape - validation failures (scheme, empty host, userinfo, query, fragment, non-empty - path), the unreachable/non-OpenCode/no-SSE/auth-mismatch failures, the - no-owned-process property (`conn.process == nil`) and the password-file rule - are unit-tested over httptest fixtures in `internal/opencode/endpoint_test.go`. - `chat_adopt` (endpoint directory/title recorded with the `operator` tag, echo, - missing id refused naming the id and recording nothing, nil payload/blank id/nil - broker/nil registry refused before any endpoint call, empty endpoint directory - refused) is unit-tested over the server fake broker in - `internal/server/chat_test.go`; the recorded-endpoint round-trip across a store - Close/Open is in `internal/pipeline/settings_test.go`; the protocol round-trip - and the CLI dispatch (`pipelines chat-adopt`) are covered in - `internal/protocol/chat_test.go` and `internal/cli/chat_adopt_test.go`. The - live scratch-server check against the operator's real endpoint (adopt, list, - prompt) was **not** run in this change; only the unit fixtures above were - executed. - -Deployment state: not deployed. This is reviewed and committed locally only; -deploying it (and restarting the server) is the operator's step. Do not describe -brokered chat as already deployed. - ## Debug and retry Debug opens a new conversation with retained failure context, exact definition and @@ -471,18 +267,11 @@ side effects remain and can happen again. Never describe retry or cancellation a rollback, or resume/replay admitted tool work just to recover formatting. Retained model-stage failures carry a fixed sanitized category. `FINAL RESULT -VALIDATION` names the exact duplicate-field/syntax detail and byte offset for a -strictly invalid final result; `HARNESS TRANSPORT` covers request, active-session -poll and final-readback failures. A validation failure never re-prompts the -admitted tool-using stage: completed edits and tool side effects are retained and -not undone, so a full retry repeats work. An explicit read-only stage is the one -exception: a malformed result there mutates nothing, so it may be re-emitted up -to `retries` with its `format_repairs` evidence retained, and a post-admission -deadline or transport failure may be retried. A failure to reach the harness -before any prompt or tool call is admitted (connecting, or creating the session) -replays nothing, so it is retried within `retries` for any stage, mutating or not. -A stage whose tools include -`edit`/`shell`, or that has no tools block (the legacy read/glob/grep/edit/shell -bundle), is never replayed. Neither category exposes provider/model bodies or -credentials. The isolated formatter's deadline is proportional to the stage -timeout and clamped so it never extends the stage's own deadline. +VALIDATION` means `output.md` had a missing, duplicate, empty or undeclared +outcome (`references/format.md`); `HARNESS TRANSPORT` covers request, session-poll +and readback failures. Completed edits and tool side effects are retained, not +undone, so a full retry repeats work. A failure before any prompt is admitted is +retried within `retries` for any stage; a post-admission deadline or transport +failure is retried only on an explicit read-only stage; a stage with `edit`/`shell` +(or no tools block) is never replayed. Neither category exposes provider/model +bodies or credentials. diff --git a/internal/author/skill/references/orchestration.md b/internal/author/skill/references/orchestration.md index 26a9c01d..d96e386e 100644 --- a/internal/author/skill/references/orchestration.md +++ b/internal/author/skill/references/orchestration.md @@ -13,11 +13,13 @@ at integration. Serialize only actual dependency or resource conflicts. ## Stage jobs -- Plan: a short approach and acceptance checks when they help. -- Code: the focused change, tests and focused checks. -- Review: the whole task diff and independent verification. -- UIReview: rendered behavior with controlled data and captures. -- Land: commit the reviewed work, integrate main and resolve conflicts. +- `plan`: a short approach and acceptance checks when they help. +- `implement`: the focused change, tests and focused checks. +- `review`: the changed hunks (server-captured changed list, `git diff --stat`, + then per-file diffs) and the reported checks; it runs no suites. +- `ui-review`: rendered behavior with controlled data and captures. +- `decide`: a human choice between edges. +- `land`: commit the reviewed work; the delivery gate checks and lands it. Names are a convention, not a requirement; a pipeline can use any clear stage names. Keep each stage's job narrow enough to judge its output. diff --git a/internal/author/skill/references/practices.md b/internal/author/skill/references/practices.md index e9df5915..7db165ba 100644 --- a/internal/author/skill/references/practices.md +++ b/internal/author/skill/references/practices.md @@ -8,22 +8,23 @@ below (errors block publishing; warnings print and do not). ## Where work belongs - **Deterministic checks belong in the server `checks` block, not the model.** - The landing stage sets `delivery=true` and lists the full check commands; the - server runs them on the committed candidate. A model running the suite puts - every log line into its context, and every later call pays for it again. -- **Code runs focused tests only**: the test file or package it touched, with - quiet or summary output. Never the full suite. -- **Review reads the diff, not the repository.** Start from the server-captured - changed-file list the runner adds to the prompt and `git diff --stat`, then read - only the changed hunks and the code around them. Review runs no suites; it judges - the reported checks and names missing coverage. -- **Land/committer relies on server checks.** With a delivery gate its prompt - commits and stops; it does not re-run what the gate runs. Do not claim "the - server lands it" unless a stage has `delivery=true` (error otherwise). + The `land` stage sets `delivery=true` and lists the full check commands; the + server runs them on the committed candidate (`references/format.md`, Delivery + gate). A model running the suite puts every log line into its context, and + every later call pays for it again. +- **`implement` runs focused tests only**: the test file or package it touched, + with quiet or summary output. Never the full suite. +- **`review` reads the changed hunks, not the repository.** Start from the + server-captured changed-file list the runner adds to the prompt and + `git diff --stat`, then read per-file diffs and only the code around them. + Review runs no suites; it judges the reported checks and names missing coverage. +- **`land` relies on server checks.** With a delivery gate its prompt commits and + stops; it does not re-run what the gate runs. Do not claim "the server lands + it" unless a stage has `delivery=true` (error otherwise). - Every stage that can modify the workspace declares `checks`, even if only - `none: ` (`references/format.md`, Checks). Checks must fail when a tool - is missing: no `|| true`, `[ ! -x tool ] || ...` or similar guards (error). - Cover every language the run can change. + `none: `. Checks must fail when a tool is missing: no `|| true`, + `[ ! -x tool ] || ...` or similar guards (error). Cover every language the run + can change. ## Prompts @@ -31,34 +32,41 @@ below (errors block publishing; warnings print and do not). warns. Shared procedure belongs in project docs or this skill. - **No runner boilerplate.** The runner already explains `output.md`, the `# outcome` and `# handoff` sections, allowed outcomes, turn files, tools and - honesty rules (`internal/stage/model_prompt.go`). Repeating them warns. + honesty rules. Repeating them warns. - **Every outcome a prompt names is a declared edge.** "Return skipped" with no `skipped` edge, or an undeclared `blocked_retryable`, fails the stage at run time; validation rejects it (unquoted outcome words such as `skipped`, `blocked`, `rejected`, `approved`, `advisory`; other back-quoted words after return/report warn). +- **Missing tools or evidence**: declare `blocked` where a stage can lack required + input; the runner tells the model to choose `blocked` when declared, else + `error`. ## Memory and handoffs -- The runner inlines the **essential memory** (`memory.md`: task, previous - stage's handoff, outcome ledger, changed files; capped) into every prompt, and - asks each stage to start its output with a short `# handoff`: what it did, - outcome, files changed, open findings, in a few lines. Do not repeat that - instruction; say only what this stage's handoff must carry if it is special. -- The **full history** (`history.md`) and full `input.md` are on demand. Never - tell a stage to read the whole history every visit (warning); say when it +- The runner inlines the essential `memory.md` into every prompt and asks each + stage for a short `# handoff` after its `# outcome` (`references/format.md`). + Do not repeat that instruction; say only what this stage's handoff must carry + if it is special. +- The full `history.md` and `input.md` are on demand for stages with `read`. + Never tell a stage to read the whole history every visit (warning); say when it needs it, e.g. "read history.md only if the handoff lacks a detail you need". +- A stage without `read` gets `input.md` inlined (up to 32 KB) on every call. Give + `read` to any stage that follows a large output so it reads detail only when + needed. ## Loops and budgets -- **Loops get limits.** An edge back to the same or an earlier stage needs - `limits` on the source outcome or the target (error). Use `max_visits` 2–3. +- **Loops get limits.** Put `{"max_visits":3,"on":["rejected"]}` on the reviewer; + leave the loop target unlimited or capped at least as high + (`references/control.md`). - **Re-entry gets only findings**: a rejected review hands back must-fix items - with location and fix, not a full re-review, and the implementer fixes those - items rather than redoing the task. Use `advisory` for non-blocking notes so - they do not buy another loop. -- Set **per-stage `budget_tokens`** (and `budget_usd` if wanted) as a runaway - guard on stages that loop or explore, sized a few times a normal visit. + with location and fix, and the implementer fixes those items rather than + redoing the task. Use `advisory` for non-blocking notes so they do not buy + another loop. +- **`budget_tokens` only when the user asks or a measured visit exists** + (`pipelines inspect RUN_ID --json`, `usage.tokens_total`). Size it 3–5× that + visit, remembering totals include cache reads, so a normal visit never trips it. ## Commands and scope diff --git a/internal/author/skill/references/verification.md b/internal/author/skill/references/verification.md index 6197cd9f..aef6ebed 100644 --- a/internal/author/skill/references/verification.md +++ b/internal/author/skill/references/verification.md @@ -1,609 +1,59 @@ -# Verification, accounting and real-provider evidence - -Historical note: model stages now use file-based turns and parse only the -`# outcome` section. The JSON/line-format envelope and format-repair ladder -described in the dated sections below are retired; see `docs/model-results.md` -for the current contract. The dated evidence remains as a record. +# Verification and evidence Structural validation is not inference, tool execution, correct task completion, -phone delivery or correct billing. Distinguish observed artifacts/command results -from model-written reports. State the command/check actually run and say **not -tested** for anything not executed. Never present mocked results as live-provider -evidence or a different backend/model as already validated. +phone delivery or correct billing. Distinguish observed artifacts and command +results from model-written reports. State the command or check actually run and +say **not tested** for anything not executed. Never present mocked results as +live-provider evidence or a different backend/model as already validated. Dated +verification logs live in `docs/verification-history.md` in the source checkout. -## Authoring useful verification +## Where verification happens in a pipeline -Preserve the facts later stages need in free-form output headings; only the -`# outcome` section is interpreted. Implementer/reviewer stages -can use appropriate permitted commands; do not accept a preceding stage's success -summary as independent proof. Actual file bytes, command exit status, retained tool -parts and server-captured diffs are stronger evidence than prose. A read failure -can prove a deliberately missing input, but is not a successful read. Tool names -mentioned in text do not prove tools executed. A reviewer stage must inspect the -actual files and run its own checks; a source-only review runs no commands and -must state that runtime behavior was not tested. Structural, mocked or -authoring-time checks are not deployed or live-provider evidence. +- The `delivery=true` stage's `checks` block is the run's deterministic suite; + the server runs it (`references/format.md`, Checks). +- The implementer runs focused checks for what it touched and reports the exact + commands and results. +- The reviewer reads the changed hunks (the server-captured changed-file list, + then `git diff --stat`, then per-file diffs), judges the reported checks and + names missing coverage. It runs no suites. Its verdict concerns the current + change only. +- Actual file bytes, command exit status, retained tool parts and server-captured + diffs are stronger evidence than prose. Tool names mentioned in text do not prove + tools executed. Do not accept a preceding stage's summary as proof of runtime + behavior. -Do not add technology-specific checks unless the task/repository requires them. -Keep runtime verification separate from authoring. Execute relevant code and fixture -checks within the authorized task; ask before unrelated chargeable calls or notifications. -Do not weaken an acceptance check just to get a passing run. -`provider-check` must run on the intended execution computer/container with its -credentials; a client-side check is not proof of a remote server's provider health. +Do not add technology-specific checks unless the task or repository requires +them, and do not weaken an acceptance check just to get a passing run. Ask before +unrelated chargeable calls or notifications. `provider-check` must run on the +intended execution computer/container with its credentials; a client-side check +is not proof of a remote server's provider health. -## Budgets and backend caveats +## Usage evidence -Defaults remain two retries and one hour per stage attempt; a run-level wall-clock -deadline is not added. Budgets are optional, positive and independent USD/token -reported-usage limits shared across attempts and repeat visits. -Accounting failures stop budget-limited stages. In-flight calls can overshoot; -these are **not provider-side hard caps or a guaranteed bill ceiling**. Reported -USD may validly be zero; do not fabricate prices/accounting from output length. - -Per-run usage is recorded as typed step evidence from the usage the stage already -measured for its budget (no second harness read and no transcript parse), and -totalled for the run and over all history. A stage revisited within a run shares -one budget, so each step records only that visit's increment of it and the run -total is not the cumulative stage budget counted again; a revisit that reports -nothing new (a failure or timeout before any usage poll, or a Pi visit that never -reaches a usage stat) records no usage for that visit rather than a known zero. -`CLI_PREFIX inspect RUN_ID --json` -exposes each model step's `tokens_total`, `tokens_in`/`tokens_out` (only when -`split_known`), `cost_usd` (only when `cost_known`) and the model it ran with, -plus the totalled `usage`; `CLI_PREFIX stats --json` adds the history-wide totals, -which equal the sum of the per-run totals. A stage that reported no cost exposes -its tokens and marks the cost unavailable rather than showing zero; a harness that -reports only a total (Pi) exposes the total and marks the in/out split -unavailable rather than presenting the total as input tokens. An upgraded or -historical database gains an empty usage table with no backfill, so old runs omit -`usage` instead of showing zero. This is current source, not deployment evidence. - -Dated current-source evidence (2026-10-17, this machine). A throwaway read-only -reader (`internal` store API) over a scratch copy of -`~/.local/share/pipelines/data/pipelines.db`, taken with -`sqlite3 "file:$HOME/.local/share/pipelines/data/pipelines.db?mode=ro" ".backup "` -so the live DB is never mutated, opened 228 runs and found **no** `step_usage` -rows: every stored run predates this change, so per-run figures are unavailable -rather than zero. Run 314 (this item's own run) and run 239 (whose implementer -step ran `opencode-go/kimi-k2.7-code`, a stage model different from the pinned -`opencode-go/deepseek-v4.1-flash`) both report `usage=(none recorded)`, and -`stats` omits the history-wide totals. The recorded figures themselves are -covered by the passing current-source round trips — `internal/pipeline/usage_test.go`, -`internal/server/usage_test.go` (including `inspect --json` over a websocket) and -the pre-upgrade/read-bounded/stats-sum cases — not by a live run, because the -deployed server predates this change and was not restarted. No live model call, -deploy or rebuild produced this evidence. - -A live V2 trial found `/generate` returned text while its retained session had no -messages and zero counters. Current source uses tracked `/prompt` → bounded -active-session polling → message readback for **budgeted V2 text-only calls, with -all tools denied**. Completed token-budgeted responses reporting zero tokens fail -closed. No new definition flags or agent tools are required for this fix. -Unbudgeted tool-free V2 generation can still have empty backend message history; -pipeline inputs, results and command audit remain retained. An unbudgeted V2 -audited agent stage makes exactly one extra bounded final session-aggregate read -to record any reported USD/token usage best-effort: missing or invalid usage stays -unknown and never turns a valid run into a failure, cumulative sessions/retries -are keyed per session so they cannot double count, and totals are never inferred -from final-assistant-only or native-partial data. Budgeted settlement and native -V1 enforcement are unchanged. Native V1 uses its existing path. - -The V2 `.../experimental/session/{id}/wait` request is **no longer used**: a dated -Docker log shows it timing out at the CLI's own request timeout after about six -minutes (run started 2026-10-10T23:07:16.547Z, `TimeoutError` logged at -2026-10-10T23:13:17.262Z, server `http.status=499`), which aborted a long -implementation stage. Instead current source polls `GET /api/session/active` -about once per second with a context-aware delay and no busy loop, until the -admitted session is absent (inactive). Missing/null/non-object `data`, or a state -other than the documented `running` type, fails closed and is never treated as -inactive. This is **mocked regression coverage** in `internal/stage` (fake CLI), -not live-provider evidence; V1 native settlement is unchanged. - -Current source reads the V2 final result with a single bounded descending -final-assistant readback (`order=desc&limit=1&type=assistant`) after the settled -poll, for both audited agent stages and budgeted V2 text-only calls, so large -retained history (including prior tool-read contents) is no longer fetched just -to select the completed result. A truncated/incomplete readback, an empty data -list, or a latest message that is neither completed `stop` nor a completed -structured tool-call fails closed with a sanitized diagnostic that omits -provider content; retained audit previews are **not** executable final results. -Session/debug evidence now reuses the same single latest-assistant readback -instead of a recent-20 all-message fetch, and returns each payload as a labelled -bounded, credential-redacted envelope with explicit preview/truncation/missing -diagnostics; a small final assistant result is therefore selected even when large -prior tool content exists. Pi evidence remains documented JSONL and is never -parsed as one JSON object. This describes current source behavior, not a guarantee -that a user's installed server includes the fix. An updated skill/client is not -deployment evidence; never rebuild/restart services or log into providers without -specific authorization. - -Retained model-stage failures now carry a fixed sanitized category. A strictly -invalid final result (a duplicate nested field, or malformed/ambiguous JSON with a -byte offset) is labeled `FINAL RESULT VALIDATION`; request, V2 active-session poll -and final-readback failures are labeled `HARNESS TRANSPORT`. A pre-admission -request failure replays nothing and is retried for any stage; a post-admission -failure is retried only on an explicit read-only stage (see the 2026-10-17 section -below). A validation failure -preserves the admitted session, transcript and evidence and performs no second tool -execution. Confirmed ordinary-endpoint failures on 2026-10-11: an all-history -readback of exactly 262144 bytes of incomplete JSON (`invalid agent messages`), a -reviewer result with a duplicate nested `performance_review` field, and an -onboarding visual-review result with `invalid character 'z' after object key:value -pair at byte 3968` where unsafe lossless repair was rejected. These are **mocked -`internal/stage` fixtures**: executable source-level evidence, not live-provider -evidence and not a deployment. OpenCode V2 prose `output_contract` does not enforce -structured output; V2 results are validated locally and no unsupported V2 format -field is sent. A review-only fresh execution is a future recovery idea, **not -implemented**. Onboarding UI visual approval remains unresolved and must not be -reported complete. - -## Bounded malformed-result recovery: 2026-10-16 - -Source-level, mocked `internal/stage` regression coverage (not live-provider -evidence and not a deployment): malformed final-result recovery now tries a -deterministic local pass first (append missing closers, drop one trailing comma, -or insert the single comma/colon the parser expected), then at most one deny-all -formatter request whose deadline is derived from the payload size, the stage -timeout and the attempt's remaining budget rather than a fixed 30-second -constant. Every path -records `repaired`, `partial` or `unrecoverable` in `format_repairs`; a `partial` -or `unrecoverable` payload is never returned as a completed result, and an -ambiguous payload (an unescaped quote/colon inside a string value) is never -guessed at. A shell-enabled stage's malformed but deterministically recoverable -result now completes without replaying admitted tool work; `safeToReplay` is -unchanged. The run-55 `ui_reviewer` payload was retrieved read-only with -`pipelines inspect 55 --json` and reproduced its exact byte offset 7837; it is the -ambiguous error class, so it is reported unrecoverable rather than repaired. This -does not update or restart any running server. - -## Harness transport reach retry and repair deadline: 2026-10-17 - -Source-level, mocked `internal/stage`, `internal/opencode` and `internal/server` -regression coverage (not live-provider evidence and not a deployment): the -harness transport path now distinguishes a pre-admission reach failure -(connecting or creating the session) from a post-admission readback/poll/agent -failure under the one `HARNESS TRANSPORT` category. A pre-admission reach failure -replays nothing, so it is retried within `retries` for any stage with a fresh -session and no replayed prompt; a post-admission failure is retried only on an -explicit read-only stage, and an `edit`/`shell`/legacy stage still never replays -admitted work. A context that expires during a request keeps both the sanitized -transport detail (operation/status/exit) and -`errors.Is(err, context.DeadlineExceeded)`; a bare attempt-boundary deadline is -not treated as a reach failure and stays terminal for a mutating stage. An -unrecognized provider execution-error name is reported as a bounded, sanitized -identifier plus any HTTP status. The isolated formatter deadline is now -proportional to the stage timeout (floored, capped, and clamped to -`remaining - margin` so it can never extend the stage/attempt deadline). These are -mocked fixtures; a deployed server is not updated by this change. - -## Failure-handling source changes: 2026-10-15 - -Source-level, mocked `internal/stage` and `internal/server` regression coverage -(not live-provider evidence): an explicit read-only tools stage now actually -retries a post-admission stage deadline or harness transport failure, creating a -fresh bounded session per attempt, while a stage with `edit`/`shell` or no tools -block still never replays admitted work; a malformed final result is re-emitted -up to `retries` only on such a read-only stage with its `format_repairs` evidence -retained. A timed-out agent stage settles knowable V2 usage and retains one -bounded, redacted latest-assistant readback plus a `retained_session` pointer in -its stored transcript. A declared outcome routed to `#failure` with no step -diagnostic now yields a run diagnostic like `stage returned outcome -"blocked"`; an existing step diagnostic is still used verbatim. - -## Dated source-level coverage: 2026-10-11 - -Source-level/mocked regression coverage (not live-provider evidence): -`resolve-releases` now uses a single bounded -`GET .../git/matching-refs/tags/v2` response that the GitHub REST docs describe as -complete with no pagination; offline fixtures cover a 100+ ref array, -`MAX_REFS`/`MAX_BODY` caps, ignored prerelease/build tags, malformed refs and a -missing newest-tag digest failing closed with no older fallback, with no -`tags/list` cross-check. Picker tests cover blink/index preservation, -ANSI-readable selection without a colliding background, and -40x16/80x24/120x36 height budgets. The compiled picker PTY check (opt-in via -`PIPELINES_HARNESS_PTY_DIR`) waits through blink intervals and asserts exact -saved/arrow-scrolled models. `setup-docker.py` accepts the actual -`opencode v2.0.26` probe token and compares source identity only on a full -install; model/login-only reconfiguration compares the verified release image -identity. Python fixtures bind temporary HOME directories so the suite passes -under the container default `/home/pipelines` without weakening mount guards. -The Docker authoring transport is also covered at source level: the shared -prefix picks `/usr/local/bin/pipelines` for a Docker-linked profile. Loopback is -recognized only via `net.ParseIP(...).IsLoopback()` and an exact case-insensitive -`localhost`, so a remote DNS name such as `127.remote.example` stays remote with -no probe. A single bounded `docker compose port` probe parses complete -`host:port` bindings, and scoped source/target aliases are embedded only when a -binding covers the requested loopback address and port; malformed or non-matching -bindings and a loopback `wss` endpoint fail closed, and `client.Dial` rewrites -only an exact source match. `provider-check` dispatches through a Docker-linked -profile's existing service with `-T`, the original cwd/model/timeout, exact -stdout/exit propagation and `--json` passed only in JSON mode (fixture tests, no -real provider), while explicit native profiles keep the host generator. The -picker uses a true `x/term` TTY check so `/dev/null` and pipes fail fast, with a -short context deadline so a regression fails rather than hangs. Real Docker -rebuild/reconfigure against live registries, a real -40x16/80x24/120x36 terminal capture, and real model inference remain **not tested** -in this change. - -### 2026-10-11 Pi sign-in handoff (source + Root PTY observation) - -Root separately exercised current-source `pipelines harness setup pi --login` -under a real Docker PTY with copied harness preferences. Root observed that -Pi 1.1.0 defaults to fullscreen and clears the Pipelines login instructions, so -the user sees a coding interface without the `/login` guidance, and that -`/login` first shows "Select authentication method" (account / API key / Radius) -rather than a provider picker. The same handler run with a temporary executable -wrapper passing `pi --tui-mode regular` retained the Pipelines header and -instructions above Pi's interface. Installed `pi --help` documents the supported -`--tui-mode regular` flag and the fullscreen default; `pi auth --help` confirms -there is no standalone login (only print/check, never run). These are source/root -observations with copied preferences: **no provider was selected, no key was -entered and no inference occurred**, and this source is **not deployed** and is -not live-provider evidence. A first reviewer capture at 80x30 found the longer -handoff plus Pi's own banner scrolled the Pipelines header off the top, so the -copy was shortened to a title plus three short lines (existing-credential reuse -folded into the `/login` line, with the account/API key/Radius method choice and -`ctrl+c`/`ctrl+d` exit keys kept) and the redundant trailing blank removed. -Independent PTY re-capture at 80x30/100x36 after this shortening is **not -tested** in this change. The matching exact login-argv and copy assertions are -covered by fixture tests in `internal/cli`/`internal/harness`. - -### 2026-10-11 server CLI fixture isolation (source-level + Root-reported run) - -Root-reported runtime evidence supplied for this change (not re-executed by the -authoring stages): the deterministic seeded-profile regression -`go test ./tmp/preview-verification -run TestServerCLIFixtureDoesNotInheritUserDockerProfile -count=1` -seeds a synthetic Docker-linked harness profile in a child -`go test ./internal/server -run '^TestCLIHeadlessRunInspectionCancellationAndRPC$'`. -It previously failed because the child inherited the caller's real -`PIPELINES_HARNESS_CONFIG`, so the CLI `debug` prefix failed closed -("refusing to redirect") against the test's ephemeral httptest endpoint. Root's -four independent preview tests for the bounded transcript helpers pass; Root's -full host race run produced no race warning and only this fixture failure, and -`go vet ./...` passed. Current source therefore isolates the server CLI fixture -with the same missing-temp-file -`t.Setenv("PIPELINES_HARNESS_CONFIG", filepath.Join(t.TempDir(), "harnesses.json"))` -pattern already used by `internal/cli`'s setup, so the host executable stays -authoritative and no Docker probe runs for an ephemeral test endpoint. The -production fail-closed redirect guard in `internal/harness/prefix.go` and its -`refusing to redirect` / loopback-wss regressions are unchanged; the failing -verification was fixed rather than removed. - -This is a current-source and Root-reported observation. At the time of that -observation, independent re-execution of the full `go build`/`go vet`/`go test` -matrix (including Docker-only race results) and the final installed-host Docker -verification were **pending**; they are now recorded in the 2026-10-11 FINAL -INSTALLED section below, so this phase-specific wording is superseded by that dated -install. The packaged reference describes source behavior; a source fix or updated -skill is not deployment evidence. - -### 2026-10-11 retained run11 reviewer re-capture (source-level, not a final install) - -Independently reading the retained run11 verification at -`tmp/maintenance-run11-verification.json` records the source-level review -sequence: an earlier reviewer PTY capture rejected the longer handoff because at -80x30 it scrolled the Pipelines header off the top; the implementer then -shortened `AuthHandoff` to a title plus three short lines; and the re-run -`reviewer` and `ui_reviewer` stages approved from real current-source Pipelines -PTY captures at 80x30 and 100x36 that showed the header and handoff visible above -installed Pi 1.1.0's regular interface. A labelled controlled fixture that -stripped `--tui-mode regular` reproduced the pre-fix fullscreen clear, and the -review stages report no provider selected, no key entered and no inference. This -is source-level reviewer evidence, read from retained output; it was not a -deployed/installed state at that time, and the final full Docker install -verification was then **pending** — it is now recorded in the 2026-10-11 FINAL -INSTALLED section below, so this phase-specific wording is superseded by that dated -install. - -Root-reported actual runs (source-level observations supplied by Root for this -change and not re-verified here): run7 showed both configured providers' exact -handoff markers; runs6/9/10 exercised live maintenance with final-assistant -polling/readback succeeding. These are Root observations, not mocked-regression -coverage and not a completed installation; no final deployed/installed state is -claimed. - -### 2026-10-11 run13 interruption, Docker race limitation and state separation - -Four states are kept explicitly separate and must not be conflated: - -1. **Current source** (this checkout, not deployed): bounded 32 KiB - (`transcriptBound = 32<<10`) redacted transcript previews produce a valid JSON - envelope with redaction applied before truncation and a rune-safe 8 KiB preview; - session/debug evidence is scoped `latest_bounded_evidence` with the single - bounded latest-assistant readback and at most the 3 most recent session-bearing - steps; an unbudgeted V2 audited agent stage records reported usage best-effort - through exactly one 2s-bounded final session-aggregate read, leaving - missing/invalid usage unknown; and the isolated server CLI fixture uses the - missing-temp-file `PIPELINES_HARNESS_CONFIG` pattern while the production - fail-closed redirect guard in `internal/harness/prefix.go` is unchanged. Pi JSONL - is never parsed as one JSON object; native V1 keeps its existing path. -2. **Host verification evidence independently recorded by Root**: after the fixture - fix, Root ran the full `go test -race -count=1 ./...` and `go vet ./...`; after - three harness-neutral evidence labels and Step wording changes Root re-ran the - affected `tui`/`author`/`cli` race and vet checks, the Python suite (80) and the - opt-in compiled picker PTY check (4). These are Root-recorded host results, not - re-executed by the authoring stages. -3. **Deployed old image**: the running Docker image predates these source changes - and does not include them. -4. **Final install (completed 2026-10-11)**: the final install and installed - authoring/provider verification were **pending** when this separation was - written and are now recorded in the 2026-10-11 FINAL INSTALLED section below; an - updated skill/client or source checkout is still not deployment evidence on its - own. - -Docker race limitation (confirmed): the Docker runtime is built with -`CGO_ENABLED=0` and has **no C compiler** (the Alpine runtime stage installs no -GCC/musl-dev), so `go test -race` cannot run there. In-Docker reviewers must use -ordinary `go build`/`go test`/`go vet` and rely on independently executed host -`-race` runs. Race checks must not be attempted in Docker and no compiler may be -installed or searched for. - -Run13 interruption (dated evidence, not a fix): the previous reviewer ran `find /` -looking for GCC; the tool timeout left a process in disk sleep, Docker could not -stop the container, and an approved Desktop restart recovered run13 as interrupted -(`executing=false`, every step `agent_active=false`) with **no automatic replay**. -This is an environment/interruption observation only: no general cancellation fix -is inferred and process termination is unchanged. Docker startup recovery already -sets running steps to `interrupted` with `agent_active=0`. - -### 2026-10-11 short-lived CLI inherited-pipe cancellation (current source) - -Current source (not deployed) adds one shared, dependency-free constructor in -`internal/opencode`: `CommandContext(ctx, executable, args...)` sets the single -finite `os/exec.Cmd.WaitDelay` constant `ShortLivedWaitDelay = 250ms` and is the -only place that value is defined. It is reused by the `internal/stage` -`OpenCodeClient.call()` API/agent path (including `Interrupt`), `opencode.Detect()` -`--help` and the native `Connection.Messages()` export fallback. It bounds -**draining of inherited output pipes only**: after the child exits os/exec waits at -most the grace for descendants holding the inherited stdout/stderr pipes, then -closes them and reports `exec.ErrWaitDelay`; it does not terminate or confirm -cleanup of descendants. - -`context.Canceled`/`context.DeadlineExceeded` identities, `exec.ErrWaitDelay` on -the sanitized errors, the ≤64 KiB request / ≤1 MiB response / 8 KiB stderr bounds, -`boundedBuffer`/`boundedOutput` drain-with-overflow behavior, redaction, strict -result validation and cleanup-unconfirmed active ownership (`ErrStageCleanup`, no -replay) are unchanged; no new module dependency is added and a parent that exits -successfully while a descendant still holds partial stdout is never treated as -success. - -Dated synthetic reproduction (`tmp/interrupt-verification`, synthetic -real-subprocess diagnostics, **not provider evidence**): with Root waiting for the -descendant itself to start before cancelling public -`OpenCodeClient{Backend:"v2"}.Interrupt`, two Docker/Linux returns took -3996/3995ms despite `context.Canceled` (`descendant-started.jsonl`); the stronger -host reproduction took 4003/4000ms (`reproduction.log`). Added source-level -regressions use real `os/exec` fixtures with start-marker barriers, bounded -watchdogs and naturally terminating children for cancel-after-descendant, -deadline identity, descendant-held pipes → `ErrWaitDelay`, ordinary -output/exit/redaction, failed-interrupt ownership/no replay, plus `Detect`/export -wait-delay and cancellation cases. This is **current source, not deployed**: -descendant termination, confirmed remote interruption and any run13 Docker disk -sleep fix are **not claimed**, and Docker race remains unavailable (no C -compiler). - -### 2026-10-11 FINAL INSTALLED verification (friend-ready preview) - -This is the dated receipt for the installed runtime, recorded by Root from -`tmp/friend-final-verification/summary.json`; it supersedes the earlier -"pending final install" statements above, which stay as phase-specific history. -A normal interactive `./install --model opencode-go/deepseek-v4.1-flash ---harness-model opencode-go/deepseek-v4.1-flash` exited **0**, resolving the current -releases live and building fresh dirty source into actual image -`sha256:1810ab50f3ea3991e7f3548c79250e65a114bb1821eac52efd872b91cf7c3136` -(OpenCode 2.0.26 / Pi 1.1.0 / Node 24.18.1) with real tool-free provider -validation, ntfy authenticated publish, server + host health, home/workspace entry, -preserved authoritative endpoint and `ok` pre-install database/home backup -integrity. Post-install preservation all passed (original definitions 1–4, history -1–16, host/container default Pi and both models/profiles, logical endpoint, -service URLs/timezone/ports/models, Pi credential hash, OpenCode connected accounts, -ntfy token); neither original-repo commit/push nor credential/data cleanup ran. -Installed host-CLI authoring with **both** Docker harnesses (separate retained -drafts, each changing only the worker prompt) performed real `ping` → `pong` and -`validate` → `validated` through the generated Linux `/usr/local/bin/pipelines` and -scoped transport aliases preserving the logical `8081` endpoint and draft metadata, -independently inspected from native OpenCode tool outputs and retained Pi JSONL -`toolResults`, with no draft submit/run/publication. Docker-proxy submit on -verification-only definition 5 published v2, a second stale v1 draft was rejected as -conflict, and **Run17** succeeded in 7 seconds with the exact -`pipelines-docker-verified` markers in OpenCode then Pi, both inactive and exactly -one completed marker read verified from retained native history (not the -latest-assistant preview); best-effort unbudgeted usage was OpenCode 4011 -tokens/$0.00029769599999999996 and Pi 1071 tokens/$0.000142002, and -`/api/session/active` was empty after exit. Installed-UI captures -(`installed-pi-*`, `pi-signin-*`, `installed-pi-models-*`) cover the actual run17 -80x30/120x36 neutral-Pi/green-both-stage renders, the Pi auth handoff/method menu -viewed/canceled/exit0 with no sign-in, and the model wizard using copied -preferences only while production preferences stayed preserved. Root's independent -checks passed: full host `go test -race -count=1 ./...` and `go vet ./...`, -Python 80 tests (1 optional PTY skipped), Node policy, isolated developer lifecycle -fixture (no inference/deploy) and author package tests; earlier independent -reviewers ran ordinary Docker build/vet/tests and focused subprocess regressions -(no Docker race, no GCC search, no global `find`). **UNVERIFIED** here: fresh -OAuth/provider login completion, phone delivery, other real providers, the -D-disk-sleep fix and active lease expiry; authentication menus and -existing-credential inference are not fresh sign-in proof. The stronger synthetic -PUBLIC `Interrupt` child-start barrier (host 252/252 ms, Docker 250/251 ms, context -canceled true, bounded true vs ~4 s before) is not proof of descendant termination, -remote cleanup or a Docker VM disk fix. Archive extraction, build-identity and -compile verification remain pending until Root executes them. - -### 2026-10-11 generated release receipt ignore (source-hygiene regression) - -Current source adds two root-anchored `.gitignore` rules for the installer's -machine-local generated artifacts: `/.pipelines-releases.json` (the receipt path -from `setup-docker.py` `release_receipt_path()`) and `/.pipelines-releases-*` (the -`tempfile.mkstemp` `prefix=".pipelines-releases-"` in `record_release_receipt()`); -no installer/runtime code changed and the pre-existing receipt and other files are -retained. A new `scripts/test_build_metadata.py` case is a controlled, isolated Git -fixture using the repository's actual `.gitignore`: a clean tracked source plus a -representative generated receipt/`.pipelines-releases-*` temporary file leave `git -status --porcelain` empty and `build_metadata.docker_args` at -`PIPELINES_DIRTY=false` with no `-dirty` version; editing the tracked source then -makes the status non-empty and `docker_args` report `PIPELINES_DIRTY=true` with a -`-dirty` version. This is **offline source-level regression coverage**, not the -recorded actual new-account install/provider login above and not live-provider or -deployment evidence; this docs-and-ignore change has no rendered UI. - -## Dated live reference validation: 2026-10-10 - -Environment: Docker Linux aarch64, OpenCode v2.0.20, -**opencode-go/deepseek-v4.1-flash**. Isolated databases/fixtures and local HTTP -notification endpoints; normal published pipelines/repositories were not changed. - -- Incident extraction → report: exact IDs/severities/counts, unchanged handoff, - retained session readback and one local success receipt passed (6.63s). -- File edit → verification: exact final release-file bytes, unchanged fixtures, - independently executed `cmp`, completed read/edit/shell evidence, retained edit - diff and no verifier edits passed (14.60s). -- Missing required input: blocked route ended failed; missing file was not invented; - retained read attempt and one local failure receipt passed (4.21s). -- Accounting/one-token budget initially failed its acceptance check because session - zeros let the run succeed. After the tracked-prompt fix, a focused live rerun - passed: **952 reported tokens, $0.00016440**, budget-exceeded failure, stopped - owned session (2.43s). This is not an independently reconciled provider bill. - -The earlier three workflows were rechecked from retained artifacts with current -assertions, without more inference. There was **no fresh full-suite live run of -all four workflows after the fix**. Do not generalize these results to all five -starter templates, other models/providers, or compatible native schema/tool usage. -Real ntfy authentication/storage, phone reception, provider outages/timeouts, -long-stream interruption and promptness of cleanup were **not tested** by this trial. -The trial ran rebuilt test binaries; it did not deploy the fix to the ordinary server. - -## Dated live reference validation: 2026-10-11 (run7) - -One existing real run7 (2026-10-11) is the retained live evidence dated for this -change: it executed both configured harnesses, OpenCode 2.0.26 and Pi 1.1.0, with -`opencode-go/deepseek-v4.1-flash` stages on the Docker server, and both steps -succeeded (6s; the retained Pi evidence panel reports 1054/10000 tokens). The -captures are retained at `tmp/friend-ui-verification/source-pi-80-*` and -`source-pi-120-*`, and a current-source TUI inspection of that run recorded actual -80x30 and 120x36 renderings, including the run graph and the bounded evidence -panel. - -This is a single real run, **not** a fresh full reference suite. Other providers, -real outages/long-stream interruption, fresh OAuth and phone delivery remain -**not tested**; the new-source installer and installed-authoring verification were -separately completed by the dated 2026-10-11 FINAL INSTALLED section above. The -[2026-10-10 OpenCode v2.0.20 trial](#dated-live-reference-validation-2026-10-10) -above remains dated history. - -## Reference suite when explicitly requested - -The source checkout's `TestReferenceWorkflowsLive` uses six model stage attempts -plus possible compatibility/format-repair calls. Retries are disabled and stages -have two-minute timeouts. It uses real existing credentials and retains private -`runs.db`, pinned definitions, fixture directories, run/session JSON and report. -The accounting case has a one-token/$0.25 budget; other cases have no spending -limit. Test notifications use a local endpoint, not the configured phone topic. - -Live inference requires explicit `PIPELINES_REFERENCE_LIVE=1`, a chosen -`PIPELINES_REFERENCE_MODEL=PROVIDER/MODEL`, and an existing private -`PIPELINES_REFERENCE_DIR`. Get consent for chargeable inference and confirm which -execution computer/account to use. Ordinary tests skip live inference. Do not -search the repository or run the suite during normal draft authoring. - -If the user explicitly asks to run the suite and provides a source checkout: - -```sh -PIPELINES_REFERENCE_LIVE=1 \ -PIPELINES_REFERENCE_MODEL=PROVIDER/MODEL \ -PIPELINES_REFERENCE_DIR=/private/approved/artifacts \ -go test ./internal/server -run '^TestReferenceWorkflowsLive$' -count=1 -v -timeout 15m -``` - -Docker credentials require running the correctly compiled test binary in that -container; do not copy host credentials or publish tests to the normal database. -`TestReferenceDefinitions` checks definitions without inference. -`TestReferenceArtifactsReview` rechecks copied run/fixture evidence without -inference using `PIPELINES_REFERENCE_REVIEW_DIR`; select only cases present in -that artifact directory. Missing cases skip and must not be reported as passed. - -## 2026-10-12 run22 success, run23 live CSV fixture and run24 interruption - -Source/structural and live-fixture evidence stays separate from runtime -deployment; nothing here is a deployment claim. - -- **run22 (structural/source checks, success):** the retained run22 verification - passed its structural/source checks of the scoped template and prompt changes. -- **run23 (live Pi CSV fixture):** a real exact-template Pi CSV run fetched the - official Python `csv` documentation through the conditional official-source - webfetch and implemented CSV parsing with the standard library's `csv.reader` - through an existing shared helper, adding no duplicate preview parser; both the - implementer and reviewer checks actually passed. Root independently verified the - full retained native tool results and protected-file hashes against its saved - baseline, rather than relying on model-written reports. -- **run24 (interrupted, not a pass):** the run24 implementation completed, but its - reviewer ignored the scope instruction and ran a prohibited filesystem-root scan - (`find /`), leaving a process in uninterruptible D-state and stalling Docker - mount/snapshot reads. Root canceled the run, made a consistent SQLite backup - (`integrity ok`), recovered Docker with the supported stop/start path, and - restored the existing ntfy/Pipelines services with the original definitions, run - history, credentials and settings verified unchanged. run24 is retained as - **interrupted**, not a success. - -- **run25 (live Pi source-only review):** the reviewer identified duplicated - implementation, a hardcoded special case, suppressed errors and test gaps. - Root inspected retained native calls: only local read/search tools ran, with no - shell/edit/network calls. Original fixture hashes were unchanged; runtime - behavior was explicitly marked untested. -- **run26 (final maintenance review, success):** the scoped plan/implement/review - workflow passed package tests, vet, build and formatting checks. Root inspected - the actual retained outputs and changes. No rendered UI change was claimed. - -The scoped prompt guidance asks stages to keep inspection/verification to relevant -repository paths and supported commands; it is a working instruction, **not** an OS -sandbox, and neither fixes Docker VM/disk bugs nor universally enforces tool -boundaries. At these recorded checkpoints the source changes had not been deployed. -Catalog deployment requires a separate check of the running server and its actual -template contents; these observations do not establish publication or deployment. -Other providers and phone delivery remain **not tested** by this record. - -## 2026-10-12 final template verification and Docker catalog deployment - -The final three-stage template ran as live Pi run27 (verification definition7, -version2) in a fresh disposable CSV fixture. Root inspected full retained native -results: the planner fetched official Python CSV documentation, the implementer -used the standard library through the existing shared parser, and implementer and -reviewer independently ran `python3 checks.py` successfully. Root reran those -checks, verified the protected fixture hashes and planner-owned review criteria, -and confirmed the preview reused the shared parser. - -The Docker server was then refreshed from clean source commit -`143f237268dbde2efd6c5ce5722a3e1fd7e08c9c`, retaining OpenCode 2.0.26, Pi 1.1.0 -and Node 24.18.1. The running server reported that exact commit and `dirty=false`; -all five served templates matched current source with the saved Pi/model defaults. -A consistent pre-deployment database backup passed integrity checks. Every stored -definition version/source, retained run ID, saved setting, endpoint and provider -credential/account checked by Root was preserved. This record is a documentation -follow-up to that deployed build; it changes no runtime code or template source. - -These bounded fixtures establish the observed behavior on this instance, not a -guarantee of every future model review, provider, task or platform. They do not -establish fresh account sign-in or phone notification reception. +`pipelines inspect RUN_ID --json` exposes each model step's `tokens_total`, +`tokens_in`/`tokens_out` (only when `split_known`), `cost_usd` (only when +`cost_known`) and the model it ran with, plus the run's totalled `usage`; +`pipelines stats --json` adds history-wide totals. Token totals include cache +reads. An unreported cost or split is shown as unavailable, never zero; runs +recorded before usage tracking omit `usage`. Budgets are reported-usage limits, +**not provider-side hard caps or a guaranteed bill ceiling**; in-flight calls can +overshoot. ## Orchestration verification habits A run's status and a stage's summary are **claims**, not evidence. When the agent -manages runs rather than making the change, its job includes independently checking -what each run claims. +manages runs rather than making the change, its job includes checking what each +run claims. -- **Read the retained evidence yourself** — `inspect RUN_ID --json` for the exact - stage outputs, outcomes, diagnostics, file changes and artifacts — instead of - trusting the model-written summary of the run. -- **Run the check.** Where an item claims a build or tests, run the exact command and - read its output; "tests pass" without a command and its observed result is not a - check. Never report a check you did not execute. -- **Read the images.** Open the screenshots the reviewer cites; do not accept a - description of an image you can open. Confirm a capture is a **real image** (magic - bytes or the reported content type), not a text file named `.png` - (`references/debugging.md`). -- **Compare the built artefact with what was delivered.** Check the md5 (or - equivalent) of what was built against what is installed or published, so a stale or - substituted file cannot pass as the artefact (`references/delivery.md`). -- **State "not tested" explicitly.** A run that `succeeded` means its stages reported - success; it is not proof that the operator's acceptance criteria were met. Name what - was not covered. -- **Report failures as failures.** Never re-run until something passes and then call - that verified; record the failure and its category. +- **Read the retained evidence yourself**: `inspect RUN_ID --json` for the exact + stage outputs, outcomes, diagnostics, file changes and artifacts. +- **Run the check.** Where an item claims a build or tests, run the exact command + and read its output. Never report a check you did not execute. +- **Read the images.** Open the screenshots a reviewer cites and confirm each is a + real image (magic bytes or content type), not a text file named `.png`. +- **Compare the built artefact with what was delivered** (md5 or equivalent), so a + stale or substituted file cannot pass (`references/delivery.md`). +- **State "not tested" explicitly.** A `succeeded` run means its stages reported + success, not that the operator's acceptance criteria were met. +- **Report failures as failures.** Never re-run until something passes and then + call that verified; record the failure and its category.