From 724de275612798c255873a5bbda31183ca360d9e Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Tue, 20 Oct 2026 23:51:14 +0100 Subject: [PATCH] feat(stage): honor per-stage reasoning effort across native harnesses --- docs/harnesses-and-control.md | 38 ++++++++ internal/author/skill/references/format.md | 19 +++- internal/pipeline/reasoning_effort_test.go | 47 +++++++++ internal/server/overrides_test.go | 7 +- internal/stage/agent.go | 9 +- internal/stage/model.go | 3 - internal/stage/opencode.go | 9 +- internal/stage/opencode_reasoning_test.go | 107 +++++++++++++++++++++ internal/stage/pi.go | 3 - internal/stage/pi_effort_test.go | 4 +- internal/stage/reasoning_effort_test.go | 7 +- 11 files changed, 233 insertions(+), 20 deletions(-) create mode 100644 internal/pipeline/reasoning_effort_test.go create mode 100644 internal/stage/opencode_reasoning_test.go diff --git a/docs/harnesses-and-control.md b/docs/harnesses-and-control.md index d5ff0904..37a52bc7 100644 --- a/docs/harnesses-and-control.md +++ b/docs/harnesses-and-control.md @@ -1,5 +1,43 @@ # Harnesses and runner-owned control +## Reasoning effort (thinking level) + +Add an optional setting to the model stage's existing `configs` block: + +````markdown +```configs +provider=openai +model=gpt-5.2 +reasoning_effort=high +``` +```` + +Use your actual configured provider/model IDs. Each stage chooses its own level; +leaving it out, or using `default`, preserves the model/harness default. + +| Setting | Behavior | +| --- | --- | +| `none` (alias `off`) | Disable reasoning, when supported | +| `minimal`, `low`, `medium`, `high`, `xhigh`, `max` | Select that native effort level, when supported | +| omitted / `default` | Leave existing defaults unchanged | + +OpenCode V2 passes this as the selected model's named **variant**, for both +text-only and tool-using stages. Variant availability is model-specific; an +unavailable level is rejected, not silently replaced. See +[OpenCode variants](https://opencode.ai/v2/docs/models#variants). +Pi uses its native **thinking level**, mapping `none` to `off`, and verifies that +the selected model actually applied it before sending the task. Unsupported or +clamped levels fail clearly; availability also depends on the installed Pi +version. OpenCode V1 cannot apply this setting and rejects explicit effort. + +This is a native runtime option, not a prompt instruction or token limit. Higher +effort can increase latency and cost; keep `budget_tokens`, `budget_usd`, and +provider spending limits separate. Existing definitions, defaults and published +versions are never rewritten. Editing the setting requires publishing a new +pipeline version as usual. + +## Harness defaults + Configure two independent model defaults: ```sh diff --git a/internal/author/skill/references/format.md b/internal/author/skill/references/format.md index 0d4b8c62..4493856f 100644 --- a/internal/author/skill/references/format.md +++ b/internal/author/skill/references/format.md @@ -79,7 +79,22 @@ authentication and inference require `pipelines provider-check` on the execution computer/container. Optional `retries` defaults to 2 (three total attempts), and accepts integers 0–100; explicit 0 disables execution retries. Optional `timeout` defaults to `1h` per attempt and accepts positive durations such as -`30m` or `90s`. Automatic retries cover retryable transport failures and +`30m` or `90s`. + +Optional `reasoning_effort=high` selects a model's reasoning/thinking level for +this stage. Values are `default`, `none` (alias `off`), `minimal`, `low`, `medium`, +`high`, `xhigh`, or `max`. Omission or `default` preserves the harness/model +default; never add a level automatically to existing workflows. Availability is +model-specific: OpenCode V2 selects the corresponding named model variant and +rejects unavailable variants; Pi uses its native thinking level (`none` becomes +`off`) and verifies the effective level before prompting, rejecting unsupported +or silently clamped levels. Explicit effort on OpenCode V1 is unsupported. +Do not instruct the agent to “think harder” as a substitute for this native +setting. Effort is not a token budget or a guaranteed quality improvement; keep +usage budgets/provider spending limits separate. Both text-only and tool-using +model stages honor the setting, without changing their tools or permissions. + +Automatic retries cover retryable transport failures and timeouts; an explicit read-only stage may additionally retry a post-admission deadline or transport failure, while a stage with `edit`/`shell` (or no tools block) never replays admitted work. Optional `budget_usd=1.50` and @@ -89,7 +104,7 @@ to this stage in the same run. Omit either budget to leave that resource unlimit Both budgets must be positive; tokens include input, output, reasoning and cache reads/writes. Accounting failures stop budget-limited stages. In-flight requests may exceed limits before usage is reported; these are not provider-side hard caps. -In current source, budgeted OpenCode V2 text-only calls use tracked prompting with +In current source, all OpenCode V2 text-only calls use tracked prompting with all tools denied, rather than the usage-less `/generate` path. Completed token-budgeted responses reporting zero tokens fail closed. This is an execution implementation detail, not a new Markdown field: never add backend/format/API diff --git a/internal/pipeline/reasoning_effort_test.go b/internal/pipeline/reasoning_effort_test.go new file mode 100644 index 00000000..b37cd799 --- /dev/null +++ b/internal/pipeline/reasoning_effort_test.go @@ -0,0 +1,47 @@ +package pipeline + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func TestReasoningEffortDefinitionRoundTrip(t *testing.T) { + source := strings.Replace(Template([]string{"worker", "review"}, true), "retries=2", "reasoning_effort=high\nretries=2", 1) + graph, err := Parse(source) + if err != nil { + t.Fatal(err) + } + if graph.Stages[0].Configs["reasoning_effort"] != "high" || graph.Stages[1].Configs["reasoning_effort"] != "" { + t.Fatal("effort leaked across stage scopes") + } + spec, err := (stage.ModelStage{}).Spec(StageDefinition(graph.Stages[0])) + if err != nil || spec.Config.ReasoningEffort != "high" { + t.Fatalf("runtime lost effort: %+v %v", spec, err) + } + s, err := Open(filepath.Join(t.TempDir(), "effort.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if _, err = s.Submit(context.Background(), "effort-fixture", source); err != nil { + t.Fatal(err) + } + items, err := s.List(context.Background()) + if err != nil { + t.Fatal(err) + } + saved, err := s.Get(context.Background(), items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + if saved.Source != source || saved.Graph.Stages[0].Configs["reasoning_effort"] != "high" { + t.Fatal("immutable definition dropped effort") + } + if _, err = Parse(strings.Replace(source, "reasoning_effort=high", "reasoning_effort=lots", 1)); err == nil { + t.Fatal("invalid effort accepted by publication validation") + } +} diff --git a/internal/server/overrides_test.go b/internal/server/overrides_test.go index 29d03b81..09fd80f3 100644 --- a/internal/server/overrides_test.go +++ b/internal/server/overrides_test.go @@ -25,7 +25,8 @@ func TestRunnerAppliesModelOverrides(t *testing.T) { t.Fatal(err) } defer store.Close() - if _, err := store.Submit(ctx, "overrides", pipeline.Template([]string{"one", "two"}, true)); err != nil { + source := strings.Replace(pipeline.Template([]string{"one", "two"}, true), "retries=2", "reasoning_effort=high\nretries=2", 1) + if _, err := store.Submit(ctx, "overrides", source); err != nil { t.Fatal(err) } items, _ := store.List(ctx) @@ -55,7 +56,7 @@ func TestRunnerAppliesModelOverrides(t *testing.T) { t.Fatal(err) } r.wg.Wait() - if got := resolved["one"]; got.Provider != "acme" || got.Model != "strong" { + if got := resolved["one"]; got.Provider != "acme" || got.Model != "strong" || got.ReasoningEffort != "high" { t.Fatalf("stage one resolved %+v, want the override", got) } if got := resolved["two"]; got.Provider != pinned["two"].Provider || got.Model != pinned["two"].Model { @@ -88,7 +89,7 @@ func TestRunnerAppliesModelOverrides(t *testing.T) { for _, s := range overlaid.Graph.Stages { configs[s.Key] = s.Configs } - if configs["one"]["provider"] != "acme" || configs["one"]["model"] != "strong" { + if configs["one"]["provider"] != "acme" || configs["one"]["model"] != "strong" || configs["one"]["reasoning_effort"] != "high" { t.Fatalf("overlaid graph stage one = %+v, want the override", configs["one"]) } if configs["two"]["provider"] != pinned["two"].Provider || configs["two"]["model"] != pinned["two"].Model { diff --git a/internal/stage/agent.go b/internal/stage/agent.go index 7d70c9f8..7bf4599a 100644 --- a/internal/stage/agent.go +++ b/internal/stage/agent.go @@ -97,8 +97,15 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s } } rules := c.permissionRules() - data, err := c.call(ctx, "post", "/api/session", map[string]any{"title": "Pipeline audited agent stage", "agent": "build", "location": map[string]string{"directory": c.Directory}, "model": map[string]string{"providerID": config.Provider, "id": config.Model}, "permissions": rules}) + model, err := openCodeModelSelection(config, c.Backend) if err != nil { + return "", err + } + data, err := c.call(ctx, "post", "/api/session", map[string]any{"title": "Pipeline audited agent stage", "agent": "build", "location": map[string]string{"directory": c.Directory}, "model": model, "permissions": rules}) + if err != nil { + if variant := model["variant"]; variant != "" { + err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err) + } // No prompt or tool call was admitted: a request failure here is a // reachable retry rather than a reason to end the run. return "", harnessUnreachable(err) diff --git a/internal/stage/model.go b/internal/stage/model.go index 4764e934..a74c39e1 100644 --- a/internal/stage/model.go +++ b/internal/stage/model.go @@ -98,9 +98,6 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { if err != nil { return ModelSpec{}, err } - if backend == "pi" && effort == "max" { - return ModelSpec{}, fmt.Errorf("reasoning_effort=max is not supported by pi; use a supported thinking level up to xhigh") - } if delivery := def.Configs["delivery"]; delivery != "" && delivery != "true" { return ModelSpec{}, fmt.Errorf("delivery must be true when set") } diff --git a/internal/stage/opencode.go b/internal/stage/opencode.go index 152d3c45..d65b39ac 100644 --- a/internal/stage/opencode.go +++ b/internal/stage/opencode.go @@ -309,13 +309,20 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp if len(encoded) > maxModelRequest { return "", fmt.Errorf("OpenCode request exceeds 64 KiB limit") } + model, err := openCodeModelSelection(config, c.Backend) + if err != nil { + return "", err + } created, err := c.call(ctx, "post", "/api/session", map[string]any{ "title": "Pipeline stage generation", "location": map[string]string{"directory": directory}, - "model": map[string]string{"providerID": config.Provider, "id": config.Model}, + "model": model, "permissions": []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}}, }) if err != nil { + if variant := model["variant"]; variant != "" { + err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err) + } // No prompt or tool call was admitted: a request failure here is a // reachable retry rather than a reason to end the run. return "", harnessUnreachable(err) diff --git a/internal/stage/opencode_reasoning_test.go b/internal/stage/opencode_reasoning_test.go new file mode 100644 index 00000000..b20b1b7b --- /dev/null +++ b/internal/stage/opencode_reasoning_test.go @@ -0,0 +1,107 @@ +package stage + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" +) + +func TestOpenCodeEffortReachesAgentAndTextSession(t *testing.T) { + for _, agent := range []bool{false, true} { + for _, effort := range []string{"", "default", "none", "off", "minimal", "low", "medium", "high", "xhigh", "max"} { + t.Run(effort+map[bool]string{true: "/agent", false: "/text"}[agent], func(t *testing.T) { + prompts := 0 + created := false + client := OpenCodeClient{Backend: "v2", Directory: t.TempDir(), sessionPollInterval: time.Millisecond} + client.command = func(_ context.Context, _ string, args ...string) ([]byte, error) { + switch { + case args[2] == "/api/session": + var body struct { + Model map[string]string `json:"model"` + } + if err := json.Unmarshal([]byte(args[4]), &body); err != nil { + t.Fatal(err) + } + want := effort + if effort == "default" { + want = "" + } + if effort == "off" { + want = "none" + } + if body.Model["providerID"] != "fixture" || body.Model["id"] != "model" || body.Model["variant"] != want { + t.Fatalf("wrong native selection: %+v", body.Model) + } + if want == "" { + if _, ok := body.Model["variant"]; ok { + t.Fatal("default unexpectedly overrides native variant") + } + } + created = true + return []byte(`{"data":{"id":"ses_effort"}}`), nil + case args[2] == "/api/session/ses_effort/prompt": + if !created { + t.Fatal("prompt before session selection") + } + prompts++ + var body map[string]any + if err := json.Unmarshal([]byte(args[4]), &body); err != nil { + t.Fatal(err) + } + if _, ok := body["model"]; ok { + t.Fatal("prompt replaced the selected native model variant") + } + return []byte(`{}`), nil + case args[2] == "/api/session/active": + return []byte(`{"data":{}}`), nil + case strings.HasPrefix(args[2], "/api/session/ses_effort/message"): + return []byte(`{"data":[{"id":"msg_result","type":"assistant","finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess"}]}]}`), nil + case args[2] == "/api/session/ses_effort": + return usageResponse("ses_effort", 0, 1), nil + default: + t.Fatalf("unexpected call: %v", args) + return nil, nil + } + } + config := ModelConfig{Provider: "fixture", Model: "model", ReasoningEffort: effort} + var result string + var err error + if agent { + result, err = (&AgentClient{OpenCodeClient: client, Tools: []string{"read"}}).Generate(context.Background(), config, "task") + } else { + result, err = client.Generate(context.Background(), config, "task") + } + if err != nil || result != "# outcome\nsuccess" || prompts != 1 { + t.Fatalf("result=%q err=%v prompts=%d", result, err, prompts) + } + }) + } + } +} + +func TestOpenCodeRejectedEffortNeverAdmitsWork(t *testing.T) { + for _, agent := range []bool{false, true} { + client := OpenCodeClient{Backend: "v2", Directory: t.TempDir()} + calls := 0 + client.command = func(_ context.Context, _ string, args ...string) ([]byte, error) { + calls++ + if args[2] != "/api/session" { + t.Fatalf("work admitted after invalid native variant: %v", args) + } + return nil, errors.New("unknown model variant high") + } + config := ModelConfig{Provider: "fixture", Model: "no-reasoning", ReasoningEffort: "high"} + var err error + if agent { + _, err = (&AgentClient{OpenCodeClient: client}).Generate(context.Background(), config, "task") + } else { + _, err = client.Generate(context.Background(), config, "task") + } + if err == nil || calls != 1 || !strings.Contains(err.Error(), "reasoning_effort=high") { + t.Fatalf("unsupported effort was ignored: %v %d", err, calls) + } + } +} diff --git a/internal/stage/pi.go b/internal/stage/pi.go index 78b845ad..6228509f 100644 --- a/internal/stage/pi.go +++ b/internal/stage/pi.go @@ -111,9 +111,6 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri if err != nil { return "", err } - if effort == "max" { - return "", fmt.Errorf("reasoning_effort=max is not supported by pi; use a supported thinking level up to xhigh") - } thinking := effort if thinking == "none" { thinking = "off" diff --git a/internal/stage/pi_effort_test.go b/internal/stage/pi_effort_test.go index 6cc0bdf3..b9149b82 100644 --- a/internal/stage/pi_effort_test.go +++ b/internal/stage/pi_effort_test.go @@ -85,6 +85,8 @@ func TestPiReasoningEffort(t *testing.T) { {name: "medium", effort: "medium", flag: "medium"}, {name: "high", effort: "high", flag: "high"}, {name: "xhigh", effort: "xhigh", flag: "xhigh"}, + {name: "max", effort: "max", flag: "max"}, + {name: "max-clamped", effort: "max", flag: "max", fixture: "clamped", wantErr: `effective "high"`}, {name: "clamped", effort: "xhigh", flag: "xhigh", fixture: "clamped", wantErr: `effective "high"`}, {name: "off-clamped", effort: "off", flag: "off", fixture: "clamped", wantErr: `effective "high"`}, {name: "nonreasoning", effort: "high", flag: "high", fixture: "no-reasoning", wantErr: "cannot confirm reasoning support"}, @@ -174,7 +176,7 @@ func TestPiReasoningEffort(t *testing.T) { } func TestPiInvalidReasoningEffortBeforeProcess(t *testing.T) { - for _, effort := range []string{"max", "invalid", "HIGH", " high "} { + for _, effort := range []string{"invalid", "HIGH", " high "} { t.Run(effort, func(t *testing.T) { client := &PiClient{Executable: "/does-not-exist", WorkStarted: true} _, err := client.Generate(context.Background(), ModelConfig{ReasoningEffort: effort}, "task") diff --git a/internal/stage/reasoning_effort_test.go b/internal/stage/reasoning_effort_test.go index 6b741ef1..204fa321 100644 --- a/internal/stage/reasoning_effort_test.go +++ b/internal/stage/reasoning_effort_test.go @@ -1,7 +1,6 @@ package stage import ( - "strings" "testing" ) @@ -31,11 +30,7 @@ func TestReasoningEffortConfig(t *testing.T) { } def.Configs["harness"] = "pi" _, err = (ModelStage{}).Spec(def) - if value == "max" { - if err == nil || !strings.Contains(err.Error(), "pi") { - t.Fatal("unsupported Pi effort accepted") - } - } else if err != nil { + if err != nil { t.Fatal(err) } })