feat(stage): honor per-stage reasoning effort across native harnesses

This commit is contained in:
2026-10-20 23:51:14 +01:00
parent 9ce2c5d143
commit 724de27561
11 changed files with 233 additions and 20 deletions
+38
View File
@@ -1,5 +1,43 @@
# Harnesses and runner-owned control
## Reasoning effort (thinking level)
Add an optional setting to the model stage's existing `configs` block:
````markdown
```configs
provider=openai
model=gpt-5.2
reasoning_effort=high
```
````
Use your actual configured provider/model IDs. Each stage chooses its own level;
leaving it out, or using `default`, preserves the model/harness default.
| Setting | Behavior |
| --- | --- |
| `none` (alias `off`) | Disable reasoning, when supported |
| `minimal`, `low`, `medium`, `high`, `xhigh`, `max` | Select that native effort level, when supported |
| omitted / `default` | Leave existing defaults unchanged |
OpenCode V2 passes this as the selected model's named **variant**, for both
text-only and tool-using stages. Variant availability is model-specific; an
unavailable level is rejected, not silently replaced. See
[OpenCode variants](https://opencode.ai/v2/docs/models#variants).
Pi uses its native **thinking level**, mapping `none` to `off`, and verifies that
the selected model actually applied it before sending the task. Unsupported or
clamped levels fail clearly; availability also depends on the installed Pi
version. OpenCode V1 cannot apply this setting and rejects explicit effort.
This is a native runtime option, not a prompt instruction or token limit. Higher
effort can increase latency and cost; keep `budget_tokens`, `budget_usd`, and
provider spending limits separate. Existing definitions, defaults and published
versions are never rewritten. Editing the setting requires publishing a new
pipeline version as usual.
## Harness defaults
Configure two independent model defaults:
```sh
+17 -2
View File
@@ -79,7 +79,22 @@ authentication and inference require `pipelines provider-check` on the execution
computer/container. Optional `retries` defaults to 2 (three total attempts),
and accepts integers 0–100; explicit 0 disables execution retries. Optional
`timeout` defaults to `1h` per attempt and accepts positive durations such as
`30m` or `90s`. Automatic retries cover retryable transport failures and
`30m` or `90s`.
Optional `reasoning_effort=high` selects a model's reasoning/thinking level for
this stage. Values are `default`, `none` (alias `off`), `minimal`, `low`, `medium`,
`high`, `xhigh`, or `max`. Omission or `default` preserves the harness/model
default; never add a level automatically to existing workflows. Availability is
model-specific: OpenCode V2 selects the corresponding named model variant and
rejects unavailable variants; Pi uses its native thinking level (`none` becomes
`off`) and verifies the effective level before prompting, rejecting unsupported
or silently clamped levels. Explicit effort on OpenCode V1 is unsupported.
Do not instruct the agent to “think harder” as a substitute for this native
setting. Effort is not a token budget or a guaranteed quality improvement; keep
usage budgets/provider spending limits separate. Both text-only and tool-using
model stages honor the setting, without changing their tools or permissions.
Automatic retries cover retryable transport failures and
timeouts; an explicit read-only stage may additionally retry a post-admission
deadline or transport failure, while a stage with `edit`/`shell` (or no tools
block) never replays admitted work. Optional `budget_usd=1.50` and
@@ -89,7 +104,7 @@ to this stage in the same run. Omit either budget to leave that resource unlimit
Both budgets must be positive; tokens include input, output, reasoning and cache
reads/writes. Accounting failures stop budget-limited stages. In-flight requests
may exceed limits before usage is reported; these are not provider-side hard caps.
In current source, budgeted OpenCode V2 text-only calls use tracked prompting with
In current source, all OpenCode V2 text-only calls use tracked prompting with
all tools denied, rather than the usage-less `/generate` path. Completed
token-budgeted responses reporting zero tokens fail closed. This is an execution
implementation detail, not a new Markdown field: never add backend/format/API
@@ -0,0 +1,47 @@
package pipeline
import (
"context"
"path/filepath"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/stage"
)
func TestReasoningEffortDefinitionRoundTrip(t *testing.T) {
source := strings.Replace(Template([]string{"worker", "review"}, true), "retries=2", "reasoning_effort=high\nretries=2", 1)
graph, err := Parse(source)
if err != nil {
t.Fatal(err)
}
if graph.Stages[0].Configs["reasoning_effort"] != "high" || graph.Stages[1].Configs["reasoning_effort"] != "" {
t.Fatal("effort leaked across stage scopes")
}
spec, err := (stage.ModelStage{}).Spec(StageDefinition(graph.Stages[0]))
if err != nil || spec.Config.ReasoningEffort != "high" {
t.Fatalf("runtime lost effort: %+v %v", spec, err)
}
s, err := Open(filepath.Join(t.TempDir(), "effort.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
if _, err = s.Submit(context.Background(), "effort-fixture", source); err != nil {
t.Fatal(err)
}
items, err := s.List(context.Background())
if err != nil {
t.Fatal(err)
}
saved, err := s.Get(context.Background(), items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
if saved.Source != source || saved.Graph.Stages[0].Configs["reasoning_effort"] != "high" {
t.Fatal("immutable definition dropped effort")
}
if _, err = Parse(strings.Replace(source, "reasoning_effort=high", "reasoning_effort=lots", 1)); err == nil {
t.Fatal("invalid effort accepted by publication validation")
}
}
+4 -3
View File
@@ -25,7 +25,8 @@ func TestRunnerAppliesModelOverrides(t *testing.T) {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "overrides", pipeline.Template([]string{"one", "two"}, true)); err != nil {
source := strings.Replace(pipeline.Template([]string{"one", "two"}, true), "retries=2", "reasoning_effort=high\nretries=2", 1)
if _, err := store.Submit(ctx, "overrides", source); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
@@ -55,7 +56,7 @@ func TestRunnerAppliesModelOverrides(t *testing.T) {
t.Fatal(err)
}
r.wg.Wait()
if got := resolved["one"]; got.Provider != "acme" || got.Model != "strong" {
if got := resolved["one"]; got.Provider != "acme" || got.Model != "strong" || got.ReasoningEffort != "high" {
t.Fatalf("stage one resolved %+v, want the override", got)
}
if got := resolved["two"]; got.Provider != pinned["two"].Provider || got.Model != pinned["two"].Model {
@@ -88,7 +89,7 @@ func TestRunnerAppliesModelOverrides(t *testing.T) {
for _, s := range overlaid.Graph.Stages {
configs[s.Key] = s.Configs
}
if configs["one"]["provider"] != "acme" || configs["one"]["model"] != "strong" {
if configs["one"]["provider"] != "acme" || configs["one"]["model"] != "strong" || configs["one"]["reasoning_effort"] != "high" {
t.Fatalf("overlaid graph stage one = %+v, want the override", configs["one"])
}
if configs["two"]["provider"] != pinned["two"].Provider || configs["two"]["model"] != pinned["two"].Model {
+8 -1
View File
@@ -97,8 +97,15 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
}
}
rules := c.permissionRules()
data, err := c.call(ctx, "post", "/api/session", map[string]any{"title": "Pipeline audited agent stage", "agent": "build", "location": map[string]string{"directory": c.Directory}, "model": map[string]string{"providerID": config.Provider, "id": config.Model}, "permissions": rules})
model, err := openCodeModelSelection(config, c.Backend)
if err != nil {
return "", err
}
data, err := c.call(ctx, "post", "/api/session", map[string]any{"title": "Pipeline audited agent stage", "agent": "build", "location": map[string]string{"directory": c.Directory}, "model": model, "permissions": rules})
if err != nil {
if variant := model["variant"]; variant != "" {
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
}
// No prompt or tool call was admitted: a request failure here is a
// reachable retry rather than a reason to end the run.
return "", harnessUnreachable(err)
-3
View File
@@ -98,9 +98,6 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
if err != nil {
return ModelSpec{}, err
}
if backend == "pi" && effort == "max" {
return ModelSpec{}, fmt.Errorf("reasoning_effort=max is not supported by pi; use a supported thinking level up to xhigh")
}
if delivery := def.Configs["delivery"]; delivery != "" && delivery != "true" {
return ModelSpec{}, fmt.Errorf("delivery must be true when set")
}
+8 -1
View File
@@ -309,13 +309,20 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
if len(encoded) > maxModelRequest {
return "", fmt.Errorf("OpenCode request exceeds 64 KiB limit")
}
model, err := openCodeModelSelection(config, c.Backend)
if err != nil {
return "", err
}
created, err := c.call(ctx, "post", "/api/session", map[string]any{
"title": "Pipeline stage generation",
"location": map[string]string{"directory": directory},
"model": map[string]string{"providerID": config.Provider, "id": config.Model},
"model": model,
"permissions": []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}},
})
if err != nil {
if variant := model["variant"]; variant != "" {
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
}
// No prompt or tool call was admitted: a request failure here is a
// reachable retry rather than a reason to end the run.
return "", harnessUnreachable(err)
+107
View File
@@ -0,0 +1,107 @@
package stage
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
)
func TestOpenCodeEffortReachesAgentAndTextSession(t *testing.T) {
for _, agent := range []bool{false, true} {
for _, effort := range []string{"", "default", "none", "off", "minimal", "low", "medium", "high", "xhigh", "max"} {
t.Run(effort+map[bool]string{true: "/agent", false: "/text"}[agent], func(t *testing.T) {
prompts := 0
created := false
client := OpenCodeClient{Backend: "v2", Directory: t.TempDir(), sessionPollInterval: time.Millisecond}
client.command = func(_ context.Context, _ string, args ...string) ([]byte, error) {
switch {
case args[2] == "/api/session":
var body struct {
Model map[string]string `json:"model"`
}
if err := json.Unmarshal([]byte(args[4]), &body); err != nil {
t.Fatal(err)
}
want := effort
if effort == "default" {
want = ""
}
if effort == "off" {
want = "none"
}
if body.Model["providerID"] != "fixture" || body.Model["id"] != "model" || body.Model["variant"] != want {
t.Fatalf("wrong native selection: %+v", body.Model)
}
if want == "" {
if _, ok := body.Model["variant"]; ok {
t.Fatal("default unexpectedly overrides native variant")
}
}
created = true
return []byte(`{"data":{"id":"ses_effort"}}`), nil
case args[2] == "/api/session/ses_effort/prompt":
if !created {
t.Fatal("prompt before session selection")
}
prompts++
var body map[string]any
if err := json.Unmarshal([]byte(args[4]), &body); err != nil {
t.Fatal(err)
}
if _, ok := body["model"]; ok {
t.Fatal("prompt replaced the selected native model variant")
}
return []byte(`{}`), nil
case args[2] == "/api/session/active":
return []byte(`{"data":{}}`), nil
case strings.HasPrefix(args[2], "/api/session/ses_effort/message"):
return []byte(`{"data":[{"id":"msg_result","type":"assistant","finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess"}]}]}`), nil
case args[2] == "/api/session/ses_effort":
return usageResponse("ses_effort", 0, 1), nil
default:
t.Fatalf("unexpected call: %v", args)
return nil, nil
}
}
config := ModelConfig{Provider: "fixture", Model: "model", ReasoningEffort: effort}
var result string
var err error
if agent {
result, err = (&AgentClient{OpenCodeClient: client, Tools: []string{"read"}}).Generate(context.Background(), config, "task")
} else {
result, err = client.Generate(context.Background(), config, "task")
}
if err != nil || result != "# outcome\nsuccess" || prompts != 1 {
t.Fatalf("result=%q err=%v prompts=%d", result, err, prompts)
}
})
}
}
}
func TestOpenCodeRejectedEffortNeverAdmitsWork(t *testing.T) {
for _, agent := range []bool{false, true} {
client := OpenCodeClient{Backend: "v2", Directory: t.TempDir()}
calls := 0
client.command = func(_ context.Context, _ string, args ...string) ([]byte, error) {
calls++
if args[2] != "/api/session" {
t.Fatalf("work admitted after invalid native variant: %v", args)
}
return nil, errors.New("unknown model variant high")
}
config := ModelConfig{Provider: "fixture", Model: "no-reasoning", ReasoningEffort: "high"}
var err error
if agent {
_, err = (&AgentClient{OpenCodeClient: client}).Generate(context.Background(), config, "task")
} else {
_, err = client.Generate(context.Background(), config, "task")
}
if err == nil || calls != 1 || !strings.Contains(err.Error(), "reasoning_effort=high") {
t.Fatalf("unsupported effort was ignored: %v %d", err, calls)
}
}
}
-3
View File
@@ -111,9 +111,6 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri
if err != nil {
return "", err
}
if effort == "max" {
return "", fmt.Errorf("reasoning_effort=max is not supported by pi; use a supported thinking level up to xhigh")
}
thinking := effort
if thinking == "none" {
thinking = "off"
+3 -1
View File
@@ -85,6 +85,8 @@ func TestPiReasoningEffort(t *testing.T) {
{name: "medium", effort: "medium", flag: "medium"},
{name: "high", effort: "high", flag: "high"},
{name: "xhigh", effort: "xhigh", flag: "xhigh"},
{name: "max", effort: "max", flag: "max"},
{name: "max-clamped", effort: "max", flag: "max", fixture: "clamped", wantErr: `effective "high"`},
{name: "clamped", effort: "xhigh", flag: "xhigh", fixture: "clamped", wantErr: `effective "high"`},
{name: "off-clamped", effort: "off", flag: "off", fixture: "clamped", wantErr: `effective "high"`},
{name: "nonreasoning", effort: "high", flag: "high", fixture: "no-reasoning", wantErr: "cannot confirm reasoning support"},
@@ -174,7 +176,7 @@ func TestPiReasoningEffort(t *testing.T) {
}
func TestPiInvalidReasoningEffortBeforeProcess(t *testing.T) {
for _, effort := range []string{"max", "invalid", "HIGH", " high "} {
for _, effort := range []string{"invalid", "HIGH", " high "} {
t.Run(effort, func(t *testing.T) {
client := &PiClient{Executable: "/does-not-exist", WorkStarted: true}
_, err := client.Generate(context.Background(), ModelConfig{ReasoningEffort: effort}, "task")
+1 -6
View File
@@ -1,7 +1,6 @@
package stage
import (
"strings"
"testing"
)
@@ -31,11 +30,7 @@ func TestReasoningEffortConfig(t *testing.T) {
}
def.Configs["harness"] = "pi"
_, err = (ModelStage{}).Spec(def)
if value == "max" {
if err == nil || !strings.Contains(err.Error(), "pi") {
t.Fatal("unsupported Pi effort accepted")
}
} else if err != nil {
if err != nil {
t.Fatal(err)
}
})