feat(stage): honor per-stage reasoning effort across native harnesses
This commit is contained in:
@@ -1,5 +1,43 @@
|
||||
# Harnesses and runner-owned control
|
||||
|
||||
## Reasoning effort (thinking level)
|
||||
|
||||
Add an optional setting to the model stage's existing `configs` block:
|
||||
|
||||
````markdown
|
||||
```configs
|
||||
provider=openai
|
||||
model=gpt-5.2
|
||||
reasoning_effort=high
|
||||
```
|
||||
````
|
||||
|
||||
Use your actual configured provider/model IDs. Each stage chooses its own level;
|
||||
leaving it out, or using `default`, preserves the model/harness default.
|
||||
|
||||
| Setting | Behavior |
|
||||
| --- | --- |
|
||||
| `none` (alias `off`) | Disable reasoning, when supported |
|
||||
| `minimal`, `low`, `medium`, `high`, `xhigh`, `max` | Select that native effort level, when supported |
|
||||
| omitted / `default` | Leave existing defaults unchanged |
|
||||
|
||||
OpenCode V2 passes this as the selected model's named **variant**, for both
|
||||
text-only and tool-using stages. Variant availability is model-specific; an
|
||||
unavailable level is rejected, not silently replaced. See
|
||||
[OpenCode variants](https://opencode.ai/v2/docs/models#variants).
|
||||
Pi uses its native **thinking level**, mapping `none` to `off`, and verifies that
|
||||
the selected model actually applied it before sending the task. Unsupported or
|
||||
clamped levels fail clearly; availability also depends on the installed Pi
|
||||
version. OpenCode V1 cannot apply this setting and rejects explicit effort.
|
||||
|
||||
This is a native runtime option, not a prompt instruction or token limit. Higher
|
||||
effort can increase latency and cost; keep `budget_tokens`, `budget_usd`, and
|
||||
provider spending limits separate. Existing definitions, defaults and published
|
||||
versions are never rewritten. Editing the setting requires publishing a new
|
||||
pipeline version as usual.
|
||||
|
||||
## Harness defaults
|
||||
|
||||
Configure two independent model defaults:
|
||||
|
||||
```sh
|
||||
|
||||
@@ -79,7 +79,22 @@ authentication and inference require `pipelines provider-check` on the execution
|
||||
computer/container. Optional `retries` defaults to 2 (three total attempts),
|
||||
and accepts integers 0–100; explicit 0 disables execution retries. Optional
|
||||
`timeout` defaults to `1h` per attempt and accepts positive durations such as
|
||||
`30m` or `90s`. Automatic retries cover retryable transport failures and
|
||||
`30m` or `90s`.
|
||||
|
||||
Optional `reasoning_effort=high` selects a model's reasoning/thinking level for
|
||||
this stage. Values are `default`, `none` (alias `off`), `minimal`, `low`, `medium`,
|
||||
`high`, `xhigh`, or `max`. Omission or `default` preserves the harness/model
|
||||
default; never add a level automatically to existing workflows. Availability is
|
||||
model-specific: OpenCode V2 selects the corresponding named model variant and
|
||||
rejects unavailable variants; Pi uses its native thinking level (`none` becomes
|
||||
`off`) and verifies the effective level before prompting, rejecting unsupported
|
||||
or silently clamped levels. Explicit effort on OpenCode V1 is unsupported.
|
||||
Do not instruct the agent to “think harder” as a substitute for this native
|
||||
setting. Effort is not a token budget or a guaranteed quality improvement; keep
|
||||
usage budgets/provider spending limits separate. Both text-only and tool-using
|
||||
model stages honor the setting, without changing their tools or permissions.
|
||||
|
||||
Automatic retries cover retryable transport failures and
|
||||
timeouts; an explicit read-only stage may additionally retry a post-admission
|
||||
deadline or transport failure, while a stage with `edit`/`shell` (or no tools
|
||||
block) never replays admitted work. Optional `budget_usd=1.50` and
|
||||
@@ -89,7 +104,7 @@ to this stage in the same run. Omit either budget to leave that resource unlimit
|
||||
Both budgets must be positive; tokens include input, output, reasoning and cache
|
||||
reads/writes. Accounting failures stop budget-limited stages. In-flight requests
|
||||
may exceed limits before usage is reported; these are not provider-side hard caps.
|
||||
In current source, budgeted OpenCode V2 text-only calls use tracked prompting with
|
||||
In current source, all OpenCode V2 text-only calls use tracked prompting with
|
||||
all tools denied, rather than the usage-less `/generate` path. Completed
|
||||
token-budgeted responses reporting zero tokens fail closed. This is an execution
|
||||
implementation detail, not a new Markdown field: never add backend/format/API
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
func TestReasoningEffortDefinitionRoundTrip(t *testing.T) {
|
||||
source := strings.Replace(Template([]string{"worker", "review"}, true), "retries=2", "reasoning_effort=high\nretries=2", 1)
|
||||
graph, err := Parse(source)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if graph.Stages[0].Configs["reasoning_effort"] != "high" || graph.Stages[1].Configs["reasoning_effort"] != "" {
|
||||
t.Fatal("effort leaked across stage scopes")
|
||||
}
|
||||
spec, err := (stage.ModelStage{}).Spec(StageDefinition(graph.Stages[0]))
|
||||
if err != nil || spec.Config.ReasoningEffort != "high" {
|
||||
t.Fatalf("runtime lost effort: %+v %v", spec, err)
|
||||
}
|
||||
s, err := Open(filepath.Join(t.TempDir(), "effort.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
if _, err = s.Submit(context.Background(), "effort-fixture", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := s.List(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved, err := s.Get(context.Background(), items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if saved.Source != source || saved.Graph.Stages[0].Configs["reasoning_effort"] != "high" {
|
||||
t.Fatal("immutable definition dropped effort")
|
||||
}
|
||||
if _, err = Parse(strings.Replace(source, "reasoning_effort=high", "reasoning_effort=lots", 1)); err == nil {
|
||||
t.Fatal("invalid effort accepted by publication validation")
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,8 @@ func TestRunnerAppliesModelOverrides(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
if _, err := store.Submit(ctx, "overrides", pipeline.Template([]string{"one", "two"}, true)); err != nil {
|
||||
source := strings.Replace(pipeline.Template([]string{"one", "two"}, true), "retries=2", "reasoning_effort=high\nretries=2", 1)
|
||||
if _, err := store.Submit(ctx, "overrides", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, _ := store.List(ctx)
|
||||
@@ -55,7 +56,7 @@ func TestRunnerAppliesModelOverrides(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.wg.Wait()
|
||||
if got := resolved["one"]; got.Provider != "acme" || got.Model != "strong" {
|
||||
if got := resolved["one"]; got.Provider != "acme" || got.Model != "strong" || got.ReasoningEffort != "high" {
|
||||
t.Fatalf("stage one resolved %+v, want the override", got)
|
||||
}
|
||||
if got := resolved["two"]; got.Provider != pinned["two"].Provider || got.Model != pinned["two"].Model {
|
||||
@@ -88,7 +89,7 @@ func TestRunnerAppliesModelOverrides(t *testing.T) {
|
||||
for _, s := range overlaid.Graph.Stages {
|
||||
configs[s.Key] = s.Configs
|
||||
}
|
||||
if configs["one"]["provider"] != "acme" || configs["one"]["model"] != "strong" {
|
||||
if configs["one"]["provider"] != "acme" || configs["one"]["model"] != "strong" || configs["one"]["reasoning_effort"] != "high" {
|
||||
t.Fatalf("overlaid graph stage one = %+v, want the override", configs["one"])
|
||||
}
|
||||
if configs["two"]["provider"] != pinned["two"].Provider || configs["two"]["model"] != pinned["two"].Model {
|
||||
|
||||
@@ -97,8 +97,15 @@ func (c *AgentClient) Generate(ctx context.Context, config ModelConfig, prompt s
|
||||
}
|
||||
}
|
||||
rules := c.permissionRules()
|
||||
data, err := c.call(ctx, "post", "/api/session", map[string]any{"title": "Pipeline audited agent stage", "agent": "build", "location": map[string]string{"directory": c.Directory}, "model": map[string]string{"providerID": config.Provider, "id": config.Model}, "permissions": rules})
|
||||
model, err := openCodeModelSelection(config, c.Backend)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
data, err := c.call(ctx, "post", "/api/session", map[string]any{"title": "Pipeline audited agent stage", "agent": "build", "location": map[string]string{"directory": c.Directory}, "model": model, "permissions": rules})
|
||||
if err != nil {
|
||||
if variant := model["variant"]; variant != "" {
|
||||
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
|
||||
}
|
||||
// No prompt or tool call was admitted: a request failure here is a
|
||||
// reachable retry rather than a reason to end the run.
|
||||
return "", harnessUnreachable(err)
|
||||
|
||||
@@ -98,9 +98,6 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
|
||||
if err != nil {
|
||||
return ModelSpec{}, err
|
||||
}
|
||||
if backend == "pi" && effort == "max" {
|
||||
return ModelSpec{}, fmt.Errorf("reasoning_effort=max is not supported by pi; use a supported thinking level up to xhigh")
|
||||
}
|
||||
if delivery := def.Configs["delivery"]; delivery != "" && delivery != "true" {
|
||||
return ModelSpec{}, fmt.Errorf("delivery must be true when set")
|
||||
}
|
||||
|
||||
@@ -309,13 +309,20 @@ func (c *OpenCodeClient) Generate(ctx context.Context, config ModelConfig, promp
|
||||
if len(encoded) > maxModelRequest {
|
||||
return "", fmt.Errorf("OpenCode request exceeds 64 KiB limit")
|
||||
}
|
||||
model, err := openCodeModelSelection(config, c.Backend)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
created, err := c.call(ctx, "post", "/api/session", map[string]any{
|
||||
"title": "Pipeline stage generation",
|
||||
"location": map[string]string{"directory": directory},
|
||||
"model": map[string]string{"providerID": config.Provider, "id": config.Model},
|
||||
"model": model,
|
||||
"permissions": []map[string]string{{"action": "*", "resource": "*", "effect": "deny"}},
|
||||
})
|
||||
if err != nil {
|
||||
if variant := model["variant"]; variant != "" {
|
||||
err = fmt.Errorf("OpenCode model selection for reasoning_effort=%s (variant %q) failed: %w", variant, variant, err)
|
||||
}
|
||||
// No prompt or tool call was admitted: a request failure here is a
|
||||
// reachable retry rather than a reason to end the run.
|
||||
return "", harnessUnreachable(err)
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestOpenCodeEffortReachesAgentAndTextSession(t *testing.T) {
|
||||
for _, agent := range []bool{false, true} {
|
||||
for _, effort := range []string{"", "default", "none", "off", "minimal", "low", "medium", "high", "xhigh", "max"} {
|
||||
t.Run(effort+map[bool]string{true: "/agent", false: "/text"}[agent], func(t *testing.T) {
|
||||
prompts := 0
|
||||
created := false
|
||||
client := OpenCodeClient{Backend: "v2", Directory: t.TempDir(), sessionPollInterval: time.Millisecond}
|
||||
client.command = func(_ context.Context, _ string, args ...string) ([]byte, error) {
|
||||
switch {
|
||||
case args[2] == "/api/session":
|
||||
var body struct {
|
||||
Model map[string]string `json:"model"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(args[4]), &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := effort
|
||||
if effort == "default" {
|
||||
want = ""
|
||||
}
|
||||
if effort == "off" {
|
||||
want = "none"
|
||||
}
|
||||
if body.Model["providerID"] != "fixture" || body.Model["id"] != "model" || body.Model["variant"] != want {
|
||||
t.Fatalf("wrong native selection: %+v", body.Model)
|
||||
}
|
||||
if want == "" {
|
||||
if _, ok := body.Model["variant"]; ok {
|
||||
t.Fatal("default unexpectedly overrides native variant")
|
||||
}
|
||||
}
|
||||
created = true
|
||||
return []byte(`{"data":{"id":"ses_effort"}}`), nil
|
||||
case args[2] == "/api/session/ses_effort/prompt":
|
||||
if !created {
|
||||
t.Fatal("prompt before session selection")
|
||||
}
|
||||
prompts++
|
||||
var body map[string]any
|
||||
if err := json.Unmarshal([]byte(args[4]), &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := body["model"]; ok {
|
||||
t.Fatal("prompt replaced the selected native model variant")
|
||||
}
|
||||
return []byte(`{}`), nil
|
||||
case args[2] == "/api/session/active":
|
||||
return []byte(`{"data":{}}`), nil
|
||||
case strings.HasPrefix(args[2], "/api/session/ses_effort/message"):
|
||||
return []byte(`{"data":[{"id":"msg_result","type":"assistant","finish":"stop","content":[{"type":"text","text":"# outcome\nsuccess"}]}]}`), nil
|
||||
case args[2] == "/api/session/ses_effort":
|
||||
return usageResponse("ses_effort", 0, 1), nil
|
||||
default:
|
||||
t.Fatalf("unexpected call: %v", args)
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
config := ModelConfig{Provider: "fixture", Model: "model", ReasoningEffort: effort}
|
||||
var result string
|
||||
var err error
|
||||
if agent {
|
||||
result, err = (&AgentClient{OpenCodeClient: client, Tools: []string{"read"}}).Generate(context.Background(), config, "task")
|
||||
} else {
|
||||
result, err = client.Generate(context.Background(), config, "task")
|
||||
}
|
||||
if err != nil || result != "# outcome\nsuccess" || prompts != 1 {
|
||||
t.Fatalf("result=%q err=%v prompts=%d", result, err, prompts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenCodeRejectedEffortNeverAdmitsWork(t *testing.T) {
|
||||
for _, agent := range []bool{false, true} {
|
||||
client := OpenCodeClient{Backend: "v2", Directory: t.TempDir()}
|
||||
calls := 0
|
||||
client.command = func(_ context.Context, _ string, args ...string) ([]byte, error) {
|
||||
calls++
|
||||
if args[2] != "/api/session" {
|
||||
t.Fatalf("work admitted after invalid native variant: %v", args)
|
||||
}
|
||||
return nil, errors.New("unknown model variant high")
|
||||
}
|
||||
config := ModelConfig{Provider: "fixture", Model: "no-reasoning", ReasoningEffort: "high"}
|
||||
var err error
|
||||
if agent {
|
||||
_, err = (&AgentClient{OpenCodeClient: client}).Generate(context.Background(), config, "task")
|
||||
} else {
|
||||
_, err = client.Generate(context.Background(), config, "task")
|
||||
}
|
||||
if err == nil || calls != 1 || !strings.Contains(err.Error(), "reasoning_effort=high") {
|
||||
t.Fatalf("unsupported effort was ignored: %v %d", err, calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -111,9 +111,6 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if effort == "max" {
|
||||
return "", fmt.Errorf("reasoning_effort=max is not supported by pi; use a supported thinking level up to xhigh")
|
||||
}
|
||||
thinking := effort
|
||||
if thinking == "none" {
|
||||
thinking = "off"
|
||||
|
||||
@@ -85,6 +85,8 @@ func TestPiReasoningEffort(t *testing.T) {
|
||||
{name: "medium", effort: "medium", flag: "medium"},
|
||||
{name: "high", effort: "high", flag: "high"},
|
||||
{name: "xhigh", effort: "xhigh", flag: "xhigh"},
|
||||
{name: "max", effort: "max", flag: "max"},
|
||||
{name: "max-clamped", effort: "max", flag: "max", fixture: "clamped", wantErr: `effective "high"`},
|
||||
{name: "clamped", effort: "xhigh", flag: "xhigh", fixture: "clamped", wantErr: `effective "high"`},
|
||||
{name: "off-clamped", effort: "off", flag: "off", fixture: "clamped", wantErr: `effective "high"`},
|
||||
{name: "nonreasoning", effort: "high", flag: "high", fixture: "no-reasoning", wantErr: "cannot confirm reasoning support"},
|
||||
@@ -174,7 +176,7 @@ func TestPiReasoningEffort(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPiInvalidReasoningEffortBeforeProcess(t *testing.T) {
|
||||
for _, effort := range []string{"max", "invalid", "HIGH", " high "} {
|
||||
for _, effort := range []string{"invalid", "HIGH", " high "} {
|
||||
t.Run(effort, func(t *testing.T) {
|
||||
client := &PiClient{Executable: "/does-not-exist", WorkStarted: true}
|
||||
_, err := client.Generate(context.Background(), ModelConfig{ReasoningEffort: effort}, "task")
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -31,11 +30,7 @@ func TestReasoningEffortConfig(t *testing.T) {
|
||||
}
|
||||
def.Configs["harness"] = "pi"
|
||||
_, err = (ModelStage{}).Spec(def)
|
||||
if value == "max" {
|
||||
if err == nil || !strings.Contains(err.Error(), "pi") {
|
||||
t.Fatal("unsupported Pi effort accepted")
|
||||
}
|
||||
} else if err != nil {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user