feat(stage): apply and verify Pi reasoning effort before prompting

This commit is contained in:
2026-10-20 23:44:26 +01:00
parent 2aaf7fe2cc
commit 9ce2c5d143
2 changed files with 219 additions and 3 deletions
+33 -3
View File
@@ -107,6 +107,17 @@ func piTools(tools []string) ([]string, error) {
func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) {
c.WorkStarted = false
effort, err := normalizeReasoningEffort(config.ReasoningEffort)
if err != nil {
return "", err
}
if effort == "max" {
return "", fmt.Errorf("reasoning_effort=max is not supported by pi; use a supported thinking level up to xhigh")
}
thinking := effort
if thinking == "none" {
thinking = "off"
}
c.ready = false
c.settled = false
c.runErr = nil
@@ -146,6 +157,9 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri
c.Executable = "pi"
}
args := []string{"--mode", "rpc", "--provider", config.Provider, "--model", config.Model, "--session-dir", c.Reference, "--no-approve", "--no-extensions", "--no-skills", "--no-context-files", "--no-prompt-templates", "--no-themes", "--extension", filepath.Join(c.Reference, "policy.js")}
if thinking != "" {
args = append(args, "--thinking", thinking)
}
if len(tools) == 0 {
args = append(args, "--no-tools")
} else {
@@ -263,9 +277,13 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri
return "", err
}
var session struct {
SessionID string `json:"sessionId"`
SessionFile string `json:"sessionFile"`
Model struct{ ID, Provider string } `json:"model"`
SessionID string `json:"sessionId"`
SessionFile string `json:"sessionFile"`
ThinkingLevel string `json:"thinkingLevel"`
Model struct {
ID, Provider string
Reasoning *bool `json:"reasoning"`
} `json:"model"`
}
ready, _ := os.ReadFile(filepath.Join(c.Reference, "ready"))
c.ready = string(ready) == "ready"
@@ -280,6 +298,18 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri
if err = os.WriteFile(filepath.Join(c.Reference, "owner.json"), owner, 0600); err != nil {
return "", err
}
// Pi clamps unsupported thinking levels during startup. Verify its effective
// state before claiming work or sending a prompt; never silently accept a
// model default or change the model. Omission leaves the existing RPC flow
// untouched. Protocol: https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/rpc-commands.md#get_state
if thinking != "" {
if thinking != "off" && (session.Model.Reasoning == nil || !*session.Model.Reasoning) {
return "", fmt.Errorf("Pi cannot confirm reasoning support for %s/%s (reasoning_effort=%s)", config.Provider, config.Model, effort)
}
if session.ThinkingLevel != thinking {
return "", fmt.Errorf("Pi did not apply reasoning_effort=%s for %s/%s: requested thinking level %q, effective %q (unsupported, clamped, or unavailable)", effort, config.Provider, config.Model, thinking, session.ThinkingLevel)
}
}
for _, command := range []string{"set_auto_retry", "set_auto_compaction"} {
if _, err = c.rpc(ctx, command, map[string]any{"enabled": false}); err != nil {
return "", err
+186
View File
@@ -0,0 +1,186 @@
package stage
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
)
// Isolated protocol fixture, not provider evidence. Mirrors Pi 1.3.0's
// --thinking and get_state (model.reasoning + effective thinkingLevel):
// https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/cli.md
// https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/rpc-commands.md
func TestPiEffortRPCFixture(t *testing.T) {
if os.Getenv("PIPELINES_PI_EFFORT_FIXTURE") != "1" {
return
}
root := filepath.Dir(os.Getenv("PIPELINES_PI_POLICY"))
args, _ := json.Marshal(os.Args)
_ = os.WriteFile(filepath.Join(root, "args.json"), args, 0600)
_ = os.WriteFile(filepath.Join(root, "ready"), []byte("ready"), 0600)
_ = os.WriteFile(filepath.Join(root, "session.jsonl"), []byte("{\"type\":\"session\",\"id\":\"effort-fixture\"}\n"), 0600)
level := "medium"
for i, arg := range os.Args {
if arg == "--thinking" && i+1 < len(os.Args) {
level = os.Args[i+1]
}
}
model := map[string]any{"id": "model", "provider": "fixture", "reasoning": true}
switch os.Getenv("PI_EFFORT_CASE") {
case "clamped":
level = "high"
case "no-reasoning":
model["reasoning"] = false
case "unknown-reasoning":
delete(model, "reasoning")
case "missing-level":
level = ""
}
message := map[string]any{"role": "assistant", "stopReason": "stop", "content": []map[string]string{{"type": "text", "text": "done"}}}
emit := func(v any) { data, _ := json.Marshal(v); fmt.Println(string(data)) }
log, _ := os.OpenFile(filepath.Join(root, "requests.jsonl"), os.O_CREATE|os.O_WRONLY, 0600)
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
_, _ = log.Write(append(append([]byte{}, scanner.Bytes()...), '\n'))
var request map[string]any
_ = json.Unmarshal(scanner.Bytes(), &request)
kind := request["type"].(string)
var data any
switch kind {
case "get_state":
data = map[string]any{"sessionId": "effort-fixture", "sessionFile": filepath.Join(root, "session.jsonl"), "model": model, "thinkingLevel": level}
case "prompt":
emit(map[string]any{"type": "message_end", "message": message})
emit(map[string]string{"type": "agent_settled"})
case "get_session_stats":
data = map[string]any{"sessionId": "effort-fixture", "assistantMessages": 1, "tokens": map[string]int{"total": 10}, "cost": 0.1}
case "get_messages":
data = map[string]any{"messages": []any{message}}
}
emit(map[string]any{"type": "response", "command": kind, "id": request["id"], "success": true, "data": data})
}
_ = log.Close()
os.Exit(0)
}
func TestPiReasoningEffort(t *testing.T) {
for _, tc := range []struct {
name, effort, fixture, flag, wantErr string
}{
{name: "omitted"},
{name: "default", effort: "default"},
{name: "none", effort: "none", flag: "off"},
{name: "off-alias", effort: "off", flag: "off"},
{name: "off-nonreasoning", effort: "none", flag: "off", fixture: "no-reasoning"},
{name: "minimal", effort: "minimal", flag: "minimal"},
{name: "low", effort: "low", flag: "low"},
{name: "medium", effort: "medium", flag: "medium"},
{name: "high", effort: "high", flag: "high"},
{name: "xhigh", effort: "xhigh", flag: "xhigh"},
{name: "clamped", effort: "xhigh", flag: "xhigh", fixture: "clamped", wantErr: `effective "high"`},
{name: "off-clamped", effort: "off", flag: "off", fixture: "clamped", wantErr: `effective "high"`},
{name: "nonreasoning", effort: "high", flag: "high", fixture: "no-reasoning", wantErr: "cannot confirm reasoning support"},
{name: "unknown-model-capability", effort: "high", flag: "high", fixture: "unknown-reasoning", wantErr: "cannot confirm reasoning support"},
{name: "missing-effective-level", effort: "high", flag: "high", fixture: "missing-level", wantErr: `effective ""`},
{name: "omitted-legacy-state", fixture: "unknown-reasoning"},
} {
t.Run(tc.name, func(t *testing.T) {
root := t.TempDir()
t.Setenv("HOME", root)
t.Setenv("XDG_CACHE_HOME", root)
t.Setenv("PIPELINES_PI_EFFORT_FIXTURE", "1")
t.Setenv("PI_EFFORT_CASE", tc.fixture)
executable, err := os.Executable()
if err != nil {
t.Fatal(err)
}
wrapper := filepath.Join(root, "pi")
if err := os.WriteFile(wrapper, []byte("#!/bin/sh\nexec '"+executable+"' -test.run=^TestPiEffortRPCFixture$ -- \"$@\"\n"), 0700); err != nil {
t.Fatal(err)
}
var active []bool
client := &PiClient{Executable: wrapper, Directory: root, Tools: []string{}, OnOwnership: func(id, ref string, value bool) error {
if id != "effort-fixture" || ref == "" {
t.Error("missing native ownership", id, ref)
}
active = append(active, value)
return nil
}}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
text, err := client.Generate(ctx, ModelConfig{Provider: "fixture", Model: "model", ReasoningEffort: tc.effort}, "task")
if tc.wantErr == "" {
if err != nil || text != "done" || !client.WorkStarted || !reflect.DeepEqual(active, []bool{true, false}) {
t.Fatal(text, err, client.WorkStarted, active)
}
} else if err == nil || !strings.Contains(err.Error(), tc.wantErr) || text != "" || client.WorkStarted || len(active) != 0 {
t.Fatal("expected pre-prompt rejection", text, err, client.WorkStarted, active)
}
if errors.Is(err, ErrStageCleanup) {
t.Fatal("cleanup failed", err)
}
if quiet, err := os.ReadFile(filepath.Join(client.Reference, "quiescent")); err != nil || string(quiet) != "quiescent" {
t.Fatal("missing quiescence", string(quiet), err)
}
if evidence, err := PiEvidence(client.Reference); err != nil || !strings.Contains(evidence, "effort-fixture") {
t.Fatal("native history lost", evidence, err)
}
argsData, err := os.ReadFile(filepath.Join(client.Reference, "args.json"))
if err != nil {
t.Fatal(err)
}
var args []string
if err := json.Unmarshal(argsData, &args); err != nil {
t.Fatal(err)
}
var flags []string
for i, arg := range args {
if arg == "--thinking" {
flags = append(flags, args[i+1])
}
}
if (tc.flag == "" && len(flags) != 0) || (tc.flag != "" && !reflect.DeepEqual(flags, []string{tc.flag})) {
t.Fatal("thinking flags", flags, "want", tc.flag)
}
requests, err := os.ReadFile(filepath.Join(client.Reference, "requests.jsonl"))
if err != nil {
t.Fatal(err)
}
var commands []string
for _, line := range strings.Split(strings.TrimSpace(string(requests)), "\n") {
var request struct{ Type string }
if err := json.Unmarshal([]byte(line), &request); err != nil {
t.Fatal(err)
}
commands = append(commands, request.Type)
}
want := []string{"get_state", "get_state"}
if tc.wantErr == "" {
want = []string{"get_state", "set_auto_retry", "set_auto_compaction", "prompt", "get_session_stats", "get_messages", "get_state"}
}
if !reflect.DeepEqual(commands, want) {
t.Fatal("RPC sequence", commands, "want", want)
}
})
}
}
func TestPiInvalidReasoningEffortBeforeProcess(t *testing.T) {
for _, effort := range []string{"max", "invalid", "HIGH", " high "} {
t.Run(effort, func(t *testing.T) {
client := &PiClient{Executable: "/does-not-exist", WorkStarted: true}
_, err := client.Generate(context.Background(), ModelConfig{ReasoningEffort: effort}, "task")
if err == nil || !strings.Contains(err.Error(), "reasoning_effort") || client.Reference != "" || client.cmd != nil || client.WorkStarted {
t.Fatal("expected rejection before process/session creation", err, client)
}
})
}
}