feat(stage): apply and verify Pi reasoning effort before prompting
This commit is contained in:
+33
-3
@@ -107,6 +107,17 @@ func piTools(tools []string) ([]string, error) {
|
||||
|
||||
func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) {
|
||||
c.WorkStarted = false
|
||||
effort, err := normalizeReasoningEffort(config.ReasoningEffort)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if effort == "max" {
|
||||
return "", fmt.Errorf("reasoning_effort=max is not supported by pi; use a supported thinking level up to xhigh")
|
||||
}
|
||||
thinking := effort
|
||||
if thinking == "none" {
|
||||
thinking = "off"
|
||||
}
|
||||
c.ready = false
|
||||
c.settled = false
|
||||
c.runErr = nil
|
||||
@@ -146,6 +157,9 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri
|
||||
c.Executable = "pi"
|
||||
}
|
||||
args := []string{"--mode", "rpc", "--provider", config.Provider, "--model", config.Model, "--session-dir", c.Reference, "--no-approve", "--no-extensions", "--no-skills", "--no-context-files", "--no-prompt-templates", "--no-themes", "--extension", filepath.Join(c.Reference, "policy.js")}
|
||||
if thinking != "" {
|
||||
args = append(args, "--thinking", thinking)
|
||||
}
|
||||
if len(tools) == 0 {
|
||||
args = append(args, "--no-tools")
|
||||
} else {
|
||||
@@ -263,9 +277,13 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri
|
||||
return "", err
|
||||
}
|
||||
var session struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
SessionFile string `json:"sessionFile"`
|
||||
Model struct{ ID, Provider string } `json:"model"`
|
||||
SessionID string `json:"sessionId"`
|
||||
SessionFile string `json:"sessionFile"`
|
||||
ThinkingLevel string `json:"thinkingLevel"`
|
||||
Model struct {
|
||||
ID, Provider string
|
||||
Reasoning *bool `json:"reasoning"`
|
||||
} `json:"model"`
|
||||
}
|
||||
ready, _ := os.ReadFile(filepath.Join(c.Reference, "ready"))
|
||||
c.ready = string(ready) == "ready"
|
||||
@@ -280,6 +298,18 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri
|
||||
if err = os.WriteFile(filepath.Join(c.Reference, "owner.json"), owner, 0600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Pi clamps unsupported thinking levels during startup. Verify its effective
|
||||
// state before claiming work or sending a prompt; never silently accept a
|
||||
// model default or change the model. Omission leaves the existing RPC flow
|
||||
// untouched. Protocol: https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/rpc-commands.md#get_state
|
||||
if thinking != "" {
|
||||
if thinking != "off" && (session.Model.Reasoning == nil || !*session.Model.Reasoning) {
|
||||
return "", fmt.Errorf("Pi cannot confirm reasoning support for %s/%s (reasoning_effort=%s)", config.Provider, config.Model, effort)
|
||||
}
|
||||
if session.ThinkingLevel != thinking {
|
||||
return "", fmt.Errorf("Pi did not apply reasoning_effort=%s for %s/%s: requested thinking level %q, effective %q (unsupported, clamped, or unavailable)", effort, config.Provider, config.Model, thinking, session.ThinkingLevel)
|
||||
}
|
||||
}
|
||||
for _, command := range []string{"set_auto_retry", "set_auto_compaction"} {
|
||||
if _, err = c.rpc(ctx, command, map[string]any{"enabled": false}); err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
package stage
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Isolated protocol fixture, not provider evidence. Mirrors Pi 1.3.0's
|
||||
// --thinking and get_state (model.reasoning + effective thinkingLevel):
|
||||
// https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/cli.md
|
||||
// https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/rpc-commands.md
|
||||
func TestPiEffortRPCFixture(t *testing.T) {
|
||||
if os.Getenv("PIPELINES_PI_EFFORT_FIXTURE") != "1" {
|
||||
return
|
||||
}
|
||||
root := filepath.Dir(os.Getenv("PIPELINES_PI_POLICY"))
|
||||
args, _ := json.Marshal(os.Args)
|
||||
_ = os.WriteFile(filepath.Join(root, "args.json"), args, 0600)
|
||||
_ = os.WriteFile(filepath.Join(root, "ready"), []byte("ready"), 0600)
|
||||
_ = os.WriteFile(filepath.Join(root, "session.jsonl"), []byte("{\"type\":\"session\",\"id\":\"effort-fixture\"}\n"), 0600)
|
||||
level := "medium"
|
||||
for i, arg := range os.Args {
|
||||
if arg == "--thinking" && i+1 < len(os.Args) {
|
||||
level = os.Args[i+1]
|
||||
}
|
||||
}
|
||||
model := map[string]any{"id": "model", "provider": "fixture", "reasoning": true}
|
||||
switch os.Getenv("PI_EFFORT_CASE") {
|
||||
case "clamped":
|
||||
level = "high"
|
||||
case "no-reasoning":
|
||||
model["reasoning"] = false
|
||||
case "unknown-reasoning":
|
||||
delete(model, "reasoning")
|
||||
case "missing-level":
|
||||
level = ""
|
||||
}
|
||||
message := map[string]any{"role": "assistant", "stopReason": "stop", "content": []map[string]string{{"type": "text", "text": "done"}}}
|
||||
emit := func(v any) { data, _ := json.Marshal(v); fmt.Println(string(data)) }
|
||||
log, _ := os.OpenFile(filepath.Join(root, "requests.jsonl"), os.O_CREATE|os.O_WRONLY, 0600)
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
for scanner.Scan() {
|
||||
_, _ = log.Write(append(append([]byte{}, scanner.Bytes()...), '\n'))
|
||||
var request map[string]any
|
||||
_ = json.Unmarshal(scanner.Bytes(), &request)
|
||||
kind := request["type"].(string)
|
||||
var data any
|
||||
switch kind {
|
||||
case "get_state":
|
||||
data = map[string]any{"sessionId": "effort-fixture", "sessionFile": filepath.Join(root, "session.jsonl"), "model": model, "thinkingLevel": level}
|
||||
case "prompt":
|
||||
emit(map[string]any{"type": "message_end", "message": message})
|
||||
emit(map[string]string{"type": "agent_settled"})
|
||||
case "get_session_stats":
|
||||
data = map[string]any{"sessionId": "effort-fixture", "assistantMessages": 1, "tokens": map[string]int{"total": 10}, "cost": 0.1}
|
||||
case "get_messages":
|
||||
data = map[string]any{"messages": []any{message}}
|
||||
}
|
||||
emit(map[string]any{"type": "response", "command": kind, "id": request["id"], "success": true, "data": data})
|
||||
}
|
||||
_ = log.Close()
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
func TestPiReasoningEffort(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, effort, fixture, flag, wantErr string
|
||||
}{
|
||||
{name: "omitted"},
|
||||
{name: "default", effort: "default"},
|
||||
{name: "none", effort: "none", flag: "off"},
|
||||
{name: "off-alias", effort: "off", flag: "off"},
|
||||
{name: "off-nonreasoning", effort: "none", flag: "off", fixture: "no-reasoning"},
|
||||
{name: "minimal", effort: "minimal", flag: "minimal"},
|
||||
{name: "low", effort: "low", flag: "low"},
|
||||
{name: "medium", effort: "medium", flag: "medium"},
|
||||
{name: "high", effort: "high", flag: "high"},
|
||||
{name: "xhigh", effort: "xhigh", flag: "xhigh"},
|
||||
{name: "clamped", effort: "xhigh", flag: "xhigh", fixture: "clamped", wantErr: `effective "high"`},
|
||||
{name: "off-clamped", effort: "off", flag: "off", fixture: "clamped", wantErr: `effective "high"`},
|
||||
{name: "nonreasoning", effort: "high", flag: "high", fixture: "no-reasoning", wantErr: "cannot confirm reasoning support"},
|
||||
{name: "unknown-model-capability", effort: "high", flag: "high", fixture: "unknown-reasoning", wantErr: "cannot confirm reasoning support"},
|
||||
{name: "missing-effective-level", effort: "high", flag: "high", fixture: "missing-level", wantErr: `effective ""`},
|
||||
{name: "omitted-legacy-state", fixture: "unknown-reasoning"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
t.Setenv("HOME", root)
|
||||
t.Setenv("XDG_CACHE_HOME", root)
|
||||
t.Setenv("PIPELINES_PI_EFFORT_FIXTURE", "1")
|
||||
t.Setenv("PI_EFFORT_CASE", tc.fixture)
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wrapper := filepath.Join(root, "pi")
|
||||
if err := os.WriteFile(wrapper, []byte("#!/bin/sh\nexec '"+executable+"' -test.run=^TestPiEffortRPCFixture$ -- \"$@\"\n"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var active []bool
|
||||
client := &PiClient{Executable: wrapper, Directory: root, Tools: []string{}, OnOwnership: func(id, ref string, value bool) error {
|
||||
if id != "effort-fixture" || ref == "" {
|
||||
t.Error("missing native ownership", id, ref)
|
||||
}
|
||||
active = append(active, value)
|
||||
return nil
|
||||
}}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
text, err := client.Generate(ctx, ModelConfig{Provider: "fixture", Model: "model", ReasoningEffort: tc.effort}, "task")
|
||||
if tc.wantErr == "" {
|
||||
if err != nil || text != "done" || !client.WorkStarted || !reflect.DeepEqual(active, []bool{true, false}) {
|
||||
t.Fatal(text, err, client.WorkStarted, active)
|
||||
}
|
||||
} else if err == nil || !strings.Contains(err.Error(), tc.wantErr) || text != "" || client.WorkStarted || len(active) != 0 {
|
||||
t.Fatal("expected pre-prompt rejection", text, err, client.WorkStarted, active)
|
||||
}
|
||||
if errors.Is(err, ErrStageCleanup) {
|
||||
t.Fatal("cleanup failed", err)
|
||||
}
|
||||
if quiet, err := os.ReadFile(filepath.Join(client.Reference, "quiescent")); err != nil || string(quiet) != "quiescent" {
|
||||
t.Fatal("missing quiescence", string(quiet), err)
|
||||
}
|
||||
if evidence, err := PiEvidence(client.Reference); err != nil || !strings.Contains(evidence, "effort-fixture") {
|
||||
t.Fatal("native history lost", evidence, err)
|
||||
}
|
||||
argsData, err := os.ReadFile(filepath.Join(client.Reference, "args.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var args []string
|
||||
if err := json.Unmarshal(argsData, &args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var flags []string
|
||||
for i, arg := range args {
|
||||
if arg == "--thinking" {
|
||||
flags = append(flags, args[i+1])
|
||||
}
|
||||
}
|
||||
if (tc.flag == "" && len(flags) != 0) || (tc.flag != "" && !reflect.DeepEqual(flags, []string{tc.flag})) {
|
||||
t.Fatal("thinking flags", flags, "want", tc.flag)
|
||||
}
|
||||
requests, err := os.ReadFile(filepath.Join(client.Reference, "requests.jsonl"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var commands []string
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(requests)), "\n") {
|
||||
var request struct{ Type string }
|
||||
if err := json.Unmarshal([]byte(line), &request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
commands = append(commands, request.Type)
|
||||
}
|
||||
want := []string{"get_state", "get_state"}
|
||||
if tc.wantErr == "" {
|
||||
want = []string{"get_state", "set_auto_retry", "set_auto_compaction", "prompt", "get_session_stats", "get_messages", "get_state"}
|
||||
}
|
||||
if !reflect.DeepEqual(commands, want) {
|
||||
t.Fatal("RPC sequence", commands, "want", want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPiInvalidReasoningEffortBeforeProcess(t *testing.T) {
|
||||
for _, effort := range []string{"max", "invalid", "HIGH", " high "} {
|
||||
t.Run(effort, func(t *testing.T) {
|
||||
client := &PiClient{Executable: "/does-not-exist", WorkStarted: true}
|
||||
_, err := client.Generate(context.Background(), ModelConfig{ReasoningEffort: effort}, "task")
|
||||
if err == nil || !strings.Contains(err.Error(), "reasoning_effort") || client.Reference != "" || client.cmd != nil || client.WorkStarted {
|
||||
t.Fatal("expected rejection before process/session creation", err, client)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user