From 9ce2c5d14373d9877c95b84073bd8e9eed86c2f6 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Tue, 20 Oct 2026 23:44:26 +0100 Subject: [PATCH] feat(stage): apply and verify Pi reasoning effort before prompting --- internal/stage/pi.go | 36 +++++- internal/stage/pi_effort_test.go | 186 +++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+), 3 deletions(-) create mode 100644 internal/stage/pi_effort_test.go diff --git a/internal/stage/pi.go b/internal/stage/pi.go index 0c2edbc8..78b845ad 100644 --- a/internal/stage/pi.go +++ b/internal/stage/pi.go @@ -107,6 +107,17 @@ func piTools(tools []string) ([]string, error) { func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) { c.WorkStarted = false + effort, err := normalizeReasoningEffort(config.ReasoningEffort) + if err != nil { + return "", err + } + if effort == "max" { + return "", fmt.Errorf("reasoning_effort=max is not supported by pi; use a supported thinking level up to xhigh") + } + thinking := effort + if thinking == "none" { + thinking = "off" + } c.ready = false c.settled = false c.runErr = nil @@ -146,6 +157,9 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri c.Executable = "pi" } args := []string{"--mode", "rpc", "--provider", config.Provider, "--model", config.Model, "--session-dir", c.Reference, "--no-approve", "--no-extensions", "--no-skills", "--no-context-files", "--no-prompt-templates", "--no-themes", "--extension", filepath.Join(c.Reference, "policy.js")} + if thinking != "" { + args = append(args, "--thinking", thinking) + } if len(tools) == 0 { args = append(args, "--no-tools") } else { @@ -263,9 +277,13 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri return "", err } var session struct { - SessionID string `json:"sessionId"` - SessionFile string `json:"sessionFile"` - Model struct{ ID, Provider string } `json:"model"` + SessionID string `json:"sessionId"` + SessionFile string `json:"sessionFile"` + ThinkingLevel string `json:"thinkingLevel"` + Model struct { + ID, Provider string + Reasoning *bool `json:"reasoning"` + } `json:"model"` } ready, _ := os.ReadFile(filepath.Join(c.Reference, "ready")) c.ready = string(ready) == "ready" @@ -280,6 +298,18 @@ func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt stri if err = os.WriteFile(filepath.Join(c.Reference, "owner.json"), owner, 0600); err != nil { return "", err } + // Pi clamps unsupported thinking levels during startup. Verify its effective + // state before claiming work or sending a prompt; never silently accept a + // model default or change the model. Omission leaves the existing RPC flow + // untouched. Protocol: https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/rpc-commands.md#get_state + if thinking != "" { + if thinking != "off" && (session.Model.Reasoning == nil || !*session.Model.Reasoning) { + return "", fmt.Errorf("Pi cannot confirm reasoning support for %s/%s (reasoning_effort=%s)", config.Provider, config.Model, effort) + } + if session.ThinkingLevel != thinking { + return "", fmt.Errorf("Pi did not apply reasoning_effort=%s for %s/%s: requested thinking level %q, effective %q (unsupported, clamped, or unavailable)", effort, config.Provider, config.Model, thinking, session.ThinkingLevel) + } + } for _, command := range []string{"set_auto_retry", "set_auto_compaction"} { if _, err = c.rpc(ctx, command, map[string]any{"enabled": false}); err != nil { return "", err diff --git a/internal/stage/pi_effort_test.go b/internal/stage/pi_effort_test.go new file mode 100644 index 00000000..6cc0bdf3 --- /dev/null +++ b/internal/stage/pi_effort_test.go @@ -0,0 +1,186 @@ +package stage + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +// Isolated protocol fixture, not provider evidence. Mirrors Pi 1.3.0's +// --thinking and get_state (model.reasoning + effective thinkingLevel): +// https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/cli.md +// https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/rpc-commands.md +func TestPiEffortRPCFixture(t *testing.T) { + if os.Getenv("PIPELINES_PI_EFFORT_FIXTURE") != "1" { + return + } + root := filepath.Dir(os.Getenv("PIPELINES_PI_POLICY")) + args, _ := json.Marshal(os.Args) + _ = os.WriteFile(filepath.Join(root, "args.json"), args, 0600) + _ = os.WriteFile(filepath.Join(root, "ready"), []byte("ready"), 0600) + _ = os.WriteFile(filepath.Join(root, "session.jsonl"), []byte("{\"type\":\"session\",\"id\":\"effort-fixture\"}\n"), 0600) + level := "medium" + for i, arg := range os.Args { + if arg == "--thinking" && i+1 < len(os.Args) { + level = os.Args[i+1] + } + } + model := map[string]any{"id": "model", "provider": "fixture", "reasoning": true} + switch os.Getenv("PI_EFFORT_CASE") { + case "clamped": + level = "high" + case "no-reasoning": + model["reasoning"] = false + case "unknown-reasoning": + delete(model, "reasoning") + case "missing-level": + level = "" + } + message := map[string]any{"role": "assistant", "stopReason": "stop", "content": []map[string]string{{"type": "text", "text": "done"}}} + emit := func(v any) { data, _ := json.Marshal(v); fmt.Println(string(data)) } + log, _ := os.OpenFile(filepath.Join(root, "requests.jsonl"), os.O_CREATE|os.O_WRONLY, 0600) + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + _, _ = log.Write(append(append([]byte{}, scanner.Bytes()...), '\n')) + var request map[string]any + _ = json.Unmarshal(scanner.Bytes(), &request) + kind := request["type"].(string) + var data any + switch kind { + case "get_state": + data = map[string]any{"sessionId": "effort-fixture", "sessionFile": filepath.Join(root, "session.jsonl"), "model": model, "thinkingLevel": level} + case "prompt": + emit(map[string]any{"type": "message_end", "message": message}) + emit(map[string]string{"type": "agent_settled"}) + case "get_session_stats": + data = map[string]any{"sessionId": "effort-fixture", "assistantMessages": 1, "tokens": map[string]int{"total": 10}, "cost": 0.1} + case "get_messages": + data = map[string]any{"messages": []any{message}} + } + emit(map[string]any{"type": "response", "command": kind, "id": request["id"], "success": true, "data": data}) + } + _ = log.Close() + os.Exit(0) +} + +func TestPiReasoningEffort(t *testing.T) { + for _, tc := range []struct { + name, effort, fixture, flag, wantErr string + }{ + {name: "omitted"}, + {name: "default", effort: "default"}, + {name: "none", effort: "none", flag: "off"}, + {name: "off-alias", effort: "off", flag: "off"}, + {name: "off-nonreasoning", effort: "none", flag: "off", fixture: "no-reasoning"}, + {name: "minimal", effort: "minimal", flag: "minimal"}, + {name: "low", effort: "low", flag: "low"}, + {name: "medium", effort: "medium", flag: "medium"}, + {name: "high", effort: "high", flag: "high"}, + {name: "xhigh", effort: "xhigh", flag: "xhigh"}, + {name: "clamped", effort: "xhigh", flag: "xhigh", fixture: "clamped", wantErr: `effective "high"`}, + {name: "off-clamped", effort: "off", flag: "off", fixture: "clamped", wantErr: `effective "high"`}, + {name: "nonreasoning", effort: "high", flag: "high", fixture: "no-reasoning", wantErr: "cannot confirm reasoning support"}, + {name: "unknown-model-capability", effort: "high", flag: "high", fixture: "unknown-reasoning", wantErr: "cannot confirm reasoning support"}, + {name: "missing-effective-level", effort: "high", flag: "high", fixture: "missing-level", wantErr: `effective ""`}, + {name: "omitted-legacy-state", fixture: "unknown-reasoning"}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + t.Setenv("HOME", root) + t.Setenv("XDG_CACHE_HOME", root) + t.Setenv("PIPELINES_PI_EFFORT_FIXTURE", "1") + t.Setenv("PI_EFFORT_CASE", tc.fixture) + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + wrapper := filepath.Join(root, "pi") + if err := os.WriteFile(wrapper, []byte("#!/bin/sh\nexec '"+executable+"' -test.run=^TestPiEffortRPCFixture$ -- \"$@\"\n"), 0700); err != nil { + t.Fatal(err) + } + var active []bool + client := &PiClient{Executable: wrapper, Directory: root, Tools: []string{}, OnOwnership: func(id, ref string, value bool) error { + if id != "effort-fixture" || ref == "" { + t.Error("missing native ownership", id, ref) + } + active = append(active, value) + return nil + }} + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + text, err := client.Generate(ctx, ModelConfig{Provider: "fixture", Model: "model", ReasoningEffort: tc.effort}, "task") + if tc.wantErr == "" { + if err != nil || text != "done" || !client.WorkStarted || !reflect.DeepEqual(active, []bool{true, false}) { + t.Fatal(text, err, client.WorkStarted, active) + } + } else if err == nil || !strings.Contains(err.Error(), tc.wantErr) || text != "" || client.WorkStarted || len(active) != 0 { + t.Fatal("expected pre-prompt rejection", text, err, client.WorkStarted, active) + } + if errors.Is(err, ErrStageCleanup) { + t.Fatal("cleanup failed", err) + } + if quiet, err := os.ReadFile(filepath.Join(client.Reference, "quiescent")); err != nil || string(quiet) != "quiescent" { + t.Fatal("missing quiescence", string(quiet), err) + } + if evidence, err := PiEvidence(client.Reference); err != nil || !strings.Contains(evidence, "effort-fixture") { + t.Fatal("native history lost", evidence, err) + } + argsData, err := os.ReadFile(filepath.Join(client.Reference, "args.json")) + if err != nil { + t.Fatal(err) + } + var args []string + if err := json.Unmarshal(argsData, &args); err != nil { + t.Fatal(err) + } + var flags []string + for i, arg := range args { + if arg == "--thinking" { + flags = append(flags, args[i+1]) + } + } + if (tc.flag == "" && len(flags) != 0) || (tc.flag != "" && !reflect.DeepEqual(flags, []string{tc.flag})) { + t.Fatal("thinking flags", flags, "want", tc.flag) + } + requests, err := os.ReadFile(filepath.Join(client.Reference, "requests.jsonl")) + if err != nil { + t.Fatal(err) + } + var commands []string + for _, line := range strings.Split(strings.TrimSpace(string(requests)), "\n") { + var request struct{ Type string } + if err := json.Unmarshal([]byte(line), &request); err != nil { + t.Fatal(err) + } + commands = append(commands, request.Type) + } + want := []string{"get_state", "get_state"} + if tc.wantErr == "" { + want = []string{"get_state", "set_auto_retry", "set_auto_compaction", "prompt", "get_session_stats", "get_messages", "get_state"} + } + if !reflect.DeepEqual(commands, want) { + t.Fatal("RPC sequence", commands, "want", want) + } + }) + } +} + +func TestPiInvalidReasoningEffortBeforeProcess(t *testing.T) { + for _, effort := range []string{"max", "invalid", "HIGH", " high "} { + t.Run(effort, func(t *testing.T) { + client := &PiClient{Executable: "/does-not-exist", WorkStarted: true} + _, err := client.Generate(context.Background(), ModelConfig{ReasoningEffort: effort}, "task") + if err == nil || !strings.Contains(err.Error(), "reasoning_effort") || client.Reference != "" || client.cmd != nil || client.WorkStarted { + t.Fatal("expected rejection before process/session creation", err, client) + } + }) + } +}