Files
pipelines/internal/stage/pi.go
T
alan 31aec05f99 Scope stage artifact access by run visit
Derive validated same-run visit artifact capabilities in the runner and pass them to OpenCode and Pi clients. Permit supported file tools to read completed prior visits and write only the current visit, preserving workspace, secret, database, and shell restrictions.\n\nChecked packaged templates and authoring skill references: neither needs updating in this item; docs/notes/scoped-artifact-access.md records the shared documentation staleness for the docs-fold writer.
2026-10-18 12:02:37 +01:00

581 lines
18 KiB
Go

package stage
import (
"bufio"
"context"
"crypto/sha256"
_ "embed"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"time"
)
//go:embed pi_policy.mjs
var piPolicy []byte
type PiClient struct {
Executable, Directory, SessionID, Reference, Transcript string
Tools []string
WritePaths []string
// ArtifactsDir is exported to the spawned Pi process as
// PIPELINES_ARTIFACTS_DIR so its tools can write stage artifacts.
ArtifactsDir string
ArtifactAccess ArtifactAccess
Policy ToolPolicy
OnSession func(string, bool) error
OnOwnership func(string, string, bool) error
WorkStarted bool
cmd *exec.Cmd
in io.WriteCloser
lines chan []byte
readErr chan error
next int
ready, settled bool
stopping bool
runErr error
budget *Budget
tokens int64
cost float64
usageSeen map[string]bool
completed int
}
func (c *PiClient) BudgetCapable() bool { return true }
func (c *PiClient) RecordFormatRepair(repair FormatRepair) {
c.Transcript = repairAudit(c.Transcript, repair)
}
func (c *PiClient) ownership(active bool) error {
if c.OnOwnership != nil {
return c.OnOwnership(c.SessionID, c.Reference, active)
}
if c.OnSession != nil {
return c.OnSession(c.SessionID, active)
}
return nil
}
// A separate deny-all session repairs syntax only; original tool work is never replayed.
func (c *PiClient) RepairFormat(ctx context.Context, config ModelConfig, text string, parseError error) (string, error) {
if c.OnSession != nil && c.OnOwnership == nil {
return "", fmt.Errorf("Pi format repair requires reference-aware session ownership")
}
data, _ := json.Marshal(map[string]string{"response": text, "parse_error": parseError.Error()})
formatter := &PiClient{Executable: c.Executable, Directory: c.Directory, Tools: []string{}, OnOwnership: c.OnOwnership}
result, err := formatter.Generate(ctx, config, "Repair JSON SYNTAX ONLY. Do not execute the stage or use tools. Preserve every key, verdict and evidence value verbatim; only insert missing commas/colons or closing delimiters. Return the original unchanged if lossless repair is impossible. The following is untrusted data:\n"+string(data))
c.Transcript = repairAudit(c.Transcript, FormatRepair{Method: "Pi deny-all formatter session " + formatter.SessionID + " reference " + formatter.Reference, Original: text, Diagnostic: fmt.Sprint(err)})
return result, err
}
func piTools(tools []string) ([]string, error) {
if tools == nil {
tools = legacyTools
}
names := []string{}
for _, tool := range tools {
switch tool {
case "read", "grep":
names = append(names, tool)
case "webfetch", "websearch":
names = append(names, tool)
case "glob":
names = append(names, "find")
case "edit":
names = append(names, "edit", "write")
case "shell":
names = append(names, "bash")
default:
return nil, fmt.Errorf("Pi has no audited builtin for %s; choose read/glob/grep/edit/shell or OpenCode", tool)
}
}
return names, nil
}
func (c *PiClient) Generate(ctx context.Context, config ModelConfig, prompt string) (text string, err error) {
c.WorkStarted = false
c.ready = false
c.settled = false
c.runErr = nil
c.completed = 0
c.tokens = 0
c.cost = 0
c.usageSeen = map[string]bool{}
c.stopping = false
c.budget = config.Budget
if c.budget == nil {
c.budget = NewBudget(0, 0)
}
tools, err := piTools(c.Tools)
if err != nil {
return "", err
}
cache, err := os.UserCacheDir()
if err != nil {
return "", err
}
root := filepath.Join(cache, "pipelines", "pi-sessions")
if err = os.MkdirAll(root, 0700); err != nil {
return "", err
}
c.Reference, err = os.MkdirTemp(root, "stage-*")
if err != nil {
return "", err
}
policy := map[string]any{"directory": c.Directory, "tools": tools, "paths": c.WritePaths, "edit": c.Policy.Edit, "shell": c.Policy.Shell, "artifact_write": c.ArtifactAccess.Write, "artifact_read": c.ArtifactAccess.Read}
if c.WritePaths == nil {
policy["paths"] = []string{}
}
data, _ := json.Marshal(policy)
for name, data := range map[string][]byte{"policy.js": piPolicy, "policy.json": data, "lease": []byte("owned")} {
if err = os.WriteFile(filepath.Join(c.Reference, name), data, 0600); err != nil {
return "", err
}
}
if c.Executable == "" {
c.Executable = "pi"
}
args := []string{"--mode", "rpc", "--provider", config.Provider, "--model", config.Model, "--session-dir", c.Reference, "--no-approve", "--no-extensions", "--no-skills", "--no-context-files", "--no-prompt-templates", "--no-themes", "--extension", filepath.Join(c.Reference, "policy.js")}
if len(tools) == 0 {
args = append(args, "--no-tools")
} else {
args = append(args, "--tools", strings.Join(tools, ","))
}
c.cmd = exec.Command(c.Executable, args...)
c.cmd.Dir = c.Directory
c.cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
c.cmd.Env = piEnvironment(c.Reference, c.ArtifactsDir)
stderr, err := os.OpenFile(filepath.Join(c.Reference, "stderr.log"), os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return "", err
}
defer stderr.Close()
c.cmd.Stderr = stderr
c.in, err = c.cmd.StdinPipe()
if err != nil {
return "", err
}
out, err := c.cmd.StdoutPipe()
if err != nil {
return "", err
}
if err = c.cmd.Start(); err != nil {
return "", fmt.Errorf("Pi startup failed: %w", err)
}
c.lines = make(chan []byte, 128)
c.readErr = make(chan error, 1)
done := make(chan struct{})
go func() {
scanner := bufio.NewScanner(out)
scanner.Buffer(make([]byte, 4096), 8<<20)
for scanner.Scan() {
line := append([]byte{}, scanner.Bytes()...)
select {
case c.lines <- line:
case <-done:
return
}
}
c.readErr <- scanner.Err()
close(c.readErr)
close(c.lines)
}()
heartbeatDone := make(chan struct{})
go func() {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-heartbeatDone:
return
case now := <-ticker.C:
_ = os.Chtimes(filepath.Join(c.Reference, "lease"), now, now)
}
}
}()
owned := false
defer func() {
c.stopping = true
cleanup, cancel := context.WithTimeout(context.Background(), 12*time.Second)
defer cancel()
quiet := !c.WorkStarted || c.settled
if !quiet {
_, abortErr := c.rpc(cleanup, "abort", nil)
if abortErr == nil {
for !c.settled {
if _, receiveErr := c.receive(cleanup); receiveErr != nil {
break
}
}
quiet = c.settled
}
}
if state, stateErr := c.rpc(cleanup, "get_state", nil); stateErr == nil {
var final struct {
SessionFile string `json:"sessionFile"`
}
if json.Unmarshal(state, &final) == nil && final.SessionFile != "" {
owner, _ := json.Marshal(map[string]any{"pid": c.cmd.Process.Pid, "session_id": c.SessionID, "session_file": final.SessionFile})
_ = os.WriteFile(filepath.Join(c.Reference, "owner.json"), owner, 0600)
}
}
_ = c.in.Close()
wait := make(chan error, 1)
go func() { wait <- c.cmd.Wait() }()
select {
case <-wait:
case <-cleanup.Done():
_ = syscall.Kill(-c.cmd.Process.Pid, syscall.SIGKILL)
<-wait
quiet = false
}
close(done)
close(heartbeatDone)
if quiet {
_ = os.WriteFile(filepath.Join(c.Reference, "quiescent"), []byte("quiescent"), 0600)
}
if !quiet {
err = errors.Join(err, ErrStageCleanup)
text = ""
} else if owned {
if saveErr := c.ownership(false); saveErr != nil {
err = errors.Join(err, saveErr, ErrStageCleanup)
text = ""
}
}
}()
state, err := c.rpc(ctx, "get_state", nil)
if err != nil {
return "", err
}
var session struct {
SessionID string `json:"sessionId"`
SessionFile string `json:"sessionFile"`
Model struct{ ID, Provider string } `json:"model"`
}
ready, _ := os.ReadFile(filepath.Join(c.Reference, "ready"))
c.ready = string(ready) == "ready"
if json.Unmarshal(state, &session) != nil || session.SessionID == "" || !c.ready {
return "", fmt.Errorf("Pi did not confirm session/policy readiness")
}
if session.Model.ID != config.Model || session.Model.Provider != config.Provider {
return "", fmt.Errorf("Pi did not select requested model %s/%s", config.Provider, config.Model)
}
c.SessionID = session.SessionID
owner, _ := json.Marshal(map[string]any{"pid": c.cmd.Process.Pid, "session_id": c.SessionID, "session_file": session.SessionFile})
if err = os.WriteFile(filepath.Join(c.Reference, "owner.json"), owner, 0600); err != nil {
return "", err
}
for _, command := range []string{"set_auto_retry", "set_auto_compaction"} {
if _, err = c.rpc(ctx, command, map[string]any{"enabled": false}); err != nil {
return "", err
}
}
if c.OnSession != nil || c.OnOwnership != nil {
if err = c.ownership(true); err != nil {
return "", err
}
owned = true
}
// The authoritative permitted-tools line is emitted once by modelPrompt. Pi
// realizes those canonical tool names through its own audited builtins, so
// append that derived mapping here rather than hardcoding Pi names. An
// explicit empty block stays text-only and gets no extra line.
if len(tools) > 0 {
prompt += "\nThis Pi harness exposes these builtin tools: " + strings.Join(tools, ", ") + "."
}
c.WorkStarted = true
if _, err = c.rpc(ctx, "prompt", map[string]any{"message": prompt}); err != nil {
return "", err
}
for !c.settled {
if _, err = c.receive(ctx); err != nil {
return "", err
}
}
stats, err := c.rpc(ctx, "get_session_stats", nil)
if err != nil {
return "", err
}
if err = c.observeStats(stats); err != nil {
return "", err
}
if c.runErr != nil {
return "", c.runErr
}
last, err := c.rpc(ctx, "get_messages", nil)
if err != nil {
return "", err
}
c.Transcript = RedactEvidence(string(last))
return piResult(last)
}
// piEnvironment builds the spawned Pi process environment, including the
// per-stage artifacts directory when the runtime provided one. Any
// PIPELINES_ARTIFACTS_DIR inherited from the process that spawns Pi is dropped
// first: the artifactsDir argument is authoritative, so an absent directory is
// never silently replaced by an ambient one (for example Pipelines' own
// PIPELINES_ARTIFACTS_DIR when a stage runs inside another stage).
func piEnvironment(reference, artifactsDir string) []string {
inherited := os.Environ()
env := make([]string, 0, len(inherited)+4)
for _, entry := range inherited {
if strings.HasPrefix(entry, ArtifactsEnv+"=") {
continue
}
env = append(env, entry)
}
env = append(env, "PI_OFFLINE=1",
"PIPELINES_PI_POLICY="+filepath.Join(reference, "policy.json"),
"PIPELINES_PI_LEASE="+filepath.Join(reference, "lease"),
"PIPELINES_PI_STOPPED="+filepath.Join(reference, "quiescent"))
if artifactsDir != "" {
env = append(env, ArtifactsEnv+"="+artifactsDir)
}
return env
}
func (c *PiClient) rpc(ctx context.Context, kind string, fields map[string]any) ([]byte, error) {
c.next++
id := fmt.Sprint(c.next)
if fields == nil {
fields = map[string]any{}
}
fields["id"] = id
fields["type"] = kind
data, _ := json.Marshal(fields)
if _, err := c.in.Write(append(data, '\n')); err != nil {
return nil, err
}
for {
record, err := c.receive(ctx)
if err != nil {
return nil, err
}
var reply struct {
Type, ID, Error string
Success bool
Data json.RawMessage
}
if json.Unmarshal(record, &reply) != nil {
return nil, fmt.Errorf("invalid Pi RPC record")
}
if reply.Type == "response" && reply.ID == id {
if !reply.Success {
return nil, fmt.Errorf("Pi %s failed: %s", kind, RedactEvidence(reply.Error))
}
return reply.Data, nil
}
}
}
func (c *PiClient) receive(ctx context.Context) ([]byte, error) {
select {
case <-ctx.Done():
return nil, ctx.Err()
case line, ok := <-c.lines:
if !ok {
err := <-c.readErr
if err == nil {
err = io.ErrUnexpectedEOF
}
return nil, fmt.Errorf("Pi process stream ended: %w", err)
}
var event struct {
Type string
Message json.RawMessage
}
if err := json.Unmarshal(line, &event); err != nil {
return nil, fmt.Errorf("invalid Pi JSONL: %w", err)
}
redacted := RedactEvidence(string(line))
if len(c.Transcript) < 32<<10 {
c.Transcript += redacted + "\n"
}
if file, err := os.OpenFile(filepath.Join(c.Reference, "rpc.jsonl"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600); err == nil {
_, _ = file.WriteString(redacted + "\n")
_ = file.Close()
}
switch event.Type {
case "pipelines_policy_ready":
c.ready = true
case "agent_settled":
c.settled = true
case "message_end":
var message struct {
Role, StopReason, ErrorMessage string
Usage *struct {
TotalTokens *int64
Cost *struct{ Total *float64 }
}
}
if json.Unmarshal(event.Message, &message) != nil {
return nil, fmt.Errorf("invalid Pi completed message")
}
if message.Role == "assistant" && c.stopping {
// Retain reported usage while draining cleanup events, without making
// accounting errors prevent the abort/settlement handshake.
key := fmt.Sprintf("%x", sha256.Sum256(event.Message))
if !c.usageSeen[key] && message.Usage != nil && message.Usage.TotalTokens != nil && message.Usage.Cost != nil && message.Usage.Cost.Total != nil {
tokens, cost := *message.Usage.TotalTokens, *message.Usage.Cost.Total
if tokens >= 0 && tokens <= math.MaxInt64-c.tokens && cost >= 0 && !math.IsInf(cost, 0) && !math.IsNaN(cost) {
c.usageSeen[key] = true
c.tokens += tokens
c.cost += cost
_ = c.observeTotals(c.tokens, c.cost)
}
}
}
if message.Role == "assistant" && !c.stopping {
if message.StopReason == "error" || message.StopReason == "aborted" {
c.runErr = fmt.Errorf("Pi provider failed: %s", RedactEvidence(message.ErrorMessage))
}
c.completed++
key := fmt.Sprintf("%x", sha256.Sum256(event.Message))
if !c.usageSeen[key] && message.Usage != nil && message.Usage.TotalTokens != nil && message.Usage.Cost != nil && message.Usage.Cost.Total != nil {
c.usageSeen[key] = true
if *message.Usage.TotalTokens < 0 || *message.Usage.TotalTokens > math.MaxInt64-c.tokens || *message.Usage.Cost.Total < 0 {
return nil, ErrBudgetAccounting
}
c.tokens += *message.Usage.TotalTokens
c.cost += *message.Usage.Cost.Total
if err := c.observeTotals(c.tokens, c.cost); err != nil {
return nil, err
}
} else if c.budget.Enabled() && !c.usageSeen[key] {
return nil, ErrBudgetAccounting
}
}
}
return line, nil
}
}
func (c *PiClient) observeTotals(tokens int64, cost float64) error {
// Pi's get_session_stats and message_end events report only a total token
// count, so account it as a total with no component split: the run record
// must expose the total and mark the in/out split unavailable rather than
// presenting the total as input tokens.
return c.budget.observe(c.SessionID, reportedUsage{USD: &cost, Tokens: &struct {
Input, Output, Reasoning *int64
Cache *struct{ Read, Write *int64 }
Total *int64
}{Total: &tokens}})
}
func (c *PiClient) observeStats(data []byte) error {
var stats struct {
SessionID string `json:"sessionId"`
AssistantMessages int `json:"assistantMessages"`
Tokens *struct{ Total *int64 }
Cost *float64
}
if json.Unmarshal(data, &stats) != nil || stats.SessionID != c.SessionID || stats.AssistantMessages < 1 || stats.Tokens == nil || stats.Tokens.Total == nil || stats.Cost == nil {
return fmt.Errorf("%w: Pi final session usage unavailable", ErrBudgetAccounting)
}
if *stats.Tokens.Total < c.tokens || *stats.Cost < c.cost {
return fmt.Errorf("%w: Pi final totals contradict retained usage", ErrBudgetAccounting)
}
if c.budget.Snapshot().LimitTokens > 0 && *stats.Tokens.Total == 0 {
return fmt.Errorf("%w: Pi completed response has zero tokens", ErrBudgetAccounting)
}
return c.observeTotals(*stats.Tokens.Total, *stats.Cost)
}
func piResult(data []byte) (string, error) {
var response struct {
Messages []struct {
Role, StopReason, ErrorMessage string
Content json.RawMessage
}
}
if err := json.Unmarshal(data, &response); err != nil {
return "", err
}
for i := len(response.Messages) - 1; i >= 0; i-- {
m := response.Messages[i]
if m.Role != "assistant" {
continue
}
if m.StopReason != "stop" {
return "", fmt.Errorf("Pi returned no completed final result (%s): %s", m.StopReason, RedactEvidence(m.ErrorMessage))
}
var text string
var parts []struct{ Type, Text string }
if err := json.Unmarshal(m.Content, &parts); err != nil {
return "", fmt.Errorf("invalid Pi assistant content: %w", err)
}
for _, p := range parts {
if p.Type == "text" {
text += p.Text
}
}
if text != "" {
return text, nil
}
break
}
return "", fmt.Errorf("Pi returned no assistant text")
}
// InterruptPi expires the retained lease. The explicit extension aborts tools and
// marks quiescence before shutting down; without that receipt recovery fails closed.
func InterruptPi(ctx context.Context, reference string) error {
if !filepath.IsAbs(reference) {
return fmt.Errorf("invalid retained Pi reference")
}
if _, err := os.Stat(filepath.Join(reference, "owner.json")); err != nil {
return errors.Join(err, ErrStageCleanup)
}
old := time.Unix(0, 0)
if err := os.Chtimes(filepath.Join(reference, "lease"), old, old); err != nil {
return err
}
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
for {
if data, err := os.ReadFile(filepath.Join(reference, "quiescent")); err == nil && string(data) == "quiescent" {
return nil
}
select {
case <-ctx.Done():
return errors.Join(ctx.Err(), ErrStageCleanup)
case <-ticker.C:
}
}
}
// PiEvidence returns the retained Pi session file as redacted JSONL text. Pi
// evidence is newline-delimited JSON and is intentionally not parsed as a single
// JSON object; callers treat it as bounded textual evidence.
func PiEvidence(reference string) (string, error) {
data, err := os.ReadFile(filepath.Join(reference, "owner.json"))
if err != nil {
return "", err
}
var owner struct {
SessionFile string `json:"session_file"`
}
if err = json.Unmarshal(data, &owner); err != nil {
return "", err
}
if !strings.HasPrefix(owner.SessionFile, reference+string(os.PathSeparator)) {
return "", fmt.Errorf("invalid Pi session evidence path")
}
file, err := os.Open(owner.SessionFile)
if err != nil {
return "", err
}
defer file.Close()
data, err = io.ReadAll(io.LimitReader(file, 8<<20))
return RedactEvidence(string(data)), err
}