Files
pipelines/internal/opencode/history.go
T
alan 18e7d073a6 Harden paged chat history loading
Close the per-thread history loading gate before launching requests, make live/history part reconciliation linear, and reject duplicate history envelope fields.

Complete task-owned history paging work remains preserved in the preceding recovery commit. Checked packaged templates and packaged authoring skill references: neither needs an update; docs/notes/large-chat-history.md records the stale shared documentation.
2026-10-18 15:21:09 +01:00

327 lines
7.5 KiB
Go

package opencode
import (
"bufio"
"context"
"encoding/json"
"fmt"
"io"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// historyJSON projects tool state BEFORE encoding/json sees it. Decoder.Token
// alone is not sufficient: it allocates an entire JSON string, even when the
// caller immediately discards it. This scanner consumes strings byte by byte
// through a fixed 32 KiB reader, including escapes and discarded detail.
//
// Workspace is O(32 KiB + 64 KiB + depth*256), depth <= 128. Retained memory is
// O(returned messages + largest projected message): answer text is never capped; each tool state
// contributes at most 64 KiB. An oversized state (including input, metadata,
// binary content or many small fields) is replaced by an honest omission card,
// NOT a raw JSON prefix. Neither wire size nor a single string size is a cap on
// history. No temporary disk copy or unbounded raw message/page is retained.
type historyJSON struct {
ctx context.Context
r *bufio.Reader
}
// Negative limits are used only for a projected message (output-proportional).
// State, cursor and key captures reuse the existing bounded output helper.
type historyBuffer = boundedOutput
func (b *boundedOutput) put(c byte) {
if b == nil {
return
}
b.WriteByte(c)
}
func readHistoryPage(ctx context.Context, sessionID string, r io.Reader) ([]protocol.ChatMessage, string, bool, error) {
p := historyJSON{ctx: ctx, r: bufio.NewReaderSize(r, 32<<10)}
messages := make([]protocol.ChatMessage, 0)
var next string
readMessages := func() error {
if err := p.take(nil, '['); err != nil {
return err
}
c, err := p.peek()
if err != nil {
return err
}
if c == ']' {
return p.take(nil, ']')
}
for {
out := &historyBuffer{limit: -1}
if err := p.value(out, 1, true); err != nil {
return err
}
var record nativeChatMessage
if err := json.Unmarshal(out.Bytes(), &record); err != nil {
return err
}
messages = append(messages, mapChatMessages(sessionID, []nativeChatMessage{record})...)
c, err := p.peek()
if err != nil {
return err
}
if c == ']' {
return p.take(nil, ']')
}
if err := p.take(nil, ','); err != nil {
return err
}
}
}
c, err := p.peek()
if err != nil {
return nil, "", false, err
}
descending := c == '{'
if c == '[' {
err = readMessages()
} else if descending {
p.take(nil, '{')
found := false
fields := make(map[string]struct{})
for err == nil {
key := &historyBuffer{limit: 256}
if err = p.str(key, nil); err != nil {
break
}
var name string
if key.overflow || json.Unmarshal(key.Bytes(), &name) != nil {
err = fmt.Errorf("invalid history envelope field")
break
}
if _, duplicate := fields[name]; duplicate {
err = fmt.Errorf("duplicate history envelope field %q", name)
break
}
fields[name] = struct{}{}
if err = p.take(nil, ':'); err != nil {
break
}
switch name {
case "data":
found = true
err = readMessages()
case "cursor":
cursor := &historyBuffer{limit: 64 << 10}
err = p.value(cursor, 1, false)
if err == nil {
var decoded struct {
Next string `json:"next"`
}
if cursor.overflow {
err = fmt.Errorf("history cursor exceeds 64 KiB")
} else {
err = json.Unmarshal(cursor.Bytes(), &decoded)
next = decoded.Next
}
}
default:
err = p.value(nil, 1, false)
}
if err != nil {
break
}
c, err = p.peek()
if err != nil || c == '}' {
if err == nil {
err = p.take(nil, '}')
}
break
}
err = p.take(nil, ',')
}
if err == nil && !found {
err = fmt.Errorf("missing history data")
}
} else {
err = fmt.Errorf("invalid history envelope")
}
if err != nil {
return nil, "", false, fmt.Errorf("invalid OpenCode history: %w", err)
}
if _, err := p.peek(); err != io.EOF {
if err == nil {
err = fmt.Errorf("trailing JSON data")
}
return nil, "", false, err
}
return messages, next, descending, nil
}
func (p *historyJSON) peek() (byte, error) {
for {
if err := p.ctx.Err(); err != nil {
return 0, err
}
b, err := p.r.Peek(1)
if err != nil {
return 0, err
}
switch b[0] {
case ' ', '\n', '\r', '\t':
p.r.ReadByte()
default:
return b[0], nil
}
}
}
func (p *historyJSON) take(out *historyBuffer, want byte) error {
c, err := p.peek()
if err != nil {
return err
}
if c != want {
return fmt.Errorf("expected JSON delimiter")
}
p.r.ReadByte()
out.put(c)
return nil
}
func (p *historyJSON) str(out, key *historyBuffer) error {
if err := p.take(out, '"'); err != nil {
return err
}
key.put('"')
escape, unicode := false, 0
for {
// Check cancellation even within a multi-gigabyte discarded string.
if err := p.ctx.Err(); err != nil {
return err
}
c, err := p.r.ReadByte()
if err != nil {
return err
}
out.put(c)
key.put(c)
if unicode > 0 {
if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F') {
return fmt.Errorf("invalid Unicode escape")
}
unicode--
} else if escape {
escape = false
switch c {
case 'u':
unicode = 4
case '"', '\\', '/', 'b', 'f', 'n', 'r', 't':
default:
return fmt.Errorf("invalid string escape")
}
} else {
switch {
case c == '"':
return nil
case c == '\\':
escape = true
case c < 0x20:
return fmt.Errorf("invalid string control character")
}
}
}
}
func (p *historyJSON) value(out *historyBuffer, depth int, project bool) error {
if depth > 128 {
return fmt.Errorf("history JSON nesting exceeds 128")
}
c, err := p.peek()
if err != nil {
return err
}
switch c {
case '"':
return p.str(out, nil)
case '{', '[':
open, close := c, byte('}')
if open == '[' {
close = ']'
}
p.take(out, open)
c, err = p.peek()
if err != nil {
return err
}
if c == close {
return p.take(out, close)
}
for {
var name string
if open == '{' {
key := &historyBuffer{limit: 256}
if err := p.str(out, key); err != nil {
return err
}
if !key.overflow {
_ = json.Unmarshal(key.Bytes(), &name)
}
if err := p.take(out, ':'); err != nil {
return err
}
}
if project && name == "state" {
state := &historyBuffer{limit: historyToolPreviewBytes}
if err := p.value(state, depth+1, false); err != nil {
return err
}
data := state.Bytes()
if state.overflow {
data = []byte(`{"truncated":true,"content":[{"type":"text","text":"Tool details omitted: stored result exceeds the 64 KiB history preview budget."}]}`)
}
for _, b := range data {
out.put(b)
}
} else if err := p.value(out, depth+1, project); err != nil {
return err
}
c, err = p.peek()
if err != nil {
return err
}
if c == close {
return p.take(out, close)
}
if err := p.take(out, ','); err != nil {
return err
}
}
default:
// JSON numbers/literals are small in this schema. Validate without ever
// accumulating an attacker-controlled unbounded primitive.
token := &historyBuffer{limit: 128}
for {
b, err := p.r.Peek(1)
if err == io.EOF {
break
}
if err != nil {
return err
}
c = b[0]
if c == ',' || c == ']' || c == '}' || c == ' ' || c == '\n' || c == '\r' || c == '\t' {
break
}
p.r.ReadByte()
token.put(c)
if token.overflow {
return fmt.Errorf("invalid history primitive")
}
}
if !json.Valid(token.Bytes()) {
return fmt.Errorf("invalid history primitive")
}
for _, b := range token.Bytes() {
out.put(b)
}
return nil
}
}