18e7d073a6
Close the per-thread history loading gate before launching requests, make live/history part reconciliation linear, and reject duplicate history envelope fields. Complete task-owned history paging work remains preserved in the preceding recovery commit. Checked packaged templates and packaged authoring skill references: neither needs an update; docs/notes/large-chat-history.md records the stale shared documentation.
327 lines
7.5 KiB
Go
327 lines
7.5 KiB
Go
package opencode
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// historyJSON projects tool state BEFORE encoding/json sees it. Decoder.Token
|
|
// alone is not sufficient: it allocates an entire JSON string, even when the
|
|
// caller immediately discards it. This scanner consumes strings byte by byte
|
|
// through a fixed 32 KiB reader, including escapes and discarded detail.
|
|
//
|
|
// Workspace is O(32 KiB + 64 KiB + depth*256), depth <= 128. Retained memory is
|
|
// O(returned messages + largest projected message): answer text is never capped; each tool state
|
|
// contributes at most 64 KiB. An oversized state (including input, metadata,
|
|
// binary content or many small fields) is replaced by an honest omission card,
|
|
// NOT a raw JSON prefix. Neither wire size nor a single string size is a cap on
|
|
// history. No temporary disk copy or unbounded raw message/page is retained.
|
|
type historyJSON struct {
|
|
ctx context.Context
|
|
r *bufio.Reader
|
|
}
|
|
|
|
// Negative limits are used only for a projected message (output-proportional).
|
|
// State, cursor and key captures reuse the existing bounded output helper.
|
|
type historyBuffer = boundedOutput
|
|
|
|
func (b *boundedOutput) put(c byte) {
|
|
if b == nil {
|
|
return
|
|
}
|
|
b.WriteByte(c)
|
|
}
|
|
|
|
func readHistoryPage(ctx context.Context, sessionID string, r io.Reader) ([]protocol.ChatMessage, string, bool, error) {
|
|
p := historyJSON{ctx: ctx, r: bufio.NewReaderSize(r, 32<<10)}
|
|
messages := make([]protocol.ChatMessage, 0)
|
|
var next string
|
|
readMessages := func() error {
|
|
if err := p.take(nil, '['); err != nil {
|
|
return err
|
|
}
|
|
c, err := p.peek()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if c == ']' {
|
|
return p.take(nil, ']')
|
|
}
|
|
for {
|
|
out := &historyBuffer{limit: -1}
|
|
if err := p.value(out, 1, true); err != nil {
|
|
return err
|
|
}
|
|
var record nativeChatMessage
|
|
if err := json.Unmarshal(out.Bytes(), &record); err != nil {
|
|
return err
|
|
}
|
|
messages = append(messages, mapChatMessages(sessionID, []nativeChatMessage{record})...)
|
|
c, err := p.peek()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if c == ']' {
|
|
return p.take(nil, ']')
|
|
}
|
|
if err := p.take(nil, ','); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
c, err := p.peek()
|
|
if err != nil {
|
|
return nil, "", false, err
|
|
}
|
|
descending := c == '{'
|
|
if c == '[' {
|
|
err = readMessages()
|
|
} else if descending {
|
|
p.take(nil, '{')
|
|
found := false
|
|
fields := make(map[string]struct{})
|
|
for err == nil {
|
|
key := &historyBuffer{limit: 256}
|
|
if err = p.str(key, nil); err != nil {
|
|
break
|
|
}
|
|
var name string
|
|
if key.overflow || json.Unmarshal(key.Bytes(), &name) != nil {
|
|
err = fmt.Errorf("invalid history envelope field")
|
|
break
|
|
}
|
|
if _, duplicate := fields[name]; duplicate {
|
|
err = fmt.Errorf("duplicate history envelope field %q", name)
|
|
break
|
|
}
|
|
fields[name] = struct{}{}
|
|
if err = p.take(nil, ':'); err != nil {
|
|
break
|
|
}
|
|
switch name {
|
|
case "data":
|
|
found = true
|
|
err = readMessages()
|
|
case "cursor":
|
|
cursor := &historyBuffer{limit: 64 << 10}
|
|
err = p.value(cursor, 1, false)
|
|
if err == nil {
|
|
var decoded struct {
|
|
Next string `json:"next"`
|
|
}
|
|
if cursor.overflow {
|
|
err = fmt.Errorf("history cursor exceeds 64 KiB")
|
|
} else {
|
|
err = json.Unmarshal(cursor.Bytes(), &decoded)
|
|
next = decoded.Next
|
|
}
|
|
}
|
|
default:
|
|
err = p.value(nil, 1, false)
|
|
}
|
|
if err != nil {
|
|
break
|
|
}
|
|
c, err = p.peek()
|
|
if err != nil || c == '}' {
|
|
if err == nil {
|
|
err = p.take(nil, '}')
|
|
}
|
|
break
|
|
}
|
|
err = p.take(nil, ',')
|
|
}
|
|
if err == nil && !found {
|
|
err = fmt.Errorf("missing history data")
|
|
}
|
|
} else {
|
|
err = fmt.Errorf("invalid history envelope")
|
|
}
|
|
if err != nil {
|
|
return nil, "", false, fmt.Errorf("invalid OpenCode history: %w", err)
|
|
}
|
|
if _, err := p.peek(); err != io.EOF {
|
|
if err == nil {
|
|
err = fmt.Errorf("trailing JSON data")
|
|
}
|
|
return nil, "", false, err
|
|
}
|
|
return messages, next, descending, nil
|
|
}
|
|
|
|
func (p *historyJSON) peek() (byte, error) {
|
|
for {
|
|
if err := p.ctx.Err(); err != nil {
|
|
return 0, err
|
|
}
|
|
b, err := p.r.Peek(1)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
switch b[0] {
|
|
case ' ', '\n', '\r', '\t':
|
|
p.r.ReadByte()
|
|
default:
|
|
return b[0], nil
|
|
}
|
|
}
|
|
}
|
|
|
|
func (p *historyJSON) take(out *historyBuffer, want byte) error {
|
|
c, err := p.peek()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if c != want {
|
|
return fmt.Errorf("expected JSON delimiter")
|
|
}
|
|
p.r.ReadByte()
|
|
out.put(c)
|
|
return nil
|
|
}
|
|
|
|
func (p *historyJSON) str(out, key *historyBuffer) error {
|
|
if err := p.take(out, '"'); err != nil {
|
|
return err
|
|
}
|
|
key.put('"')
|
|
escape, unicode := false, 0
|
|
for {
|
|
// Check cancellation even within a multi-gigabyte discarded string.
|
|
if err := p.ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
c, err := p.r.ReadByte()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
out.put(c)
|
|
key.put(c)
|
|
if unicode > 0 {
|
|
if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F') {
|
|
return fmt.Errorf("invalid Unicode escape")
|
|
}
|
|
unicode--
|
|
} else if escape {
|
|
escape = false
|
|
switch c {
|
|
case 'u':
|
|
unicode = 4
|
|
case '"', '\\', '/', 'b', 'f', 'n', 'r', 't':
|
|
default:
|
|
return fmt.Errorf("invalid string escape")
|
|
}
|
|
} else {
|
|
switch {
|
|
case c == '"':
|
|
return nil
|
|
case c == '\\':
|
|
escape = true
|
|
case c < 0x20:
|
|
return fmt.Errorf("invalid string control character")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (p *historyJSON) value(out *historyBuffer, depth int, project bool) error {
|
|
if depth > 128 {
|
|
return fmt.Errorf("history JSON nesting exceeds 128")
|
|
}
|
|
c, err := p.peek()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch c {
|
|
case '"':
|
|
return p.str(out, nil)
|
|
case '{', '[':
|
|
open, close := c, byte('}')
|
|
if open == '[' {
|
|
close = ']'
|
|
}
|
|
p.take(out, open)
|
|
c, err = p.peek()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if c == close {
|
|
return p.take(out, close)
|
|
}
|
|
for {
|
|
var name string
|
|
if open == '{' {
|
|
key := &historyBuffer{limit: 256}
|
|
if err := p.str(out, key); err != nil {
|
|
return err
|
|
}
|
|
if !key.overflow {
|
|
_ = json.Unmarshal(key.Bytes(), &name)
|
|
}
|
|
if err := p.take(out, ':'); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if project && name == "state" {
|
|
state := &historyBuffer{limit: historyToolPreviewBytes}
|
|
if err := p.value(state, depth+1, false); err != nil {
|
|
return err
|
|
}
|
|
data := state.Bytes()
|
|
if state.overflow {
|
|
data = []byte(`{"truncated":true,"content":[{"type":"text","text":"Tool details omitted: stored result exceeds the 64 KiB history preview budget."}]}`)
|
|
}
|
|
for _, b := range data {
|
|
out.put(b)
|
|
}
|
|
} else if err := p.value(out, depth+1, project); err != nil {
|
|
return err
|
|
}
|
|
c, err = p.peek()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if c == close {
|
|
return p.take(out, close)
|
|
}
|
|
if err := p.take(out, ','); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
default:
|
|
// JSON numbers/literals are small in this schema. Validate without ever
|
|
// accumulating an attacker-controlled unbounded primitive.
|
|
token := &historyBuffer{limit: 128}
|
|
for {
|
|
b, err := p.r.Peek(1)
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
c = b[0]
|
|
if c == ',' || c == ']' || c == '}' || c == ' ' || c == '\n' || c == '\r' || c == '\t' {
|
|
break
|
|
}
|
|
p.r.ReadByte()
|
|
token.put(c)
|
|
if token.overflow {
|
|
return fmt.Errorf("invalid history primitive")
|
|
}
|
|
}
|
|
if !json.Valid(token.Bytes()) {
|
|
return fmt.Errorf("invalid history primitive")
|
|
}
|
|
for _, b := range token.Bytes() {
|
|
out.put(b)
|
|
}
|
|
return nil
|
|
}
|
|
}
|