Files
alan a3fdecbbcb Keep long Claude stream records valid in the retained evidence
RedactEvidence truncates at 16 KB and appends a line break, which cut long
stream-json records in two. The session export parses that stream, so the
first real Claude stage (run 346) parked on "Session export pending". The
stream is now redacted record by record without truncation.
2026-11-11 10:48:42 +00:00

381 lines
13 KiB
Go

package stage
import (
"encoding/json"
"git.a-wan.net/alan/pipelines/internal/opencode"
"regexp"
"strings"
"time"
"unicode/utf8"
)
// Bound the retained transcript written to SQLite. This matches the previous
// 32 KiB last-line-of-defense cut, but above it a >32 KiB transcript becomes a
// valid, bounded JSON envelope instead of a malformed JSON prefix. The preview
// and each preserved diagnostic field are separately bounded so metadata can
// never turn the envelope itself into unbounded evidence.
const (
transcriptBound = 32 << 10
transcriptPreview = 8 << 10
transcriptFieldBound = 3 << 10
)
// Failure evidence is bounded and credential-redacted; request bodies (which
// contain prompts) and the subprocess environment are never recorded here.
type OperationEvidence struct {
Operation string `json:"operation"`
Backend string `json:"backend"`
Executable string `json:"executable"`
Directory string `json:"directory"`
Provider string `json:"provider"`
Model string `json:"model"`
StartedAt string `json:"started_at"`
DurationMS int64 `json:"duration_ms"`
ExitCode int `json:"exit_code"`
HTTPStatus int `json:"http_status,omitempty"`
Diagnostic string `json:"diagnostic"`
Stdout string `json:"stdout,omitempty"`
Stderr string `json:"stderr,omitempty"`
Transport string `json:"transport,omitempty"`
NativeOperation string `json:"native_operation,omitempty"`
ExitKind string `json:"exit_kind,omitempty"`
}
type OperationLog struct {
Provider, Model string
Failures []OperationEvidence
FormatChecks []opencode.FormatProbe
}
var sensitiveKey = regexp.MustCompile(`(?i)^(authorization|password|secret|api[-_]?key|access[-_]?token|refresh[-_]?token|token|cookie|set-cookie|client[-_]?secret)$`)
var credentialText = regexp.MustCompile(`(?i)(bearer\s+)[^\s"'<>]+|\b(?:sk-[a-zA-Z0-9_-]+|tk_[a-zA-Z0-9]{29})\b|((?:api[-_]?key|access[-_]?token|refresh[-_]?token|password|client[-_]?secret|authorization|token)\s*[=:]\s*["']?)[^\s"'&,<>]+`)
var authorizationLine = regexp.MustCompile(`(?im)(authorization\s*[:=]\s*)[^\r\n]+`)
var credentialURL = regexp.MustCompile(`(https?://)[^/\s:@]+:[^/\s@]+@`)
func RedactEvidence(text string) string {
var data any
decoder := json.NewDecoder(strings.NewReader(text))
decoder.UseNumber()
if json.Valid([]byte(text)) && decoder.Decode(&data) == nil {
var clean func(any)
clean = func(v any) {
switch x := v.(type) {
case map[string]any:
for k, v := range x {
if sensitiveKey.MatchString(k) {
x[k] = "[REDACTED]"
} else {
clean(v)
}
}
case []any:
for _, v := range x {
clean(v)
}
}
}
clean(data)
encoded, _ := json.Marshal(data)
text = string(encoded)
}
text = credentialURL.ReplaceAllString(text, "${1}[REDACTED]@")
if !json.Valid([]byte(text)) {
text = authorizationLine.ReplaceAllString(text, "${1}[REDACTED]")
}
text = credentialText.ReplaceAllStringFunc(text, func(match string) string {
parts := credentialText.FindStringSubmatch(match)
if parts[1] != "" {
return parts[1] + "[REDACTED]"
}
if parts[2] != "" {
return parts[2] + "[REDACTED]"
}
return "[REDACTED]"
})
// Keep terminal control sequences out of diagnostic presentations.
text = strings.Map(func(r rune) rune {
if r < 32 && r != '\n' && r != '\t' {
return -1
}
return r
}, text)
if len(text) > 16<<10 {
text = text[:16<<10] + "\n[truncated]"
}
return text
}
// RedactEvidenceJSON redacts one JSON record and keeps it one valid JSON
// record: sensitive keys and credentials inside string values are replaced as
// in [RedactEvidence], but nothing is truncated and no line break is added,
// so a retained JSONL stream stays parseable however long a record is. A line
// that is not JSON falls back to [RedactEvidence].
func RedactEvidenceJSON(line []byte) string {
var data any
decoder := json.NewDecoder(strings.NewReader(string(line)))
decoder.UseNumber()
if !json.Valid(line) || decoder.Decode(&data) != nil {
return RedactEvidence(string(line))
}
redactText := func(text string) string {
text = credentialURL.ReplaceAllString(text, "${1}[REDACTED]@")
return credentialText.ReplaceAllStringFunc(text, func(match string) string {
parts := credentialText.FindStringSubmatch(match)
if parts[1] != "" {
return parts[1] + "[REDACTED]"
}
if parts[2] != "" {
return parts[2] + "[REDACTED]"
}
return "[REDACTED]"
})
}
var clean func(any) any
clean = func(v any) any {
switch x := v.(type) {
case map[string]any:
for k, v := range x {
if sensitiveKey.MatchString(k) {
x[k] = "[REDACTED]"
} else {
x[k] = clean(v)
}
}
return x
case []any:
for i, v := range x {
x[i] = clean(v)
}
return x
case string:
return redactText(x)
}
return v
}
encoded, err := json.Marshal(clean(data))
if err != nil {
return RedactEvidence(string(line))
}
return string(encoded)
}
// TruncateRunes is the exported form of truncateRunes. Callers outside the stage
// package that must bound text (for example the one-line summariser) reuse this
// single UTF-8-safe truncation instead of re-implementing rune handling.
func TruncateRunes(s string, limit int) (string, int) {
return truncateRunes(s, limit)
}
// truncateRunes caps s to at most limit bytes without splitting a UTF-8 rune,
// reporting how many bytes were dropped.
func truncateRunes(s string, limit int) (string, int) {
if limit < 0 {
limit = 0
}
if len(s) <= limit {
return s, 0
}
cut := limit
for cut > 0 && !utf8.RuneStart(s[cut]) {
cut--
}
return s[:cut], len(s) - cut
}
// boundedJSON redacts a raw JSON value and bounds it. Values that stay small
// remain structured JSON; oversized or non-JSON values become a bounded JSON
// string so the surrounding envelope is always valid JSON. RedactEvidence parses
// and cleans valid JSON first, so sensitive keys are redacted even when the
// field is later truncated.
func boundedJSON(raw json.RawMessage, limit int) json.RawMessage {
if limit < 2 {
limit = 2
}
redacted := RedactEvidence(string(raw))
if json.Valid([]byte(redacted)) && len(redacted) <= limit {
return json.RawMessage(redacted)
}
// Bound the encoded form, not the raw string length: JSON string escaping can
// expand quotes, backslashes and control runes, so shrink the string until
// json.Marshal(head) itself fits the cap. RedactEvidence already cleaned any
// sensitive keys before truncation, so the fallback text stays redacted.
head, _ := truncateRunes(redacted, limit)
for {
encoded, err := json.Marshal(head)
if err != nil {
return json.RawMessage(`""`)
}
if len(encoded) <= limit {
return encoded
}
if len(head) == 0 {
return json.RawMessage(`""`)
}
next := len(head) * limit / len(encoded)
if next >= len(head) {
next = len(head) - 1
}
head, _ = truncateRunes(head, next)
}
}
// BoundTranscript returns the bounded, credential-redacted representation stored
// for a stage transcript. At or under the bound the input is returned byte-for-
// byte unchanged. Above it a valid JSON object becomes an envelope that preserves
// available stage_limits and opencode_failures/native_format_checks/format_repairs
// diagnostics while adding explicit truncation, the original byte size, a
// rune-safe bounded preview, an omission note and a retained session pointer.
// Any other transcript (plain text or invalid JSON) becomes an honest bounded
// text envelope rather than a failed JSON parse. Exact stage output is stored
// separately and never rewritten here.
func BoundTranscript(transcript, sessionID string) string {
if len(transcript) <= transcriptBound {
return transcript
}
// Redact the full transcript first so sensitive JSON keys are cleaned before
// any head is taken; the preview is then a bounded, credential-redacted view
// of the original serialized transcript.
redacted := RedactEvidence(transcript)
previewText, _ := truncateRunes(redacted, transcriptPreview)
omitted := len(transcript) - len(previewText)
if omitted < 0 {
omitted = 0
}
envelope := map[string]any{
"format": "text",
"scope": "bounded_preview",
"truncated": true,
"original_bytes": len(transcript),
"preview_bytes": len(previewText),
"omitted_bytes": omitted,
"preview": previewText,
"omitted": "Transcript exceeded the 32 KiB retained-evidence bound; exact stage output is stored separately and the full transcript remains in the retained session.",
}
if sessionID != "" {
envelope["retained_session"] = sessionID
}
var data map[string]json.RawMessage
if json.Unmarshal([]byte(transcript), &data) == nil && data != nil {
envelope["format"] = "json"
for _, key := range []string{"stage_limits", "opencode_failures", "native_format_checks", "format_repairs"} {
if raw, ok := data[key]; ok && len(raw) > 0 && string(raw) != "null" {
envelope[key] = boundedJSON(raw, transcriptFieldBound)
}
}
}
encoded, err := json.Marshal(envelope)
if err != nil {
return `{"format":"text","scope":"bounded_preview","truncated":true,"omitted":"transcript could not be encoded"}`
}
// Each preserved field is bounded by its encoded size and the preview is
// bounded in runes, but JSON escaping can still expand the assembled object.
// Shrink the preview, then omit optional diagnostics as a final guard, until
// the whole envelope fits the fixed cap.
for len(encoded) > transcriptBound {
if len(previewText) > 0 {
previewText, _ = truncateRunes(previewText, len(previewText)/2)
envelope["preview"] = previewText
envelope["preview_bytes"] = len(previewText)
} else if _, ok := envelope["stage_limits"]; ok {
delete(envelope, "stage_limits")
} else if _, ok := envelope["opencode_failures"]; ok {
delete(envelope, "opencode_failures")
} else if _, ok := envelope["native_format_checks"]; ok {
delete(envelope, "native_format_checks")
} else if _, ok := envelope["format_repairs"]; ok {
delete(envelope, "format_repairs")
} else {
break
}
encoded, err = json.Marshal(envelope)
if err != nil {
return `{"format":"text","scope":"bounded_preview","truncated":true,"omitted":"transcript could not be encoded"}`
}
}
return string(encoded)
}
// RetainSession adds a retained_session pointer to an already-JSON-object
// transcript so a failed or timed-out step carries the session ID even below the
// 32 KiB BoundTranscript threshold. Non-object and empty transcripts are
// returned unchanged and the value is JSON-encoded, so the result stays valid.
func RetainSession(transcript, sessionID string) string {
if sessionID == "" {
return transcript
}
var data map[string]json.RawMessage
if json.Unmarshal([]byte(transcript), &data) != nil || data == nil {
return transcript
}
encoded, err := json.Marshal(sessionID)
if err != nil {
return transcript
}
data["retained_session"] = encoded
merged, err := json.Marshal(data)
if err != nil {
return transcript
}
return string(merged)
}
// Record appends one pre-sanitized operation evidence to the log, reusing the
// same provider/model stamping, credential redaction and 32-record bound as the
// stage-internal record path. Packages outside stage (for example the one-line
// summariser) use it to leave bounded diagnostics without re-implementing it.
func (l *OperationLog) Record(e OperationEvidence, started time.Time) {
l.record(e, started)
}
func (l *OperationLog) record(e OperationEvidence, started time.Time) {
if l == nil {
return
}
e.Provider, e.Model = l.Provider, l.Model
if e.Backend == "v1" {
e.Transport = "native_http"
e.ExitKind = "not a subprocess exit"
} else {
e.Transport = "cli_subprocess"
e.ExitKind = "subprocess exit code"
}
e.StartedAt = started.UTC().Format(time.RFC3339Nano)
e.DurationMS = time.Since(started).Milliseconds()
e.Diagnostic = RedactEvidence(e.Diagnostic)
e.Stdout = RedactEvidence(e.Stdout)
e.Stderr = RedactEvidence(e.Stderr)
if len(l.Failures) < 32 {
l.Failures = append(l.Failures, e)
}
}
func (l *OperationLog) Transcript(original string) string {
if l == nil || (len(l.Failures) == 0 && len(l.FormatChecks) == 0) {
return original
}
var data map[string]any
if json.Unmarshal([]byte(original), &data) != nil || data == nil {
data = map[string]any{}
if original != "" {
data["original_transcript"] = original
}
}
if len(l.Failures) > 0 {
data["opencode_failures"] = l.Failures
}
if len(l.FormatChecks) > 0 {
data["native_format_checks"] = l.FormatChecks
}
encoded, _ := json.Marshal(data)
return string(encoded)
}
func (l *OperationLog) recordFormatProbe(probe opencode.FormatProbe) {
if l == nil || len(l.FormatChecks) >= 4 {
return
}
probe.Response = RedactEvidence(probe.Response)
probe.Diagnostic = RedactEvidence(probe.Diagnostic)
l.FormatChecks = append(l.FormatChecks, probe)
}