package stage import ( "encoding/json" "git.a-wan.net/alan/pipelines/internal/opencode" "regexp" "strings" "time" "unicode/utf8" ) // Bound the retained transcript written to SQLite. This matches the previous // 32 KiB last-line-of-defense cut, but above it a >32 KiB transcript becomes a // valid, bounded JSON envelope instead of a malformed JSON prefix. The preview // and each preserved diagnostic field are separately bounded so metadata can // never turn the envelope itself into unbounded evidence. const ( transcriptBound = 32 << 10 transcriptPreview = 8 << 10 transcriptFieldBound = 3 << 10 ) // Failure evidence is bounded and credential-redacted; request bodies (which // contain prompts) and the subprocess environment are never recorded here. type OperationEvidence struct { Operation string `json:"operation"` Backend string `json:"backend"` Executable string `json:"executable"` Directory string `json:"directory"` Provider string `json:"provider"` Model string `json:"model"` StartedAt string `json:"started_at"` DurationMS int64 `json:"duration_ms"` ExitCode int `json:"exit_code"` HTTPStatus int `json:"http_status,omitempty"` Diagnostic string `json:"diagnostic"` Stdout string `json:"stdout,omitempty"` Stderr string `json:"stderr,omitempty"` Transport string `json:"transport,omitempty"` NativeOperation string `json:"native_operation,omitempty"` ExitKind string `json:"exit_kind,omitempty"` } type OperationLog struct { Provider, Model string Failures []OperationEvidence FormatChecks []opencode.FormatProbe } var sensitiveKey = regexp.MustCompile(`(?i)^(authorization|password|secret|api[-_]?key|access[-_]?token|refresh[-_]?token|token|cookie|set-cookie|client[-_]?secret)$`) var credentialText = regexp.MustCompile(`(?i)(bearer\s+)[^\s"'<>]+|\b(?:sk-[a-zA-Z0-9_-]+|tk_[a-zA-Z0-9]{29})\b|((?:api[-_]?key|access[-_]?token|refresh[-_]?token|password|client[-_]?secret|authorization|token)\s*[=:]\s*["']?)[^\s"'&,<>]+`) var authorizationLine = regexp.MustCompile(`(?im)(authorization\s*[:=]\s*)[^\r\n]+`) var credentialURL = regexp.MustCompile(`(https?://)[^/\s:@]+:[^/\s@]+@`) func RedactEvidence(text string) string { var data any decoder := json.NewDecoder(strings.NewReader(text)) decoder.UseNumber() if json.Valid([]byte(text)) && decoder.Decode(&data) == nil { var clean func(any) clean = func(v any) { switch x := v.(type) { case map[string]any: for k, v := range x { if sensitiveKey.MatchString(k) { x[k] = "[REDACTED]" } else { clean(v) } } case []any: for _, v := range x { clean(v) } } } clean(data) encoded, _ := json.Marshal(data) text = string(encoded) } text = credentialURL.ReplaceAllString(text, "${1}[REDACTED]@") if !json.Valid([]byte(text)) { text = authorizationLine.ReplaceAllString(text, "${1}[REDACTED]") } text = credentialText.ReplaceAllStringFunc(text, func(match string) string { parts := credentialText.FindStringSubmatch(match) if parts[1] != "" { return parts[1] + "[REDACTED]" } if parts[2] != "" { return parts[2] + "[REDACTED]" } return "[REDACTED]" }) // Keep terminal control sequences out of diagnostic presentations. text = strings.Map(func(r rune) rune { if r < 32 && r != '\n' && r != '\t' { return -1 } return r }, text) if len(text) > 16<<10 { text = text[:16<<10] + "\n[truncated]" } return text } // TruncateRunes is the exported form of truncateRunes. Callers outside the stage // package that must bound text (for example the one-line summariser) reuse this // single UTF-8-safe truncation instead of re-implementing rune handling. func TruncateRunes(s string, limit int) (string, int) { return truncateRunes(s, limit) } // truncateRunes caps s to at most limit bytes without splitting a UTF-8 rune, // reporting how many bytes were dropped. func truncateRunes(s string, limit int) (string, int) { if limit < 0 { limit = 0 } if len(s) <= limit { return s, 0 } cut := limit for cut > 0 && !utf8.RuneStart(s[cut]) { cut-- } return s[:cut], len(s) - cut } // boundedJSON redacts a raw JSON value and bounds it. Values that stay small // remain structured JSON; oversized or non-JSON values become a bounded JSON // string so the surrounding envelope is always valid JSON. RedactEvidence parses // and cleans valid JSON first, so sensitive keys are redacted even when the // field is later truncated. func boundedJSON(raw json.RawMessage, limit int) json.RawMessage { if limit < 2 { limit = 2 } redacted := RedactEvidence(string(raw)) if json.Valid([]byte(redacted)) && len(redacted) <= limit { return json.RawMessage(redacted) } // Bound the encoded form, not the raw string length: JSON string escaping can // expand quotes, backslashes and control runes, so shrink the string until // json.Marshal(head) itself fits the cap. RedactEvidence already cleaned any // sensitive keys before truncation, so the fallback text stays redacted. head, _ := truncateRunes(redacted, limit) for { encoded, err := json.Marshal(head) if err != nil { return json.RawMessage(`""`) } if len(encoded) <= limit { return encoded } if len(head) == 0 { return json.RawMessage(`""`) } next := len(head) * limit / len(encoded) if next >= len(head) { next = len(head) - 1 } head, _ = truncateRunes(head, next) } } // BoundTranscript returns the bounded, credential-redacted representation stored // for a stage transcript. At or under the bound the input is returned byte-for- // byte unchanged. Above it a valid JSON object becomes an envelope that preserves // available stage_limits and opencode_failures/native_format_checks/format_repairs // diagnostics while adding explicit truncation, the original byte size, a // rune-safe bounded preview, an omission note and a retained session pointer. // Any other transcript (plain text or invalid JSON) becomes an honest bounded // text envelope rather than a failed JSON parse. Exact stage output is stored // separately and never rewritten here. func BoundTranscript(transcript, sessionID string) string { if len(transcript) <= transcriptBound { return transcript } // Redact the full transcript first so sensitive JSON keys are cleaned before // any head is taken; the preview is then a bounded, credential-redacted view // of the original serialized transcript. redacted := RedactEvidence(transcript) previewText, _ := truncateRunes(redacted, transcriptPreview) omitted := len(transcript) - len(previewText) if omitted < 0 { omitted = 0 } envelope := map[string]any{ "format": "text", "scope": "bounded_preview", "truncated": true, "original_bytes": len(transcript), "preview_bytes": len(previewText), "omitted_bytes": omitted, "preview": previewText, "omitted": "Transcript exceeded the 32 KiB retained-evidence bound; exact stage output is stored separately and the full transcript remains in the retained session.", } if sessionID != "" { envelope["retained_session"] = sessionID } var data map[string]json.RawMessage if json.Unmarshal([]byte(transcript), &data) == nil && data != nil { envelope["format"] = "json" for _, key := range []string{"stage_limits", "opencode_failures", "native_format_checks", "format_repairs"} { if raw, ok := data[key]; ok && len(raw) > 0 && string(raw) != "null" { envelope[key] = boundedJSON(raw, transcriptFieldBound) } } } encoded, err := json.Marshal(envelope) if err != nil { return `{"format":"text","scope":"bounded_preview","truncated":true,"omitted":"transcript could not be encoded"}` } // Each preserved field is bounded by its encoded size and the preview is // bounded in runes, but JSON escaping can still expand the assembled object. // Shrink the preview, then omit optional diagnostics as a final guard, until // the whole envelope fits the fixed cap. for len(encoded) > transcriptBound { if len(previewText) > 0 { previewText, _ = truncateRunes(previewText, len(previewText)/2) envelope["preview"] = previewText envelope["preview_bytes"] = len(previewText) } else if _, ok := envelope["stage_limits"]; ok { delete(envelope, "stage_limits") } else if _, ok := envelope["opencode_failures"]; ok { delete(envelope, "opencode_failures") } else if _, ok := envelope["native_format_checks"]; ok { delete(envelope, "native_format_checks") } else if _, ok := envelope["format_repairs"]; ok { delete(envelope, "format_repairs") } else { break } encoded, err = json.Marshal(envelope) if err != nil { return `{"format":"text","scope":"bounded_preview","truncated":true,"omitted":"transcript could not be encoded"}` } } return string(encoded) } // RetainSession adds a retained_session pointer to an already-JSON-object // transcript so a failed or timed-out step carries the session ID even below the // 32 KiB BoundTranscript threshold. Non-object and empty transcripts are // returned unchanged and the value is JSON-encoded, so the result stays valid. func RetainSession(transcript, sessionID string) string { if sessionID == "" { return transcript } var data map[string]json.RawMessage if json.Unmarshal([]byte(transcript), &data) != nil || data == nil { return transcript } encoded, err := json.Marshal(sessionID) if err != nil { return transcript } data["retained_session"] = encoded merged, err := json.Marshal(data) if err != nil { return transcript } return string(merged) } // Record appends one pre-sanitized operation evidence to the log, reusing the // same provider/model stamping, credential redaction and 32-record bound as the // stage-internal record path. Packages outside stage (for example the one-line // summariser) use it to leave bounded diagnostics without re-implementing it. func (l *OperationLog) Record(e OperationEvidence, started time.Time) { l.record(e, started) } func (l *OperationLog) record(e OperationEvidence, started time.Time) { if l == nil { return } e.Provider, e.Model = l.Provider, l.Model if e.Backend == "v1" { e.Transport = "native_http" e.ExitKind = "not a subprocess exit" } else { e.Transport = "cli_subprocess" e.ExitKind = "subprocess exit code" } e.StartedAt = started.UTC().Format(time.RFC3339Nano) e.DurationMS = time.Since(started).Milliseconds() e.Diagnostic = RedactEvidence(e.Diagnostic) e.Stdout = RedactEvidence(e.Stdout) e.Stderr = RedactEvidence(e.Stderr) if len(l.Failures) < 32 { l.Failures = append(l.Failures, e) } } func (l *OperationLog) Transcript(original string) string { if l == nil || (len(l.Failures) == 0 && len(l.FormatChecks) == 0) { return original } var data map[string]any if json.Unmarshal([]byte(original), &data) != nil || data == nil { data = map[string]any{} if original != "" { data["original_transcript"] = original } } if len(l.Failures) > 0 { data["opencode_failures"] = l.Failures } if len(l.FormatChecks) > 0 { data["native_format_checks"] = l.FormatChecks } encoded, _ := json.Marshal(data) return string(encoded) } func (l *OperationLog) recordFormatProbe(probe opencode.FormatProbe) { if l == nil || len(l.FormatChecks) >= 4 { return } probe.Response = RedactEvidence(probe.Response) probe.Diagnostic = RedactEvidence(probe.Diagnostic) l.FormatChecks = append(l.FormatChecks, probe) }