From a3fdecbbcba9b901f0eef7dee9f2bb8ad73212b6 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Wed, 11 Nov 2026 10:48:42 +0000 Subject: [PATCH] Keep long Claude stream records valid in the retained evidence RedactEvidence truncates at 16 KB and appends a line break, which cut long stream-json records in two. The session export parses that stream, so the first real Claude stage (run 346) parked on "Session export pending". The stream is now redacted record by record without truncation. --- internal/stage/claude_stage.go | 3 +- internal/stage/claude_stage_test.go | 19 ++++++++++ internal/stage/diagnostics.go | 54 +++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/internal/stage/claude_stage.go b/internal/stage/claude_stage.go index 6d919ddd..eec4dead 100644 --- a/internal/stage/claude_stage.go +++ b/internal/stage/claude_stage.go @@ -357,7 +357,8 @@ func (c *ClaudeStageClient) run(ctx context.Context, env, args []string, model, if json.Unmarshal(line, &record) != nil { continue } - redacted := RedactEvidence(string(line)) + // One valid JSON record per line: the export parses this stream. + redacted := RedactEvidenceJSON(line) _, _ = stream.WriteString(redacted + "\n") if len(c.Transcript) < 32<<10 { c.Transcript += redacted + "\n" diff --git a/internal/stage/claude_stage_test.go b/internal/stage/claude_stage_test.go index 2f9db2f2..8404d090 100644 --- a/internal/stage/claude_stage_test.go +++ b/internal/stage/claude_stage_test.go @@ -2,6 +2,7 @@ package stage import ( "context" + "encoding/json" "errors" "os" "path/filepath" @@ -365,3 +366,21 @@ func TestClaudeHarnessValidation(t *testing.T) { } } } + +// A long stream record (a file read, a diff) must stay one valid JSON line in +// the retained evidence: the session export parses that stream, and a record +// cut at 16 KB parked a real run on "Session export pending". +func TestRedactEvidenceJSONKeepsLongRecordsValid(t *testing.T) { + long := strings.Repeat("x", 40<<10) + line := []byte(`{"type":"user","message":{"role":"user","content":"` + long + ` https://user:secret@example.test/x"},"api_key":"sk-live"}`) + out := RedactEvidenceJSON(line) + if strings.Contains(out, "\n") || !json.Valid([]byte(out)) { + t.Fatalf("not one valid JSON line (len %d)", len(out)) + } + if !strings.Contains(out, long) { + t.Fatal("the record was truncated") + } + if strings.Contains(out, "secret@") || strings.Contains(out, "sk-live") { + t.Fatal("a credential survived redaction") + } +} diff --git a/internal/stage/diagnostics.go b/internal/stage/diagnostics.go index dc7b9a2e..c36f997f 100644 --- a/internal/stage/diagnostics.go +++ b/internal/stage/diagnostics.go @@ -105,6 +105,60 @@ func RedactEvidence(text string) string { return text } +// RedactEvidenceJSON redacts one JSON record and keeps it one valid JSON +// record: sensitive keys and credentials inside string values are replaced as +// in [RedactEvidence], but nothing is truncated and no line break is added, +// so a retained JSONL stream stays parseable however long a record is. A line +// that is not JSON falls back to [RedactEvidence]. +func RedactEvidenceJSON(line []byte) string { + var data any + decoder := json.NewDecoder(strings.NewReader(string(line))) + decoder.UseNumber() + if !json.Valid(line) || decoder.Decode(&data) != nil { + return RedactEvidence(string(line)) + } + redactText := func(text string) string { + text = credentialURL.ReplaceAllString(text, "${1}[REDACTED]@") + return credentialText.ReplaceAllStringFunc(text, func(match string) string { + parts := credentialText.FindStringSubmatch(match) + if parts[1] != "" { + return parts[1] + "[REDACTED]" + } + if parts[2] != "" { + return parts[2] + "[REDACTED]" + } + return "[REDACTED]" + }) + } + var clean func(any) any + clean = func(v any) any { + switch x := v.(type) { + case map[string]any: + for k, v := range x { + if sensitiveKey.MatchString(k) { + x[k] = "[REDACTED]" + } else { + x[k] = clean(v) + } + } + return x + case []any: + for i, v := range x { + x[i] = clean(v) + } + return x + case string: + return redactText(x) + } + return v + } + encoded, err := json.Marshal(clean(data)) + if err != nil { + return RedactEvidence(string(line)) + } + return string(encoded) +} + // TruncateRunes is the exported form of truncateRunes. Callers outside the stage // package that must bound text (for example the one-line summariser) reuse this // single UTF-8-safe truncation instead of re-implementing rune handling.