Files
pipelines/internal/stage/diagnostics.go
T
alan 05ae530442 Make Pipelines Docker-only with OpenCode and Pi
Install current verified harness releases inside Docker, preserve persistent accounts and settings, and provide separate interactive and pipeline model defaults with terminal onboarding and fuzzy pickers.

Add Docker-aware authoring transport, Pi stage execution, bounded workflow controls, reliable final-result readback and usage settlement, and finite short-lived CLI pipe draining. Keep dated installation evidence and packaged authoring guidance consistent.

Verified with a real Docker installation, both harness stages and interactive authors, the built-in plan/implement/review starter, independent host race/vet checks, installer tests, and packaged skill tests. Generated release receipts stay ignored in clean checkouts.
2026-10-11 23:00:25 +01:00

288 lines
9.8 KiB
Go

package stage
import (
"encoding/json"
"git.a-wan.net/alan/pipelines/internal/opencode"
"regexp"
"strings"
"time"
"unicode/utf8"
)
// Bound the retained transcript written to SQLite. This matches the previous
// 32 KiB last-line-of-defense cut, but above it a >32 KiB transcript becomes a
// valid, bounded JSON envelope instead of a malformed JSON prefix. The preview
// and each preserved diagnostic field are separately bounded so metadata can
// never turn the envelope itself into unbounded evidence.
const (
transcriptBound = 32 << 10
transcriptPreview = 8 << 10
transcriptFieldBound = 3 << 10
)
// Failure evidence is bounded and credential-redacted; request bodies (which
// contain prompts) and the subprocess environment are never recorded here.
type OperationEvidence struct {
Operation string `json:"operation"`
Backend string `json:"backend"`
Executable string `json:"executable"`
Directory string `json:"directory"`
Provider string `json:"provider"`
Model string `json:"model"`
StartedAt string `json:"started_at"`
DurationMS int64 `json:"duration_ms"`
ExitCode int `json:"exit_code"`
HTTPStatus int `json:"http_status,omitempty"`
Diagnostic string `json:"diagnostic"`
Stdout string `json:"stdout,omitempty"`
Stderr string `json:"stderr,omitempty"`
Transport string `json:"transport,omitempty"`
NativeOperation string `json:"native_operation,omitempty"`
ExitKind string `json:"exit_kind,omitempty"`
}
type OperationLog struct {
Provider, Model string
Failures []OperationEvidence
FormatChecks []opencode.FormatProbe
}
var sensitiveKey = regexp.MustCompile(`(?i)^(authorization|password|secret|api[-_]?key|access[-_]?token|refresh[-_]?token|token|cookie|set-cookie|client[-_]?secret)$`)
var credentialText = regexp.MustCompile(`(?i)(bearer\s+)[^\s"'<>]+|\b(?:sk-[a-zA-Z0-9_-]+|tk_[a-zA-Z0-9]{29})\b|((?:api[-_]?key|access[-_]?token|refresh[-_]?token|password|client[-_]?secret|authorization|token)\s*[=:]\s*["']?)[^\s"'&,<>]+`)
var authorizationLine = regexp.MustCompile(`(?im)(authorization\s*[:=]\s*)[^\r\n]+`)
var credentialURL = regexp.MustCompile(`(https?://)[^/\s:@]+:[^/\s@]+@`)
func RedactEvidence(text string) string {
var data any
decoder := json.NewDecoder(strings.NewReader(text))
decoder.UseNumber()
if json.Valid([]byte(text)) && decoder.Decode(&data) == nil {
var clean func(any)
clean = func(v any) {
switch x := v.(type) {
case map[string]any:
for k, v := range x {
if sensitiveKey.MatchString(k) {
x[k] = "[REDACTED]"
} else {
clean(v)
}
}
case []any:
for _, v := range x {
clean(v)
}
}
}
clean(data)
encoded, _ := json.Marshal(data)
text = string(encoded)
}
text = credentialURL.ReplaceAllString(text, "${1}[REDACTED]@")
if !json.Valid([]byte(text)) {
text = authorizationLine.ReplaceAllString(text, "${1}[REDACTED]")
}
text = credentialText.ReplaceAllStringFunc(text, func(match string) string {
parts := credentialText.FindStringSubmatch(match)
if parts[1] != "" {
return parts[1] + "[REDACTED]"
}
if parts[2] != "" {
return parts[2] + "[REDACTED]"
}
return "[REDACTED]"
})
// Keep terminal control sequences out of diagnostic presentations.
text = strings.Map(func(r rune) rune {
if r < 32 && r != '\n' && r != '\t' {
return -1
}
return r
}, text)
if len(text) > 16<<10 {
text = text[:16<<10] + "\n[truncated]"
}
return text
}
// truncateRunes caps s to at most limit bytes without splitting a UTF-8 rune,
// reporting how many bytes were dropped.
func truncateRunes(s string, limit int) (string, int) {
if limit < 0 {
limit = 0
}
if len(s) <= limit {
return s, 0
}
cut := limit
for cut > 0 && !utf8.RuneStart(s[cut]) {
cut--
}
return s[:cut], len(s) - cut
}
// boundedJSON redacts a raw JSON value and bounds it. Values that stay small
// remain structured JSON; oversized or non-JSON values become a bounded JSON
// string so the surrounding envelope is always valid JSON. RedactEvidence parses
// and cleans valid JSON first, so sensitive keys are redacted even when the
// field is later truncated.
func boundedJSON(raw json.RawMessage, limit int) json.RawMessage {
if limit < 2 {
limit = 2
}
redacted := RedactEvidence(string(raw))
if json.Valid([]byte(redacted)) && len(redacted) <= limit {
return json.RawMessage(redacted)
}
// Bound the encoded form, not the raw string length: JSON string escaping can
// expand quotes, backslashes and control runes, so shrink the string until
// json.Marshal(head) itself fits the cap. RedactEvidence already cleaned any
// sensitive keys before truncation, so the fallback text stays redacted.
head, _ := truncateRunes(redacted, limit)
for {
encoded, err := json.Marshal(head)
if err != nil {
return json.RawMessage(`""`)
}
if len(encoded) <= limit {
return encoded
}
if len(head) == 0 {
return json.RawMessage(`""`)
}
next := len(head) * limit / len(encoded)
if next >= len(head) {
next = len(head) - 1
}
head, _ = truncateRunes(head, next)
}
}
// BoundTranscript returns the bounded, credential-redacted representation stored
// for a stage transcript. At or under the bound the input is returned byte-for-
// byte unchanged. Above it a valid JSON object becomes an envelope that preserves
// available stage_limits and opencode_failures/native_format_checks/format_repairs
// diagnostics while adding explicit truncation, the original byte size, a
// rune-safe bounded preview, an omission note and a retained session pointer.
// Any other transcript (plain text or invalid JSON) becomes an honest bounded
// text envelope rather than a failed JSON parse. Exact stage output is stored
// separately and never rewritten here.
func BoundTranscript(transcript, sessionID string) string {
if len(transcript) <= transcriptBound {
return transcript
}
// Redact the full transcript first so sensitive JSON keys are cleaned before
// any head is taken; the preview is then a bounded, credential-redacted view
// of the original serialized transcript.
redacted := RedactEvidence(transcript)
previewText, _ := truncateRunes(redacted, transcriptPreview)
omitted := len(transcript) - len(previewText)
if omitted < 0 {
omitted = 0
}
envelope := map[string]any{
"format": "text",
"scope": "bounded_preview",
"truncated": true,
"original_bytes": len(transcript),
"preview_bytes": len(previewText),
"omitted_bytes": omitted,
"preview": previewText,
"omitted": "Transcript exceeded the 32 KiB retained-evidence bound; exact stage output is stored separately and the full transcript remains in the retained session.",
}
if sessionID != "" {
envelope["retained_session"] = sessionID
}
var data map[string]json.RawMessage
if json.Unmarshal([]byte(transcript), &data) == nil && data != nil {
envelope["format"] = "json"
for _, key := range []string{"stage_limits", "opencode_failures", "native_format_checks", "format_repairs"} {
if raw, ok := data[key]; ok && len(raw) > 0 && string(raw) != "null" {
envelope[key] = boundedJSON(raw, transcriptFieldBound)
}
}
}
encoded, err := json.Marshal(envelope)
if err != nil {
return `{"format":"text","scope":"bounded_preview","truncated":true,"omitted":"transcript could not be encoded"}`
}
// Each preserved field is bounded by its encoded size and the preview is
// bounded in runes, but JSON escaping can still expand the assembled object.
// Shrink the preview, then omit optional diagnostics as a final guard, until
// the whole envelope fits the fixed cap.
for len(encoded) > transcriptBound {
if len(previewText) > 0 {
previewText, _ = truncateRunes(previewText, len(previewText)/2)
envelope["preview"] = previewText
envelope["preview_bytes"] = len(previewText)
} else if _, ok := envelope["stage_limits"]; ok {
delete(envelope, "stage_limits")
} else if _, ok := envelope["opencode_failures"]; ok {
delete(envelope, "opencode_failures")
} else if _, ok := envelope["native_format_checks"]; ok {
delete(envelope, "native_format_checks")
} else if _, ok := envelope["format_repairs"]; ok {
delete(envelope, "format_repairs")
} else {
break
}
encoded, err = json.Marshal(envelope)
if err != nil {
return `{"format":"text","scope":"bounded_preview","truncated":true,"omitted":"transcript could not be encoded"}`
}
}
return string(encoded)
}
func (l *OperationLog) record(e OperationEvidence, started time.Time) {
if l == nil {
return
}
e.Provider, e.Model = l.Provider, l.Model
if e.Backend == "v1" {
e.Transport = "native_http"
e.ExitKind = "not a subprocess exit"
} else {
e.Transport = "cli_subprocess"
e.ExitKind = "subprocess exit code"
}
e.StartedAt = started.UTC().Format(time.RFC3339Nano)
e.DurationMS = time.Since(started).Milliseconds()
e.Diagnostic = RedactEvidence(e.Diagnostic)
e.Stdout = RedactEvidence(e.Stdout)
e.Stderr = RedactEvidence(e.Stderr)
if len(l.Failures) < 32 {
l.Failures = append(l.Failures, e)
}
}
func (l *OperationLog) Transcript(original string) string {
if l == nil || (len(l.Failures) == 0 && len(l.FormatChecks) == 0) {
return original
}
var data map[string]any
if json.Unmarshal([]byte(original), &data) != nil || data == nil {
data = map[string]any{}
if original != "" {
data["original_transcript"] = original
}
}
if len(l.Failures) > 0 {
data["opencode_failures"] = l.Failures
}
if len(l.FormatChecks) > 0 {
data["native_format_checks"] = l.FormatChecks
}
encoded, _ := json.Marshal(data)
return string(encoded)
}
func (l *OperationLog) recordFormatProbe(probe opencode.FormatProbe) {
if l == nil || len(l.FormatChecks) >= 4 {
return
}
probe.Response = RedactEvidence(probe.Response)
probe.Diagnostic = RedactEvidence(probe.Diagnostic)
l.FormatChecks = append(l.FormatChecks, probe)
}