05ae530442
Install current verified harness releases inside Docker, preserve persistent accounts and settings, and provide separate interactive and pipeline model defaults with terminal onboarding and fuzzy pickers. Add Docker-aware authoring transport, Pi stage execution, bounded workflow controls, reliable final-result readback and usage settlement, and finite short-lived CLI pipe draining. Keep dated installation evidence and packaged authoring guidance consistent. Verified with a real Docker installation, both harness stages and interactive authors, the built-in plan/implement/review starter, independent host race/vet checks, installer tests, and packaged skill tests. Generated release receipts stay ignored in clean checkouts.
288 lines
9.8 KiB
Go
288 lines
9.8 KiB
Go
package stage
|
|
|
|
import (
|
|
"encoding/json"
|
|
"git.a-wan.net/alan/pipelines/internal/opencode"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// Bound the retained transcript written to SQLite. This matches the previous
|
|
// 32 KiB last-line-of-defense cut, but above it a >32 KiB transcript becomes a
|
|
// valid, bounded JSON envelope instead of a malformed JSON prefix. The preview
|
|
// and each preserved diagnostic field are separately bounded so metadata can
|
|
// never turn the envelope itself into unbounded evidence.
|
|
const (
|
|
transcriptBound = 32 << 10
|
|
transcriptPreview = 8 << 10
|
|
transcriptFieldBound = 3 << 10
|
|
)
|
|
|
|
// Failure evidence is bounded and credential-redacted; request bodies (which
|
|
// contain prompts) and the subprocess environment are never recorded here.
|
|
type OperationEvidence struct {
|
|
Operation string `json:"operation"`
|
|
Backend string `json:"backend"`
|
|
Executable string `json:"executable"`
|
|
Directory string `json:"directory"`
|
|
Provider string `json:"provider"`
|
|
Model string `json:"model"`
|
|
StartedAt string `json:"started_at"`
|
|
DurationMS int64 `json:"duration_ms"`
|
|
ExitCode int `json:"exit_code"`
|
|
HTTPStatus int `json:"http_status,omitempty"`
|
|
Diagnostic string `json:"diagnostic"`
|
|
Stdout string `json:"stdout,omitempty"`
|
|
Stderr string `json:"stderr,omitempty"`
|
|
Transport string `json:"transport,omitempty"`
|
|
NativeOperation string `json:"native_operation,omitempty"`
|
|
ExitKind string `json:"exit_kind,omitempty"`
|
|
}
|
|
|
|
type OperationLog struct {
|
|
Provider, Model string
|
|
Failures []OperationEvidence
|
|
FormatChecks []opencode.FormatProbe
|
|
}
|
|
|
|
var sensitiveKey = regexp.MustCompile(`(?i)^(authorization|password|secret|api[-_]?key|access[-_]?token|refresh[-_]?token|token|cookie|set-cookie|client[-_]?secret)$`)
|
|
var credentialText = regexp.MustCompile(`(?i)(bearer\s+)[^\s"'<>]+|\b(?:sk-[a-zA-Z0-9_-]+|tk_[a-zA-Z0-9]{29})\b|((?:api[-_]?key|access[-_]?token|refresh[-_]?token|password|client[-_]?secret|authorization|token)\s*[=:]\s*["']?)[^\s"'&,<>]+`)
|
|
var authorizationLine = regexp.MustCompile(`(?im)(authorization\s*[:=]\s*)[^\r\n]+`)
|
|
var credentialURL = regexp.MustCompile(`(https?://)[^/\s:@]+:[^/\s@]+@`)
|
|
|
|
func RedactEvidence(text string) string {
|
|
var data any
|
|
decoder := json.NewDecoder(strings.NewReader(text))
|
|
decoder.UseNumber()
|
|
if json.Valid([]byte(text)) && decoder.Decode(&data) == nil {
|
|
var clean func(any)
|
|
clean = func(v any) {
|
|
switch x := v.(type) {
|
|
case map[string]any:
|
|
for k, v := range x {
|
|
if sensitiveKey.MatchString(k) {
|
|
x[k] = "[REDACTED]"
|
|
} else {
|
|
clean(v)
|
|
}
|
|
}
|
|
case []any:
|
|
for _, v := range x {
|
|
clean(v)
|
|
}
|
|
}
|
|
}
|
|
clean(data)
|
|
encoded, _ := json.Marshal(data)
|
|
text = string(encoded)
|
|
}
|
|
text = credentialURL.ReplaceAllString(text, "${1}[REDACTED]@")
|
|
if !json.Valid([]byte(text)) {
|
|
text = authorizationLine.ReplaceAllString(text, "${1}[REDACTED]")
|
|
}
|
|
text = credentialText.ReplaceAllStringFunc(text, func(match string) string {
|
|
parts := credentialText.FindStringSubmatch(match)
|
|
if parts[1] != "" {
|
|
return parts[1] + "[REDACTED]"
|
|
}
|
|
if parts[2] != "" {
|
|
return parts[2] + "[REDACTED]"
|
|
}
|
|
return "[REDACTED]"
|
|
})
|
|
// Keep terminal control sequences out of diagnostic presentations.
|
|
text = strings.Map(func(r rune) rune {
|
|
if r < 32 && r != '\n' && r != '\t' {
|
|
return -1
|
|
}
|
|
return r
|
|
}, text)
|
|
if len(text) > 16<<10 {
|
|
text = text[:16<<10] + "\n[truncated]"
|
|
}
|
|
return text
|
|
}
|
|
|
|
// truncateRunes caps s to at most limit bytes without splitting a UTF-8 rune,
|
|
// reporting how many bytes were dropped.
|
|
func truncateRunes(s string, limit int) (string, int) {
|
|
if limit < 0 {
|
|
limit = 0
|
|
}
|
|
if len(s) <= limit {
|
|
return s, 0
|
|
}
|
|
cut := limit
|
|
for cut > 0 && !utf8.RuneStart(s[cut]) {
|
|
cut--
|
|
}
|
|
return s[:cut], len(s) - cut
|
|
}
|
|
|
|
// boundedJSON redacts a raw JSON value and bounds it. Values that stay small
|
|
// remain structured JSON; oversized or non-JSON values become a bounded JSON
|
|
// string so the surrounding envelope is always valid JSON. RedactEvidence parses
|
|
// and cleans valid JSON first, so sensitive keys are redacted even when the
|
|
// field is later truncated.
|
|
func boundedJSON(raw json.RawMessage, limit int) json.RawMessage {
|
|
if limit < 2 {
|
|
limit = 2
|
|
}
|
|
redacted := RedactEvidence(string(raw))
|
|
if json.Valid([]byte(redacted)) && len(redacted) <= limit {
|
|
return json.RawMessage(redacted)
|
|
}
|
|
// Bound the encoded form, not the raw string length: JSON string escaping can
|
|
// expand quotes, backslashes and control runes, so shrink the string until
|
|
// json.Marshal(head) itself fits the cap. RedactEvidence already cleaned any
|
|
// sensitive keys before truncation, so the fallback text stays redacted.
|
|
head, _ := truncateRunes(redacted, limit)
|
|
for {
|
|
encoded, err := json.Marshal(head)
|
|
if err != nil {
|
|
return json.RawMessage(`""`)
|
|
}
|
|
if len(encoded) <= limit {
|
|
return encoded
|
|
}
|
|
if len(head) == 0 {
|
|
return json.RawMessage(`""`)
|
|
}
|
|
next := len(head) * limit / len(encoded)
|
|
if next >= len(head) {
|
|
next = len(head) - 1
|
|
}
|
|
head, _ = truncateRunes(head, next)
|
|
}
|
|
}
|
|
|
|
// BoundTranscript returns the bounded, credential-redacted representation stored
|
|
// for a stage transcript. At or under the bound the input is returned byte-for-
|
|
// byte unchanged. Above it a valid JSON object becomes an envelope that preserves
|
|
// available stage_limits and opencode_failures/native_format_checks/format_repairs
|
|
// diagnostics while adding explicit truncation, the original byte size, a
|
|
// rune-safe bounded preview, an omission note and a retained session pointer.
|
|
// Any other transcript (plain text or invalid JSON) becomes an honest bounded
|
|
// text envelope rather than a failed JSON parse. Exact stage output is stored
|
|
// separately and never rewritten here.
|
|
func BoundTranscript(transcript, sessionID string) string {
|
|
if len(transcript) <= transcriptBound {
|
|
return transcript
|
|
}
|
|
// Redact the full transcript first so sensitive JSON keys are cleaned before
|
|
// any head is taken; the preview is then a bounded, credential-redacted view
|
|
// of the original serialized transcript.
|
|
redacted := RedactEvidence(transcript)
|
|
previewText, _ := truncateRunes(redacted, transcriptPreview)
|
|
omitted := len(transcript) - len(previewText)
|
|
if omitted < 0 {
|
|
omitted = 0
|
|
}
|
|
envelope := map[string]any{
|
|
"format": "text",
|
|
"scope": "bounded_preview",
|
|
"truncated": true,
|
|
"original_bytes": len(transcript),
|
|
"preview_bytes": len(previewText),
|
|
"omitted_bytes": omitted,
|
|
"preview": previewText,
|
|
"omitted": "Transcript exceeded the 32 KiB retained-evidence bound; exact stage output is stored separately and the full transcript remains in the retained session.",
|
|
}
|
|
if sessionID != "" {
|
|
envelope["retained_session"] = sessionID
|
|
}
|
|
var data map[string]json.RawMessage
|
|
if json.Unmarshal([]byte(transcript), &data) == nil && data != nil {
|
|
envelope["format"] = "json"
|
|
for _, key := range []string{"stage_limits", "opencode_failures", "native_format_checks", "format_repairs"} {
|
|
if raw, ok := data[key]; ok && len(raw) > 0 && string(raw) != "null" {
|
|
envelope[key] = boundedJSON(raw, transcriptFieldBound)
|
|
}
|
|
}
|
|
}
|
|
encoded, err := json.Marshal(envelope)
|
|
if err != nil {
|
|
return `{"format":"text","scope":"bounded_preview","truncated":true,"omitted":"transcript could not be encoded"}`
|
|
}
|
|
// Each preserved field is bounded by its encoded size and the preview is
|
|
// bounded in runes, but JSON escaping can still expand the assembled object.
|
|
// Shrink the preview, then omit optional diagnostics as a final guard, until
|
|
// the whole envelope fits the fixed cap.
|
|
for len(encoded) > transcriptBound {
|
|
if len(previewText) > 0 {
|
|
previewText, _ = truncateRunes(previewText, len(previewText)/2)
|
|
envelope["preview"] = previewText
|
|
envelope["preview_bytes"] = len(previewText)
|
|
} else if _, ok := envelope["stage_limits"]; ok {
|
|
delete(envelope, "stage_limits")
|
|
} else if _, ok := envelope["opencode_failures"]; ok {
|
|
delete(envelope, "opencode_failures")
|
|
} else if _, ok := envelope["native_format_checks"]; ok {
|
|
delete(envelope, "native_format_checks")
|
|
} else if _, ok := envelope["format_repairs"]; ok {
|
|
delete(envelope, "format_repairs")
|
|
} else {
|
|
break
|
|
}
|
|
encoded, err = json.Marshal(envelope)
|
|
if err != nil {
|
|
return `{"format":"text","scope":"bounded_preview","truncated":true,"omitted":"transcript could not be encoded"}`
|
|
}
|
|
}
|
|
return string(encoded)
|
|
}
|
|
|
|
func (l *OperationLog) record(e OperationEvidence, started time.Time) {
|
|
if l == nil {
|
|
return
|
|
}
|
|
e.Provider, e.Model = l.Provider, l.Model
|
|
if e.Backend == "v1" {
|
|
e.Transport = "native_http"
|
|
e.ExitKind = "not a subprocess exit"
|
|
} else {
|
|
e.Transport = "cli_subprocess"
|
|
e.ExitKind = "subprocess exit code"
|
|
}
|
|
e.StartedAt = started.UTC().Format(time.RFC3339Nano)
|
|
e.DurationMS = time.Since(started).Milliseconds()
|
|
e.Diagnostic = RedactEvidence(e.Diagnostic)
|
|
e.Stdout = RedactEvidence(e.Stdout)
|
|
e.Stderr = RedactEvidence(e.Stderr)
|
|
if len(l.Failures) < 32 {
|
|
l.Failures = append(l.Failures, e)
|
|
}
|
|
}
|
|
|
|
func (l *OperationLog) Transcript(original string) string {
|
|
if l == nil || (len(l.Failures) == 0 && len(l.FormatChecks) == 0) {
|
|
return original
|
|
}
|
|
var data map[string]any
|
|
if json.Unmarshal([]byte(original), &data) != nil || data == nil {
|
|
data = map[string]any{}
|
|
if original != "" {
|
|
data["original_transcript"] = original
|
|
}
|
|
}
|
|
if len(l.Failures) > 0 {
|
|
data["opencode_failures"] = l.Failures
|
|
}
|
|
if len(l.FormatChecks) > 0 {
|
|
data["native_format_checks"] = l.FormatChecks
|
|
}
|
|
encoded, _ := json.Marshal(data)
|
|
return string(encoded)
|
|
}
|
|
|
|
func (l *OperationLog) recordFormatProbe(probe opencode.FormatProbe) {
|
|
if l == nil || len(l.FormatChecks) >= 4 {
|
|
return
|
|
}
|
|
probe.Response = RedactEvidence(probe.Response)
|
|
probe.Diagnostic = RedactEvidence(probe.Diagnostic)
|
|
l.FormatChecks = append(l.FormatChecks, probe)
|
|
}
|