Files
pipelines/internal/pipeline/lint.go
T

309 lines
12 KiB
Go

package pipeline
import (
"errors"
"fmt"
"regexp"
"slices"
"strings"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// Authoring lint applies only to new publications (ParseDefinition), never to
// stored immutable versions, which Parse keeps reading and running unchanged.
// Errors are structural or textual mistakes that are unambiguous; warnings are
// token-cost and clarity problems the author should usually fix.
// MaxPromptBytes is the stage-prompt length above which lint warns. The runner
// adds its own turn-file, outcome and tool instructions around every prompt.
const MaxPromptBytes = 2000
// LintFinding is one authoring-policy result for a stage.
type LintFinding struct {
Stage string `json:"stage"`
Rule string `json:"rule"`
Message string `json:"message"`
}
func (f LintFinding) String() string {
return fmt.Sprintf("stage %s: %s: %s", f.Stage, f.Rule, f.Message)
}
// Lint returns the authoring errors and warnings for a parsed graph.
func Lint(g Graph) (errs, warnings []LintFinding) {
index := map[string]int{}
delivery := false
for i, s := range g.Stages {
index[s.Key] = i
if s.Configs["delivery"] == "true" {
delivery = true
}
}
add := func(list *[]LintFinding, s Stage, rule, format string, args ...any) {
*list = append(*list, LintFinding{Stage: s.Key, Rule: rule, Message: fmt.Sprintf(format, args...)})
}
for i, s := range g.Stages {
model := stage.EffectiveType(s.Type) == stage.ModelType
if model {
if reason := mutatingReason(s); reason != "" {
if _, present := s.Blocks["checks"]; !present {
if s.Configs["delivery"] == "true" {
add(&errs, s, "checks", "delivery stage needs a checks block listing the commands the server runs before landing, one per line")
} else {
add(&errs, s, "checks", "stage can modify the workspace (%s) but has no checks block; add one: \"none: <reason>\" (for example \"none: focused tests only; the Land delivery gate runs the suite\"), or set delivery=true on the landing stage and list its check commands", reason)
}
}
}
for _, word := range undeclaredOutcomes(s.Prompt, s.Edges) {
if word.strong {
add(&errs, s, "outcome", "prompt tells the stage to %s %q but it is not a declared edge; add \"%s -> <target>\" to edges or remove it from the prompt", word.verb, word.name, word.name)
} else {
add(&warnings, s, "outcome", "prompt tells the stage to %s `%s`, which is not a declared edge; declare it or reword the prompt if it is not an outcome", word.verb, word.name)
}
}
if !delivery {
if claim := deliveryClaim.FindString(s.Prompt); claim != "" {
add(&errs, s, "delivery", "prompt says %q but no stage sets delivery=true; add delivery=true and a checks block to the landing stage, or state that the stage itself (or the operator) lands the change", claim)
}
}
for _, phrase := range runnerBoilerplate {
if phrase.MatchString(s.Prompt) {
add(&warnings, s, "boilerplate", "prompt repeats runner-injected instructions (%q); the runner already explains output.md, outcomes, turn files and tools, so delete it", phrase.FindString(s.Prompt))
break
}
}
if match := unconditionalHistoryRead(s.Prompt); match != "" {
add(&warnings, s, "history", "prompt asks for %q on every visit; the runner inlines the essential memory (memory.md: task, previous handoff, outcome ledger), so read history.md only for a specific need, e.g. \"read history.md only if the handoff lacks a detail you need\"", match)
}
if n := len(strings.TrimSpace(s.Prompt)); n > MaxPromptBytes {
add(&warnings, s, "prompt-length", "prompt is %d bytes (over %d); keep stage prompts to the goal and real constraints, and move shared procedure into the skill or project docs", n, MaxPromptBytes)
}
}
if text, present := s.Blocks["checks"]; present {
if _, none := stage.ChecksNone(text); !none {
for _, line := range strings.Split(text, "\n") {
if guard := silentGuard(line); guard != "" {
add(&errs, s, "checks", "check %q can pass without running anything (%s); a missing tool must fail the check, so remove the guard", strings.TrimSpace(line), guard)
}
}
}
}
if stage.EffectiveType(s.Type) == stage.HumanType {
continue // a person bounds a human stage's loop
}
for _, outcome := range sortedOutcomes(s.Edges) {
target := s.Edges[outcome]
if protocol.IsTerminal(target) || index[target] > i {
continue
}
if !loopLimited(s, outcome, g.Stages[index[target]]) {
add(&errs, s, "limits", "edge %s -> %s loops back without a runner limit; add a limits block such as {\"max_visits\":3,\"on\":[%q]} to %s, or {\"max_visits\":3} to %s", outcome, target, outcome, s.Key, target)
}
}
}
return errs, warnings
}
// mutatingReason names why a model stage can change the workspace, or "".
func mutatingReason(s Stage) string {
if s.Configs["delivery"] == "true" {
return "delivery=true"
}
tools, explicit, err := stage.ToolNames(StageDefinition(s))
if err != nil {
return ""
}
if !explicit {
return "no tools block grants edit and shell"
}
var found []string
for _, tool := range tools {
if tool == "edit" || tool == "shell" {
found = append(found, tool)
}
}
if len(found) == 0 {
return ""
}
return "tools " + strings.Join(found, ", ")
}
func loopLimited(s Stage, outcome string, target Stage) bool {
if target.Limits != nil {
return true
}
return s.Limits != nil && (len(s.Limits.On) == 0 || slices.Contains(s.Limits.On, outcome))
}
func sortedOutcomes(edges map[string]string) []string {
keys := make([]string, 0, len(edges))
for key := range edges {
keys = append(keys, key)
}
slices.Sort(keys)
return keys
}
// outcomeWords are edge names that read as outcomes even unquoted. Words that
// commonly appear in prose after "report" (failed, pass, done, failure) are
// deliberately absent; they are still checked when quoted.
var outcomeWords = map[string]bool{
"success": true, "blocked": true, "blocked_retryable": true, "skipped": true,
"rejected": true, "approved": true, "advisory": true, "revise": true,
"abort": true, "noop": true, "unchanged": true, "changes_requested": true,
"needs_changes": true, "retryable": true,
}
var (
outcomeToken = "[`'\"]?#?[A-Za-z][A-Za-z0-9_-]*[`'\"]?"
outcomePhrase = regexp.MustCompile("(?i)\\b(return(?:s|ing)?|report(?:s|ing)?|choose|outcome(?:\\s+is)?)\\s+(" + outcomeToken + "(?:\\s*(?:,|/|\\bor\\b|\\|)\\s*" + outcomeToken + ")*)")
outcomeTokenRe = regexp.MustCompile(outcomeToken)
negated = regexp.MustCompile(`(?i)\b(?:not|never|no|don't|do not|nor)\s+$`)
)
type outcomeMention struct {
verb, name string
strong bool
}
// undeclaredOutcomes finds "return blocked" / "report `skipped`" style
// instructions naming an outcome the stage does not declare. Negated phrases
// ("never return success") are ignored.
func undeclaredOutcomes(prompt string, edges map[string]string) []outcomeMention {
var found []outcomeMention
seen := map[string]bool{}
for _, match := range outcomePhrase.FindAllStringSubmatchIndex(prompt, -1) {
start := match[0] - 12
if start < 0 {
start = 0
}
if negated.MatchString(prompt[start:match[0]]) {
continue
}
verb := strings.ToLower(strings.Fields(prompt[match[2]:match[3]])[0])
outcomeVerb := strings.HasPrefix(verb, "outcome")
for _, token := range outcomeTokenRe.FindAllString(prompt[match[4]:match[5]], -1) {
quoted := strings.ContainsAny(token, "`'\"")
if strings.Contains(token, "#") {
continue
}
name := strings.Trim(token, "`'\"")
lower := strings.ToLower(name)
if lower == "or" || lower == "error" || seen[name] {
continue
}
if _, declared := edges[name]; declared {
continue
}
if !quoted && !outcomeWords[lower] {
continue
}
seen[name] = true
found = append(found, outcomeMention{verb: verb, name: name, strong: outcomeWords[lower] || (quoted && outcomeVerb)})
}
}
return found
}
// deliveryClaim matches affirmative statements that the server lands work.
var deliveryClaim = regexp.MustCompile(`(?i)\bthe server (?:(?:then|will|automatically|itself|checks|verifies|tests|validates|and|,)\s*){0,4}(?:lands?|merges?|delivers?)\b|\b(?:landed|merged|delivered) by the server\b|\bserver[- ]side (?:landing|delivery)\b|\bthe (?:server's )?delivery gate\b`)
// runnerBoilerplate are phrases the runner itself injects around every model
// prompt (internal/stage/model_prompt.go) or that restate its output contract.
var runnerBoilerplate = func() []*regexp.Regexp {
var list []*regexp.Regexp
for _, phrase := range []string{
`rely on this essential memory first`,
`start (?:it|your output|the output) with a short ` + "`?#\\s*handoff",
`return its complete content as your final message`,
`(?:the )?only required section is(?: the)? ` + "`?#?\\s*outcome",
"only `?#\\s*outcome`? (?:section )?is required",
`everything else is free-form markdown`,
`do not wrap it in a json envelope`,
`allowed outcomes:`,
`do not invent file edits`,
`never claim unperformed work succeeded`,
`write the complete output\.md`,
`return the complete output\.md`,
} {
list = append(list, regexp.MustCompile(`(?i)`+strings.ReplaceAll(phrase, " ", `\s+`)))
}
return list
}()
// fullHistoryRead matches prompts that ask for the whole run history
// (history.md, the on-demand record of every turn) instead of reading it only
// for a specific need. memory.md is the capped essential memory the runner
// inlines into every prompt, so naming it is redundant but never costly.
var fullHistoryRead = regexp.MustCompile(`(?i)\b(?:read|review|load|open|check|consult|go through)\b(?:[^.\n]|\.[A-Za-z]){0,30}(?:\bhistory\.md\b|\b(?:entire|full|whole|complete|all of the|all of)\s+(?:run\s+)?(?:history|transcript)\b)`)
var conditional = regexp.MustCompile(`(?i)\b(?:only|when|if|unless)\b`)
// unconditionalHistoryRead returns the first full-history instruction whose
// sentence does not make it conditional ("only when the handoff lacks ...").
func unconditionalHistoryRead(prompt string) string {
for _, loc := range fullHistoryRead.FindAllStringIndex(prompt, -1) {
start := strings.LastIndexAny(prompt[:loc[0]], ".\n") + 1
end := len(prompt)
if i := strings.IndexAny(prompt[loc[1]:], ".\n"); i >= 0 {
end = loc[1] + i
}
if !conditional.MatchString(prompt[start:end]) {
return prompt[loc[0]:loc[1]]
}
}
return ""
}
var silentGuards = []struct {
re *regexp.Regexp
name string
}{
{regexp.MustCompile(`\|\|\s*(?:true\b|:(?:\s|;|\)|$)|exit\s+0\b|echo\b[^|&;()]*(?:;|$))`), "an || fallback that succeeds"},
{regexp.MustCompile(`(?:\[\[?|\btest)\s+!\s+-[a-zA-Z]\s+[^|&;]+?\]?\]?\s*\|\|`), "a negated file test that skips the command"},
{regexp.MustCompile(`!\s*command\s+-v\s+\S+[^|&;]*\|\|`), "a missing-tool guard that skips the command"},
{regexp.MustCompile(`;\s*(?:true|exit\s+0)\s*$`), "a trailing command that always succeeds"},
{regexp.MustCompile(`^\s*if\s.*\bthen\b.*\bfi\b`), "an if without else"},
{regexp.MustCompile(`(?:^|\s)set\s+\+e\b`), "set +e"},
}
var elseBranch = regexp.MustCompile(`\belse\b|\bexit\s+[1-9]`)
func silentGuard(line string) string {
for _, guard := range silentGuards {
if !guard.re.MatchString(line) {
continue
}
if guard.name == "an if without else" && elseBranch.MatchString(line) {
continue
}
return guard.name
}
return ""
}
func lintError(errs []LintFinding) error {
if len(errs) == 0 {
return nil
}
list := make([]error, 0, len(errs))
for _, finding := range errs {
list = append(list, errors.New(finding.String()))
}
return errors.Join(list...)
}
// LintStrings renders findings for protocol replies.
func LintStrings(findings []LintFinding) []string {
if len(findings) == 0 {
return nil
}
out := make([]string, 0, len(findings))
for _, finding := range findings {
out = append(out, finding.String())
}
return out
}