309 lines
12 KiB
Go
309 lines
12 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// Authoring lint applies only to new publications (ParseDefinition), never to
|
|
// stored immutable versions, which Parse keeps reading and running unchanged.
|
|
// Errors are structural or textual mistakes that are unambiguous; warnings are
|
|
// token-cost and clarity problems the author should usually fix.
|
|
|
|
// MaxPromptBytes is the stage-prompt length above which lint warns. The runner
|
|
// adds its own turn-file, outcome and tool instructions around every prompt.
|
|
const MaxPromptBytes = 2000
|
|
|
|
// LintFinding is one authoring-policy result for a stage.
|
|
type LintFinding struct {
|
|
Stage string `json:"stage"`
|
|
Rule string `json:"rule"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
func (f LintFinding) String() string {
|
|
return fmt.Sprintf("stage %s: %s: %s", f.Stage, f.Rule, f.Message)
|
|
}
|
|
|
|
// Lint returns the authoring errors and warnings for a parsed graph.
|
|
func Lint(g Graph) (errs, warnings []LintFinding) {
|
|
index := map[string]int{}
|
|
delivery := false
|
|
for i, s := range g.Stages {
|
|
index[s.Key] = i
|
|
if s.Configs["delivery"] == "true" {
|
|
delivery = true
|
|
}
|
|
}
|
|
add := func(list *[]LintFinding, s Stage, rule, format string, args ...any) {
|
|
*list = append(*list, LintFinding{Stage: s.Key, Rule: rule, Message: fmt.Sprintf(format, args...)})
|
|
}
|
|
for i, s := range g.Stages {
|
|
model := stage.EffectiveType(s.Type) == stage.ModelType
|
|
if model {
|
|
if reason := mutatingReason(s); reason != "" {
|
|
if _, present := s.Blocks["checks"]; !present {
|
|
if s.Configs["delivery"] == "true" {
|
|
add(&errs, s, "checks", "delivery stage needs a checks block listing the commands the server runs before landing, one per line")
|
|
} else {
|
|
add(&errs, s, "checks", "stage can modify the workspace (%s) but has no checks block; add one: \"none: <reason>\" (for example \"none: focused tests only; the Land delivery gate runs the suite\"), or set delivery=true on the landing stage and list its check commands", reason)
|
|
}
|
|
}
|
|
}
|
|
for _, word := range undeclaredOutcomes(s.Prompt, s.Edges) {
|
|
if word.strong {
|
|
add(&errs, s, "outcome", "prompt tells the stage to %s %q but it is not a declared edge; add \"%s -> <target>\" to edges or remove it from the prompt", word.verb, word.name, word.name)
|
|
} else {
|
|
add(&warnings, s, "outcome", "prompt tells the stage to %s `%s`, which is not a declared edge; declare it or reword the prompt if it is not an outcome", word.verb, word.name)
|
|
}
|
|
}
|
|
if !delivery {
|
|
if claim := deliveryClaim.FindString(s.Prompt); claim != "" {
|
|
add(&errs, s, "delivery", "prompt says %q but no stage sets delivery=true; add delivery=true and a checks block to the landing stage, or state that the stage itself (or the operator) lands the change", claim)
|
|
}
|
|
}
|
|
for _, phrase := range runnerBoilerplate {
|
|
if phrase.MatchString(s.Prompt) {
|
|
add(&warnings, s, "boilerplate", "prompt repeats runner-injected instructions (%q); the runner already explains output.md, outcomes, turn files and tools, so delete it", phrase.FindString(s.Prompt))
|
|
break
|
|
}
|
|
}
|
|
if match := unconditionalHistoryRead(s.Prompt); match != "" {
|
|
add(&warnings, s, "history", "prompt asks for %q on every visit; the runner inlines the essential memory (memory.md: task, previous handoff, outcome ledger), so read history.md only for a specific need, e.g. \"read history.md only if the handoff lacks a detail you need\"", match)
|
|
}
|
|
if n := len(strings.TrimSpace(s.Prompt)); n > MaxPromptBytes {
|
|
add(&warnings, s, "prompt-length", "prompt is %d bytes (over %d); keep stage prompts to the goal and real constraints, and move shared procedure into the skill or project docs", n, MaxPromptBytes)
|
|
}
|
|
}
|
|
if text, present := s.Blocks["checks"]; present {
|
|
if _, none := stage.ChecksNone(text); !none {
|
|
for _, line := range strings.Split(text, "\n") {
|
|
if guard := silentGuard(line); guard != "" {
|
|
add(&errs, s, "checks", "check %q can pass without running anything (%s); a missing tool must fail the check, so remove the guard", strings.TrimSpace(line), guard)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if stage.EffectiveType(s.Type) == stage.HumanType {
|
|
continue // a person bounds a human stage's loop
|
|
}
|
|
for _, outcome := range sortedOutcomes(s.Edges) {
|
|
target := s.Edges[outcome]
|
|
if protocol.IsTerminal(target) || index[target] > i {
|
|
continue
|
|
}
|
|
if !loopLimited(s, outcome, g.Stages[index[target]]) {
|
|
add(&errs, s, "limits", "edge %s -> %s loops back without a runner limit; add a limits block such as {\"max_visits\":3,\"on\":[%q]} to %s, or {\"max_visits\":3} to %s", outcome, target, outcome, s.Key, target)
|
|
}
|
|
}
|
|
}
|
|
return errs, warnings
|
|
}
|
|
|
|
// mutatingReason names why a model stage can change the workspace, or "".
|
|
func mutatingReason(s Stage) string {
|
|
if s.Configs["delivery"] == "true" {
|
|
return "delivery=true"
|
|
}
|
|
tools, explicit, err := stage.ToolNames(StageDefinition(s))
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
if !explicit {
|
|
return "no tools block grants edit and shell"
|
|
}
|
|
var found []string
|
|
for _, tool := range tools {
|
|
if tool == "edit" || tool == "shell" {
|
|
found = append(found, tool)
|
|
}
|
|
}
|
|
if len(found) == 0 {
|
|
return ""
|
|
}
|
|
return "tools " + strings.Join(found, ", ")
|
|
}
|
|
|
|
func loopLimited(s Stage, outcome string, target Stage) bool {
|
|
if target.Limits != nil {
|
|
return true
|
|
}
|
|
return s.Limits != nil && (len(s.Limits.On) == 0 || slices.Contains(s.Limits.On, outcome))
|
|
}
|
|
|
|
func sortedOutcomes(edges map[string]string) []string {
|
|
keys := make([]string, 0, len(edges))
|
|
for key := range edges {
|
|
keys = append(keys, key)
|
|
}
|
|
slices.Sort(keys)
|
|
return keys
|
|
}
|
|
|
|
// outcomeWords are edge names that read as outcomes even unquoted. Words that
|
|
// commonly appear in prose after "report" (failed, pass, done, failure) are
|
|
// deliberately absent; they are still checked when quoted.
|
|
var outcomeWords = map[string]bool{
|
|
"success": true, "blocked": true, "blocked_retryable": true, "skipped": true,
|
|
"rejected": true, "approved": true, "advisory": true, "revise": true,
|
|
"abort": true, "noop": true, "unchanged": true, "changes_requested": true,
|
|
"needs_changes": true, "retryable": true,
|
|
}
|
|
|
|
var (
|
|
outcomeToken = "[`'\"]?#?[A-Za-z][A-Za-z0-9_-]*[`'\"]?"
|
|
outcomePhrase = regexp.MustCompile("(?i)\\b(return(?:s|ing)?|report(?:s|ing)?|choose|outcome(?:\\s+is)?)\\s+(" + outcomeToken + "(?:\\s*(?:,|/|\\bor\\b|\\|)\\s*" + outcomeToken + ")*)")
|
|
outcomeTokenRe = regexp.MustCompile(outcomeToken)
|
|
negated = regexp.MustCompile(`(?i)\b(?:not|never|no|don't|do not|nor)\s+$`)
|
|
)
|
|
|
|
type outcomeMention struct {
|
|
verb, name string
|
|
strong bool
|
|
}
|
|
|
|
// undeclaredOutcomes finds "return blocked" / "report `skipped`" style
|
|
// instructions naming an outcome the stage does not declare. Negated phrases
|
|
// ("never return success") are ignored.
|
|
func undeclaredOutcomes(prompt string, edges map[string]string) []outcomeMention {
|
|
var found []outcomeMention
|
|
seen := map[string]bool{}
|
|
for _, match := range outcomePhrase.FindAllStringSubmatchIndex(prompt, -1) {
|
|
start := match[0] - 12
|
|
if start < 0 {
|
|
start = 0
|
|
}
|
|
if negated.MatchString(prompt[start:match[0]]) {
|
|
continue
|
|
}
|
|
verb := strings.ToLower(strings.Fields(prompt[match[2]:match[3]])[0])
|
|
outcomeVerb := strings.HasPrefix(verb, "outcome")
|
|
for _, token := range outcomeTokenRe.FindAllString(prompt[match[4]:match[5]], -1) {
|
|
quoted := strings.ContainsAny(token, "`'\"")
|
|
if strings.Contains(token, "#") {
|
|
continue
|
|
}
|
|
name := strings.Trim(token, "`'\"")
|
|
lower := strings.ToLower(name)
|
|
if lower == "or" || lower == "error" || seen[name] {
|
|
continue
|
|
}
|
|
if _, declared := edges[name]; declared {
|
|
continue
|
|
}
|
|
if !quoted && !outcomeWords[lower] {
|
|
continue
|
|
}
|
|
seen[name] = true
|
|
found = append(found, outcomeMention{verb: verb, name: name, strong: outcomeWords[lower] || (quoted && outcomeVerb)})
|
|
}
|
|
}
|
|
return found
|
|
}
|
|
|
|
// deliveryClaim matches affirmative statements that the server lands work.
|
|
var deliveryClaim = regexp.MustCompile(`(?i)\bthe server (?:(?:then|will|automatically|itself|checks|verifies|tests|validates|and|,)\s*){0,4}(?:lands?|merges?|delivers?)\b|\b(?:landed|merged|delivered) by the server\b|\bserver[- ]side (?:landing|delivery)\b|\bthe (?:server's )?delivery gate\b`)
|
|
|
|
// runnerBoilerplate are phrases the runner itself injects around every model
|
|
// prompt (internal/stage/model_prompt.go) or that restate its output contract.
|
|
var runnerBoilerplate = func() []*regexp.Regexp {
|
|
var list []*regexp.Regexp
|
|
for _, phrase := range []string{
|
|
`rely on this essential memory first`,
|
|
`start (?:it|your output|the output) with a short ` + "`?#\\s*handoff",
|
|
`return its complete content as your final message`,
|
|
`(?:the )?only required section is(?: the)? ` + "`?#?\\s*outcome",
|
|
"only `?#\\s*outcome`? (?:section )?is required",
|
|
`everything else is free-form markdown`,
|
|
`do not wrap it in a json envelope`,
|
|
`allowed outcomes:`,
|
|
`do not invent file edits`,
|
|
`never claim unperformed work succeeded`,
|
|
`write the complete output\.md`,
|
|
`return the complete output\.md`,
|
|
} {
|
|
list = append(list, regexp.MustCompile(`(?i)`+strings.ReplaceAll(phrase, " ", `\s+`)))
|
|
}
|
|
return list
|
|
}()
|
|
|
|
// fullHistoryRead matches prompts that ask for the whole run history
|
|
// (history.md, the on-demand record of every turn) instead of reading it only
|
|
// for a specific need. memory.md is the capped essential memory the runner
|
|
// inlines into every prompt, so naming it is redundant but never costly.
|
|
var fullHistoryRead = regexp.MustCompile(`(?i)\b(?:read|review|load|open|check|consult|go through)\b(?:[^.\n]|\.[A-Za-z]){0,30}(?:\bhistory\.md\b|\b(?:entire|full|whole|complete|all of the|all of)\s+(?:run\s+)?(?:history|transcript)\b)`)
|
|
|
|
var conditional = regexp.MustCompile(`(?i)\b(?:only|when|if|unless)\b`)
|
|
|
|
// unconditionalHistoryRead returns the first full-history instruction whose
|
|
// sentence does not make it conditional ("only when the handoff lacks ...").
|
|
func unconditionalHistoryRead(prompt string) string {
|
|
for _, loc := range fullHistoryRead.FindAllStringIndex(prompt, -1) {
|
|
start := strings.LastIndexAny(prompt[:loc[0]], ".\n") + 1
|
|
end := len(prompt)
|
|
if i := strings.IndexAny(prompt[loc[1]:], ".\n"); i >= 0 {
|
|
end = loc[1] + i
|
|
}
|
|
if !conditional.MatchString(prompt[start:end]) {
|
|
return prompt[loc[0]:loc[1]]
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
var silentGuards = []struct {
|
|
re *regexp.Regexp
|
|
name string
|
|
}{
|
|
{regexp.MustCompile(`\|\|\s*(?:true\b|:(?:\s|;|\)|$)|exit\s+0\b|echo\b[^|&;()]*(?:;|$))`), "an || fallback that succeeds"},
|
|
{regexp.MustCompile(`(?:\[\[?|\btest)\s+!\s+-[a-zA-Z]\s+[^|&;]+?\]?\]?\s*\|\|`), "a negated file test that skips the command"},
|
|
{regexp.MustCompile(`!\s*command\s+-v\s+\S+[^|&;]*\|\|`), "a missing-tool guard that skips the command"},
|
|
{regexp.MustCompile(`;\s*(?:true|exit\s+0)\s*$`), "a trailing command that always succeeds"},
|
|
{regexp.MustCompile(`^\s*if\s.*\bthen\b.*\bfi\b`), "an if without else"},
|
|
{regexp.MustCompile(`(?:^|\s)set\s+\+e\b`), "set +e"},
|
|
}
|
|
|
|
var elseBranch = regexp.MustCompile(`\belse\b|\bexit\s+[1-9]`)
|
|
|
|
func silentGuard(line string) string {
|
|
for _, guard := range silentGuards {
|
|
if !guard.re.MatchString(line) {
|
|
continue
|
|
}
|
|
if guard.name == "an if without else" && elseBranch.MatchString(line) {
|
|
continue
|
|
}
|
|
return guard.name
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func lintError(errs []LintFinding) error {
|
|
if len(errs) == 0 {
|
|
return nil
|
|
}
|
|
list := make([]error, 0, len(errs))
|
|
for _, finding := range errs {
|
|
list = append(list, errors.New(finding.String()))
|
|
}
|
|
return errors.Join(list...)
|
|
}
|
|
|
|
// LintStrings renders findings for protocol replies.
|
|
func LintStrings(findings []LintFinding) []string {
|
|
if len(findings) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]string, 0, len(findings))
|
|
for _, finding := range findings {
|
|
out = append(out, finding.String())
|
|
}
|
|
return out
|
|
}
|