From 37e941bbbab7c4792056fa8a5c12dc67679c0223 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Wed, 4 Nov 2026 13:04:11 +0000 Subject: [PATCH] Authoring lint: required checks, undeclared outcomes, delivery claims, loop limits, silent guards, boilerplate --- examples/debug.md | 46 ++-- examples/task.md | 30 ++- examples/three-stage.md | 40 ++-- internal/cli/app.go | 3 + internal/pipeline/definition.go | 2 +- internal/pipeline/definition_test.go | 4 +- internal/pipeline/lint.go | 306 +++++++++++++++++++++++++++ internal/pipeline/lint_test.go | 236 +++++++++++++++++++++ internal/pipeline/schedule.go | 13 ++ internal/pipeline/store.go | 17 +- internal/protocol/message.go | 1 + internal/server/websocket.go | 6 +- internal/stage/checks_test.go | 37 ++++ internal/stage/model.go | 4 +- internal/stage/tools.go | 42 ++++ internal/tui/pipeline_test.go | 2 + 16 files changed, 745 insertions(+), 44 deletions(-) create mode 100644 internal/pipeline/lint.go create mode 100644 internal/pipeline/lint_test.go create mode 100644 internal/stage/checks_test.go diff --git a/examples/debug.md b/examples/debug.md index 5cf1a50d..5e9eb087 100644 --- a/examples/debug.md +++ b/examples/debug.md @@ -2,10 +2,11 @@ ```prompt Investigate the reported problem in input.md without editing. Read the relevant -code and logs, reproduce the failure with safe commands where possible, and find -the cause rather than the first symptom. Record the evidence, the smallest -credible fix, and anything you could not verify. Return blocked when the problem -cannot be reproduced or the needed evidence is unavailable. +code and logs (search first, then read file ranges, not whole logs), reproduce the +failure with the narrowest safe command, and find the cause rather than the first +symptom. Record the evidence, the smallest credible fix, and anything you could +not verify. Return blocked when the problem cannot be reproduced or the needed +evidence is unavailable. ``` ```tools @@ -15,6 +16,10 @@ grep shell ``` +```checks +none: read-only diagnosis; shell runs only the reproduction +``` + ```configs provider=default model=default @@ -33,7 +38,8 @@ error -> #failure ```prompt Fix the diagnosed cause with the smallest coherent change. Follow the repository's conventions, preserve unrelated work, and add or update a regression -test that would fail before the fix. Run the relevant checks and report the exact +test that would fail before the fix. Run that test and other focused checks for +the touched code; the committer runs the full checks once. Report the exact commands and actual results. Do not commit, push or deploy. ``` @@ -45,6 +51,10 @@ edit shell ``` +```checks +none: focused checks only; the committer runs the project's required checks once +``` + ```configs provider=default model=default @@ -61,10 +71,10 @@ error -> #failure # stage verify ```prompt -Independently verify the fix. Reproduce the original problem, run the relevant -checks, and inspect the changed files. Do not edit. Return success only when the -original failure is gone and the checks pass; otherwise return rejected with -concrete evidence. +Independently verify the fix. Re-run the original reproduction and the new +regression test, and read the diff (`git diff --stat` first). Do not run the full +suite and do not edit. Return success only when the original failure is gone; +otherwise return rejected with concrete evidence. ``` ```tools @@ -74,6 +84,10 @@ grep shell ``` +```checks +none: read-only verification; runs only the reproduction and regression test +``` + ```configs provider=default model=default @@ -90,10 +104,12 @@ error -> #failure # stage committer ```prompt -Commit this run's change locally. Inspect `git status` and the full diff; stage -only this run's work, never secrets or generated files. Run the relevant checks -and report the actual results. If nothing changed, say so rather than creating an -empty commit. Do not push, tag, deploy or rewrite history. +Commit this run's change locally. Inspect `git status` and `git diff --stat`, +then the staged diff; stage only this run's work, never secrets or generated +files. Run the project's required checks once with quiet or summary output and +report the commands and results; return blocked with the failing output when one +fails. If nothing changed, say so rather than creating an empty commit. Do not +push, tag, deploy or rewrite history. ``` ```tools @@ -103,6 +119,10 @@ grep shell ``` +```checks +none: generic starter has no project commands; with delivery=true list them here for the server to run +``` + ```configs provider=default model=default diff --git a/examples/task.md b/examples/task.md index 1d7b8db2..81290d3a 100644 --- a/examples/task.md +++ b/examples/task.md @@ -1,13 +1,13 @@ # stage worker ```prompt -Make the small code change requested in input.md. Inspect the project first and follow its -conventions. Keep the change focused, preserve unrelated work, and run the -relevant checks. Report what changed, the exact checks you ran, and anything you -could not verify. Do not commit, push or deploy. +Make the small code change requested in input.md. Inspect only the files the +change needs and follow the project's conventions. Keep the change focused and +preserve unrelated work. Run focused checks for the touched code (one package or +test file, quiet or summary output), not the whole suite. Report what changed, the +exact checks you ran, and anything you could not verify. Do not commit, push or deploy. Do not run destructive commands, install dependencies or change external services. If the scope or required access is missing, return blocked with the missing input. -Return a concise Markdown change summary with paths and verification evidence. ``` ```tools @@ -18,6 +18,10 @@ edit shell ``` +```checks +none: starter leaves the change uncommitted; the worker runs focused checks and the user runs the project suite +``` + ```configs provider=default model=default @@ -35,12 +39,12 @@ error -> #failure ```prompt Independently review the actual changed files and full diff against the request -and project conventions, not just the worker's summary. Check correctness, -regressions, test coverage and unrelated changes. Inspect status and diffs and -run relevant existing project checks when safe; never install dependencies or -modify source, commit, push or deploy. Return a concise Markdown review with -findings, paths, verification evidence and any unverified behavior. Return blocked -if there are material findings or insufficient evidence to approve the change. +and project conventions, not just the worker's summary. Start with `git status` +and `git diff --stat`, then read the changed hunks and only the surrounding code +you need. Check correctness, regressions, test coverage and unrelated changes. Do +not run test suites or modify files; rely on the worker's reported checks and name +any gap. Report findings with paths and evidence. Return blocked if there are +material findings or insufficient evidence to approve the change. ``` ```tools @@ -50,6 +54,10 @@ grep shell ``` +```checks +none: read-only review; shell is used only for git status and diff +``` + ```configs provider=default model=default diff --git a/examples/three-stage.md b/examples/three-stage.md index 7d240020..13b97bd1 100644 --- a/examples/three-stage.md +++ b/examples/three-stage.md @@ -33,8 +33,9 @@ error -> #failure ```prompt Implement the plan from input.md. Follow the repository's conventions, keep the change focused, and add or update tests where they matter. Preserve unrelated -work. Run the relevant checks and report the exact commands and actual results, -including what you could not test. Do not commit, push or deploy. +work. Run focused checks for the touched code only (quiet or summary output); +the committer runs the full checks once. Report the exact commands and actual +results, including what you could not test. Do not commit, push or deploy. ``` ```tools @@ -45,6 +46,10 @@ edit shell ``` +```checks +none: focused checks only; the committer runs the project's required checks once +``` + ```configs provider=default model=default @@ -61,12 +66,13 @@ error -> #failure # stage reviewer ```prompt -Independently review the actual change against the task and plan. Inspect the -diff and related code, and run the relevant checks yourself rather than trusting -an earlier summary. Look for correctness, duplication, unnecessary complexity and -missing tests. Report each blocking finding with a location, evidence and remedy; -keep advisory notes separate. Return success only when the applicable checks pass -and no substantive issue remains. Do not edit or commit. +Independently review the actual change against the task and plan. Start with +`git diff --stat`, then read the changed hunks and only the related code you +need. Do not run test suites; judge the implementer's reported checks and name +any missing coverage. Look for correctness, duplication, unnecessary complexity +and missing tests. Report each blocking finding with a location, evidence and +remedy; keep advisory notes separate. Return success only when no substantive +issue remains. Do not edit or commit. ``` ```tools @@ -78,6 +84,10 @@ websearch webfetch ``` +```checks +none: read-only review; shell is used only for git diff +``` + ```configs provider=default model=default @@ -94,10 +104,12 @@ error -> #failure # stage committer ```prompt -Commit this run's change locally. Inspect `git status` and the full diff; stage -only this run's work, never secrets or generated files. Run the relevant checks -and report the actual results. If nothing changed, say so rather than creating an -empty commit. Do not push, tag, deploy or rewrite history. +Commit this run's change locally. Inspect `git status` and `git diff --stat`, +then the staged diff; stage only this run's work, never secrets or generated +files. Run the project's required checks once with quiet or summary output and +report the commands and results; return blocked with the failing output when one +fails. If nothing changed, say so rather than creating an empty commit. Do not +push, tag, deploy or rewrite history. ``` ```tools @@ -107,6 +119,10 @@ grep shell ``` +```checks +none: generic starter has no project commands; with delivery=true list them here for the server to run +``` + ```configs provider=default model=default diff --git a/internal/cli/app.go b/internal/cli/app.go index 7cd88615..665ac36d 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -746,6 +746,9 @@ func (a App) Run(ctx context.Context, args []string) error { if o.json { return a.output(r) } + for _, warning := range r.Warnings { + fmt.Fprintln(a.Err, "warning: "+warning) + } if command == "validate" { fmt.Fprintln(a.Out, "Valid draft (server validation). No changes submitted.") } else { diff --git a/internal/pipeline/definition.go b/internal/pipeline/definition.go index 610c1283..94cdf5d4 100644 --- a/internal/pipeline/definition.go +++ b/internal/pipeline/definition.go @@ -33,7 +33,7 @@ func Template(names []string, linear bool) string { if linear && i+1 < len(names) { next = names[i+1] } - fmt.Fprintf(&b, "# stage %s\n\n```prompt\nDescribe the task. Write the complete output.md markdown; the only required section is `# outcome`.\n```\n\n```tools\n```\n\n```edges\nsuccess -> %s\nerror -> %s\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=2\ntimeout=1h\n```\n\n", name, next, protocol.FailureEnd) + fmt.Fprintf(&b, "# stage %s\n\n```prompt\nDescribe the task.\n```\n\n```tools\n```\n\n```edges\nsuccess -> %s\nerror -> %s\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=2\ntimeout=1h\n```\n\n", name, next, protocol.FailureEnd) } return b.String() } diff --git a/internal/pipeline/definition_test.go b/internal/pipeline/definition_test.go index ebf20255..6b8918de 100644 --- a/internal/pipeline/definition_test.go +++ b/internal/pipeline/definition_test.go @@ -36,9 +36,9 @@ func TestValidation(t *testing.T) { } } for name, source := range map[string]string{ - "missing block": strings.Replace(valid, "```prompt\nDescribe the task. Write the complete output.md markdown; the only required section is `# outcome`.\n```", "", 1), + "missing block": strings.Replace(valid, "```prompt\nDescribe the task.\n```", "", 1), "retired input": valid + "```input\nold contract\n```\n", - "empty block": strings.Replace(valid, "Describe the task. Write the complete output.md markdown; the only required section is `# outcome`.", " ", 1), + "empty block": strings.Replace(valid, "Describe the task.", " ", 1), "duplicate block": valid + "```prompt\nagain\n```\n", "unknown block": valid + "```foo\nx\n```\n", "unclosed": valid + "```input\nx", diff --git a/internal/pipeline/lint.go b/internal/pipeline/lint.go new file mode 100644 index 00000000..33be5122 --- /dev/null +++ b/internal/pipeline/lint.go @@ -0,0 +1,306 @@ +package pipeline + +import ( + "errors" + "fmt" + "regexp" + "slices" + "strings" + + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +// Authoring lint applies only to new publications (ParseDefinition), never to +// stored immutable versions, which Parse keeps reading and running unchanged. +// Errors are structural or textual mistakes that are unambiguous; warnings are +// token-cost and clarity problems the author should usually fix. + +// MaxPromptBytes is the stage-prompt length above which lint warns. The runner +// adds its own turn-file, outcome and tool instructions around every prompt. +const MaxPromptBytes = 2000 + +// LintFinding is one authoring-policy result for a stage. +type LintFinding struct { + Stage string `json:"stage"` + Rule string `json:"rule"` + Message string `json:"message"` +} + +func (f LintFinding) String() string { + return fmt.Sprintf("stage %s: %s: %s", f.Stage, f.Rule, f.Message) +} + +// Lint returns the authoring errors and warnings for a parsed graph. +func Lint(g Graph) (errs, warnings []LintFinding) { + index := map[string]int{} + delivery := false + for i, s := range g.Stages { + index[s.Key] = i + if s.Configs["delivery"] == "true" { + delivery = true + } + } + add := func(list *[]LintFinding, s Stage, rule, format string, args ...any) { + *list = append(*list, LintFinding{Stage: s.Key, Rule: rule, Message: fmt.Sprintf(format, args...)}) + } + for i, s := range g.Stages { + model := stage.EffectiveType(s.Type) == stage.ModelType + if model { + if reason := mutatingReason(s); reason != "" { + if _, present := s.Blocks["checks"]; !present { + if s.Configs["delivery"] == "true" { + add(&errs, s, "checks", "delivery stage needs a checks block listing the commands the server runs before landing, one per line") + } else { + add(&errs, s, "checks", "stage can modify the workspace (%s) but has no checks block; add one: \"none: \" (for example \"none: focused tests only; the Land delivery gate runs the suite\"), or set delivery=true on the landing stage and list its check commands", reason) + } + } + } + for _, word := range undeclaredOutcomes(s.Prompt, s.Edges) { + if word.strong { + add(&errs, s, "outcome", "prompt tells the stage to %s %q but it is not a declared edge; add \"%s -> \" to edges or remove it from the prompt", word.verb, word.name, word.name) + } else { + add(&warnings, s, "outcome", "prompt tells the stage to %s `%s`, which is not a declared edge; declare it or reword the prompt if it is not an outcome", word.verb, word.name) + } + } + if !delivery { + if claim := deliveryClaim.FindString(s.Prompt); claim != "" { + add(&errs, s, "delivery", "prompt says %q but no stage sets delivery=true; add delivery=true and a checks block to the landing stage, or state that the stage itself (or the operator) lands the change", claim) + } + } + for _, phrase := range runnerBoilerplate { + if phrase.MatchString(s.Prompt) { + add(&warnings, s, "boilerplate", "prompt repeats runner-injected instructions (%q); the runner already explains output.md, outcomes, turn files and tools, so delete it", phrase.FindString(s.Prompt)) + break + } + } + if match := unconditionalHistoryRead(s.Prompt); match != "" { + add(&warnings, s, "history", "prompt asks for %q; stages get the essential memory automatically and should read the full history the runner provides only for a specific need", match) + } + if n := len(strings.TrimSpace(s.Prompt)); n > MaxPromptBytes { + add(&warnings, s, "prompt-length", "prompt is %d bytes (over %d); keep stage prompts to the goal and real constraints, and move shared procedure into the skill or project docs", n, MaxPromptBytes) + } + } + if text, present := s.Blocks["checks"]; present { + if _, none := stage.ChecksNone(text); !none { + for _, line := range strings.Split(text, "\n") { + if guard := silentGuard(line); guard != "" { + add(&errs, s, "checks", "check %q can pass without running anything (%s); a missing tool must fail the check, so remove the guard", strings.TrimSpace(line), guard) + } + } + } + } + if stage.EffectiveType(s.Type) == stage.HumanType { + continue // a person bounds a human stage's loop + } + for _, outcome := range sortedOutcomes(s.Edges) { + target := s.Edges[outcome] + if protocol.IsTerminal(target) || index[target] > i { + continue + } + if !loopLimited(s, outcome, g.Stages[index[target]]) { + add(&errs, s, "limits", "edge %s -> %s loops back without a runner limit; add a limits block such as {\"max_visits\":3,\"on\":[%q]} to %s, or {\"max_visits\":3} to %s", outcome, target, outcome, s.Key, target) + } + } + } + return errs, warnings +} + +// mutatingReason names why a model stage can change the workspace, or "". +func mutatingReason(s Stage) string { + if s.Configs["delivery"] == "true" { + return "delivery=true" + } + tools, explicit, err := stage.ToolNames(StageDefinition(s)) + if err != nil { + return "" + } + if !explicit { + return "no tools block grants edit and shell" + } + var found []string + for _, tool := range tools { + if tool == "edit" || tool == "shell" { + found = append(found, tool) + } + } + if len(found) == 0 { + return "" + } + return "tools " + strings.Join(found, ", ") +} + +func loopLimited(s Stage, outcome string, target Stage) bool { + if target.Limits != nil { + return true + } + return s.Limits != nil && (len(s.Limits.On) == 0 || slices.Contains(s.Limits.On, outcome)) +} + +func sortedOutcomes(edges map[string]string) []string { + keys := make([]string, 0, len(edges)) + for key := range edges { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +// outcomeWords are edge names that read as outcomes even unquoted. Words that +// commonly appear in prose after "report" (failed, pass, done, failure) are +// deliberately absent; they are still checked when quoted. +var outcomeWords = map[string]bool{ + "success": true, "blocked": true, "blocked_retryable": true, "skipped": true, + "rejected": true, "approved": true, "advisory": true, "revise": true, + "abort": true, "noop": true, "unchanged": true, "changes_requested": true, + "needs_changes": true, "retryable": true, +} + +var ( + outcomeToken = "[`'\"]?#?[A-Za-z][A-Za-z0-9_-]*[`'\"]?" + outcomePhrase = regexp.MustCompile("(?i)\\b(return(?:s|ing)?|report(?:s|ing)?|choose|outcome(?:\\s+is)?)\\s+(" + outcomeToken + "(?:\\s*(?:,|/|\\bor\\b|\\|)\\s*" + outcomeToken + ")*)") + outcomeTokenRe = regexp.MustCompile(outcomeToken) + negated = regexp.MustCompile(`(?i)\b(?:not|never|no|don't|do not|nor)\s+$`) +) + +type outcomeMention struct { + verb, name string + strong bool +} + +// undeclaredOutcomes finds "return blocked" / "report `skipped`" style +// instructions naming an outcome the stage does not declare. Negated phrases +// ("never return success") are ignored. +func undeclaredOutcomes(prompt string, edges map[string]string) []outcomeMention { + var found []outcomeMention + seen := map[string]bool{} + for _, match := range outcomePhrase.FindAllStringSubmatchIndex(prompt, -1) { + start := match[0] - 12 + if start < 0 { + start = 0 + } + if negated.MatchString(prompt[start:match[0]]) { + continue + } + verb := strings.ToLower(strings.Fields(prompt[match[2]:match[3]])[0]) + outcomeVerb := strings.HasPrefix(verb, "outcome") + for _, token := range outcomeTokenRe.FindAllString(prompt[match[4]:match[5]], -1) { + quoted := strings.ContainsAny(token, "`'\"") + if strings.Contains(token, "#") { + continue + } + name := strings.Trim(token, "`'\"") + lower := strings.ToLower(name) + if lower == "or" || lower == "error" || seen[name] { + continue + } + if _, declared := edges[name]; declared { + continue + } + if !quoted && !outcomeWords[lower] { + continue + } + seen[name] = true + found = append(found, outcomeMention{verb: verb, name: name, strong: outcomeWords[lower] || (quoted && outcomeVerb)}) + } + } + return found +} + +// deliveryClaim matches affirmative statements that the server lands work. +var deliveryClaim = regexp.MustCompile(`(?i)\bthe server (?:then |will |automatically |itself )?(?:lands?|merges?|delivers?)\b|\b(?:landed|merged|delivered) by the server\b|\bserver[- ]side (?:landing|delivery)\b|\bthe (?:server's )?delivery gate\b`) + +// runnerBoilerplate are phrases the runner itself injects around every model +// prompt (internal/stage/model_prompt.go) or that restate its output contract. +var runnerBoilerplate = func() []*regexp.Regexp { + var list []*regexp.Regexp + for _, phrase := range []string{ + `read these two files before working`, + `memory is the accumulated markdown`, + `return its complete content as your final message`, + `(?:the )?only required section is(?: the)? ` + "`?#?\\s*outcome", + "only `?#\\s*outcome`? (?:section )?is required", + `everything else is free-form markdown`, + `do not wrap it in a json envelope`, + `allowed outcomes:`, + `do not invent file edits`, + `never claim unperformed work succeeded`, + `write the complete output\.md`, + `return the complete output\.md`, + } { + list = append(list, regexp.MustCompile(`(?i)`+strings.ReplaceAll(phrase, " ", `\s+`))) + } + return list +}() + +// fullHistoryRead matches prompts that ask for the whole run history on every +// visit instead of reading it only for a specific need. +var fullHistoryRead = regexp.MustCompile(`(?i)\b(?:read|review|load|open)\b[^.\n]{0,30}\b(?:entire|full|whole|complete|all of the|all of)\s+(?:run\s+)?(?:memory(?:\.md)?|history|transcript)\b|\bread\s+(?:the\s+)?(?:input\.md\s+and\s+)?memory\.md\b`) + +var conditional = regexp.MustCompile(`(?i)\b(?:only|when|if|unless)\b`) + +// unconditionalHistoryRead returns the first full-history instruction whose +// sentence does not make it conditional ("only when the handoff lacks ..."). +func unconditionalHistoryRead(prompt string) string { + for _, loc := range fullHistoryRead.FindAllStringIndex(prompt, -1) { + start := strings.LastIndexAny(prompt[:loc[0]], ".\n") + 1 + end := len(prompt) + if i := strings.IndexAny(prompt[loc[1]:], ".\n"); i >= 0 { + end = loc[1] + i + } + if !conditional.MatchString(prompt[start:end]) { + return prompt[loc[0]:loc[1]] + } + } + return "" +} + +var silentGuards = []struct { + re *regexp.Regexp + name string +}{ + {regexp.MustCompile(`\|\|\s*(?:true\b|:(?:\s|;|\)|$)|exit\s+0\b|echo\b)`), "an || fallback that succeeds"}, + {regexp.MustCompile(`(?:\[\[?|\btest)\s+!\s+-[a-zA-Z]\s+[^|&;]+?\]?\]?\s*\|\|`), "a negated file test that skips the command"}, + {regexp.MustCompile(`!\s*command\s+-v\s+\S+[^|&;]*\|\|`), "a missing-tool guard that skips the command"}, + {regexp.MustCompile(`;\s*(?:true|exit\s+0)\s*$`), "a trailing command that always succeeds"}, + {regexp.MustCompile(`^\s*if\s.*\bthen\b.*\bfi\b`), "an if without else"}, + {regexp.MustCompile(`(?:^|\s)set\s+\+e\b`), "set +e"}, +} + +var elseBranch = regexp.MustCompile(`\belse\b|\bexit\s+[1-9]`) + +func silentGuard(line string) string { + for _, guard := range silentGuards { + if !guard.re.MatchString(line) { + continue + } + if guard.name == "an if without else" && elseBranch.MatchString(line) { + continue + } + return guard.name + } + return "" +} + +func lintError(errs []LintFinding) error { + if len(errs) == 0 { + return nil + } + list := make([]error, 0, len(errs)) + for _, finding := range errs { + list = append(list, errors.New(finding.String())) + } + return errors.Join(list...) +} + +// LintStrings renders findings for protocol replies. +func LintStrings(findings []LintFinding) []string { + if len(findings) == 0 { + return nil + } + out := make([]string, 0, len(findings)) + for _, finding := range findings { + out = append(out, finding.String()) + } + return out +} diff --git a/internal/pipeline/lint_test.go b/internal/pipeline/lint_test.go new file mode 100644 index 00000000..eea7bbbf --- /dev/null +++ b/internal/pipeline/lint_test.go @@ -0,0 +1,236 @@ +package pipeline + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// lintStage builds a one- or two-stage definition around the given blocks. +func lintSource(stages ...string) string { + return strings.Join(stages, "\n") +} + +func modelStage(key, prompt, tools, edges, extra string) string { + b := "# stage " + key + "\n\n```prompt\n" + prompt + "\n```\n\n```configs\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\n```\n\n```tools\n" + tools + "\n```\n\n```edges\n" + edges + "\n```\n" + if extra != "" { + b += "\n" + extra + "\n" + } + return b +} + +func lintRules(t *testing.T, source string) (errs, warns map[string][]string) { + t.Helper() + g, err := ParseDefinition(source) + if err != nil { + t.Fatalf("structural parse: %v", err) + } + e, w := Lint(g) + errs, warns = map[string][]string{}, map[string][]string{} + for _, f := range e { + errs[f.Rule] = append(errs[f.Rule], f.String()) + } + for _, f := range w { + warns[f.Rule] = append(warns[f.Rule], f.String()) + } + return errs, warns +} + +const okEdges = "success -> #success\nblocked -> #failure\nerror -> #failure" + +func TestLintRequiresChecksOnMutatingStages(t *testing.T) { + for name, tc := range map[string]struct { + tools, extra string + want bool + }{ + "edit+shell without checks": {tools: "read\nedit\nshell", want: true}, + "shell only without checks": {tools: "read\nshell", want: true}, + "read-only needs nothing": {tools: "read\nglob\ngrep"}, + "text-only needs nothing": {tools: ""}, + "none with reason": {tools: "read\nedit", extra: "```checks\nnone: focused tests only\n```"}, + } { + t.Run(name, func(t *testing.T) { + errs, _ := lintRules(t, modelStage("Code", "Make the change.", tc.tools, okEdges, tc.extra)) + if got := len(errs["checks"]) > 0; got != tc.want { + t.Fatalf("checks error = %v, want %v: %v", got, tc.want, errs) + } + }) + } + // An absent tools block is the legacy edit/shell bundle. + source := "# stage Code\n\n```prompt\nDo it.\n```\n\n```configs\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\n```\n\n```edges\n" + okEdges + "\n```\n" + if errs, _ := lintRules(t, source); len(errs["checks"]) == 0 { + t.Fatal("stage without a tools block was not required to declare checks") + } + // A delivery stage without checks is told to list the commands. + delivery := strings.Replace(modelStage("Land", "Commit.", "read\nshell", okEdges, ""), "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1) + if errs, _ := lintRules(t, delivery); len(errs["checks"]) == 0 || !strings.Contains(errs["checks"][0], "listing the commands") { + t.Fatalf("delivery stage without checks: %v", errs) + } +} + +func TestLintDeliveryChecksStructural(t *testing.T) { + if _, err := ParseDefinition(modelStage("Land", "Commit.", "read\nshell", okEdges, "```checks\ngo test ./...\n```")); err == nil || !strings.Contains(err.Error(), "only on delivery stages") { + t.Fatalf("checks commands on a non-delivery stage = %v", err) + } +} + +func TestLintUndeclaredOutcomes(t *testing.T) { + for name, tc := range map[string]struct { + prompt string + err, warn bool + edgesExtra string + }{ + "report skipped undeclared": {prompt: "If nothing to do, report skipped.", err: true}, + "return blocked_retryable": {prompt: "On a flaky device return blocked_retryable.", err: true}, + "quoted vocab after outcome": {prompt: "Set the outcome is `rejected` when wrong.", err: true}, + "declared word is fine": {prompt: "Return blocked when input is missing."}, + "prose after report is fine": {prompt: "Report failed tests and the findings."}, + "negated is fine": {prompt: "Never return skipped for real work."}, + "quoted non-vocab warns": {prompt: "Return `partial` when half done.", warn: true}, + "list after return": {prompt: "Return success or `skipped`.", err: true}, + "declared custom edge": {prompt: "Return skipped when nothing changed.", edgesExtra: "\nskipped -> #success"}, + "error is implicit": {prompt: "Return error on failure."}, + "terminal reference is ignored": {prompt: "Return `#success` via the edge."}, + } { + t.Run(name, func(t *testing.T) { + errs, warns := lintRules(t, modelStage("Plan", tc.prompt, "read", okEdges+tc.edgesExtra, "")) + if got := len(errs["outcome"]) > 0; got != tc.err { + t.Fatalf("outcome error = %v, want %v: %v", got, tc.err, errs) + } + if got := len(warns["outcome"]) > 0; got != tc.warn { + t.Fatalf("outcome warning = %v, want %v: %v", got, tc.warn, warns) + } + }) + } +} + +func TestLintDeliveryClaimWithoutDeliveryStage(t *testing.T) { + claim := modelStage("Land", "Commit the change; the server lands it after checks.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```") + if errs, _ := lintRules(t, claim); len(errs["delivery"]) == 0 { + t.Fatal("delivery claim without delivery=true was accepted") + } + honest := modelStage("Land", "Commit the change locally; the operator lands it.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```") + if errs, _ := lintRules(t, honest); len(errs["delivery"]) != 0 { + t.Fatalf("honest committer flagged: %v", errs) + } + negated := modelStage("Land", "Commit locally. Without a delivery gate nothing is landed.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```") + if errs, _ := lintRules(t, negated); len(errs["delivery"]) != 0 { + t.Fatalf("negated claim flagged: %v", errs) + } + delivered := strings.Replace(claim, "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1) + delivered = strings.Replace(delivered, "none: commits locally only", "go test ./...", 1) + if errs, _ := lintRules(t, delivered); len(errs) != 0 { + t.Fatalf("delivery stage claim flagged: %v", errs) + } +} + +func TestLintFeedbackLoopNeedsLimits(t *testing.T) { + code := modelStage("Code", "Make the change.", "read\nedit\nshell", "success -> Review\nerror -> #failure", "```checks\nnone: focused tests only\n```") + review := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "") + if errs, _ := lintRules(t, lintSource(code, review)); len(errs["limits"]) == 0 { + t.Fatal("unbounded review loop accepted") + } + limited := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "```limits\n{\"max_visits\":3,\"on\":[\"rejected\"]}\n```") + if errs, _ := lintRules(t, lintSource(code, limited)); len(errs["limits"]) != 0 { + t.Fatalf("limited loop flagged: %v", errs) + } + targetLimited := strings.Replace(code, "```checks", "```limits\n{\"max_visits\":3}\n```\n\n```checks", 1) + if errs, _ := lintRules(t, lintSource(targetLimited, review)); len(errs["limits"]) != 0 { + t.Fatalf("target-limited loop flagged: %v", errs) + } + otherOutcome := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "```limits\n{\"max_visits\":3,\"on\":[\"approved\"]}\n```") + if _, err := ParseDefinition(lintSource(code, otherOutcome)); err == nil { + t.Fatal("expected structural rejection of a terminal limit outcome") + } + human := "# stage Gate\n\n```prompt\nChoose.\n```\n\n```configs\ntype=human\n```\n\n```edges\ndeploy -> #success\nrevise -> Code\nerror -> #failure\n```\n" + codeToGate := strings.Replace(code, "success -> Review", "success -> Gate", 1) + if errs, _ := lintRules(t, lintSource(codeToGate, human)); len(errs["limits"]) != 0 { + t.Fatalf("human-driven loop flagged: %v", errs) + } +} + +func TestLintSilentCheckGuards(t *testing.T) { + for check, silent := range map[string]bool{ + "go test ./...": false, + "go vet ./... || true": true, + "[ ! -x tool/flutter ] || (cd app && flutter test)": true, + "test ! -f pubspec.yaml || flutter test": true, + "! command -v flutter || flutter test": true, + "if [ -f pubspec.yaml ]; then flutter test; fi": true, + "if [ -f pubspec.yaml ]; then flutter test; else exit 1; fi": false, + "make check || echo skipped": true, + "go test ./... && go vet ./...": false, + "gofmt -l . | (! grep .)": false, + } { + if got := silentGuard(check) != ""; got != silent { + t.Errorf("silentGuard(%q) = %v, want %v", check, got, silent) + } + } + source := modelStage("Land", "Commit.", "read\nshell", okEdges, "```checks\ngo test ./... || true\n```") + source = strings.Replace(source, "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1) + if errs, _ := lintRules(t, source); len(errs["checks"]) == 0 { + t.Fatal("silent delivery check accepted") + } +} + +func TestLintWarnings(t *testing.T) { + for name, tc := range map[string]struct { + prompt, rule string + }{ + "runner output boilerplate": {prompt: "Do it. Write the complete output.md markdown; only `# outcome` is required.", rule: "boilerplate"}, + "allowed outcomes list": {prompt: "Do it. Allowed outcomes: success, blocked.", rule: "boilerplate"}, + "full history read": {prompt: "First read the entire memory.md, then do it.", rule: "history"}, + "memory file read": {prompt: "Read input.md and memory.md, then do it.", rule: "history"}, + "long prompt": {prompt: strings.Repeat("Keep it short. ", 200), rule: "prompt-length"}, + } { + t.Run(name, func(t *testing.T) { + _, warns := lintRules(t, modelStage("Plan", tc.prompt, "read", okEdges, "")) + if len(warns[tc.rule]) == 0 { + t.Fatalf("no %s warning: %v", tc.rule, warns) + } + }) + } + if _, warns := lintRules(t, modelStage("Plan", "Plan the task. Read the full history only when the handoff lacks a detail.", "read", okEdges, "")); len(warns) != 0 { + t.Fatalf("clean prompt warned: %v", warns) + } +} + +func TestParseDefinitionReportAggregatesErrorsAndKeepsStoredParse(t *testing.T) { + source := modelStage("Code", "Report skipped when done.", "read\nedit\nshell", okEdges, "") + _, _, err := ParseDefinitionReport(source) + if err == nil || !strings.Contains(err.Error(), "checks") || !strings.Contains(err.Error(), "skipped") { + t.Fatalf("lint errors not reported together: %v", err) + } + // Stored versions keep loading and running: Parse/ParseDefinition do not lint. + if _, err := Parse(source); err != nil { + t.Fatalf("stored version parse changed: %v", err) + } + if _, err := ParseDefinition(source); err != nil { + t.Fatalf("structural validation changed: %v", err) + } +} + +func TestExamplesPassLint(t *testing.T) { + paths, err := filepath.Glob(filepath.Join("..", "..", "examples", "*.md")) + if err != nil { + t.Fatal(err) + } + for _, path := range paths { + if filepath.Base(path) == "README.md" { + continue + } + source, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + source = []byte(WithDefaultModel(string(source), "opencode-go/deepseek-v4.1-flash")) + _, warnings, err := ParseDefinitionReport(string(source)) + if err != nil { + t.Errorf("%s: %v", path, err) + } + if len(warnings) != 0 { + t.Errorf("%s warnings: %v", path, warnings) + } + } +} diff --git a/internal/pipeline/schedule.go b/internal/pipeline/schedule.go index b185298d..4013e74c 100644 --- a/internal/pipeline/schedule.go +++ b/internal/pipeline/schedule.go @@ -11,6 +11,7 @@ var weekdays = []string{"sun", "mon", "tue", "wed", "thu", "fri", "sat"} // ParseDefinition validates new publications. Parse deliberately remains a // historical reader for immutable versions containing deprecated settings. +// Authoring lint is separate: see ParseDefinitionReport. func ParseDefinition(source string) (protocol.Graph, error) { g, err := Parse(source) if err == nil && g.Settings != nil { @@ -19,6 +20,18 @@ func ParseDefinition(source string) (protocol.Graph, error) { return g, err } +// ParseDefinitionReport is ParseDefinition plus the authoring lint (Lint) the +// publication API applies to new drafts: lint errors reject the draft, all +// reported together; warnings are returned for the author and never block it. +func ParseDefinitionReport(source string) (protocol.Graph, []string, error) { + g, err := ParseDefinition(source) + if err != nil { + return g, nil, err + } + errs, warnings := Lint(g) + return g, LintStrings(warnings), lintError(errs) +} + func ValidateSchedule(rule protocol.Schedule) error { text := "" if len(rule.Days) > 0 { diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index d3cb9aff..40c68bf3 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -290,11 +290,26 @@ func (s *Store) Submit(ctx context.Context, name, source string, createOnly ...b } // SubmitChecked guards draft edits against concurrent accepted versions atomically. +// It applies structural validation only; PublishChecked adds the authoring lint. func (s *Store) SubmitChecked(ctx context.Context, name, source string, createOnly bool, expectedVersion int) (int, error) { + return s.submit(ctx, name, source, createOnly, expectedVersion, ParseDefinition) +} + +// PublishChecked is SubmitChecked for the publication API (CLI, TUI, web and +// import submit through it): lint errors (Lint) reject the version and are +// recorded as the submission diagnostic like any other validation failure. +func (s *Store) PublishChecked(ctx context.Context, name, source string, createOnly bool, expectedVersion int) (int, error) { + return s.submit(ctx, name, source, createOnly, expectedVersion, func(source string) (Graph, error) { + g, _, err := ParseDefinitionReport(source) + return g, err + }) +} + +func (s *Store) submit(ctx context.Context, name, source string, createOnly bool, expectedVersion int, parse func(string) (Graph, error)) (int, error) { s.mu.Lock() defer s.mu.Unlock() name = strings.TrimSpace(name) - g, validation := ParseDefinition(source) + g, validation := parse(source) if expectedVersion < 0 { validation = fmt.Errorf("expected version must be nonnegative") } diff --git a/internal/protocol/message.go b/internal/protocol/message.go index e8353824..8d4c46bb 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -136,6 +136,7 @@ type Message struct { ID string `json:"id,omitempty"` Time string `json:"time,omitempty"` Error string `json:"error,omitempty"` + Warnings []string `json:"warnings,omitempty"` // non-blocking authoring lint for validate/submit Name string `json:"name,omitempty"` Source string `json:"source,omitempty"` Title string `json:"title,omitempty"` diff --git a/internal/server/websocket.go b/internal/server/websocket.go index 319e6b6f..215dd369 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -281,8 +281,9 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin response.Templates[i].Source = pipeline.WithDefaults(response.Templates[i].Source, model, backend) } } else if request.Type == protocol.Validate { - graph, err := pipeline.ParseDefinition(request.Source) + graph, warnings, err := pipeline.ParseDefinitionReport(request.Source) response.Type = protocol.Validated + response.Warnings = warnings if err != nil { response.Type = protocol.Error response.Error = err.Error() @@ -318,7 +319,8 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin } else if request.Type == protocol.Submit && len(stores) > 0 { response.Type = protocol.Submitted response.Name = strings.TrimSpace(request.Name) - version, err := stores[0].SubmitChecked(ctx, request.Name, request.Source, request.CreateOnly, request.ExpectedVersion) + version, err := stores[0].PublishChecked(ctx, request.Name, request.Source, request.CreateOnly, request.ExpectedVersion) + _, response.Warnings, _ = pipeline.ParseDefinitionReport(request.Source) if err != nil { response.Type = protocol.Error response.Error = err.Error() diff --git a/internal/stage/checks_test.go b/internal/stage/checks_test.go new file mode 100644 index 00000000..7d3c104a --- /dev/null +++ b/internal/stage/checks_test.go @@ -0,0 +1,37 @@ +package stage + +import "testing" + +func TestChecksNoneDeclaration(t *testing.T) { + for name, tc := range map[string]struct { + checks string + delivery bool + ok bool + }{ + "none on mutating stage": {checks: "none: read-only review", ok: true}, + "none case insensitive": {checks: "None: focused tests only", ok: true}, + "none without reason": {checks: "none:"}, + "none with extra lines": {checks: "none: reason\ngo test ./..."}, + "none on delivery stage": {checks: "none: trust me", delivery: true}, + "commands on non-delivery": {checks: "go test ./..."}, + "commands on delivery": {checks: "go test ./...", delivery: true, ok: true}, + } { + t.Run(name, func(t *testing.T) { + def := modelDefinition() + def.Blocks["checks"] = tc.checks + if tc.delivery { + def.Configs["delivery"] = "true" + } + _, err := (ModelStage{}).Spec(def) + if (err == nil) != tc.ok { + t.Fatalf("Spec(%q) error = %v, want ok=%v", tc.checks, err, tc.ok) + } + }) + } + def := modelDefinition() + def.Configs["delivery"] = "true" + def.Blocks["checks"] = "none: skip" + if _, err := DeliveryChecks(def); err == nil { + t.Fatal("delivery gate would run none: as a command") + } +} diff --git a/internal/stage/model.go b/internal/stage/model.go index a74c39e1..d933de1c 100644 --- a/internal/stage/model.go +++ b/internal/stage/model.go @@ -104,8 +104,8 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) { if def.Configs["delivery"] == "true" && backend != "opencode" { return ModelSpec{}, fmt.Errorf("delivery is supported only by the opencode harness") } - if _, present := def.Blocks["checks"]; present && def.Configs["delivery"] != "true" { - return ModelSpec{}, fmt.Errorf("checks block is supported only by delivery stages") + if err := ValidateChecksBlock(def); err != nil { + return ModelSpec{}, err } if backend == "pi" { tools, _, _ := ToolNames(def) diff --git a/internal/stage/tools.go b/internal/stage/tools.go index 53ce3043..b29d25db 100644 --- a/internal/stage/tools.go +++ b/internal/stage/tools.go @@ -72,6 +72,9 @@ func DeliveryChecks(def Definition) ([]string, error) { if text == "" { return nil, fmt.Errorf("delivery stages require a nonempty checks block") } + if _, none := ChecksNone(text); none { + return nil, fmt.Errorf("delivery stages run their checks before landing; list the check commands instead of none") + } var checks []string for _, line := range strings.Split(text, "\n") { check := strings.TrimSpace(line) @@ -89,6 +92,45 @@ func DeliveryChecks(def Definition) ([]string, error) { return checks, nil } +// ChecksNone reports whether a checks block uses the explicit +// "none: " form. It matches on the leading none: alone, so a malformed +// declaration (no reason, extra lines) is rejected rather than run as a command. +func ChecksNone(text string) (reason string, ok bool) { + text = strings.TrimSpace(text) + if !strings.HasPrefix(strings.ToLower(text), "none:") { + return "", false + } + return strings.TrimSpace(text[len("none:"):]), true +} + +// ValidateChecksBlock is the checks syntax shared by stored and new +// definitions. Only a delivery stage executes commands (the server's delivery +// gate); every other stage may declare only "none: ", so a listed +// command is never accepted and then silently left unexecuted. +func ValidateChecksBlock(def Definition) error { + text, present := def.Blocks["checks"] + if !present { + return nil + } + if reason, none := ChecksNone(text); none { + if strings.Contains(strings.TrimSpace(text), "\n") { + return fmt.Errorf("checks none must be a single line: none: ") + } + if len(reason) < 3 || len(reason) > 300 { + return fmt.Errorf("checks none needs a short reason (3-300 bytes), for example \"none: read-only review\"") + } + if def.Configs["delivery"] == "true" { + return fmt.Errorf("delivery stages run their checks before landing; list the check commands instead of none") + } + return nil + } + if def.Configs["delivery"] != "true" { + return fmt.Errorf("checks commands run only on delivery stages (delivery=true); write \"none: \" here and declare the commands on the delivery stage") + } + _, err := DeliveryChecks(def) + return err +} + // ToolNames validates model-stage permissions, not general graph semantics. // A present empty block explicitly denies all tools. func ToolNames(def Definition) ([]string, bool, error) { diff --git a/internal/tui/pipeline_test.go b/internal/tui/pipeline_test.go index 882afda0..e06b238c 100644 --- a/internal/tui/pipeline_test.go +++ b/internal/tui/pipeline_test.go @@ -49,6 +49,8 @@ func TestSearchVersionsAndEditSubmission(t *testing.T) { defer store.Close() source := pipeline.Template([]string{"implementer", "reviewer"}, true) source = strings.Replace(source, "success -> #success", "bad_implementation -> implementer\nsuccess -> #success", 1) + // reviewer is the last stage; its feedback loop needs a runner limit to publish. + source += "```limits\n{\"max_visits\":3,\"on\":[\"bad_implementation\"]}\n```\n" if _, err := store.Submit(ctx, "demo", source); err != nil { t.Fatal(err) }