Files
pipelines/internal/pipeline/lint_test.go
T

253 lines
12 KiB
Go

package pipeline
import (
"os"
"path/filepath"
"strings"
"testing"
)
// lintStage builds a one- or two-stage definition around the given blocks.
func lintSource(stages ...string) string {
return strings.Join(stages, "\n")
}
func modelStage(key, prompt, tools, edges, extra string) string {
b := "# stage " + key + "\n\n```prompt\n" + prompt + "\n```\n\n```configs\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\n```\n\n```tools\n" + tools + "\n```\n\n```edges\n" + edges + "\n```\n"
if extra != "" {
b += "\n" + extra + "\n"
}
return b
}
func lintRules(t *testing.T, source string) (errs, warns map[string][]string) {
t.Helper()
g, err := ParseDefinition(source)
if err != nil {
t.Fatalf("structural parse: %v", err)
}
e, w := Lint(g)
errs, warns = map[string][]string{}, map[string][]string{}
for _, f := range e {
errs[f.Rule] = append(errs[f.Rule], f.String())
}
for _, f := range w {
warns[f.Rule] = append(warns[f.Rule], f.String())
}
return errs, warns
}
const okEdges = "success -> #success\nblocked -> #failure\nerror -> #failure"
func TestLintRequiresChecksOnMutatingStages(t *testing.T) {
for name, tc := range map[string]struct {
tools, extra string
want bool
}{
"edit+shell without checks": {tools: "read\nedit\nshell", want: true},
"shell only without checks": {tools: "read\nshell", want: true},
"read-only needs nothing": {tools: "read\nglob\ngrep"},
"text-only needs nothing": {tools: ""},
"none with reason": {tools: "read\nedit", extra: "```checks\nnone: focused tests only\n```"},
} {
t.Run(name, func(t *testing.T) {
errs, _ := lintRules(t, modelStage("Code", "Make the change.", tc.tools, okEdges, tc.extra))
if got := len(errs["checks"]) > 0; got != tc.want {
t.Fatalf("checks error = %v, want %v: %v", got, tc.want, errs)
}
})
}
// An absent tools block is the legacy edit/shell bundle.
source := "# stage Code\n\n```prompt\nDo it.\n```\n\n```configs\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\n```\n\n```edges\n" + okEdges + "\n```\n"
if errs, _ := lintRules(t, source); len(errs["checks"]) == 0 {
t.Fatal("stage without a tools block was not required to declare checks")
}
// A delivery stage without checks is told to list the commands.
delivery := strings.Replace(modelStage("Land", "Commit.", "read\nshell", okEdges, ""), "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1)
if errs, _ := lintRules(t, delivery); len(errs["checks"]) == 0 || !strings.Contains(errs["checks"][0], "listing the commands") {
t.Fatalf("delivery stage without checks: %v", errs)
}
}
func TestLintDeliveryChecksStructural(t *testing.T) {
if _, err := ParseDefinition(modelStage("Land", "Commit.", "read\nshell", okEdges, "```checks\ngo test ./...\n```")); err == nil || !strings.Contains(err.Error(), "only on delivery stages") {
t.Fatalf("checks commands on a non-delivery stage = %v", err)
}
}
func TestLintUndeclaredOutcomes(t *testing.T) {
for name, tc := range map[string]struct {
prompt string
err, warn bool
edgesExtra string
}{
"report skipped undeclared": {prompt: "If nothing to do, report skipped.", err: true},
"return blocked_retryable": {prompt: "On a flaky device return blocked_retryable.", err: true},
"quoted vocab after outcome": {prompt: "Set the outcome is `rejected` when wrong.", err: true},
"declared word is fine": {prompt: "Return blocked when input is missing."},
"prose after report is fine": {prompt: "Report failed tests and the findings."},
"negated is fine": {prompt: "Never return skipped for real work."},
"quoted non-vocab warns": {prompt: "Return `partial` when half done.", warn: true},
"list after return": {prompt: "Return success or `skipped`.", err: true},
"declared custom edge": {prompt: "Return skipped when nothing changed.", edgesExtra: "\nskipped -> #success"},
"error is implicit": {prompt: "Return error on failure."},
"terminal reference is ignored": {prompt: "Return `#success` via the edge."},
} {
t.Run(name, func(t *testing.T) {
errs, warns := lintRules(t, modelStage("Plan", tc.prompt, "read", okEdges+tc.edgesExtra, ""))
if got := len(errs["outcome"]) > 0; got != tc.err {
t.Fatalf("outcome error = %v, want %v: %v", got, tc.err, errs)
}
if got := len(warns["outcome"]) > 0; got != tc.warn {
t.Fatalf("outcome warning = %v, want %v: %v", got, tc.warn, warns)
}
})
}
}
func TestLintDeliveryClaimWithoutDeliveryStage(t *testing.T) {
claim := modelStage("Land", "Commit the change; the server lands it after checks.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
if errs, _ := lintRules(t, claim); len(errs["delivery"]) == 0 {
t.Fatal("delivery claim without delivery=true was accepted")
}
checksAndLands := modelStage("Land", "Report success; the server checks and lands it.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
if errs, _ := lintRules(t, checksAndLands); len(errs["delivery"]) == 0 {
t.Fatal("\"the server checks and lands it\" without delivery=true was accepted")
}
never := modelStage("Land", "Commit locally; the server never lands this.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
if errs, _ := lintRules(t, never); len(errs["delivery"]) != 0 {
t.Fatalf("negated claim flagged: %v", errs)
}
honest := modelStage("Land", "Commit the change locally; the operator lands it.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
if errs, _ := lintRules(t, honest); len(errs["delivery"]) != 0 {
t.Fatalf("honest committer flagged: %v", errs)
}
negated := modelStage("Land", "Commit locally. Without a delivery gate nothing is landed.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
if errs, _ := lintRules(t, negated); len(errs["delivery"]) != 0 {
t.Fatalf("negated claim flagged: %v", errs)
}
delivered := strings.Replace(claim, "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1)
delivered = strings.Replace(delivered, "none: commits locally only", "go test ./...", 1)
if errs, _ := lintRules(t, delivered); len(errs) != 0 {
t.Fatalf("delivery stage claim flagged: %v", errs)
}
}
func TestLintFeedbackLoopNeedsLimits(t *testing.T) {
code := modelStage("Code", "Make the change.", "read\nedit\nshell", "success -> Review\nerror -> #failure", "```checks\nnone: focused tests only\n```")
review := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "")
if errs, _ := lintRules(t, lintSource(code, review)); len(errs["limits"]) == 0 {
t.Fatal("unbounded review loop accepted")
}
limited := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "```limits\n{\"max_visits\":3,\"on\":[\"rejected\"]}\n```")
if errs, _ := lintRules(t, lintSource(code, limited)); len(errs["limits"]) != 0 {
t.Fatalf("limited loop flagged: %v", errs)
}
targetLimited := strings.Replace(code, "```checks", "```limits\n{\"max_visits\":3}\n```\n\n```checks", 1)
if errs, _ := lintRules(t, lintSource(targetLimited, review)); len(errs["limits"]) != 0 {
t.Fatalf("target-limited loop flagged: %v", errs)
}
otherOutcome := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "```limits\n{\"max_visits\":3,\"on\":[\"approved\"]}\n```")
if _, err := ParseDefinition(lintSource(code, otherOutcome)); err == nil {
t.Fatal("expected structural rejection of a terminal limit outcome")
}
human := "# stage Gate\n\n```prompt\nChoose.\n```\n\n```configs\ntype=human\n```\n\n```edges\ndeploy -> #success\nrevise -> Code\nerror -> #failure\n```\n"
codeToGate := strings.Replace(code, "success -> Review", "success -> Gate", 1)
if errs, _ := lintRules(t, lintSource(codeToGate, human)); len(errs["limits"]) != 0 {
t.Fatalf("human-driven loop flagged: %v", errs)
}
}
func TestLintSilentCheckGuards(t *testing.T) {
for check, silent := range map[string]bool{
"go test ./...": false,
"go vet ./... || true": true,
"[ ! -x tool/flutter ] || (cd app && flutter test)": true,
"test ! -f pubspec.yaml || flutter test": true,
"! command -v flutter || flutter test": true,
"if [ -f pubspec.yaml ]; then flutter test; fi": true,
"if [ -f pubspec.yaml ]; then flutter test; else exit 1; fi": false,
"make check || echo skipped": true,
"go test ./... && go vet ./...": false,
`F=$(command -v flutter || echo /opt/flutter) && "$F" test`: false,
"gofmt -l . | (! grep .)": false,
} {
if got := silentGuard(check) != ""; got != silent {
t.Errorf("silentGuard(%q) = %v, want %v", check, got, silent)
}
}
source := modelStage("Land", "Commit.", "read\nshell", okEdges, "```checks\ngo test ./... || true\n```")
source = strings.Replace(source, "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1)
if errs, _ := lintRules(t, source); len(errs["checks"]) == 0 {
t.Fatal("silent delivery check accepted")
}
}
func TestLintWarnings(t *testing.T) {
for name, tc := range map[string]struct {
prompt, rule string
}{
"runner output boilerplate": {prompt: "Do it. Write the complete output.md markdown; only `# outcome` is required.", rule: "boilerplate"},
"allowed outcomes list": {prompt: "Do it. Allowed outcomes: success, blocked.", rule: "boilerplate"},
"handoff instruction": {prompt: "Do it. Start your output with a short # handoff section.", rule: "boilerplate"},
"full history read": {prompt: "First read the entire history, then do it.", rule: "history"},
"history file read": {prompt: "Read input.md and history.md, then do it.", rule: "history"},
"long prompt": {prompt: strings.Repeat("Keep it short. ", 200), rule: "prompt-length"},
} {
t.Run(name, func(t *testing.T) {
_, warns := lintRules(t, modelStage("Plan", tc.prompt, "read", okEdges, ""))
if len(warns[tc.rule]) == 0 {
t.Fatalf("no %s warning: %v", tc.rule, warns)
}
})
}
for _, clean := range []string{
"Plan the task. Read history.md only when the handoff lacks a detail.",
"Plan the request in memory.md.",
"Read memory.md for the original request, then plan.",
} {
if _, warns := lintRules(t, modelStage("Plan", clean, "read", okEdges, "")); len(warns) != 0 {
t.Fatalf("clean prompt %q warned: %v", clean, warns)
}
}
}
func TestParseDefinitionReportAggregatesErrorsAndKeepsStoredParse(t *testing.T) {
source := modelStage("Code", "Report skipped when done.", "read\nedit\nshell", okEdges, "")
_, _, err := ParseDefinitionReport(source)
if err == nil || !strings.Contains(err.Error(), "checks") || !strings.Contains(err.Error(), "skipped") {
t.Fatalf("lint errors not reported together: %v", err)
}
// Stored versions keep loading and running: Parse/ParseDefinition do not lint.
if _, err := Parse(source); err != nil {
t.Fatalf("stored version parse changed: %v", err)
}
if _, err := ParseDefinition(source); err != nil {
t.Fatalf("structural validation changed: %v", err)
}
}
func TestExamplesPassLint(t *testing.T) {
paths, err := filepath.Glob(filepath.Join("..", "..", "examples", "*.md"))
if err != nil {
t.Fatal(err)
}
for _, path := range paths {
if filepath.Base(path) == "README.md" {
continue
}
source, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
source = []byte(WithDefaultModel(string(source), "opencode-go/deepseek-v4.1-flash"))
_, warnings, err := ParseDefinitionReport(string(source))
if err != nil {
t.Errorf("%s: %v", path, err)
}
if len(warnings) != 0 {
t.Errorf("%s warnings: %v", path, warnings)
}
}
}