package pipeline import ( "os" "path/filepath" "strings" "testing" ) // lintStage builds a one- or two-stage definition around the given blocks. func lintSource(stages ...string) string { return strings.Join(stages, "\n") } func modelStage(key, prompt, tools, edges, extra string) string { b := "# stage " + key + "\n\n```prompt\n" + prompt + "\n```\n\n```configs\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\n```\n\n```tools\n" + tools + "\n```\n\n```edges\n" + edges + "\n```\n" if extra != "" { b += "\n" + extra + "\n" } return b } func lintRules(t *testing.T, source string) (errs, warns map[string][]string) { t.Helper() g, err := ParseDefinition(source) if err != nil { t.Fatalf("structural parse: %v", err) } e, w := Lint(g) errs, warns = map[string][]string{}, map[string][]string{} for _, f := range e { errs[f.Rule] = append(errs[f.Rule], f.String()) } for _, f := range w { warns[f.Rule] = append(warns[f.Rule], f.String()) } return errs, warns } const okEdges = "success -> #success\nblocked -> #failure\nerror -> #failure" func TestLintRequiresChecksOnMutatingStages(t *testing.T) { for name, tc := range map[string]struct { tools, extra string want bool }{ "edit+shell without checks": {tools: "read\nedit\nshell", want: true}, "shell only without checks": {tools: "read\nshell", want: true}, "read-only needs nothing": {tools: "read\nglob\ngrep"}, "text-only needs nothing": {tools: ""}, "none with reason": {tools: "read\nedit", extra: "```checks\nnone: focused tests only\n```"}, } { t.Run(name, func(t *testing.T) { errs, _ := lintRules(t, modelStage("Code", "Make the change.", tc.tools, okEdges, tc.extra)) if got := len(errs["checks"]) > 0; got != tc.want { t.Fatalf("checks error = %v, want %v: %v", got, tc.want, errs) } }) } // An absent tools block is the legacy edit/shell bundle. source := "# stage Code\n\n```prompt\nDo it.\n```\n\n```configs\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\n```\n\n```edges\n" + okEdges + "\n```\n" if errs, _ := lintRules(t, source); len(errs["checks"]) == 0 { t.Fatal("stage without a tools block was not required to declare checks") } // A delivery stage without checks is told to list the commands. delivery := strings.Replace(modelStage("Land", "Commit.", "read\nshell", okEdges, ""), "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1) if errs, _ := lintRules(t, delivery); len(errs["checks"]) == 0 || !strings.Contains(errs["checks"][0], "listing the commands") { t.Fatalf("delivery stage without checks: %v", errs) } } func TestLintDeliveryChecksStructural(t *testing.T) { if _, err := ParseDefinition(modelStage("Land", "Commit.", "read\nshell", okEdges, "```checks\ngo test ./...\n```")); err == nil || !strings.Contains(err.Error(), "only on delivery stages") { t.Fatalf("checks commands on a non-delivery stage = %v", err) } } func TestLintUndeclaredOutcomes(t *testing.T) { for name, tc := range map[string]struct { prompt string err, warn bool edgesExtra string }{ "report skipped undeclared": {prompt: "If nothing to do, report skipped.", err: true}, "return blocked_retryable": {prompt: "On a flaky device return blocked_retryable.", err: true}, "quoted vocab after outcome": {prompt: "Set the outcome is `rejected` when wrong.", err: true}, "declared word is fine": {prompt: "Return blocked when input is missing."}, "prose after report is fine": {prompt: "Report failed tests and the findings."}, "negated is fine": {prompt: "Never return skipped for real work."}, "quoted non-vocab warns": {prompt: "Return `partial` when half done.", warn: true}, "list after return": {prompt: "Return success or `skipped`.", err: true}, "declared custom edge": {prompt: "Return skipped when nothing changed.", edgesExtra: "\nskipped -> #success"}, "error is implicit": {prompt: "Return error on failure."}, "terminal reference is ignored": {prompt: "Return `#success` via the edge."}, } { t.Run(name, func(t *testing.T) { errs, warns := lintRules(t, modelStage("Plan", tc.prompt, "read", okEdges+tc.edgesExtra, "")) if got := len(errs["outcome"]) > 0; got != tc.err { t.Fatalf("outcome error = %v, want %v: %v", got, tc.err, errs) } if got := len(warns["outcome"]) > 0; got != tc.warn { t.Fatalf("outcome warning = %v, want %v: %v", got, tc.warn, warns) } }) } } func TestLintDeliveryClaimWithoutDeliveryStage(t *testing.T) { claim := modelStage("Land", "Commit the change; the server lands it after checks.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```") if errs, _ := lintRules(t, claim); len(errs["delivery"]) == 0 { t.Fatal("delivery claim without delivery=true was accepted") } checksAndLands := modelStage("Land", "Report success; the server checks and lands it.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```") if errs, _ := lintRules(t, checksAndLands); len(errs["delivery"]) == 0 { t.Fatal("\"the server checks and lands it\" without delivery=true was accepted") } never := modelStage("Land", "Commit locally; the server never lands this.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```") if errs, _ := lintRules(t, never); len(errs["delivery"]) != 0 { t.Fatalf("negated claim flagged: %v", errs) } honest := modelStage("Land", "Commit the change locally; the operator lands it.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```") if errs, _ := lintRules(t, honest); len(errs["delivery"]) != 0 { t.Fatalf("honest committer flagged: %v", errs) } negated := modelStage("Land", "Commit locally. Without a delivery gate nothing is landed.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```") if errs, _ := lintRules(t, negated); len(errs["delivery"]) != 0 { t.Fatalf("negated claim flagged: %v", errs) } delivered := strings.Replace(claim, "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1) delivered = strings.Replace(delivered, "none: commits locally only", "go test ./...", 1) if errs, _ := lintRules(t, delivered); len(errs) != 0 { t.Fatalf("delivery stage claim flagged: %v", errs) } } func TestLintFeedbackLoopNeedsLimits(t *testing.T) { code := modelStage("Code", "Make the change.", "read\nedit\nshell", "success -> Review\nerror -> #failure", "```checks\nnone: focused tests only\n```") review := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "") if errs, _ := lintRules(t, lintSource(code, review)); len(errs["limits"]) == 0 { t.Fatal("unbounded review loop accepted") } limited := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "```limits\n{\"max_visits\":3,\"on\":[\"rejected\"]}\n```") if errs, _ := lintRules(t, lintSource(code, limited)); len(errs["limits"]) != 0 { t.Fatalf("limited loop flagged: %v", errs) } targetLimited := strings.Replace(code, "```checks", "```limits\n{\"max_visits\":3}\n```\n\n```checks", 1) if errs, _ := lintRules(t, lintSource(targetLimited, review)); len(errs["limits"]) != 0 { t.Fatalf("target-limited loop flagged: %v", errs) } otherOutcome := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "```limits\n{\"max_visits\":3,\"on\":[\"approved\"]}\n```") if _, err := ParseDefinition(lintSource(code, otherOutcome)); err == nil { t.Fatal("expected structural rejection of a terminal limit outcome") } human := "# stage Gate\n\n```prompt\nChoose.\n```\n\n```configs\ntype=human\n```\n\n```edges\ndeploy -> #success\nrevise -> Code\nerror -> #failure\n```\n" codeToGate := strings.Replace(code, "success -> Review", "success -> Gate", 1) if errs, _ := lintRules(t, lintSource(codeToGate, human)); len(errs["limits"]) != 0 { t.Fatalf("human-driven loop flagged: %v", errs) } } func TestLintSilentCheckGuards(t *testing.T) { for check, silent := range map[string]bool{ "go test ./...": false, "go vet ./... || true": true, "[ ! -x tool/flutter ] || (cd app && flutter test)": true, "test ! -f pubspec.yaml || flutter test": true, "! command -v flutter || flutter test": true, "if [ -f pubspec.yaml ]; then flutter test; fi": true, "if [ -f pubspec.yaml ]; then flutter test; else exit 1; fi": false, "make check || echo skipped": true, "go test ./... && go vet ./...": false, `F=$(command -v flutter || echo /opt/flutter) && "$F" test`: false, "gofmt -l . | (! grep .)": false, } { if got := silentGuard(check) != ""; got != silent { t.Errorf("silentGuard(%q) = %v, want %v", check, got, silent) } } source := modelStage("Land", "Commit.", "read\nshell", okEdges, "```checks\ngo test ./... || true\n```") source = strings.Replace(source, "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1) if errs, _ := lintRules(t, source); len(errs["checks"]) == 0 { t.Fatal("silent delivery check accepted") } } func TestLintWarnings(t *testing.T) { for name, tc := range map[string]struct { prompt, rule string }{ "runner output boilerplate": {prompt: "Do it. Write the complete output.md markdown; only `# outcome` is required.", rule: "boilerplate"}, "allowed outcomes list": {prompt: "Do it. Allowed outcomes: success, blocked.", rule: "boilerplate"}, "handoff instruction": {prompt: "Do it. Start your output with a short # handoff section.", rule: "boilerplate"}, "full history read": {prompt: "First read the entire history, then do it.", rule: "history"}, "history file read": {prompt: "Read input.md and history.md, then do it.", rule: "history"}, "long prompt": {prompt: strings.Repeat("Keep it short. ", 200), rule: "prompt-length"}, } { t.Run(name, func(t *testing.T) { _, warns := lintRules(t, modelStage("Plan", tc.prompt, "read", okEdges, "")) if len(warns[tc.rule]) == 0 { t.Fatalf("no %s warning: %v", tc.rule, warns) } }) } for _, clean := range []string{ "Plan the task. Read history.md only when the handoff lacks a detail.", "Plan the request in memory.md.", "Read memory.md for the original request, then plan.", } { if _, warns := lintRules(t, modelStage("Plan", clean, "read", okEdges, "")); len(warns) != 0 { t.Fatalf("clean prompt %q warned: %v", clean, warns) } } } func TestParseDefinitionReportAggregatesErrorsAndKeepsStoredParse(t *testing.T) { source := modelStage("Code", "Report skipped when done.", "read\nedit\nshell", okEdges, "") _, _, err := ParseDefinitionReport(source) if err == nil || !strings.Contains(err.Error(), "checks") || !strings.Contains(err.Error(), "skipped") { t.Fatalf("lint errors not reported together: %v", err) } // Stored versions keep loading and running: Parse/ParseDefinition do not lint. if _, err := Parse(source); err != nil { t.Fatalf("stored version parse changed: %v", err) } if _, err := ParseDefinition(source); err != nil { t.Fatalf("structural validation changed: %v", err) } } func TestExamplesPassLint(t *testing.T) { paths, err := filepath.Glob(filepath.Join("..", "..", "examples", "*.md")) if err != nil { t.Fatal(err) } for _, path := range paths { if filepath.Base(path) == "README.md" { continue } source, err := os.ReadFile(path) if err != nil { t.Fatal(err) } source = []byte(WithDefaultModel(string(source), "opencode-go/deepseek-v4.1-flash")) _, warnings, err := ParseDefinitionReport(string(source)) if err != nil { t.Errorf("%s: %v", path, err) } if len(warnings) != 0 { t.Errorf("%s warnings: %v", path, warnings) } } }