253 lines
12 KiB
Go
253 lines
12 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// lintStage builds a one- or two-stage definition around the given blocks.
|
|
func lintSource(stages ...string) string {
|
|
return strings.Join(stages, "\n")
|
|
}
|
|
|
|
func modelStage(key, prompt, tools, edges, extra string) string {
|
|
b := "# stage " + key + "\n\n```prompt\n" + prompt + "\n```\n\n```configs\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\n```\n\n```tools\n" + tools + "\n```\n\n```edges\n" + edges + "\n```\n"
|
|
if extra != "" {
|
|
b += "\n" + extra + "\n"
|
|
}
|
|
return b
|
|
}
|
|
|
|
func lintRules(t *testing.T, source string) (errs, warns map[string][]string) {
|
|
t.Helper()
|
|
g, err := ParseDefinition(source)
|
|
if err != nil {
|
|
t.Fatalf("structural parse: %v", err)
|
|
}
|
|
e, w := Lint(g)
|
|
errs, warns = map[string][]string{}, map[string][]string{}
|
|
for _, f := range e {
|
|
errs[f.Rule] = append(errs[f.Rule], f.String())
|
|
}
|
|
for _, f := range w {
|
|
warns[f.Rule] = append(warns[f.Rule], f.String())
|
|
}
|
|
return errs, warns
|
|
}
|
|
|
|
const okEdges = "success -> #success\nblocked -> #failure\nerror -> #failure"
|
|
|
|
func TestLintRequiresChecksOnMutatingStages(t *testing.T) {
|
|
for name, tc := range map[string]struct {
|
|
tools, extra string
|
|
want bool
|
|
}{
|
|
"edit+shell without checks": {tools: "read\nedit\nshell", want: true},
|
|
"shell only without checks": {tools: "read\nshell", want: true},
|
|
"read-only needs nothing": {tools: "read\nglob\ngrep"},
|
|
"text-only needs nothing": {tools: ""},
|
|
"none with reason": {tools: "read\nedit", extra: "```checks\nnone: focused tests only\n```"},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
errs, _ := lintRules(t, modelStage("Code", "Make the change.", tc.tools, okEdges, tc.extra))
|
|
if got := len(errs["checks"]) > 0; got != tc.want {
|
|
t.Fatalf("checks error = %v, want %v: %v", got, tc.want, errs)
|
|
}
|
|
})
|
|
}
|
|
// An absent tools block is the legacy edit/shell bundle.
|
|
source := "# stage Code\n\n```prompt\nDo it.\n```\n\n```configs\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\n```\n\n```edges\n" + okEdges + "\n```\n"
|
|
if errs, _ := lintRules(t, source); len(errs["checks"]) == 0 {
|
|
t.Fatal("stage without a tools block was not required to declare checks")
|
|
}
|
|
// A delivery stage without checks is told to list the commands.
|
|
delivery := strings.Replace(modelStage("Land", "Commit.", "read\nshell", okEdges, ""), "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1)
|
|
if errs, _ := lintRules(t, delivery); len(errs["checks"]) == 0 || !strings.Contains(errs["checks"][0], "listing the commands") {
|
|
t.Fatalf("delivery stage without checks: %v", errs)
|
|
}
|
|
}
|
|
|
|
func TestLintDeliveryChecksStructural(t *testing.T) {
|
|
if _, err := ParseDefinition(modelStage("Land", "Commit.", "read\nshell", okEdges, "```checks\ngo test ./...\n```")); err == nil || !strings.Contains(err.Error(), "only on delivery stages") {
|
|
t.Fatalf("checks commands on a non-delivery stage = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestLintUndeclaredOutcomes(t *testing.T) {
|
|
for name, tc := range map[string]struct {
|
|
prompt string
|
|
err, warn bool
|
|
edgesExtra string
|
|
}{
|
|
"report skipped undeclared": {prompt: "If nothing to do, report skipped.", err: true},
|
|
"return blocked_retryable": {prompt: "On a flaky device return blocked_retryable.", err: true},
|
|
"quoted vocab after outcome": {prompt: "Set the outcome is `rejected` when wrong.", err: true},
|
|
"declared word is fine": {prompt: "Return blocked when input is missing."},
|
|
"prose after report is fine": {prompt: "Report failed tests and the findings."},
|
|
"negated is fine": {prompt: "Never return skipped for real work."},
|
|
"quoted non-vocab warns": {prompt: "Return `partial` when half done.", warn: true},
|
|
"list after return": {prompt: "Return success or `skipped`.", err: true},
|
|
"declared custom edge": {prompt: "Return skipped when nothing changed.", edgesExtra: "\nskipped -> #success"},
|
|
"error is implicit": {prompt: "Return error on failure."},
|
|
"terminal reference is ignored": {prompt: "Return `#success` via the edge."},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
errs, warns := lintRules(t, modelStage("Plan", tc.prompt, "read", okEdges+tc.edgesExtra, ""))
|
|
if got := len(errs["outcome"]) > 0; got != tc.err {
|
|
t.Fatalf("outcome error = %v, want %v: %v", got, tc.err, errs)
|
|
}
|
|
if got := len(warns["outcome"]) > 0; got != tc.warn {
|
|
t.Fatalf("outcome warning = %v, want %v: %v", got, tc.warn, warns)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLintDeliveryClaimWithoutDeliveryStage(t *testing.T) {
|
|
claim := modelStage("Land", "Commit the change; the server lands it after checks.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
|
|
if errs, _ := lintRules(t, claim); len(errs["delivery"]) == 0 {
|
|
t.Fatal("delivery claim without delivery=true was accepted")
|
|
}
|
|
checksAndLands := modelStage("Land", "Report success; the server checks and lands it.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
|
|
if errs, _ := lintRules(t, checksAndLands); len(errs["delivery"]) == 0 {
|
|
t.Fatal("\"the server checks and lands it\" without delivery=true was accepted")
|
|
}
|
|
never := modelStage("Land", "Commit locally; the server never lands this.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
|
|
if errs, _ := lintRules(t, never); len(errs["delivery"]) != 0 {
|
|
t.Fatalf("negated claim flagged: %v", errs)
|
|
}
|
|
honest := modelStage("Land", "Commit the change locally; the operator lands it.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
|
|
if errs, _ := lintRules(t, honest); len(errs["delivery"]) != 0 {
|
|
t.Fatalf("honest committer flagged: %v", errs)
|
|
}
|
|
negated := modelStage("Land", "Commit locally. Without a delivery gate nothing is landed.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
|
|
if errs, _ := lintRules(t, negated); len(errs["delivery"]) != 0 {
|
|
t.Fatalf("negated claim flagged: %v", errs)
|
|
}
|
|
delivered := strings.Replace(claim, "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1)
|
|
delivered = strings.Replace(delivered, "none: commits locally only", "go test ./...", 1)
|
|
if errs, _ := lintRules(t, delivered); len(errs) != 0 {
|
|
t.Fatalf("delivery stage claim flagged: %v", errs)
|
|
}
|
|
}
|
|
|
|
func TestLintFeedbackLoopNeedsLimits(t *testing.T) {
|
|
code := modelStage("Code", "Make the change.", "read\nedit\nshell", "success -> Review\nerror -> #failure", "```checks\nnone: focused tests only\n```")
|
|
review := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "")
|
|
if errs, _ := lintRules(t, lintSource(code, review)); len(errs["limits"]) == 0 {
|
|
t.Fatal("unbounded review loop accepted")
|
|
}
|
|
limited := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "```limits\n{\"max_visits\":3,\"on\":[\"rejected\"]}\n```")
|
|
if errs, _ := lintRules(t, lintSource(code, limited)); len(errs["limits"]) != 0 {
|
|
t.Fatalf("limited loop flagged: %v", errs)
|
|
}
|
|
targetLimited := strings.Replace(code, "```checks", "```limits\n{\"max_visits\":3}\n```\n\n```checks", 1)
|
|
if errs, _ := lintRules(t, lintSource(targetLimited, review)); len(errs["limits"]) != 0 {
|
|
t.Fatalf("target-limited loop flagged: %v", errs)
|
|
}
|
|
otherOutcome := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "```limits\n{\"max_visits\":3,\"on\":[\"approved\"]}\n```")
|
|
if _, err := ParseDefinition(lintSource(code, otherOutcome)); err == nil {
|
|
t.Fatal("expected structural rejection of a terminal limit outcome")
|
|
}
|
|
human := "# stage Gate\n\n```prompt\nChoose.\n```\n\n```configs\ntype=human\n```\n\n```edges\ndeploy -> #success\nrevise -> Code\nerror -> #failure\n```\n"
|
|
codeToGate := strings.Replace(code, "success -> Review", "success -> Gate", 1)
|
|
if errs, _ := lintRules(t, lintSource(codeToGate, human)); len(errs["limits"]) != 0 {
|
|
t.Fatalf("human-driven loop flagged: %v", errs)
|
|
}
|
|
}
|
|
|
|
func TestLintSilentCheckGuards(t *testing.T) {
|
|
for check, silent := range map[string]bool{
|
|
"go test ./...": false,
|
|
"go vet ./... || true": true,
|
|
"[ ! -x tool/flutter ] || (cd app && flutter test)": true,
|
|
"test ! -f pubspec.yaml || flutter test": true,
|
|
"! command -v flutter || flutter test": true,
|
|
"if [ -f pubspec.yaml ]; then flutter test; fi": true,
|
|
"if [ -f pubspec.yaml ]; then flutter test; else exit 1; fi": false,
|
|
"make check || echo skipped": true,
|
|
"go test ./... && go vet ./...": false,
|
|
`F=$(command -v flutter || echo /opt/flutter) && "$F" test`: false,
|
|
"gofmt -l . | (! grep .)": false,
|
|
} {
|
|
if got := silentGuard(check) != ""; got != silent {
|
|
t.Errorf("silentGuard(%q) = %v, want %v", check, got, silent)
|
|
}
|
|
}
|
|
source := modelStage("Land", "Commit.", "read\nshell", okEdges, "```checks\ngo test ./... || true\n```")
|
|
source = strings.Replace(source, "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1)
|
|
if errs, _ := lintRules(t, source); len(errs["checks"]) == 0 {
|
|
t.Fatal("silent delivery check accepted")
|
|
}
|
|
}
|
|
|
|
func TestLintWarnings(t *testing.T) {
|
|
for name, tc := range map[string]struct {
|
|
prompt, rule string
|
|
}{
|
|
"runner output boilerplate": {prompt: "Do it. Write the complete output.md markdown; only `# outcome` is required.", rule: "boilerplate"},
|
|
"allowed outcomes list": {prompt: "Do it. Allowed outcomes: success, blocked.", rule: "boilerplate"},
|
|
"handoff instruction": {prompt: "Do it. Start your output with a short # handoff section.", rule: "boilerplate"},
|
|
"full history read": {prompt: "First read the entire history, then do it.", rule: "history"},
|
|
"history file read": {prompt: "Read input.md and history.md, then do it.", rule: "history"},
|
|
"long prompt": {prompt: strings.Repeat("Keep it short. ", 200), rule: "prompt-length"},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
_, warns := lintRules(t, modelStage("Plan", tc.prompt, "read", okEdges, ""))
|
|
if len(warns[tc.rule]) == 0 {
|
|
t.Fatalf("no %s warning: %v", tc.rule, warns)
|
|
}
|
|
})
|
|
}
|
|
for _, clean := range []string{
|
|
"Plan the task. Read history.md only when the handoff lacks a detail.",
|
|
"Plan the request in memory.md.",
|
|
"Read memory.md for the original request, then plan.",
|
|
} {
|
|
if _, warns := lintRules(t, modelStage("Plan", clean, "read", okEdges, "")); len(warns) != 0 {
|
|
t.Fatalf("clean prompt %q warned: %v", clean, warns)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestParseDefinitionReportAggregatesErrorsAndKeepsStoredParse(t *testing.T) {
|
|
source := modelStage("Code", "Report skipped when done.", "read\nedit\nshell", okEdges, "")
|
|
_, _, err := ParseDefinitionReport(source)
|
|
if err == nil || !strings.Contains(err.Error(), "checks") || !strings.Contains(err.Error(), "skipped") {
|
|
t.Fatalf("lint errors not reported together: %v", err)
|
|
}
|
|
// Stored versions keep loading and running: Parse/ParseDefinition do not lint.
|
|
if _, err := Parse(source); err != nil {
|
|
t.Fatalf("stored version parse changed: %v", err)
|
|
}
|
|
if _, err := ParseDefinition(source); err != nil {
|
|
t.Fatalf("structural validation changed: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestExamplesPassLint(t *testing.T) {
|
|
paths, err := filepath.Glob(filepath.Join("..", "..", "examples", "*.md"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, path := range paths {
|
|
if filepath.Base(path) == "README.md" {
|
|
continue
|
|
}
|
|
source, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
source = []byte(WithDefaultModel(string(source), "opencode-go/deepseek-v4.1-flash"))
|
|
_, warnings, err := ParseDefinitionReport(string(source))
|
|
if err != nil {
|
|
t.Errorf("%s: %v", path, err)
|
|
}
|
|
if len(warnings) != 0 {
|
|
t.Errorf("%s warnings: %v", path, warnings)
|
|
}
|
|
}
|
|
}
|