Merge: required checks, authoring lint, good-practices skill page
This commit is contained in:
+13
@@ -279,6 +279,19 @@ pipelines validate /path/to/pipeline-123.md --json
|
||||
pipelines submit /path/to/pipeline-123.md --json
|
||||
```
|
||||
|
||||
`validate` and `submit` apply the authoring lint to new drafts (stored versions
|
||||
keep loading and running unchanged). Errors reject the draft and are listed
|
||||
together, one `stage KEY: RULE: message` per line: a stage that can modify the
|
||||
workspace without a `checks` block (`none: <reason>` or, on a `delivery=true`
|
||||
stage, the commands), a prompt naming an undeclared outcome ("report skipped"),
|
||||
a "the server lands it" claim with no `delivery=true` stage, a loop edge without
|
||||
`limits`, and a check guard that passes without running (`|| true`,
|
||||
`[ ! -x tool ] || ...`). Warnings print as `warning: ...` on stderr (the
|
||||
`warnings` array with `--json`) and do not block: runner boilerplate in a prompt,
|
||||
a prompt over 2000 bytes, an instruction to read the full history every visit,
|
||||
and back-quoted non-outcome words after return/report. The skill's
|
||||
`references/practices.md` explains each rule.
|
||||
|
||||
`--json` never opens an editor or picker; provide a name explicitly. It returns
|
||||
draft metadata including `path`, `name`, `server`, `create_only`, and `base_version`.
|
||||
`--no-open` prints the path and review commands. `$PIPELINES_DRAFT_DIR` overrides
|
||||
|
||||
@@ -30,6 +30,13 @@ routes to `blocked` (or `rejected`) and fails the run with an explanation. Each
|
||||
stage returns Markdown; only its `# outcome` section is interpreted (see
|
||||
`docs/notes/stage-turns.md`).
|
||||
|
||||
Stages that can change files (`edit` or `shell`) declare a `checks` block. These
|
||||
examples leave changes uncommitted, so each says `none: <reason>`: implementers
|
||||
run focused checks only and reviewers read the diff without running suites. A
|
||||
landing stage would set `delivery=true` and list the project's check commands
|
||||
there for the server to run. Every example passes `pipelines validate` with no
|
||||
errors or warnings.
|
||||
|
||||
## On a new install
|
||||
|
||||
A brand-new database starts with these four as ordinary pipelines, bound to the
|
||||
|
||||
@@ -3,9 +3,10 @@
|
||||
```prompt
|
||||
Make the small change requested in the run prompt (see memory.md). On a later
|
||||
visit, input.md holds the reviewer's findings: fix each must-fix finding. Keep the
|
||||
change focused, follow the project's conventions and run its relevant checks.
|
||||
Leave the changes uncommitted; never commit, push or deploy. Report changed
|
||||
paths and checks run. Return blocked if the request cannot be met.
|
||||
change focused and follow the project's conventions. Run focused checks for the
|
||||
touched code only (quiet or summary output), not the whole suite. Leave the
|
||||
changes uncommitted; never commit, push or deploy. Report changed paths and
|
||||
checks run. Return blocked if the request cannot be met.
|
||||
```
|
||||
|
||||
```tools
|
||||
@@ -16,6 +17,10 @@ edit
|
||||
shell
|
||||
```
|
||||
|
||||
```checks
|
||||
none: changes stay uncommitted; the stage runs focused checks and the user runs the full suite
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=default
|
||||
model=default
|
||||
@@ -32,8 +37,9 @@ error -> #failure
|
||||
# stage review
|
||||
|
||||
```prompt
|
||||
Independently review the actual change (git status and git diff) against the
|
||||
original request in memory.md, not just the implementer's summary. Return
|
||||
Independently review the actual change against the original request in
|
||||
memory.md, not just the implementer's summary: `git diff --stat` first, then the
|
||||
changed hunks and only the code around them. Run no test suites. Return
|
||||
rejected only for a must-fix problem, giving its location, evidence and fix.
|
||||
Otherwise return approved, with any non-blocking notes in the body. Do not edit,
|
||||
commit or push.
|
||||
@@ -46,6 +52,10 @@ grep
|
||||
shell
|
||||
```
|
||||
|
||||
```checks
|
||||
none: read-only review; shell is used only for git status and diff
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=default
|
||||
model=default
|
||||
|
||||
@@ -57,8 +57,9 @@ error -> #failure
|
||||
```prompt
|
||||
Carry out the plan the person approved (see memory.md), including any feedback
|
||||
given with the decision. Change only local files in the workspace and leave them
|
||||
uncommitted; never commit, push or deploy. Report changed paths and checks run.
|
||||
Return blocked if the plan cannot be followed.
|
||||
uncommitted; never commit, push or deploy. Run focused checks for the touched
|
||||
code only. Report changed paths and checks run. Return blocked if the plan
|
||||
cannot be followed.
|
||||
```
|
||||
|
||||
```tools
|
||||
@@ -69,6 +70,10 @@ edit
|
||||
shell
|
||||
```
|
||||
|
||||
```checks
|
||||
none: changes stay uncommitted; the stage runs focused checks and the user runs the full suite
|
||||
```
|
||||
|
||||
```configs
|
||||
provider=default
|
||||
model=default
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
var skillFiles = []string{"SKILL.md", "references/orchestration.md", "references/work.md", "references/queue.md", "references/debugging.md", "references/delivery.md", "references/format.md", "references/notifications.md", "references/operations.md", "references/transfer.md", "references/verification.md", "references/control.md", "references/harnesses.md", "references/access.md", "references/schedules.md"}
|
||||
var skillFiles = []string{"SKILL.md", "references/orchestration.md", "references/work.md", "references/queue.md", "references/debugging.md", "references/delivery.md", "references/format.md", "references/notifications.md", "references/operations.md", "references/transfer.md", "references/verification.md", "references/control.md", "references/harnesses.md", "references/access.md", "references/schedules.md", "references/practices.md"}
|
||||
|
||||
func SkillDirectory() (string, error) {
|
||||
dir := os.Getenv("XDG_CONFIG_HOME")
|
||||
|
||||
@@ -35,8 +35,11 @@ explain the gap; ask before authoring. Do not add a task-specific pipeline as a
|
||||
## Authoring
|
||||
|
||||
A definition is Markdown: `# stage <Name>` sections with `prompt`, `configs`,
|
||||
`tools`, `edges`, and optional `limits`/`checks`. Start from
|
||||
`examples/`. Use stage names that describe the work:
|
||||
`tools`, `edges`, and `limits`/`checks` where required: every stage that can
|
||||
modify the workspace declares `checks` (commands on the `delivery=true` stage,
|
||||
otherwise `none: <reason>`), and every loop has `limits`. Start from
|
||||
`examples/` and read `references/practices.md` before writing prompts. Use stage
|
||||
names that describe the work:
|
||||
|
||||
`Plan` · `Code` · `Review` · `UIReview` · `Land` (`Prepare` and `Survey` are
|
||||
useful variants).
|
||||
@@ -46,8 +49,10 @@ constraints. The runner enforces turns, tools and limits; prompts do not need to
|
||||
repeat them. Only `# outcome` is parsed from `output.md`; everything else is
|
||||
free markdown. Use `links` and `memory` only when they clarify a handoff.
|
||||
|
||||
`pipelines validate DRAFT` checks a draft; `pipelines submit DRAFT` publishes a
|
||||
new immutable version. `pipelines help <command>` has flags.
|
||||
`pipelines validate DRAFT` checks a draft, including the authoring rules in
|
||||
`references/practices.md` (errors block submit; `warning:` lines do not but should
|
||||
be fixed); `pipelines submit DRAFT` publishes a new immutable version.
|
||||
`pipelines help <command>` has flags.
|
||||
|
||||
Schedules are separate saved run configurations: pipeline/version, prompt,
|
||||
workspace and other run options, plus timing. Never add `# settings` or schedule
|
||||
@@ -73,6 +78,7 @@ Read only what the task needs:
|
||||
| Need | Reference |
|
||||
| --- | --- |
|
||||
| Definition syntax and turn files | `references/format.md` |
|
||||
| Good practices (tokens, checks, prompts, loops) | `references/practices.md` |
|
||||
| Limits, tools and permissions | `references/control.md` |
|
||||
| Stages, parallel work, recovery | `references/orchestration.md` |
|
||||
| CLI, storage and evidence | `references/operations.md` |
|
||||
|
||||
@@ -15,7 +15,7 @@ Each model stage requires nonempty `prompt`, `configs`, and `edges` blocks:
|
||||
# stage worker
|
||||
|
||||
```prompt
|
||||
Perform the task. Write the complete output.md markdown; only `# outcome` is required.
|
||||
Perform the task.
|
||||
```
|
||||
|
||||
```configs
|
||||
@@ -42,7 +42,33 @@ An omitted error edge defaults to failure. Execution is sequential; forks choose
|
||||
one outcome, never execute parallel branches. Cycles are allowed; every stage
|
||||
must be reachable from entry and have a possible path to a terminal. Use bounded
|
||||
runner-owned `limits` for feedback loops, never revision counters in model
|
||||
prompts/outputs. Read `references/control.md` for the exact format.
|
||||
prompts/outputs: publishing rejects an edge to the same or an earlier stage
|
||||
(definition order) unless the source stage limits that outcome or the target
|
||||
stage has `limits` (a human stage's choices are exempt). Read
|
||||
`references/control.md` for the exact format.
|
||||
|
||||
## Checks
|
||||
|
||||
Every model stage that can modify the workspace (it declares `edit` or `shell`,
|
||||
has no `tools` block, or sets `delivery=true`) must declare a `checks` block, so
|
||||
the author decides where verification happens. Read-only stages need none.
|
||||
|
||||
- A `delivery=true` stage lists 1–20 commands, one per line. The server runs them
|
||||
on the committed candidate (and again on a merged tree) before landing; a
|
||||
failing command blocks landing. These are the run's deterministic checks.
|
||||
- Every other stage writes exactly one line, `none: <reason>`, naming where
|
||||
verification happens instead, for example
|
||||
`none: focused tests only; the Land delivery gate runs the suite` or
|
||||
`none: read-only review; shell is used only for git diff`. Commands on a
|
||||
non-delivery stage are rejected because nothing would run them.
|
||||
|
||||
A check must fail when it cannot run. Validation rejects guards that pass
|
||||
without running anything: `|| true`, `|| :`, `|| exit 0`, `|| echo`,
|
||||
`[ ! -x tool ] || ...`, `test ! -f ... || ...`, `! command -v tool || ...`,
|
||||
an `if ...; then ...; fi` with no failing else, and `set +e`. Cover every
|
||||
language the run can change (a Go-only check list does not verify Flutter work).
|
||||
`pipelines validate` reports these with the remaining authoring rules
|
||||
(`references/practices.md`).
|
||||
|
||||
## Result format
|
||||
|
||||
@@ -233,6 +259,10 @@ grep
|
||||
shell
|
||||
```
|
||||
|
||||
```checks
|
||||
none: commits locally; the manager lands and runs the suite
|
||||
```
|
||||
|
||||
```edges
|
||||
success -> #success
|
||||
blocked -> #failure
|
||||
@@ -240,6 +270,10 @@ error -> #failure
|
||||
```
|
||||
````
|
||||
|
||||
A landing committer instead sets `delivery=true` in `configs` and lists the
|
||||
repository's required check commands in `checks`; the server runs them, so its
|
||||
prompt does not run the suite again.
|
||||
|
||||
State this pre-commit checklist in the prompt:
|
||||
|
||||
1. Inspect `git status` and the full diff. Stage this run's change, including
|
||||
@@ -247,9 +281,10 @@ State this pre-commit checklist in the prompt:
|
||||
exclusion. Do not rely on a model-declared file list.
|
||||
2. Inspect the staged diff before committing; never commit unrelated work, secrets
|
||||
or generated artifacts.
|
||||
3. Run the repository's required checks and record the exact commands and observed
|
||||
results as evidence. Do not commit when a required check fails; return `blocked`
|
||||
with the failing output instead.
|
||||
3. Without a delivery gate, run the repository's required checks once (summary
|
||||
output) and record the exact commands and observed results as evidence. Do not
|
||||
commit when a required check fails; return `blocked` with the failing output
|
||||
instead. With `delivery=true` the server runs the `checks` block; skip this step.
|
||||
4. Create no empty commit: when nothing is staged, return `success` with a clear
|
||||
note.
|
||||
5. Commit with a concise message naming the run's change and report the commit hash;
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
# Good practices
|
||||
|
||||
Pipelines are written by models through this skill, and a run's cost is roughly
|
||||
model calls × context per call. Most waste comes from stages that re-read or
|
||||
re-generate large output. `pipelines validate` enforces the mechanical rules
|
||||
below (errors block publishing; warnings print and do not).
|
||||
|
||||
## Where work belongs
|
||||
|
||||
- **Deterministic checks belong in the server `checks` block, not the model.**
|
||||
The landing stage sets `delivery=true` and lists the full check commands; the
|
||||
server runs them on the committed candidate. A model running the suite puts
|
||||
every log line into its context, and every later call pays for it again.
|
||||
- **Code runs focused tests only**: the test file or package it touched, with
|
||||
quiet or summary output. Never the full suite.
|
||||
- **Review reads the diff, not the repository.** Start from the server-captured
|
||||
changed-file list the runner adds to the prompt and `git diff --stat`, then read
|
||||
only the changed hunks and the code around them. Review runs no suites; it judges
|
||||
the reported checks and names missing coverage.
|
||||
- **Land/committer relies on server checks.** With a delivery gate its prompt
|
||||
commits and stops; it does not re-run what the gate runs. Do not claim "the
|
||||
server lands it" unless a stage has `delivery=true` (error otherwise).
|
||||
- Every stage that can modify the workspace declares `checks`, even if only
|
||||
`none: <reason>` (`references/format.md`, Checks). Checks must fail when a tool
|
||||
is missing: no `|| true`, `[ ! -x tool ] || ...` or similar guards (error).
|
||||
Cover every language the run can change.
|
||||
|
||||
## Prompts
|
||||
|
||||
- **Short stage prompts**: the goal and the real constraints. Over 2000 bytes
|
||||
warns. Shared procedure belongs in project docs or this skill.
|
||||
- **No runner boilerplate.** The runner already explains `output.md`, the
|
||||
`# outcome` and `# handoff` sections, allowed outcomes, turn files, tools and
|
||||
honesty rules (`internal/stage/model_prompt.go`). Repeating them warns.
|
||||
- **Every outcome a prompt names is a declared edge.** "Return skipped" with no
|
||||
`skipped` edge, or an undeclared `blocked_retryable`, fails the stage at run
|
||||
time; validation rejects it (unquoted outcome words such as `skipped`,
|
||||
`blocked`, `rejected`, `approved`, `advisory`; other back-quoted words after
|
||||
return/report warn).
|
||||
|
||||
## Memory and handoffs
|
||||
|
||||
- The runner inlines the **essential memory** (`memory.md`: task, previous
|
||||
stage's handoff, outcome ledger, changed files; capped) into every prompt, and
|
||||
asks each stage to start its output with a short `# handoff`: what it did,
|
||||
outcome, files changed, open findings, in a few lines. Do not repeat that
|
||||
instruction; say only what this stage's handoff must carry if it is special.
|
||||
- The **full history** (`history.md`) and full `input.md` are on demand. Never
|
||||
tell a stage to read the whole history every visit (warning); say when it
|
||||
needs it, e.g. "read history.md only if the handoff lacks a detail you need".
|
||||
|
||||
## Loops and budgets
|
||||
|
||||
- **Loops get limits.** An edge back to the same or an earlier stage needs
|
||||
`limits` on the source outcome or the target (error). Use `max_visits` 2–3.
|
||||
- **Re-entry gets only findings**: a rejected review hands back must-fix items
|
||||
with location and fix, not a full re-review, and the implementer fixes those
|
||||
items rather than redoing the task. Use `advisory` for non-blocking notes so
|
||||
they do not buy another loop.
|
||||
- Set **per-stage `budget_tokens`** (and `budget_usd` if wanted) as a runaway
|
||||
guard on stages that loop or explore, sized a few times a normal visit.
|
||||
|
||||
## Commands and scope
|
||||
|
||||
- **Quiet commands**: summary flags, `git diff --stat` before a full diff,
|
||||
`grep`/`glob` before `read`, read file ranges rather than whole files, tail or
|
||||
filter long logs, never print generated or vendored files.
|
||||
- **Small task scope**: one independently testable change per run (roughly a
|
||||
10–15 minute change). Split larger work into several runs in parallel.
|
||||
@@ -746,6 +746,9 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
if o.json {
|
||||
return a.output(r)
|
||||
}
|
||||
for _, warning := range r.Warnings {
|
||||
fmt.Fprintln(a.Err, "warning: "+warning)
|
||||
}
|
||||
if command == "validate" {
|
||||
fmt.Fprintln(a.Out, "Valid draft (server validation). No changes submitted.")
|
||||
} else {
|
||||
|
||||
@@ -33,7 +33,7 @@ func Template(names []string, linear bool) string {
|
||||
if linear && i+1 < len(names) {
|
||||
next = names[i+1]
|
||||
}
|
||||
fmt.Fprintf(&b, "# stage %s\n\n```prompt\nDescribe the task. Write the complete output.md markdown; the only required section is `# outcome`.\n```\n\n```tools\n```\n\n```edges\nsuccess -> %s\nerror -> %s\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=2\ntimeout=1h\n```\n\n", name, next, protocol.FailureEnd)
|
||||
fmt.Fprintf(&b, "# stage %s\n\n```prompt\nDescribe the task.\n```\n\n```tools\n```\n\n```edges\nsuccess -> %s\nerror -> %s\n```\n\n```configs\ntype=model\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\nretries=2\ntimeout=1h\n```\n\n", name, next, protocol.FailureEnd)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
@@ -36,9 +36,9 @@ func TestValidation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
for name, source := range map[string]string{
|
||||
"missing block": strings.Replace(valid, "```prompt\nDescribe the task. Write the complete output.md markdown; the only required section is `# outcome`.\n```", "", 1),
|
||||
"missing block": strings.Replace(valid, "```prompt\nDescribe the task.\n```", "", 1),
|
||||
"retired input": valid + "```input\nold contract\n```\n",
|
||||
"empty block": strings.Replace(valid, "Describe the task. Write the complete output.md markdown; the only required section is `# outcome`.", " ", 1),
|
||||
"empty block": strings.Replace(valid, "Describe the task.", " ", 1),
|
||||
"duplicate block": valid + "```prompt\nagain\n```\n",
|
||||
"unknown block": valid + "```foo\nx\n```\n",
|
||||
"unclosed": valid + "```input\nx",
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
"git.a-wan.net/alan/pipelines/internal/stage"
|
||||
)
|
||||
|
||||
// Authoring lint applies only to new publications (ParseDefinition), never to
|
||||
// stored immutable versions, which Parse keeps reading and running unchanged.
|
||||
// Errors are structural or textual mistakes that are unambiguous; warnings are
|
||||
// token-cost and clarity problems the author should usually fix.
|
||||
|
||||
// MaxPromptBytes is the stage-prompt length above which lint warns. The runner
|
||||
// adds its own turn-file, outcome and tool instructions around every prompt.
|
||||
const MaxPromptBytes = 2000
|
||||
|
||||
// LintFinding is one authoring-policy result for a stage.
|
||||
type LintFinding struct {
|
||||
Stage string `json:"stage"`
|
||||
Rule string `json:"rule"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func (f LintFinding) String() string {
|
||||
return fmt.Sprintf("stage %s: %s: %s", f.Stage, f.Rule, f.Message)
|
||||
}
|
||||
|
||||
// Lint returns the authoring errors and warnings for a parsed graph.
|
||||
func Lint(g Graph) (errs, warnings []LintFinding) {
|
||||
index := map[string]int{}
|
||||
delivery := false
|
||||
for i, s := range g.Stages {
|
||||
index[s.Key] = i
|
||||
if s.Configs["delivery"] == "true" {
|
||||
delivery = true
|
||||
}
|
||||
}
|
||||
add := func(list *[]LintFinding, s Stage, rule, format string, args ...any) {
|
||||
*list = append(*list, LintFinding{Stage: s.Key, Rule: rule, Message: fmt.Sprintf(format, args...)})
|
||||
}
|
||||
for i, s := range g.Stages {
|
||||
model := stage.EffectiveType(s.Type) == stage.ModelType
|
||||
if model {
|
||||
if reason := mutatingReason(s); reason != "" {
|
||||
if _, present := s.Blocks["checks"]; !present {
|
||||
if s.Configs["delivery"] == "true" {
|
||||
add(&errs, s, "checks", "delivery stage needs a checks block listing the commands the server runs before landing, one per line")
|
||||
} else {
|
||||
add(&errs, s, "checks", "stage can modify the workspace (%s) but has no checks block; add one: \"none: <reason>\" (for example \"none: focused tests only; the Land delivery gate runs the suite\"), or set delivery=true on the landing stage and list its check commands", reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, word := range undeclaredOutcomes(s.Prompt, s.Edges) {
|
||||
if word.strong {
|
||||
add(&errs, s, "outcome", "prompt tells the stage to %s %q but it is not a declared edge; add \"%s -> <target>\" to edges or remove it from the prompt", word.verb, word.name, word.name)
|
||||
} else {
|
||||
add(&warnings, s, "outcome", "prompt tells the stage to %s `%s`, which is not a declared edge; declare it or reword the prompt if it is not an outcome", word.verb, word.name)
|
||||
}
|
||||
}
|
||||
if !delivery {
|
||||
if claim := deliveryClaim.FindString(s.Prompt); claim != "" {
|
||||
add(&errs, s, "delivery", "prompt says %q but no stage sets delivery=true; add delivery=true and a checks block to the landing stage, or state that the stage itself (or the operator) lands the change", claim)
|
||||
}
|
||||
}
|
||||
for _, phrase := range runnerBoilerplate {
|
||||
if phrase.MatchString(s.Prompt) {
|
||||
add(&warnings, s, "boilerplate", "prompt repeats runner-injected instructions (%q); the runner already explains output.md, outcomes, turn files and tools, so delete it", phrase.FindString(s.Prompt))
|
||||
break
|
||||
}
|
||||
}
|
||||
if match := unconditionalHistoryRead(s.Prompt); match != "" {
|
||||
add(&warnings, s, "history", "prompt asks for %q on every visit; the runner inlines the essential memory (memory.md: task, previous handoff, outcome ledger), so read history.md only for a specific need, e.g. \"read history.md only if the handoff lacks a detail you need\"", match)
|
||||
}
|
||||
if n := len(strings.TrimSpace(s.Prompt)); n > MaxPromptBytes {
|
||||
add(&warnings, s, "prompt-length", "prompt is %d bytes (over %d); keep stage prompts to the goal and real constraints, and move shared procedure into the skill or project docs", n, MaxPromptBytes)
|
||||
}
|
||||
}
|
||||
if text, present := s.Blocks["checks"]; present {
|
||||
if _, none := stage.ChecksNone(text); !none {
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
if guard := silentGuard(line); guard != "" {
|
||||
add(&errs, s, "checks", "check %q can pass without running anything (%s); a missing tool must fail the check, so remove the guard", strings.TrimSpace(line), guard)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if stage.EffectiveType(s.Type) == stage.HumanType {
|
||||
continue // a person bounds a human stage's loop
|
||||
}
|
||||
for _, outcome := range sortedOutcomes(s.Edges) {
|
||||
target := s.Edges[outcome]
|
||||
if protocol.IsTerminal(target) || index[target] > i {
|
||||
continue
|
||||
}
|
||||
if !loopLimited(s, outcome, g.Stages[index[target]]) {
|
||||
add(&errs, s, "limits", "edge %s -> %s loops back without a runner limit; add a limits block such as {\"max_visits\":3,\"on\":[%q]} to %s, or {\"max_visits\":3} to %s", outcome, target, outcome, s.Key, target)
|
||||
}
|
||||
}
|
||||
}
|
||||
return errs, warnings
|
||||
}
|
||||
|
||||
// mutatingReason names why a model stage can change the workspace, or "".
|
||||
func mutatingReason(s Stage) string {
|
||||
if s.Configs["delivery"] == "true" {
|
||||
return "delivery=true"
|
||||
}
|
||||
tools, explicit, err := stage.ToolNames(StageDefinition(s))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
if !explicit {
|
||||
return "no tools block grants edit and shell"
|
||||
}
|
||||
var found []string
|
||||
for _, tool := range tools {
|
||||
if tool == "edit" || tool == "shell" {
|
||||
found = append(found, tool)
|
||||
}
|
||||
}
|
||||
if len(found) == 0 {
|
||||
return ""
|
||||
}
|
||||
return "tools " + strings.Join(found, ", ")
|
||||
}
|
||||
|
||||
func loopLimited(s Stage, outcome string, target Stage) bool {
|
||||
if target.Limits != nil {
|
||||
return true
|
||||
}
|
||||
return s.Limits != nil && (len(s.Limits.On) == 0 || slices.Contains(s.Limits.On, outcome))
|
||||
}
|
||||
|
||||
func sortedOutcomes(edges map[string]string) []string {
|
||||
keys := make([]string, 0, len(edges))
|
||||
for key := range edges {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
slices.Sort(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
// outcomeWords are edge names that read as outcomes even unquoted. Words that
|
||||
// commonly appear in prose after "report" (failed, pass, done, failure) are
|
||||
// deliberately absent; they are still checked when quoted.
|
||||
var outcomeWords = map[string]bool{
|
||||
"success": true, "blocked": true, "blocked_retryable": true, "skipped": true,
|
||||
"rejected": true, "approved": true, "advisory": true, "revise": true,
|
||||
"abort": true, "noop": true, "unchanged": true, "changes_requested": true,
|
||||
"needs_changes": true, "retryable": true,
|
||||
}
|
||||
|
||||
var (
|
||||
outcomeToken = "[`'\"]?#?[A-Za-z][A-Za-z0-9_-]*[`'\"]?"
|
||||
outcomePhrase = regexp.MustCompile("(?i)\\b(return(?:s|ing)?|report(?:s|ing)?|choose|outcome(?:\\s+is)?)\\s+(" + outcomeToken + "(?:\\s*(?:,|/|\\bor\\b|\\|)\\s*" + outcomeToken + ")*)")
|
||||
outcomeTokenRe = regexp.MustCompile(outcomeToken)
|
||||
negated = regexp.MustCompile(`(?i)\b(?:not|never|no|don't|do not|nor)\s+$`)
|
||||
)
|
||||
|
||||
type outcomeMention struct {
|
||||
verb, name string
|
||||
strong bool
|
||||
}
|
||||
|
||||
// undeclaredOutcomes finds "return blocked" / "report `skipped`" style
|
||||
// instructions naming an outcome the stage does not declare. Negated phrases
|
||||
// ("never return success") are ignored.
|
||||
func undeclaredOutcomes(prompt string, edges map[string]string) []outcomeMention {
|
||||
var found []outcomeMention
|
||||
seen := map[string]bool{}
|
||||
for _, match := range outcomePhrase.FindAllStringSubmatchIndex(prompt, -1) {
|
||||
start := match[0] - 12
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
if negated.MatchString(prompt[start:match[0]]) {
|
||||
continue
|
||||
}
|
||||
verb := strings.ToLower(strings.Fields(prompt[match[2]:match[3]])[0])
|
||||
outcomeVerb := strings.HasPrefix(verb, "outcome")
|
||||
for _, token := range outcomeTokenRe.FindAllString(prompt[match[4]:match[5]], -1) {
|
||||
quoted := strings.ContainsAny(token, "`'\"")
|
||||
if strings.Contains(token, "#") {
|
||||
continue
|
||||
}
|
||||
name := strings.Trim(token, "`'\"")
|
||||
lower := strings.ToLower(name)
|
||||
if lower == "or" || lower == "error" || seen[name] {
|
||||
continue
|
||||
}
|
||||
if _, declared := edges[name]; declared {
|
||||
continue
|
||||
}
|
||||
if !quoted && !outcomeWords[lower] {
|
||||
continue
|
||||
}
|
||||
seen[name] = true
|
||||
found = append(found, outcomeMention{verb: verb, name: name, strong: outcomeWords[lower] || (quoted && outcomeVerb)})
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
// deliveryClaim matches affirmative statements that the server lands work.
|
||||
var deliveryClaim = regexp.MustCompile(`(?i)\bthe server (?:(?:then|will|automatically|itself|checks|verifies|tests|validates|and|,)\s*){0,4}(?:lands?|merges?|delivers?)\b|\b(?:landed|merged|delivered) by the server\b|\bserver[- ]side (?:landing|delivery)\b|\bthe (?:server's )?delivery gate\b`)
|
||||
|
||||
// runnerBoilerplate are phrases the runner itself injects around every model
|
||||
// prompt (internal/stage/model_prompt.go) or that restate its output contract.
|
||||
var runnerBoilerplate = func() []*regexp.Regexp {
|
||||
var list []*regexp.Regexp
|
||||
for _, phrase := range []string{
|
||||
`rely on this essential memory first`,
|
||||
`start (?:it|your output|the output) with a short ` + "`?#\\s*handoff",
|
||||
`return its complete content as your final message`,
|
||||
`(?:the )?only required section is(?: the)? ` + "`?#?\\s*outcome",
|
||||
"only `?#\\s*outcome`? (?:section )?is required",
|
||||
`everything else is free-form markdown`,
|
||||
`do not wrap it in a json envelope`,
|
||||
`allowed outcomes:`,
|
||||
`do not invent file edits`,
|
||||
`never claim unperformed work succeeded`,
|
||||
`write the complete output\.md`,
|
||||
`return the complete output\.md`,
|
||||
} {
|
||||
list = append(list, regexp.MustCompile(`(?i)`+strings.ReplaceAll(phrase, " ", `\s+`)))
|
||||
}
|
||||
return list
|
||||
}()
|
||||
|
||||
// fullHistoryRead matches prompts that ask for the whole run history
|
||||
// (history.md, the on-demand record of every turn) instead of reading it only
|
||||
// for a specific need. memory.md is the capped essential memory the runner
|
||||
// inlines into every prompt, so naming it is redundant but never costly.
|
||||
var fullHistoryRead = regexp.MustCompile(`(?i)\b(?:read|review|load|open|check|consult|go through)\b(?:[^.\n]|\.[A-Za-z]){0,30}(?:\bhistory\.md\b|\b(?:entire|full|whole|complete|all of the|all of)\s+(?:run\s+)?(?:history|transcript)\b)`)
|
||||
|
||||
var conditional = regexp.MustCompile(`(?i)\b(?:only|when|if|unless)\b`)
|
||||
|
||||
// unconditionalHistoryRead returns the first full-history instruction whose
|
||||
// sentence does not make it conditional ("only when the handoff lacks ...").
|
||||
func unconditionalHistoryRead(prompt string) string {
|
||||
for _, loc := range fullHistoryRead.FindAllStringIndex(prompt, -1) {
|
||||
start := strings.LastIndexAny(prompt[:loc[0]], ".\n") + 1
|
||||
end := len(prompt)
|
||||
if i := strings.IndexAny(prompt[loc[1]:], ".\n"); i >= 0 {
|
||||
end = loc[1] + i
|
||||
}
|
||||
if !conditional.MatchString(prompt[start:end]) {
|
||||
return prompt[loc[0]:loc[1]]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var silentGuards = []struct {
|
||||
re *regexp.Regexp
|
||||
name string
|
||||
}{
|
||||
{regexp.MustCompile(`\|\|\s*(?:true\b|:(?:\s|;|\)|$)|exit\s+0\b|echo\b[^|&;()]*(?:;|$))`), "an || fallback that succeeds"},
|
||||
{regexp.MustCompile(`(?:\[\[?|\btest)\s+!\s+-[a-zA-Z]\s+[^|&;]+?\]?\]?\s*\|\|`), "a negated file test that skips the command"},
|
||||
{regexp.MustCompile(`!\s*command\s+-v\s+\S+[^|&;]*\|\|`), "a missing-tool guard that skips the command"},
|
||||
{regexp.MustCompile(`;\s*(?:true|exit\s+0)\s*$`), "a trailing command that always succeeds"},
|
||||
{regexp.MustCompile(`^\s*if\s.*\bthen\b.*\bfi\b`), "an if without else"},
|
||||
{regexp.MustCompile(`(?:^|\s)set\s+\+e\b`), "set +e"},
|
||||
}
|
||||
|
||||
var elseBranch = regexp.MustCompile(`\belse\b|\bexit\s+[1-9]`)
|
||||
|
||||
func silentGuard(line string) string {
|
||||
for _, guard := range silentGuards {
|
||||
if !guard.re.MatchString(line) {
|
||||
continue
|
||||
}
|
||||
if guard.name == "an if without else" && elseBranch.MatchString(line) {
|
||||
continue
|
||||
}
|
||||
return guard.name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func lintError(errs []LintFinding) error {
|
||||
if len(errs) == 0 {
|
||||
return nil
|
||||
}
|
||||
list := make([]error, 0, len(errs))
|
||||
for _, finding := range errs {
|
||||
list = append(list, errors.New(finding.String()))
|
||||
}
|
||||
return errors.Join(list...)
|
||||
}
|
||||
|
||||
// LintStrings renders findings for protocol replies.
|
||||
func LintStrings(findings []LintFinding) []string {
|
||||
if len(findings) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(findings))
|
||||
for _, finding := range findings {
|
||||
out = append(out, finding.String())
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// lintStage builds a one- or two-stage definition around the given blocks.
|
||||
func lintSource(stages ...string) string {
|
||||
return strings.Join(stages, "\n")
|
||||
}
|
||||
|
||||
func modelStage(key, prompt, tools, edges, extra string) string {
|
||||
b := "# stage " + key + "\n\n```prompt\n" + prompt + "\n```\n\n```configs\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\n```\n\n```tools\n" + tools + "\n```\n\n```edges\n" + edges + "\n```\n"
|
||||
if extra != "" {
|
||||
b += "\n" + extra + "\n"
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func lintRules(t *testing.T, source string) (errs, warns map[string][]string) {
|
||||
t.Helper()
|
||||
g, err := ParseDefinition(source)
|
||||
if err != nil {
|
||||
t.Fatalf("structural parse: %v", err)
|
||||
}
|
||||
e, w := Lint(g)
|
||||
errs, warns = map[string][]string{}, map[string][]string{}
|
||||
for _, f := range e {
|
||||
errs[f.Rule] = append(errs[f.Rule], f.String())
|
||||
}
|
||||
for _, f := range w {
|
||||
warns[f.Rule] = append(warns[f.Rule], f.String())
|
||||
}
|
||||
return errs, warns
|
||||
}
|
||||
|
||||
const okEdges = "success -> #success\nblocked -> #failure\nerror -> #failure"
|
||||
|
||||
func TestLintRequiresChecksOnMutatingStages(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
tools, extra string
|
||||
want bool
|
||||
}{
|
||||
"edit+shell without checks": {tools: "read\nedit\nshell", want: true},
|
||||
"shell only without checks": {tools: "read\nshell", want: true},
|
||||
"read-only needs nothing": {tools: "read\nglob\ngrep"},
|
||||
"text-only needs nothing": {tools: ""},
|
||||
"none with reason": {tools: "read\nedit", extra: "```checks\nnone: focused tests only\n```"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
errs, _ := lintRules(t, modelStage("Code", "Make the change.", tc.tools, okEdges, tc.extra))
|
||||
if got := len(errs["checks"]) > 0; got != tc.want {
|
||||
t.Fatalf("checks error = %v, want %v: %v", got, tc.want, errs)
|
||||
}
|
||||
})
|
||||
}
|
||||
// An absent tools block is the legacy edit/shell bundle.
|
||||
source := "# stage Code\n\n```prompt\nDo it.\n```\n\n```configs\nprovider=opencode-go\nmodel=deepseek-v4.1-flash\n```\n\n```edges\n" + okEdges + "\n```\n"
|
||||
if errs, _ := lintRules(t, source); len(errs["checks"]) == 0 {
|
||||
t.Fatal("stage without a tools block was not required to declare checks")
|
||||
}
|
||||
// A delivery stage without checks is told to list the commands.
|
||||
delivery := strings.Replace(modelStage("Land", "Commit.", "read\nshell", okEdges, ""), "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1)
|
||||
if errs, _ := lintRules(t, delivery); len(errs["checks"]) == 0 || !strings.Contains(errs["checks"][0], "listing the commands") {
|
||||
t.Fatalf("delivery stage without checks: %v", errs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLintDeliveryChecksStructural(t *testing.T) {
|
||||
if _, err := ParseDefinition(modelStage("Land", "Commit.", "read\nshell", okEdges, "```checks\ngo test ./...\n```")); err == nil || !strings.Contains(err.Error(), "only on delivery stages") {
|
||||
t.Fatalf("checks commands on a non-delivery stage = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLintUndeclaredOutcomes(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
prompt string
|
||||
err, warn bool
|
||||
edgesExtra string
|
||||
}{
|
||||
"report skipped undeclared": {prompt: "If nothing to do, report skipped.", err: true},
|
||||
"return blocked_retryable": {prompt: "On a flaky device return blocked_retryable.", err: true},
|
||||
"quoted vocab after outcome": {prompt: "Set the outcome is `rejected` when wrong.", err: true},
|
||||
"declared word is fine": {prompt: "Return blocked when input is missing."},
|
||||
"prose after report is fine": {prompt: "Report failed tests and the findings."},
|
||||
"negated is fine": {prompt: "Never return skipped for real work."},
|
||||
"quoted non-vocab warns": {prompt: "Return `partial` when half done.", warn: true},
|
||||
"list after return": {prompt: "Return success or `skipped`.", err: true},
|
||||
"declared custom edge": {prompt: "Return skipped when nothing changed.", edgesExtra: "\nskipped -> #success"},
|
||||
"error is implicit": {prompt: "Return error on failure."},
|
||||
"terminal reference is ignored": {prompt: "Return `#success` via the edge."},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
errs, warns := lintRules(t, modelStage("Plan", tc.prompt, "read", okEdges+tc.edgesExtra, ""))
|
||||
if got := len(errs["outcome"]) > 0; got != tc.err {
|
||||
t.Fatalf("outcome error = %v, want %v: %v", got, tc.err, errs)
|
||||
}
|
||||
if got := len(warns["outcome"]) > 0; got != tc.warn {
|
||||
t.Fatalf("outcome warning = %v, want %v: %v", got, tc.warn, warns)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLintDeliveryClaimWithoutDeliveryStage(t *testing.T) {
|
||||
claim := modelStage("Land", "Commit the change; the server lands it after checks.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
|
||||
if errs, _ := lintRules(t, claim); len(errs["delivery"]) == 0 {
|
||||
t.Fatal("delivery claim without delivery=true was accepted")
|
||||
}
|
||||
checksAndLands := modelStage("Land", "Report success; the server checks and lands it.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
|
||||
if errs, _ := lintRules(t, checksAndLands); len(errs["delivery"]) == 0 {
|
||||
t.Fatal("\"the server checks and lands it\" without delivery=true was accepted")
|
||||
}
|
||||
never := modelStage("Land", "Commit locally; the server never lands this.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
|
||||
if errs, _ := lintRules(t, never); len(errs["delivery"]) != 0 {
|
||||
t.Fatalf("negated claim flagged: %v", errs)
|
||||
}
|
||||
honest := modelStage("Land", "Commit the change locally; the operator lands it.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
|
||||
if errs, _ := lintRules(t, honest); len(errs["delivery"]) != 0 {
|
||||
t.Fatalf("honest committer flagged: %v", errs)
|
||||
}
|
||||
negated := modelStage("Land", "Commit locally. Without a delivery gate nothing is landed.", "read\nshell", okEdges, "```checks\nnone: commits locally only\n```")
|
||||
if errs, _ := lintRules(t, negated); len(errs["delivery"]) != 0 {
|
||||
t.Fatalf("negated claim flagged: %v", errs)
|
||||
}
|
||||
delivered := strings.Replace(claim, "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1)
|
||||
delivered = strings.Replace(delivered, "none: commits locally only", "go test ./...", 1)
|
||||
if errs, _ := lintRules(t, delivered); len(errs) != 0 {
|
||||
t.Fatalf("delivery stage claim flagged: %v", errs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLintFeedbackLoopNeedsLimits(t *testing.T) {
|
||||
code := modelStage("Code", "Make the change.", "read\nedit\nshell", "success -> Review\nerror -> #failure", "```checks\nnone: focused tests only\n```")
|
||||
review := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "")
|
||||
if errs, _ := lintRules(t, lintSource(code, review)); len(errs["limits"]) == 0 {
|
||||
t.Fatal("unbounded review loop accepted")
|
||||
}
|
||||
limited := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "```limits\n{\"max_visits\":3,\"on\":[\"rejected\"]}\n```")
|
||||
if errs, _ := lintRules(t, lintSource(code, limited)); len(errs["limits"]) != 0 {
|
||||
t.Fatalf("limited loop flagged: %v", errs)
|
||||
}
|
||||
targetLimited := strings.Replace(code, "```checks", "```limits\n{\"max_visits\":3}\n```\n\n```checks", 1)
|
||||
if errs, _ := lintRules(t, lintSource(targetLimited, review)); len(errs["limits"]) != 0 {
|
||||
t.Fatalf("target-limited loop flagged: %v", errs)
|
||||
}
|
||||
otherOutcome := modelStage("Review", "Read the diff.", "read", "approved -> #success\nrejected -> Code\nerror -> #failure", "```limits\n{\"max_visits\":3,\"on\":[\"approved\"]}\n```")
|
||||
if _, err := ParseDefinition(lintSource(code, otherOutcome)); err == nil {
|
||||
t.Fatal("expected structural rejection of a terminal limit outcome")
|
||||
}
|
||||
human := "# stage Gate\n\n```prompt\nChoose.\n```\n\n```configs\ntype=human\n```\n\n```edges\ndeploy -> #success\nrevise -> Code\nerror -> #failure\n```\n"
|
||||
codeToGate := strings.Replace(code, "success -> Review", "success -> Gate", 1)
|
||||
if errs, _ := lintRules(t, lintSource(codeToGate, human)); len(errs["limits"]) != 0 {
|
||||
t.Fatalf("human-driven loop flagged: %v", errs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLintSilentCheckGuards(t *testing.T) {
|
||||
for check, silent := range map[string]bool{
|
||||
"go test ./...": false,
|
||||
"go vet ./... || true": true,
|
||||
"[ ! -x tool/flutter ] || (cd app && flutter test)": true,
|
||||
"test ! -f pubspec.yaml || flutter test": true,
|
||||
"! command -v flutter || flutter test": true,
|
||||
"if [ -f pubspec.yaml ]; then flutter test; fi": true,
|
||||
"if [ -f pubspec.yaml ]; then flutter test; else exit 1; fi": false,
|
||||
"make check || echo skipped": true,
|
||||
"go test ./... && go vet ./...": false,
|
||||
`F=$(command -v flutter || echo /opt/flutter) && "$F" test`: false,
|
||||
"gofmt -l . | (! grep .)": false,
|
||||
} {
|
||||
if got := silentGuard(check) != ""; got != silent {
|
||||
t.Errorf("silentGuard(%q) = %v, want %v", check, got, silent)
|
||||
}
|
||||
}
|
||||
source := modelStage("Land", "Commit.", "read\nshell", okEdges, "```checks\ngo test ./... || true\n```")
|
||||
source = strings.Replace(source, "model=deepseek-v4.1-flash", "model=deepseek-v4.1-flash\ndelivery=true", 1)
|
||||
if errs, _ := lintRules(t, source); len(errs["checks"]) == 0 {
|
||||
t.Fatal("silent delivery check accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLintWarnings(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
prompt, rule string
|
||||
}{
|
||||
"runner output boilerplate": {prompt: "Do it. Write the complete output.md markdown; only `# outcome` is required.", rule: "boilerplate"},
|
||||
"allowed outcomes list": {prompt: "Do it. Allowed outcomes: success, blocked.", rule: "boilerplate"},
|
||||
"handoff instruction": {prompt: "Do it. Start your output with a short # handoff section.", rule: "boilerplate"},
|
||||
"full history read": {prompt: "First read the entire history, then do it.", rule: "history"},
|
||||
"history file read": {prompt: "Read input.md and history.md, then do it.", rule: "history"},
|
||||
"long prompt": {prompt: strings.Repeat("Keep it short. ", 200), rule: "prompt-length"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, warns := lintRules(t, modelStage("Plan", tc.prompt, "read", okEdges, ""))
|
||||
if len(warns[tc.rule]) == 0 {
|
||||
t.Fatalf("no %s warning: %v", tc.rule, warns)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, clean := range []string{
|
||||
"Plan the task. Read history.md only when the handoff lacks a detail.",
|
||||
"Plan the request in memory.md.",
|
||||
"Read memory.md for the original request, then plan.",
|
||||
} {
|
||||
if _, warns := lintRules(t, modelStage("Plan", clean, "read", okEdges, "")); len(warns) != 0 {
|
||||
t.Fatalf("clean prompt %q warned: %v", clean, warns)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDefinitionReportAggregatesErrorsAndKeepsStoredParse(t *testing.T) {
|
||||
source := modelStage("Code", "Report skipped when done.", "read\nedit\nshell", okEdges, "")
|
||||
_, _, err := ParseDefinitionReport(source)
|
||||
if err == nil || !strings.Contains(err.Error(), "checks") || !strings.Contains(err.Error(), "skipped") {
|
||||
t.Fatalf("lint errors not reported together: %v", err)
|
||||
}
|
||||
// Stored versions keep loading and running: Parse/ParseDefinition do not lint.
|
||||
if _, err := Parse(source); err != nil {
|
||||
t.Fatalf("stored version parse changed: %v", err)
|
||||
}
|
||||
if _, err := ParseDefinition(source); err != nil {
|
||||
t.Fatalf("structural validation changed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExamplesPassLint(t *testing.T) {
|
||||
paths, err := filepath.Glob(filepath.Join("..", "..", "examples", "*.md"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, path := range paths {
|
||||
if filepath.Base(path) == "README.md" {
|
||||
continue
|
||||
}
|
||||
source, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source = []byte(WithDefaultModel(string(source), "opencode-go/deepseek-v4.1-flash"))
|
||||
_, warnings, err := ParseDefinitionReport(string(source))
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", path, err)
|
||||
}
|
||||
if len(warnings) != 0 {
|
||||
t.Errorf("%s warnings: %v", path, warnings)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ var weekdays = []string{"sun", "mon", "tue", "wed", "thu", "fri", "sat"}
|
||||
|
||||
// ParseDefinition validates new publications. Parse deliberately remains a
|
||||
// historical reader for immutable versions containing deprecated settings.
|
||||
// Authoring lint is separate: see ParseDefinitionReport.
|
||||
func ParseDefinition(source string) (protocol.Graph, error) {
|
||||
g, err := Parse(source)
|
||||
if err == nil && g.Settings != nil {
|
||||
@@ -19,6 +20,18 @@ func ParseDefinition(source string) (protocol.Graph, error) {
|
||||
return g, err
|
||||
}
|
||||
|
||||
// ParseDefinitionReport is ParseDefinition plus the authoring lint (Lint) the
|
||||
// publication API applies to new drafts: lint errors reject the draft, all
|
||||
// reported together; warnings are returned for the author and never block it.
|
||||
func ParseDefinitionReport(source string) (protocol.Graph, []string, error) {
|
||||
g, err := ParseDefinition(source)
|
||||
if err != nil {
|
||||
return g, nil, err
|
||||
}
|
||||
errs, warnings := Lint(g)
|
||||
return g, LintStrings(warnings), lintError(errs)
|
||||
}
|
||||
|
||||
func ValidateSchedule(rule protocol.Schedule) error {
|
||||
text := ""
|
||||
if len(rule.Days) > 0 {
|
||||
|
||||
@@ -298,11 +298,26 @@ func (s *Store) Submit(ctx context.Context, name, source string, createOnly ...b
|
||||
}
|
||||
|
||||
// SubmitChecked guards draft edits against concurrent accepted versions atomically.
|
||||
// It applies structural validation only; PublishChecked adds the authoring lint.
|
||||
func (s *Store) SubmitChecked(ctx context.Context, name, source string, createOnly bool, expectedVersion int) (int, error) {
|
||||
return s.submit(ctx, name, source, createOnly, expectedVersion, ParseDefinition)
|
||||
}
|
||||
|
||||
// PublishChecked is SubmitChecked for the publication API (CLI, TUI, web and
|
||||
// import submit through it): lint errors (Lint) reject the version and are
|
||||
// recorded as the submission diagnostic like any other validation failure.
|
||||
func (s *Store) PublishChecked(ctx context.Context, name, source string, createOnly bool, expectedVersion int) (int, error) {
|
||||
return s.submit(ctx, name, source, createOnly, expectedVersion, func(source string) (Graph, error) {
|
||||
g, _, err := ParseDefinitionReport(source)
|
||||
return g, err
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) submit(ctx context.Context, name, source string, createOnly bool, expectedVersion int, parse func(string) (Graph, error)) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
name = strings.TrimSpace(name)
|
||||
g, validation := ParseDefinition(source)
|
||||
g, validation := parse(source)
|
||||
if expectedVersion < 0 {
|
||||
validation = fmt.Errorf("expected version must be nonnegative")
|
||||
}
|
||||
|
||||
@@ -136,6 +136,7 @@ type Message struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Time string `json:"time,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Warnings []string `json:"warnings,omitempty"` // non-blocking authoring lint for validate/submit
|
||||
Name string `json:"name,omitempty"`
|
||||
Source string `json:"source,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
|
||||
@@ -281,8 +281,9 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
response.Templates[i].Source = pipeline.WithDefaults(response.Templates[i].Source, model, backend)
|
||||
}
|
||||
} else if request.Type == protocol.Validate {
|
||||
graph, err := pipeline.ParseDefinition(request.Source)
|
||||
graph, warnings, err := pipeline.ParseDefinitionReport(request.Source)
|
||||
response.Type = protocol.Validated
|
||||
response.Warnings = warnings
|
||||
if err != nil {
|
||||
response.Type = protocol.Error
|
||||
response.Error = err.Error()
|
||||
@@ -318,7 +319,8 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
} else if request.Type == protocol.Submit && len(stores) > 0 {
|
||||
response.Type = protocol.Submitted
|
||||
response.Name = strings.TrimSpace(request.Name)
|
||||
version, err := stores[0].SubmitChecked(ctx, request.Name, request.Source, request.CreateOnly, request.ExpectedVersion)
|
||||
version, err := stores[0].PublishChecked(ctx, request.Name, request.Source, request.CreateOnly, request.ExpectedVersion)
|
||||
_, response.Warnings, _ = pipeline.ParseDefinitionReport(request.Source)
|
||||
if err != nil {
|
||||
response.Type = protocol.Error
|
||||
response.Error = err.Error()
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package stage
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestChecksNoneDeclaration(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
checks string
|
||||
delivery bool
|
||||
ok bool
|
||||
}{
|
||||
"none on mutating stage": {checks: "none: read-only review", ok: true},
|
||||
"none case insensitive": {checks: "None: focused tests only", ok: true},
|
||||
"none without reason": {checks: "none:"},
|
||||
"none with extra lines": {checks: "none: reason\ngo test ./..."},
|
||||
"none on delivery stage": {checks: "none: trust me", delivery: true},
|
||||
"commands on non-delivery": {checks: "go test ./..."},
|
||||
"commands on delivery": {checks: "go test ./...", delivery: true, ok: true},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
def := modelDefinition()
|
||||
def.Blocks["checks"] = tc.checks
|
||||
if tc.delivery {
|
||||
def.Configs["delivery"] = "true"
|
||||
}
|
||||
_, err := (ModelStage{}).Spec(def)
|
||||
if (err == nil) != tc.ok {
|
||||
t.Fatalf("Spec(%q) error = %v, want ok=%v", tc.checks, err, tc.ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
def := modelDefinition()
|
||||
def.Configs["delivery"] = "true"
|
||||
def.Blocks["checks"] = "none: skip"
|
||||
if _, err := DeliveryChecks(def); err == nil {
|
||||
t.Fatal("delivery gate would run none: as a command")
|
||||
}
|
||||
}
|
||||
@@ -109,8 +109,8 @@ func (ModelStage) Spec(def Definition) (ModelSpec, error) {
|
||||
if def.Configs["delivery"] == "true" && backend != "opencode" {
|
||||
return ModelSpec{}, fmt.Errorf("delivery is supported only by the opencode harness")
|
||||
}
|
||||
if _, present := def.Blocks["checks"]; present && def.Configs["delivery"] != "true" {
|
||||
return ModelSpec{}, fmt.Errorf("checks block is supported only by delivery stages")
|
||||
if err := ValidateChecksBlock(def); err != nil {
|
||||
return ModelSpec{}, err
|
||||
}
|
||||
if backend == "pi" {
|
||||
tools, _, _ := ToolNames(def)
|
||||
|
||||
@@ -72,6 +72,9 @@ func DeliveryChecks(def Definition) ([]string, error) {
|
||||
if text == "" {
|
||||
return nil, fmt.Errorf("delivery stages require a nonempty checks block")
|
||||
}
|
||||
if _, none := ChecksNone(text); none {
|
||||
return nil, fmt.Errorf("delivery stages run their checks before landing; list the check commands instead of none")
|
||||
}
|
||||
var checks []string
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
check := strings.TrimSpace(line)
|
||||
@@ -89,6 +92,45 @@ func DeliveryChecks(def Definition) ([]string, error) {
|
||||
return checks, nil
|
||||
}
|
||||
|
||||
// ChecksNone reports whether a checks block uses the explicit
|
||||
// "none: <reason>" form. It matches on the leading none: alone, so a malformed
|
||||
// declaration (no reason, extra lines) is rejected rather than run as a command.
|
||||
func ChecksNone(text string) (reason string, ok bool) {
|
||||
text = strings.TrimSpace(text)
|
||||
if !strings.HasPrefix(strings.ToLower(text), "none:") {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimSpace(text[len("none:"):]), true
|
||||
}
|
||||
|
||||
// ValidateChecksBlock is the checks syntax shared by stored and new
|
||||
// definitions. Only a delivery stage executes commands (the server's delivery
|
||||
// gate); every other stage may declare only "none: <reason>", so a listed
|
||||
// command is never accepted and then silently left unexecuted.
|
||||
func ValidateChecksBlock(def Definition) error {
|
||||
text, present := def.Blocks["checks"]
|
||||
if !present {
|
||||
return nil
|
||||
}
|
||||
if reason, none := ChecksNone(text); none {
|
||||
if strings.Contains(strings.TrimSpace(text), "\n") {
|
||||
return fmt.Errorf("checks none must be a single line: none: <reason>")
|
||||
}
|
||||
if len(reason) < 3 || len(reason) > 300 {
|
||||
return fmt.Errorf("checks none needs a short reason (3-300 bytes), for example \"none: read-only review\"")
|
||||
}
|
||||
if def.Configs["delivery"] == "true" {
|
||||
return fmt.Errorf("delivery stages run their checks before landing; list the check commands instead of none")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if def.Configs["delivery"] != "true" {
|
||||
return fmt.Errorf("checks commands run only on delivery stages (delivery=true); write \"none: <reason>\" here and declare the commands on the delivery stage")
|
||||
}
|
||||
_, err := DeliveryChecks(def)
|
||||
return err
|
||||
}
|
||||
|
||||
// ToolNames validates model-stage permissions, not general graph semantics.
|
||||
// A present empty block explicitly denies all tools.
|
||||
func ToolNames(def Definition) ([]string, bool, error) {
|
||||
|
||||
@@ -49,6 +49,8 @@ func TestSearchVersionsAndEditSubmission(t *testing.T) {
|
||||
defer store.Close()
|
||||
source := pipeline.Template([]string{"implementer", "reviewer"}, true)
|
||||
source = strings.Replace(source, "success -> #success", "bad_implementation -> implementer\nsuccess -> #success", 1)
|
||||
// reviewer is the last stage; its feedback loop needs a runner limit to publish.
|
||||
source += "```limits\n{\"max_visits\":3,\"on\":[\"bad_implementation\"]}\n```\n"
|
||||
if _, err := store.Submit(ctx, "demo", source); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user