a48e4a5b6a
The template catalog is now Example 1-4: one stage, two stages (handoff), feedback loop (runner-owned limits, retries, timeout) and human decision (type=human approve/revise/abort with notify hooks). All read-only or leave changes uncommitted; none commits, pushes or deploys. A brand-new database (schema created by this Open) gets the examples once at server start, bound to the configured default model and harness. A server_settings marker records the decision, so removed examples never come back and existing installs are marked without any pipeline being added or rewritten.
218 lines
7.6 KiB
Go
218 lines
7.6 KiB
Go
package examples_test
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/examples"
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// The graded set, in reading order, with its stage keys.
|
|
var wantStages = []struct {
|
|
key string
|
|
stages []string
|
|
}{
|
|
{"one-stage", []string{"review"}},
|
|
{"two-stages", []string{"draft", "check"}},
|
|
{"feedback-loop", []string{"implement", "review"}},
|
|
{"human-decision", []string{"plan", "decide", "act"}},
|
|
}
|
|
|
|
func TestCatalogIsTheGradedExampleSet(t *testing.T) {
|
|
items := examples.Catalog()
|
|
if len(items) != len(wantStages) {
|
|
t.Fatal("unexpected catalog size", len(items))
|
|
}
|
|
for i, item := range items {
|
|
// Numbered names keep the reading order in name-sorted pipeline lists.
|
|
if item.Key != wantStages[i].key || !strings.HasPrefix(item.Name, fmt.Sprintf("Example %d: ", i+1)) || item.Description == "" || item.Scheduled {
|
|
t.Fatal("invalid or out-of-order example metadata", item.Key, item.Name)
|
|
}
|
|
graph, err := pipeline.ParseDefinition(item.Source)
|
|
if err != nil {
|
|
t.Fatal(item.Key, err)
|
|
}
|
|
if graph.Settings != nil {
|
|
t.Fatal("examples must be manual-only", item.Key)
|
|
}
|
|
var keys []string
|
|
for _, s := range graph.Stages {
|
|
keys = append(keys, s.Key)
|
|
if strings.TrimSpace(s.Prompt) == "" || s.FallbackError || s.Edges["error"] != protocol.FailureEnd {
|
|
t.Fatalf("%s/%s needs a prompt and an explicit error -> #failure edge", item.Key, s.Key)
|
|
}
|
|
}
|
|
if !slices.Equal(keys, wantStages[i].stages) {
|
|
t.Fatalf("%s stages = %v, want %v", item.Key, keys, wantStages[i].stages)
|
|
}
|
|
}
|
|
}
|
|
|
|
func parsedCatalog(t *testing.T) map[string]protocol.Graph {
|
|
t.Helper()
|
|
parsed := map[string]protocol.Graph{}
|
|
for _, item := range examples.Catalog() {
|
|
graph, err := pipeline.Parse(item.Source)
|
|
if err != nil {
|
|
t.Fatal(item.Key, err)
|
|
}
|
|
parsed[item.Key] = graph
|
|
}
|
|
return parsed
|
|
}
|
|
|
|
func stageByKey(t *testing.T, graph protocol.Graph, key string) protocol.Stage {
|
|
t.Helper()
|
|
for _, s := range graph.Stages {
|
|
if s.Key == key {
|
|
return s
|
|
}
|
|
}
|
|
t.Fatalf("stage %s not found", key)
|
|
return protocol.Stage{}
|
|
}
|
|
|
|
// Every model stage declares an explicit role allowlist. Only the stages that
|
|
// make the requested change may edit; nothing has web tools.
|
|
var exampleTools = map[string]map[string][]string{
|
|
"one-stage": {"review": {"read", "glob", "grep"}},
|
|
"two-stages": {"draft": {"read", "glob", "grep"}, "check": {"read", "glob", "grep"}},
|
|
"feedback-loop": {"implement": {"read", "glob", "grep", "edit", "shell"}, "review": {"read", "glob", "grep", "shell"}},
|
|
"human-decision": {"plan": {"read", "glob", "grep"}, "act": {"read", "glob", "grep", "edit", "shell"}},
|
|
}
|
|
|
|
func TestExampleRoleToolRestrictions(t *testing.T) {
|
|
parsed := parsedCatalog(t)
|
|
for key, stages := range exampleTools {
|
|
graph := parsed[key]
|
|
for stageKey, want := range stages {
|
|
tools, explicit, err := stage.ToolNames(pipeline.StageDefinition(stageByKey(t, graph, stageKey)))
|
|
if err != nil || !explicit {
|
|
t.Fatalf("%s/%s must declare an explicit tools block: %v", key, stageKey, err)
|
|
}
|
|
slices.Sort(tools)
|
|
sortedWant := slices.Clone(want)
|
|
slices.Sort(sortedWant)
|
|
if !slices.Equal(tools, sortedWant) {
|
|
t.Fatalf("%s/%s tools = %v, want %v", key, stageKey, tools, sortedWant)
|
|
}
|
|
}
|
|
}
|
|
for key, graph := range parsed {
|
|
for _, s := range graph.Stages {
|
|
if s.Type == stage.ModelType {
|
|
if _, ok := exampleTools[key][s.Key]; !ok {
|
|
t.Fatalf("%s/%s needs a role tool expectation", key, s.Key)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAllExampleDefinitionsAreRegistered(t *testing.T) {
|
|
files, err := filepath.Glob("*.md")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
registered := map[string]string{}
|
|
for _, item := range examples.Catalog() {
|
|
registered[item.Key+".md"] = item.Source
|
|
}
|
|
for _, file := range files {
|
|
if file == "README.md" {
|
|
continue
|
|
}
|
|
data, err := os.ReadFile(file)
|
|
if err != nil || registered[file] != string(data) {
|
|
t.Fatalf("%s must ship in the embedded catalog: %v", file, err)
|
|
}
|
|
}
|
|
if len(files)-1 != len(registered) {
|
|
t.Fatal("every catalog entry needs a checked-in file")
|
|
}
|
|
}
|
|
|
|
func TestExamplesBindConfiguredDefaults(t *testing.T) {
|
|
for _, item := range examples.Catalog() {
|
|
for _, backend := range []string{"opencode", "pi"} {
|
|
raw, err := pipeline.Parse(item.Source)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, s := range raw.Stages {
|
|
if s.Type != stage.ModelType {
|
|
continue
|
|
}
|
|
if s.Configs["provider"] != "default" || s.Configs["model"] != "default" || s.Configs["harness"] != "" || s.Configs["delivery"] != "" || len(s.Memory) != 0 || s.Output != "" {
|
|
t.Fatalf("%s/%s must inherit model and harness without delivery or retired blocks: %v", item.Key, s.Key, s.Configs)
|
|
}
|
|
}
|
|
graph, err := pipeline.ParseDefinition(pipeline.WithDefaults(item.Source, "test-provider/test-model", backend))
|
|
if err != nil {
|
|
t.Fatal(item.Key, err)
|
|
}
|
|
for _, s := range graph.Stages {
|
|
if s.Type != stage.ModelType {
|
|
continue
|
|
}
|
|
if s.Configs["provider"] != "test-provider" || s.Configs["model"] != "test-model" || s.Configs["harness"] != backend {
|
|
t.Fatalf("%s/%s did not bind configured defaults: %v", item.Key, s.Key, s.Configs)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// No example commits, pushes or deploys; editing examples leave changes uncommitted.
|
|
func TestExamplesAreSafe(t *testing.T) {
|
|
for key, graph := range parsedCatalog(t) {
|
|
for _, s := range graph.Stages {
|
|
prompt := strings.ToLower(s.Prompt)
|
|
edits := slices.Contains(exampleTools[key][s.Key], "edit")
|
|
if edits && !strings.Contains(prompt, "uncommitted") {
|
|
t.Fatalf("%s/%s must leave its changes uncommitted", key, s.Key)
|
|
}
|
|
if !edits && s.Type == stage.ModelType && !strings.Contains(prompt, "do not edit") && !strings.Contains(prompt, "change nothing") {
|
|
t.Fatalf("%s/%s must say it changes nothing", key, s.Key)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestFeedbackLoopUsesRunnerOwnedLimitsAndRetries(t *testing.T) {
|
|
graph := parsedCatalog(t)["feedback-loop"]
|
|
implement, review := stageByKey(t, graph, "implement"), stageByKey(t, graph, "review")
|
|
if implement.Edges["success"] != "review" || review.Edges["rejected"] != "implement" || review.Edges["approved"] != protocol.SuccessEnd {
|
|
t.Fatal("feedback loop must route implement -> review, rejected -> implement", implement.Edges, review.Edges)
|
|
}
|
|
if review.Limits == nil || review.Limits.MaxVisits != 3 || !slices.Equal(review.Limits.On, []string{"rejected"}) {
|
|
t.Fatalf("review must cap rejected loops with runner-owned limits: %+v", review.Limits)
|
|
}
|
|
if implement.Configs["retries"] != "1" || implement.Configs["timeout"] != "30m" || review.Configs["retries"] != "2" || review.Configs["timeout"] != "15m" {
|
|
t.Fatal("feedback loop must show explicit retries and timeouts", implement.Configs, review.Configs)
|
|
}
|
|
}
|
|
|
|
func TestHumanDecisionWaitsAndNotifies(t *testing.T) {
|
|
graph := parsedCatalog(t)["human-decision"]
|
|
plan, decide, act := stageByKey(t, graph, "plan"), stageByKey(t, graph, "decide"), stageByKey(t, graph, "act")
|
|
if decide.Type != stage.HumanType || decide.Edges["approve"] != "act" || decide.Edges["revise"] != "plan" || decide.Edges["abort"] != protocol.FailureEnd {
|
|
t.Fatalf("decide must be a human approve/revise/abort stage: %+v", decide)
|
|
}
|
|
for _, tc := range []struct {
|
|
s protocol.Stage
|
|
want []string
|
|
}{{plan, []string{"failure"}}, {decide, []string{"waiting"}}, {act, []string{"success", "failure"}}} {
|
|
if tc.s.Notify == nil || !slices.Equal(tc.s.Notify.On, tc.want) {
|
|
t.Fatalf("%s notify = %+v, want on=%v", tc.s.Key, tc.s.Notify, tc.want)
|
|
}
|
|
}
|
|
}
|