Files
pipelines/internal/pipeline/search_test.go
T

610 lines
21 KiB
Go

package pipeline
import (
"context"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
"unicode/utf8"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// searchFixture builds one pipeline whose name, source, run, stage, field text
// and artifact all contain "zephyr", so a single free-text query exercises
// every kind.
func searchFixture(t *testing.T) (*Store, int64) {
t.Helper()
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "search.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { store.Close() })
if _, err := store.Submit(ctx, "Zephyr Build", Template([]string{"zephyr"}, true)); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
definition, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
run, _, err := store.CreateRun(ctx, definition, "zephyr-key", "/work/zephyr", false, nil, map[string]any{"prompt": "zephyr task"})
if err != nil {
t.Fatal(err)
}
if err := store.BeginStep(ctx, run, 1, "zephyr", map[string]any{"prompt": "zephyr task"}); err != nil {
t.Fatal(err)
}
artifactURL := "/artifacts/" + strconv.FormatInt(run, 10) + "/1/zephyr.txt"
if err := store.FinishStep(ctx, run, protocol.RunStep{
Sequence: 1, Status: "completed", Outcome: "success", Target: protocol.SuccessEnd,
Output: map[string]any{"summary": "zephyr completed"},
Control: &protocol.ControlEvidence{Visit: 1, MaxVisits: 2, ModelTarget: "zephyr-model"},
Artifacts: []protocol.StageArtifact{{Name: "zephyr.txt", SizeBytes: 12, ContentType: "text/plain", URL: artifactURL}},
}); err != nil {
t.Fatal(err)
}
if err := store.FinishRun(ctx, run, "succeeded", "success", "", map[string]any{"summary": "zephyr completed"}); err != nil {
t.Fatal(err)
}
return store, run
}
func searchKinds(response *protocol.SearchResponse) map[string]bool {
kinds := map[string]bool{}
for _, hit := range response.Results {
kinds[hit.Kind] = true
}
return kinds
}
func TestSearchReturnsEveryKind(t *testing.T) {
store, run := searchFixture(t)
response, err := store.Search(context.Background(), protocol.SearchRequest{Query: "zephyr"}, time.Now())
if err != nil {
t.Fatal(err)
}
kinds := searchKinds(response)
for _, kind := range protocol.SearchKinds {
if !kinds[kind] {
t.Fatalf("kind %q missing from %+v", kind, response.Results)
}
}
if response.Cap != protocol.SearchDefaultLimit || response.ScanBoundBytes != protocol.SearchMaxScanBytes {
t.Fatalf("bounds not reported: %+v", response)
}
if response.Truncated {
t.Fatalf("a tiny store must not report truncation: %+v", response)
}
// Every target must point at an addressable entity an existing command resolves.
ctx := context.Background()
for _, hit := range response.Results {
switch hit.Kind {
case protocol.SearchKindPipeline:
id := targetInt(t, hit.Target, "pipeline:")
if _, err := store.Get(ctx, id, 0); err != nil {
t.Fatalf("pipeline target %q does not resolve: %v", hit.Target, err)
}
case protocol.SearchKindVersion:
parts := strings.Split(strings.TrimPrefix(hit.Target, "pipeline:"), ":v")
if len(parts) != 2 {
t.Fatalf("bad version target %q", hit.Target)
}
if _, err := store.Get(ctx, parseInt(t, parts[0]), int(parseInt(t, parts[1]))); err != nil {
t.Fatalf("version target %q does not resolve: %v", hit.Target, err)
}
case protocol.SearchKindRun:
if _, err := store.GetRun(ctx, targetInt(t, hit.Target, "run:")); err != nil {
t.Fatalf("run target %q does not resolve: %v", hit.Target, err)
}
case protocol.SearchKindStep, protocol.SearchKindField:
fields := strings.Split(strings.TrimPrefix(hit.Target, "run:"), ":")
if len(fields) < 2 {
t.Fatalf("bad step/field target %q", hit.Target)
}
detail, err := store.GetRun(ctx, parseInt(t, fields[0]))
if err != nil {
t.Fatalf("step/field target %q does not resolve: %v", hit.Target, err)
}
sequence := parseInt(t, fields[1])
found := false
for _, step := range detail.Steps {
if int64(step.Sequence) == sequence {
found = true
}
}
if !found {
t.Fatalf("step/field target %q names no recorded step", hit.Target)
}
case protocol.SearchKindArtifact:
if !strings.HasPrefix(hit.Target, "/artifacts/") {
t.Fatalf("artifact target %q is not a served URL", hit.Target)
}
}
}
if run <= 0 {
t.Fatal("fixture run missing")
}
}
func targetInt(t *testing.T, target, prefix string) int64 {
t.Helper()
if !strings.HasPrefix(target, prefix) {
t.Fatalf("target %q lacks prefix %q", target, prefix)
}
return parseInt(t, strings.TrimPrefix(target, prefix))
}
func parseInt(t *testing.T, value string) int64 {
t.Helper()
n, err := parsePositiveInt(value)
if err != nil {
t.Fatalf("bad number %q: %v", value, err)
}
return n
}
func TestSearchFieldTextCoversInputOutputEvidence(t *testing.T) {
store, _ := searchFixture(t)
response, err := store.Search(context.Background(), protocol.SearchRequest{Query: "zephyr", Kind: protocol.SearchKindField}, time.Now())
if err != nil {
t.Fatal(err)
}
fields := map[string]bool{}
for _, hit := range response.Results {
if hit.Field == nil {
t.Fatalf("field kind without a field payload: %+v", hit)
}
fields[hit.Field.Field] = true
if !strings.Contains(hit.Field.Snippet, "zephyr") {
t.Fatalf("snippet lacks the match: %+v", hit.Field)
}
}
for _, field := range []string{"input", "output", "evidence"} {
if !fields[field] {
t.Fatalf("field text search missed %q: %+v", field, response.Results)
}
}
}
func TestSearchRankingExactSubstringBodyAndRecency(t *testing.T) {
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "ranking.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
submit := func(name string) {
if _, err := store.Submit(ctx, name, Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
}
submit("alpha")
submit("alphabet")
items, _ := store.List(ctx)
definition, _ := store.Get(ctx, items[0].ID, 1)
run, _, err := store.CreateRun(ctx, definition, "alpha-key", "/work", false, nil, map[string]any{"prompt": "body alpha match"})
if err != nil {
t.Fatal(err)
}
if err := store.BeginStep(ctx, run, 1, "one", map[string]any{"prompt": "body alpha match"}); err != nil {
t.Fatal(err)
}
if err := store.FinishStep(ctx, run, protocol.RunStep{Sequence: 1, Status: "completed", Outcome: "success", Target: protocol.SuccessEnd}); err != nil {
t.Fatal(err)
}
response, err := store.Search(ctx, protocol.SearchRequest{Query: "alpha"}, time.Now())
if err != nil {
t.Fatal(err)
}
seen := map[int]bool{}
for i, hit := range response.Results {
seen[hit.Score] = true
if i > 0 && response.Results[i-1].Score < hit.Score {
t.Fatalf("scores not descending: %+v", response.Results)
}
}
if !seen[3] || !seen[2] || !seen[1] {
t.Fatalf("expected exact/substring/body tiers: %+v", response.Results)
}
// The exact pipeline match outranks the substring pipeline match.
exactIndex, substringIndex := -1, -1
for i, hit := range response.Results {
if hit.Pipeline == nil {
continue
}
if hit.Pipeline.Name == "alpha" {
exactIndex = i
}
if hit.Pipeline.Name == "alphabet" {
substringIndex = i
}
if hit.Pipeline.Name == "alpha" && hit.Score != 3 {
t.Fatalf("exact name scored %d", hit.Score)
}
if hit.Pipeline.Name == "alphabet" && hit.Score != 2 {
t.Fatalf("substring name scored %d", hit.Score)
}
}
if exactIndex < 0 || substringIndex < 0 || exactIndex > substringIndex {
t.Fatalf("exact name must outrank a name substring: %+v", response.Results)
}
// Equal-tier ties break by recency.
older, _, err := store.CreateRun(ctx, definition, "alpha-older", "/work", false, nil, map[string]any{"prompt": "alpha"})
if err != nil {
t.Fatal(err)
}
newer, _, err := store.CreateRun(ctx, definition, "alpha-newer", "/work", false, nil, map[string]any{"prompt": "alpha"})
if err != nil {
t.Fatal(err)
}
setCreated(t, store, older, "2026-01-01 00:00:00")
setCreated(t, store, newer, "2026-06-01 00:00:00")
response, err = store.Search(ctx, protocol.SearchRequest{Query: "alpha"}, time.Now())
if err != nil {
t.Fatal(err)
}
var olderIndex, newerIndex = -1, -1
for i, hit := range response.Results {
if hit.Run == nil || hit.Run.ID == 0 {
continue
}
if hit.Run.ID == older {
olderIndex = i
}
if hit.Run.ID == newer {
newerIndex = i
}
}
if olderIndex < 0 || newerIndex < 0 || newerIndex > olderIndex {
t.Fatalf("recency tie-break lost: older=%d newer=%d", olderIndex, newerIndex)
}
}
func TestSearchStatusWordsSelectRuns(t *testing.T) {
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "statuses.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "statuses", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, _ := store.Get(ctx, items[0].ID, 1)
// One run per accepted status word, each with a distinct created_at so a
// date selector can never be mistaken for the status selector.
byStatus := map[string]int64{}
for i, status := range protocol.SearchStatuses {
run, _, err := store.CreateRun(ctx, definition, "status-"+status, "/work", false, nil, nil)
if err != nil {
t.Fatal(err)
}
if _, err := store.db.Exec("UPDATE pipeline_runs SET status=?,created_at=? WHERE id=?", status, "2026-01-0"+strconv.Itoa(i+1)+" 09:00:00", run); err != nil {
t.Fatal(err)
}
byStatus[status] = run
}
now := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC)
for _, status := range protocol.SearchStatuses {
response, err := store.Search(ctx, protocol.SearchRequest{Query: status}, now)
if err != nil {
t.Fatal(err)
}
var runs []int64
for _, hit := range response.Results {
if hit.Run == nil {
t.Fatalf("status %q returned a non-run kind %q", status, hit.Kind)
}
runs = append(runs, hit.Run.ID)
}
if len(runs) != 1 || runs[0] != byStatus[status] {
t.Fatalf("status %q selected %v, want [%d]", status, runs, byStatus[status])
}
}
}
func TestSearchStatusAndDateSelectors(t *testing.T) {
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "selector.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "selectors", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, _ := store.Get(ctx, items[0].ID, 1)
yesterdayRun, _, err := store.CreateRun(ctx, definition, "yesterday", "/work", false, nil, nil)
if err != nil {
t.Fatal(err)
}
todayRun, _, err := store.CreateRun(ctx, definition, "today", "/work", false, nil, nil)
if err != nil {
t.Fatal(err)
}
if err := store.FinishRun(ctx, yesterdayRun, "failed", "error", "boom", nil); err != nil {
t.Fatal(err)
}
setCreated(t, store, yesterdayRun, "2026-10-15 09:00:00")
setCreated(t, store, todayRun, "2026-10-16 09:00:00")
now := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC)
selects := func(query string) map[int64]bool {
t.Helper()
response, err := store.Search(ctx, protocol.SearchRequest{Query: query}, now)
if err != nil {
t.Fatal(err)
}
ids := map[int64]bool{}
for _, hit := range response.Results {
if hit.Run != nil {
ids[hit.Run.ID] = true
}
}
return ids
}
if ids := selects("failed"); !ids[yesterdayRun] || ids[todayRun] {
t.Fatalf("status selector: %+v", ids)
}
if ids := selects("today"); !ids[todayRun] || ids[yesterdayRun] {
t.Fatalf("today selector: %+v", ids)
}
if ids := selects("yesterday"); !ids[yesterdayRun] || ids[todayRun] {
t.Fatalf("yesterday selector: %+v", ids)
}
if ids := selects("2026-10-15"); !ids[yesterdayRun] || ids[todayRun] {
t.Fatalf("literal date selector: %+v", ids)
}
if ids := selects("this week"); !ids[yesterdayRun] || !ids[todayRun] {
t.Fatalf("this week selector: %+v", ids)
}
// Non-selector text stays free text.
if ids := selects("selectors"); !ids[yesterdayRun] || !ids[todayRun] {
t.Fatalf("free text was misread as a selector: %+v", ids)
}
}
func TestParseSearchTerms(t *testing.T) {
now := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC)
cases := []struct {
query string
status string
hasDate bool
from, to string
text string
}{
{query: "failed", status: "failed"},
{query: " RUNNING ", status: "running"},
{query: "today", hasDate: true, from: "2026-10-16 00:00:00", to: "2026-10-17 00:00:00"},
{query: "yesterday", hasDate: true, from: "2026-10-15 00:00:00", to: "2026-10-16 00:00:00"},
{query: "this week", hasDate: true, from: "2026-10-12 00:00:00", to: "2026-10-19 00:00:00"},
{query: "2026-10-05", hasDate: true, from: "2026-10-05 00:00:00", to: "2026-10-06 00:00:00"},
{query: "failed to build", text: "failed to build"},
{query: "build", text: "build"},
}
for _, test := range cases {
terms := parseSearchTerms(test.query, now)
if terms.Status != test.status || terms.Text != test.text || terms.HasDate != test.hasDate {
t.Fatalf("parse %q = %+v", test.query, terms)
}
if test.hasDate {
if got := terms.From.Format(searchSQLTime); got != test.from {
t.Fatalf("parse %q from = %s, want %s", test.query, got, test.from)
}
if got := terms.To.Format(searchSQLTime); got != test.to {
t.Fatalf("parse %q to = %s, want %s", test.query, got, test.to)
}
}
}
}
func TestSearchKindFilterAndLimits(t *testing.T) {
store, _ := searchFixture(t)
ctx := context.Background()
response, err := store.Search(ctx, protocol.SearchRequest{Query: "zephyr", Kind: protocol.SearchKindRun}, time.Now())
if err != nil {
t.Fatal(err)
}
for _, hit := range response.Results {
if hit.Kind != protocol.SearchKindRun {
t.Fatalf("kind filter leaked %q", hit.Kind)
}
}
if _, err := store.Search(ctx, protocol.SearchRequest{Query: "zephyr", Kind: "bogus"}, time.Now()); err == nil {
t.Fatal("invalid kind must return a clear error")
}
for _, test := range []struct {
limit int
want int
}{{limit: 0, want: protocol.SearchDefaultLimit}, {limit: -5, want: protocol.SearchDefaultLimit}, {limit: 1, want: 1}, {limit: 500, want: protocol.SearchMaxLimit}} {
response, err := store.Search(ctx, protocol.SearchRequest{Query: "zephyr", Limit: test.limit}, time.Now())
if err != nil {
t.Fatal(err)
}
if response.Cap != test.want {
t.Fatalf("limit %d => cap %d, want %d", test.limit, response.Cap, test.want)
}
if len(response.Results) > response.Cap {
t.Fatalf("limit %d returned %d results", test.limit, len(response.Results))
}
}
// A cap below the candidate count drops matches and must report Truncated.
truncated, err := store.Search(ctx, protocol.SearchRequest{Query: "zephyr", Limit: 1}, time.Now())
if err != nil {
t.Fatal(err)
}
if len(truncated.Results) != 1 || !truncated.Truncated {
t.Fatalf("result cap must truncate: %+v", truncated)
}
}
func TestSearchEdgeCases(t *testing.T) {
store, run := searchFixture(t)
ctx := context.Background()
empty, err := store.Search(ctx, protocol.SearchRequest{Query: " "}, time.Now())
if err != nil || len(empty.Results) != 0 {
t.Fatalf("empty query: %+v %v", empty, err)
}
none, err := store.Search(ctx, protocol.SearchRequest{Query: "no-such-token-xyzzy"}, time.Now())
if err != nil || len(none.Results) != 0 {
t.Fatalf("no-match query: %+v %v", none, err)
}
if none.Results == nil {
t.Fatal("results must be an empty slice, not nil")
}
// A run without a finish reports no duration.
setFinishedNull(t, store, run)
response, err := store.Search(ctx, protocol.SearchRequest{Query: "zephyr", Kind: protocol.SearchKindRun}, time.Now())
if err != nil {
t.Fatal(err)
}
for _, hit := range response.Results {
if hit.Run != nil && hit.Run.FinishedAt != "" {
t.Fatalf("null finished_at surfaced a value: %+v", hit.Run)
}
}
}
func TestSearchSnippetBoundaries(t *testing.T) {
if got := searchSnippet("prefix café résumé naïve suffix", "naïve"); !strings.Contains(got, "naïve") || !utf8.ValidString(got) {
t.Fatalf("unicode snippet malformed: %q", got)
}
if got := searchSnippet("line one\n\tline two\r\n zephyr here", "zephyr"); strings.ContainsAny(got, "\n\r\t") || !strings.Contains(got, "zephyr") {
t.Fatalf("whitespace snippet not normalized: %q", got)
}
if got := searchSnippet(strings.Repeat("é", 200)+"needle"+strings.Repeat("é", 200), "needle"); !utf8.ValidString(got) || !strings.Contains(got, "needle") {
t.Fatalf("boundary snippet malformed: %q", got)
}
if got := searchSnippet("no match here", "absent"); got == "" || !utf8.ValidString(got) {
t.Fatalf("no-match snippet must stay a clean line: %q", got)
}
}
func TestSearchScanBoundTruncates(t *testing.T) {
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "scan.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "scan", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, _ := store.Get(ctx, items[0].ID, 1)
// An older step that does match, and a newer step whose body alone exceeds
// the scan budget: the budget must stop the scan and be reported.
older, _, err := store.CreateRun(ctx, definition, "scan-older", "/work", false, nil, nil)
if err != nil {
t.Fatal(err)
}
if err := store.BeginStep(ctx, older, 1, "one", nil); err != nil {
t.Fatal(err)
}
if err := store.FinishStep(ctx, older, protocol.RunStep{Sequence: 1, Status: "completed", Outcome: "success", Target: protocol.SuccessEnd, Output: map[string]any{"summary": "needle in older step"}}); err != nil {
t.Fatal(err)
}
newer, _, err := store.CreateRun(ctx, definition, "scan-newer", "/work", false, nil, nil)
if err != nil {
t.Fatal(err)
}
if err := store.BeginStep(ctx, newer, 1, "one", nil); err != nil {
t.Fatal(err)
}
blob := strings.Repeat("x", protocol.SearchMaxScanBytes+1024)
if err := store.FinishStep(ctx, newer, protocol.RunStep{Sequence: 1, Status: "completed", Outcome: "success", Target: protocol.SuccessEnd, Output: map[string]any{"blob": blob}}); err != nil {
t.Fatal(err)
}
response, err := store.Search(ctx, protocol.SearchRequest{Query: "needle"}, time.Now())
if err != nil {
t.Fatal(err)
}
if !response.Truncated {
t.Fatalf("scan bound not reported: %+v", response)
}
for _, hit := range response.Results {
if hit.Field != nil {
t.Fatalf("scan passed the bound and matched: %+v", hit.Field)
}
}
recent, err := store.Search(ctx, protocol.SearchRequest{Kind: protocol.SearchKindField}, time.Now())
if err != nil || !recent.Truncated || len(recent.Results) != 0 || recent.ScannedBytes > protocol.SearchMaxScanBytes {
t.Fatalf("kind-only fields bypassed the byte budget: %+v %v", recent, err)
}
}
// TestSearchScanBoundTruncatesByRowCount proves the step-count bound is
// observable: more recorded steps than the budget sets Truncated even though
// every body is tiny and fits the byte budget.
func TestSearchScanBoundTruncatesByRowCount(t *testing.T) {
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), "scanrows.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "scanrows", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, _ := store.Get(ctx, items[0].ID, 1)
run, _, err := store.CreateRun(ctx, definition, "scanrows", "/work", false, nil, nil)
if err != nil {
t.Fatal(err)
}
tx, err := store.db.Begin()
if err != nil {
t.Fatal(err)
}
for sequence := 1; sequence <= protocol.SearchMaxScanSteps+1; sequence++ {
if _, err := tx.Exec(`INSERT INTO run_steps(run_id,sequence,stage_key,input_json) VALUES(?,?,?,?)`, run, sequence, "one", "{}"); err != nil {
t.Fatal(err)
}
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
response, err := store.Search(ctx, protocol.SearchRequest{Query: "needle"}, time.Now())
if err != nil {
t.Fatal(err)
}
if !response.Truncated {
t.Fatalf("step-count bound not reported: scanned_steps=%d total=%d", response.ScannedSteps, protocol.SearchMaxScanSteps+1)
}
if response.ScannedSteps != protocol.SearchMaxScanSteps {
t.Fatalf("scanned_steps=%d, want %d", response.ScannedSteps, protocol.SearchMaxScanSteps)
}
if response.ScannedBytes > protocol.SearchMaxScanBytes {
t.Fatalf("scanned_bytes=%d exceeds bound %d", response.ScannedBytes, protocol.SearchMaxScanBytes)
}
recent, err := store.Search(ctx, protocol.SearchRequest{Kind: protocol.SearchKindField}, time.Now())
if err != nil || !recent.Truncated || recent.ScannedSteps != protocol.SearchMaxScanSteps {
t.Fatalf("kind-only fields bypassed the step budget: %+v %v", recent, err)
}
}
func setCreated(t *testing.T, store *Store, runID int64, timestamp string) {
t.Helper()
if _, err := store.db.Exec("UPDATE pipeline_runs SET created_at=? WHERE id=?", timestamp, runID); err != nil {
t.Fatal(err)
}
}
func setFinishedNull(t *testing.T, store *Store, runID int64) {
t.Helper()
if _, err := store.db.Exec("UPDATE pipeline_runs SET finished_at=NULL,status='running' WHERE id=?", runID); err != nil {
t.Fatal(err)
}
}