Files
pipelines/internal/pipeline/search.go
T
alan 80232a4ffe Add omni search: ranked mixed results in the TUI and CLI
Back the TUI "/" popup with the server's search command and add a matching
pipelines search CLI command.

- protocol: SearchRequest/SearchResponse and per-kind SearchResult payloads,
  one shared Label for the six-kind badge dispatch and RuneIndexFold for
  case-insensitive matching.
- pipeline: the read-only, ranked, bounded Store.Search implementation.
- cli: search QUERY [--type KIND] [--limit N] [--json], printing the shared
  label plus a text-field snippet when present.
- tui: type to search, up/down to choose, Enter to open and Esc to cancel over
  one mixed ranked list, with short type badges and a bracketed non-colour
  match marker; Enter navigates to the matched pipeline, run, step, field
  section or artifact entry.
- tests: protocol and TUI coverage, including an env-gated live test against
  the running server.
2026-10-16 23:03:17 +01:00

670 lines
21 KiB
Go

package pipeline
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"sort"
"strconv"
"strings"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// Search answers one ranked, bounded, read-only query over the existing tables.
// It scans no artifact contents and makes no follow-up calls: each kind runs at
// most one query, field text is decoded with decodeValues over a fixed scan
// budget, and ranking/tie-breaking happens in memory over the candidate set.
func (s *Store) Search(ctx context.Context, req protocol.SearchRequest, now time.Time) (*protocol.SearchResponse, error) {
kind := strings.TrimSpace(req.Kind)
if kind != "" && searchKindIndex(kind) < 0 {
return nil, fmt.Errorf("unknown search kind %q; use one of %s", req.Kind, strings.Join(protocol.SearchKinds, ", "))
}
response := &protocol.SearchResponse{
Query: req.Query,
Kind: kind,
Results: []protocol.SearchResult{},
Cap: searchLimit(req.Limit),
ScanBoundBytes: protocol.SearchMaxScanBytes,
}
// An empty query is a well-defined no-op, never a match-everything scan.
if strings.TrimSpace(req.Query) == "" {
return response, nil
}
terms := parseSearchTerms(req.Query, now)
// A bare status word or date selects runs; it is never treated as body text.
selectorOnly := terms.Text == ""
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
if err != nil {
return nil, err
}
defer tx.Rollback()
var results []protocol.SearchResult
var scan searchScan
for i, candidate := range protocol.SearchKinds {
if kind != "" && searchKindIndex(kind) != i {
continue
}
if selectorOnly && candidate != protocol.SearchKindRun {
continue
}
var hits []protocol.SearchResult
var err error
switch candidate {
case protocol.SearchKindPipeline:
hits, err = searchPipelines(ctx, tx, terms, &scan)
case protocol.SearchKindVersion:
hits, err = searchVersions(ctx, tx, terms, &scan)
case protocol.SearchKindRun:
hits, err = searchRuns(ctx, tx, terms, &scan)
case protocol.SearchKindStep:
hits, err = searchSteps(ctx, tx, terms, &scan)
case protocol.SearchKindField:
hits, err = searchFields(ctx, tx, terms, &scan)
case protocol.SearchKindArtifact:
hits, err = searchArtifacts(ctx, tx, terms, &scan)
}
if err != nil {
return nil, err
}
results = append(results, hits...)
}
if err := tx.Commit(); err != nil {
return nil, err
}
response.ScannedSteps = scan.steps
response.ScannedBytes = scan.bytes
if scan.truncated {
response.Truncated = true
}
sortSearchResults(results)
if len(results) > response.Cap {
results = results[:response.Cap]
response.Truncated = true
}
if results == nil {
results = []protocol.SearchResult{}
}
response.Results = results
return response, nil
}
func searchKindIndex(kind string) int {
for i, k := range protocol.SearchKinds {
if k == kind {
return i
}
}
return -1
}
// searchLimit clamps a requested cap. Zero, negative or over-max requests fall
// back to the default or the maximum, so the cap is always enforced server-side.
func searchLimit(limit int) int {
if limit <= 0 {
return protocol.SearchDefaultLimit
}
if limit > protocol.SearchMaxLimit {
return protocol.SearchMaxLimit
}
return limit
}
// searchTerms is the parsed query. Text carries free text; Status and the
// From/To window carry a bare status word or a date expression that selects runs.
type searchTerms struct {
Text string
Status string
HasDate bool
From time.Time
To time.Time
}
// parseSearchTerms recognises a lone status word and a lone date expression and
// leaves everything else as free text, so ordinary words are never misread as a
// selector. Relative dates resolve against the injected now in its own location;
// the comparison window is converted to UTC to match the stored timestamps.
func parseSearchTerms(query string, now time.Time) searchTerms {
trimmed := strings.TrimSpace(query)
lower := strings.ToLower(trimmed)
for _, status := range protocol.SearchStatuses {
if lower == status {
return searchTerms{Status: status}
}
}
location := now.Location()
day := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, location)
switch lower {
case "today":
return dateTerms(day, day.AddDate(0, 0, 1))
case "yesterday":
return dateTerms(day.AddDate(0, 0, -1), day)
case "this week":
// Monday 00:00 local is the start of the week.
start := day.AddDate(0, 0, -((int(day.Weekday()) + 6) % 7))
return dateTerms(start, start.AddDate(0, 0, 7))
}
if parsed, err := time.ParseInLocation("2006-01-02", trimmed, location); err == nil {
return dateTerms(parsed, parsed.AddDate(0, 0, 1))
}
return searchTerms{Text: trimmed}
}
func dateTerms(from, to time.Time) searchTerms {
return searchTerms{HasDate: true, From: from.UTC(), To: to.UTC()}
}
const searchSQLTime = "2006-01-02 15:04:05"
// searchStepJoin is the shared run_steps/step_control source and projections the
// step and field scans build on, so those queries cannot drift apart.
const (
searchStepJoin = `FROM run_steps s LEFT JOIN step_control c ON c.run_id=s.run_id AND c.sequence=s.sequence`
searchStepEvidence = `COALESCE(c.evidence_json,'')`
searchStepRecency = `COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',s.started_at),'')`
)
// searchRowBudget bounds how many raw rows one non-body kind scans before it
// stops and reports Truncated, so a LIKE scan over a very large table cannot
// scale without limit. Field text uses the separate SearchMaxScanSteps budget.
const searchRowBudget = 2000
// searchLike builds a case-insensitive substring pattern, escaping LIKE
// metacharacters so a query can never broaden the pattern.
func searchLike(text string) string {
escaped := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(strings.ToLower(text))
return "%" + escaped + "%"
}
// searchTier ranks a "name" match: exact beats a substring. The third tier (a
// body-text match) is assigned by the caller.
func searchTier(value, text string) (int, bool) {
if text == "" {
return 0, false
}
if strings.EqualFold(value, text) {
return 3, true
}
if strings.Contains(strings.ToLower(value), strings.ToLower(text)) {
return 2, true
}
return 0, false
}
func searchContains(value, text string) bool {
return text != "" && strings.Contains(strings.ToLower(value), strings.ToLower(text))
}
func sortSearchResults(results []protocol.SearchResult) {
sort.SliceStable(results, func(i, j int) bool {
a, b := results[i], results[j]
if a.Score != b.Score {
return a.Score > b.Score
}
if a.Recency != b.Recency {
return a.Recency > b.Recency
}
if a.Kind != b.Kind {
return a.Kind < b.Kind
}
return a.Target < b.Target
})
}
func searchPipelines(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *searchScan) ([]protocol.SearchResult, error) {
if terms.Text == "" {
return nil, nil
}
rows, err := tx.QueryContext(ctx, `SELECT p.id,p.name,v.version,json_array_length(v.graph_json,'$.stages'),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',v.created_at),'')
FROM pipelines p JOIN pipeline_versions v ON v.pipeline_id=p.id
WHERE v.version=(SELECT MAX(version) FROM pipeline_versions WHERE pipeline_id=p.id)
AND lower(p.name) LIKE ? ESCAPE '\'
ORDER BY p.name,p.id LIMIT ?`, searchLike(terms.Text), searchRowBudget+1)
if err != nil {
return nil, err
}
defer rows.Close()
var results []protocol.SearchResult
scanned := 0
for rows.Next() {
if scanned == searchRowBudget {
scan.truncated = true
break
}
scanned++
var id int64
var name, recency string
var version, stages int
if err := rows.Scan(&id, &name, &version, &stages, &recency); err != nil {
return nil, err
}
score, ok := searchTier(name, terms.Text)
if !ok {
continue
}
results = append(results, protocol.SearchResult{
Kind: protocol.SearchKindPipeline, Target: searchPipelineTarget(id),
Score: score, MatchedField: "name", Recency: recency,
Pipeline: &protocol.SearchPipeline{ID: id, Name: name, Version: version, Stages: stages},
})
}
return results, rows.Err()
}
func searchVersions(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *searchScan) ([]protocol.SearchResult, error) {
if terms.Text == "" {
return nil, nil
}
query := `SELECT v.pipeline_id,v.version,v.content_hash,COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',v.created_at),''),v.source
FROM pipeline_versions v WHERE lower(v.source) LIKE ? ESCAPE '\'`
args := []any{searchLike(terms.Text)}
var wanted int64
numeric := false
if n, err := parsePositiveInt(terms.Text); err == nil {
query += " OR v.version=?"
args = append(args, n)
wanted, numeric = n, true
}
query += " ORDER BY v.pipeline_id,v.version DESC LIMIT ?"
args = append(args, searchRowBudget+1)
rows, err := tx.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var results []protocol.SearchResult
scanned := 0
for rows.Next() {
if scanned == searchRowBudget {
scan.truncated = true
break
}
scanned++
var pipelineID int64
var version int
var hash, recency, source string
if err := rows.Scan(&pipelineID, &version, &hash, &recency, &source); err != nil {
return nil, err
}
score, matched := 1, "source"
if numeric && int64(version) == wanted {
score, matched = 3, "version"
} else if !searchContains(source, terms.Text) {
continue
}
results = append(results, protocol.SearchResult{
Kind: protocol.SearchKindVersion, Target: searchVersionTarget(pipelineID, version),
Score: score, MatchedField: matched, Recency: recency,
Version: &protocol.SearchVersion{PipelineID: pipelineID, Version: version, ContentHash: hash, CreatedAt: recency},
})
}
return results, rows.Err()
}
func searchRuns(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *searchScan) ([]protocol.SearchResult, error) {
// Reuse runSelect's join and timestamp projection rather than restating the
// run-row SELECT; only the fields search needs are added.
base := `SELECT r.id,p.name,r.status,r.workspace,` + runStamps + `,r.diagnostic ` + runFrom
var query string
var args []any
switch {
case terms.Status != "":
query = base + " WHERE r.status=?"
args = append(args, terms.Status)
case terms.HasDate:
query = base + " WHERE r.created_at>=? AND r.created_at<?"
args = append(args, terms.From.Format(searchSQLTime), terms.To.Format(searchSQLTime))
default:
query = base + ` WHERE lower(p.name) LIKE ? ESCAPE '\' OR lower(r.workspace) LIKE ? ESCAPE '\' OR lower(r.status) LIKE ? ESCAPE '\' OR lower(r.diagnostic) LIKE ? ESCAPE '\' OR CAST(r.id AS TEXT)=?`
pattern := searchLike(terms.Text)
args = append(args, pattern, pattern, pattern, pattern, terms.Text)
}
query += " ORDER BY r.id DESC LIMIT ?"
args = append(args, searchRowBudget+1)
rows, err := tx.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var results []protocol.SearchResult
scanned := 0
for rows.Next() {
if scanned == searchRowBudget {
scan.truncated = true
break
}
scanned++
var id int64
var name, status, workspace, created, finished, diagnostic string
if err := rows.Scan(&id, &name, &status, &workspace, &created, &finished, &diagnostic); err != nil {
return nil, err
}
score, matched := 1, "status"
if terms.Status != "" {
matched = "status"
} else if terms.HasDate {
matched = "created_at"
} else if s, ok := searchTier(name, terms.Text); ok {
score, matched = s, "pipeline_name"
} else if numericEqual(terms.Text, id) {
score, matched = 3, "id"
} else if searchContains(workspace, terms.Text) {
matched = "workspace"
} else if searchContains(status, terms.Text) {
matched = "status"
} else if searchContains(diagnostic, terms.Text) {
matched = "diagnostic"
} else {
continue
}
run := &protocol.SearchRun{ID: id, PipelineName: name, Status: status, Workspace: workspace, CreatedAt: created, FinishedAt: finished}
if created != "" && finished != "" {
if secs, ok := durationSeconds(created, finished); ok {
run.DurationSeconds = secs
}
}
results = append(results, protocol.SearchResult{
Kind: protocol.SearchKindRun, Target: searchRunTarget(id),
Score: score, MatchedField: matched, Recency: created, Run: run,
})
}
return results, rows.Err()
}
func searchSteps(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *searchScan) ([]protocol.SearchResult, error) {
if terms.Text == "" {
return nil, nil
}
pattern := searchLike(terms.Text)
rows, err := tx.QueryContext(ctx, `SELECT s.run_id,s.sequence,s.stage_key,s.outcome,s.target,s.diagnostic,`+searchStepRecency+`,`+searchStepEvidence+`
`+searchStepJoin+`
WHERE lower(s.stage_key) LIKE ? ESCAPE '\' OR lower(s.outcome) LIKE ? ESCAPE '\' OR lower(s.target) LIKE ? ESCAPE '\' OR lower(s.diagnostic) LIKE ? ESCAPE '\'
ORDER BY s.run_id DESC,s.sequence DESC LIMIT ?`, pattern, pattern, pattern, pattern, searchRowBudget+1)
if err != nil {
return nil, err
}
defer rows.Close()
var results []protocol.SearchResult
scanned := 0
for rows.Next() {
if scanned == searchRowBudget {
scan.truncated = true
break
}
scanned++
var runID int64
var sequence int
var stage, outcome, target, diagnostic, recency, control string
if err := rows.Scan(&runID, &sequence, &stage, &outcome, &target, &diagnostic, &recency, &control); err != nil {
return nil, err
}
score, matched := 1, "diagnostic"
if s, ok := searchTier(stage, terms.Text); ok {
score, matched = s, "stage"
} else if searchContains(outcome, terms.Text) {
matched = "outcome"
} else if searchContains(target, terms.Text) {
matched = "target"
} else if searchContains(diagnostic, terms.Text) {
matched = "diagnostic"
} else {
continue
}
visit := 0
if control != "" && control != "null" {
var evidence protocol.ControlEvidence
if err := json.Unmarshal([]byte(control), &evidence); err == nil {
visit = evidence.Visit
}
}
results = append(results, protocol.SearchResult{
Kind: protocol.SearchKindStep, Target: searchStepTarget(runID, sequence),
Score: score, MatchedField: matched, Recency: recency,
Step: &protocol.SearchStep{RunID: runID, Sequence: sequence, Stage: stage, Visit: visit, Outcome: outcome, Target: target, Diagnostic: diagnostic},
})
}
return results, rows.Err()
}
// searchScan accumulates the shared bounded-scan budget so the response can
// report how much body text was inspected and whether any bound was reached.
type searchScan struct {
steps int
bytes int
truncated bool
}
func searchFields(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *searchScan) ([]protocol.SearchResult, error) {
if terms.Text == "" {
return nil, nil
}
// One extra step is requested so a scan that hits the step bound can see that
// more rows remain and report Truncated rather than ending silently.
rows, err := tx.QueryContext(ctx, `SELECT s.run_id,s.sequence,s.input_json,s.output_json,s.transcript,`+searchStepEvidence+`,`+searchStepRecency+`
`+searchStepJoin+`
ORDER BY s.run_id DESC,s.sequence DESC LIMIT ?`, protocol.SearchMaxScanSteps+1)
if err != nil {
return nil, err
}
defer rows.Close()
var results []protocol.SearchResult
for rows.Next() {
if scan.steps >= protocol.SearchMaxScanSteps {
scan.truncated = true
break
}
var runID int64
var sequence int
var input, output, transcript, evidence, recency string
if err := rows.Scan(&runID, &sequence, &input, &output, &transcript, &evidence, &recency); err != nil {
return nil, err
}
size := len(input) + len(output) + len(transcript) + len(evidence)
if scan.bytes+size > protocol.SearchMaxScanBytes {
scan.truncated = true
break
}
scan.bytes += size
scan.steps++
for _, field := range []struct {
name string
raw string
}{
{name: "input", raw: input},
{name: "output", raw: output},
{name: "evidence", raw: evidence},
} {
if field.raw == "" {
continue
}
var values map[string]any
// A malformed retained row cannot match anything, so it is skipped
// rather than failing the whole query and hiding every other hit.
if err := decodeValues(field.raw, &values); err != nil {
continue
}
text, ok := findMatch(values, terms.Text)
if !ok {
continue
}
results = append(results, protocol.SearchResult{
Kind: protocol.SearchKindField, Target: searchFieldTarget(runID, sequence, field.name),
Score: 1, MatchedField: field.name, Recency: recency,
Field: &protocol.SearchField{RunID: runID, Sequence: sequence, Field: field.name, Snippet: searchSnippet(text, terms.Text)},
})
}
if searchContains(transcript, terms.Text) {
results = append(results, protocol.SearchResult{
Kind: protocol.SearchKindField, Target: searchFieldTarget(runID, sequence, "transcript"),
Score: 1, MatchedField: "transcript", Recency: recency,
Field: &protocol.SearchField{RunID: runID, Sequence: sequence, Field: "transcript", Snippet: searchSnippet(transcript, terms.Text)},
})
}
}
return results, rows.Err()
}
func searchArtifacts(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *searchScan) ([]protocol.SearchResult, error) {
if terms.Text == "" {
return nil, nil
}
pattern := searchLike(terms.Text)
rows, err := tx.QueryContext(ctx, `SELECT a.run_id,a.sequence,a.artifacts_json,COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),'')
FROM step_artifacts a JOIN pipeline_runs r ON r.id=a.run_id
WHERE lower(a.artifacts_json) LIKE ? ESCAPE '\'
ORDER BY a.run_id DESC,a.sequence DESC LIMIT ?`, pattern, searchRowBudget+1)
if err != nil {
return nil, err
}
defer rows.Close()
var results []protocol.SearchResult
scanned := 0
for rows.Next() {
if scanned == searchRowBudget {
scan.truncated = true
break
}
scanned++
var runID int64
var sequence int
var artifacts, recency string
if err := rows.Scan(&runID, &sequence, &artifacts, &recency); err != nil {
return nil, err
}
var evidence stageArtifactsEvidence
// A malformed retained row cannot match anything, so it is skipped
// rather than failing the whole query and hiding every other hit.
if err := json.Unmarshal([]byte(artifacts), &evidence); err != nil {
continue
}
for _, file := range evidence.Files {
score, matched := 1, "content_type"
if s, ok := searchTier(file.Name, terms.Text); ok {
score, matched = s, "name"
} else if !searchContains(file.ContentType, terms.Text) {
continue
}
results = append(results, protocol.SearchResult{
Kind: protocol.SearchKindArtifact, Target: file.URL,
Score: score, MatchedField: matched, Recency: recency,
Artifact: &protocol.SearchArtifact{RunID: runID, Sequence: sequence, Name: file.Name, ContentType: file.ContentType, SizeBytes: file.SizeBytes, URL: file.URL},
})
}
}
return results, rows.Err()
}
// findMatch walks decoded JSON values in a stable key order and returns the
// first string leaf containing the query, so a hit and its snippet are
// deterministic rather than map-iteration dependent.
func findMatch(value any, text string) (string, bool) {
switch typed := value.(type) {
case string:
if searchContains(typed, text) {
return typed, true
}
case map[string]any:
keys := make([]string, 0, len(typed))
for key := range typed {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if match, ok := findMatch(typed[key], text); ok {
return match, true
}
}
case []any:
for _, item := range typed {
if match, ok := findMatch(item, text); ok {
return match, true
}
}
}
return "", false
}
// searchSnippet reduces text to a single clean line and centers a short window
// on the first case-insensitive match, folding runes so UTF-8 boundaries are
// never split and the result is always well-formed.
func searchSnippet(text, query string) string {
normalized := protocol.NormalizeText(text)
runes := []rune(normalized)
index := protocol.RuneIndexFold(runes, []rune(query))
if index < 0 {
return truncateRunes(normalized, 80)
}
const margin = 40
start := index - margin
if start < 0 {
start = 0
}
end := index + len([]rune(query)) + margin
if end > len(runes) {
end = len(runes)
}
snippet := string(runes[start:end])
if start > 0 {
snippet = "…" + snippet
}
if end < len(runes) {
snippet = snippet + "…"
}
return snippet
}
func truncateRunes(text string, limit int) string {
runes := []rune(text)
if len(runes) <= limit {
return text
}
return string(runes[:limit]) + "…"
}
func searchPipelineTarget(id int64) string { return fmt.Sprintf("pipeline:%d", id) }
func searchVersionTarget(id int64, version int) string {
return fmt.Sprintf("pipeline:%d:v%d", id, version)
}
func searchRunTarget(id int64) string { return fmt.Sprintf("run:%d", id) }
func searchStepTarget(id int64, sequence int) string { return fmt.Sprintf("run:%d:%d", id, sequence) }
func searchFieldTarget(id int64, sequence int, field string) string {
return fmt.Sprintf("run:%d:%d:%s", id, sequence, field)
}
func parsePositiveInt(text string) (int64, error) {
n, err := strconv.ParseInt(strings.TrimSpace(text), 10, 64)
if err != nil || n <= 0 {
return 0, fmt.Errorf("not a positive integer")
}
return n, nil
}
func numericEqual(text string, id int64) bool {
n, err := parsePositiveInt(text)
return err == nil && n == id
}
func durationSeconds(created, finished string) (float64, bool) {
start, err := time.Parse(time.RFC3339, created)
if err != nil {
return 0, false
}
end, err := time.Parse(time.RFC3339, finished)
if err != nil {
return 0, false
}
if end.Before(start) {
return 0, false
}
return end.Sub(start).Seconds(), true
}