search: browse bounded recent items with kind-only queries
This commit is contained in:
+151
-39
@@ -29,13 +29,17 @@ func (s *Store) Search(ctx context.Context, req protocol.SearchRequest, now time
|
||||
Cap: searchLimit(req.Limit),
|
||||
ScanBoundBytes: protocol.SearchMaxScanBytes,
|
||||
}
|
||||
// An empty query is a well-defined no-op, never a match-everything scan.
|
||||
if strings.TrimSpace(req.Query) == "" {
|
||||
// Empty text without a kind remains a no-op. A validated kind explicitly
|
||||
// requests bounded recent canonical rows, never a wildcard text search.
|
||||
browse := strings.TrimSpace(req.Query) == ""
|
||||
if browse && kind == "" {
|
||||
return response, nil
|
||||
}
|
||||
terms := parseSearchTerms(req.Query, now)
|
||||
terms.Browse = browse
|
||||
terms.Limit = response.Cap
|
||||
// A bare status word or date selects runs; it is never treated as body text.
|
||||
selectorOnly := terms.Text == ""
|
||||
selectorOnly := terms.Text == "" && !browse
|
||||
|
||||
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
|
||||
if err != nil {
|
||||
@@ -82,7 +86,14 @@ func (s *Store) Search(ctx context.Context, req protocol.SearchRequest, now time
|
||||
response.Truncated = true
|
||||
}
|
||||
|
||||
sortSearchResults(results)
|
||||
if browse {
|
||||
for i := range results {
|
||||
results[i].Score = 0
|
||||
}
|
||||
sortRecentSearchResults(results)
|
||||
} else {
|
||||
sortSearchResults(results)
|
||||
}
|
||||
if len(results) > response.Cap {
|
||||
results = results[:response.Cap]
|
||||
response.Truncated = true
|
||||
@@ -118,6 +129,8 @@ func searchLimit(limit int) int {
|
||||
// searchTerms is the parsed query. Text carries free text; Status and the
|
||||
// From/To window carry a bare status word or a date expression that selects runs.
|
||||
type searchTerms struct {
|
||||
Browse bool
|
||||
Limit int
|
||||
Text string
|
||||
Status string
|
||||
HasDate bool
|
||||
@@ -174,6 +187,13 @@ const (
|
||||
// scale without limit. Field text uses the separate SearchMaxScanSteps budget.
|
||||
const searchRowBudget = 2000
|
||||
|
||||
func searchCandidateLimit(terms searchTerms) int {
|
||||
if terms.Browse {
|
||||
return searchLimit(terms.Limit) + 1
|
||||
}
|
||||
return searchRowBudget + 1
|
||||
}
|
||||
|
||||
// searchLike builds a case-insensitive substring pattern, escaping LIKE
|
||||
// metacharacters so a query can never broaden the pattern.
|
||||
func searchLike(text string) string {
|
||||
@@ -216,15 +236,59 @@ func sortSearchResults(results []protocol.SearchResult) {
|
||||
})
|
||||
}
|
||||
|
||||
// Browsing has no relevance score. Numeric entity IDs, then visit/version IDs,
|
||||
// break equal timestamps deterministically (not lexical "run:9" vs "run:10").
|
||||
func sortRecentSearchResults(results []protocol.SearchResult) {
|
||||
identity := func(hit protocol.SearchResult) (int64, int) {
|
||||
switch {
|
||||
case hit.Pipeline != nil:
|
||||
return hit.Pipeline.ID, hit.Pipeline.Version
|
||||
case hit.Version != nil:
|
||||
return hit.Version.PipelineID, hit.Version.Version
|
||||
case hit.Run != nil:
|
||||
return hit.Run.ID, 0
|
||||
case hit.Step != nil:
|
||||
return hit.Step.RunID, hit.Step.Sequence
|
||||
case hit.Field != nil:
|
||||
return hit.Field.RunID, hit.Field.Sequence
|
||||
case hit.Artifact != nil:
|
||||
return hit.Artifact.RunID, hit.Artifact.Sequence
|
||||
}
|
||||
return 0, 0
|
||||
}
|
||||
sort.SliceStable(results, func(i, j int) bool {
|
||||
a, b := results[i], results[j]
|
||||
if a.Recency != b.Recency {
|
||||
return a.Recency > b.Recency
|
||||
}
|
||||
ai, as := identity(a)
|
||||
bi, bs := identity(b)
|
||||
if ai != bi {
|
||||
return ai > bi
|
||||
}
|
||||
if as != bs {
|
||||
return as > bs
|
||||
}
|
||||
return a.Target < b.Target
|
||||
})
|
||||
}
|
||||
|
||||
func searchPipelines(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *searchScan) ([]protocol.SearchResult, error) {
|
||||
if terms.Text == "" {
|
||||
if terms.Text == "" && !terms.Browse {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, `SELECT p.id,p.name,v.version,json_array_length(v.graph_json,'$.stages'),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',v.created_at),'')
|
||||
query := `SELECT p.id,p.name,v.version,json_array_length(v.graph_json,'$.stages'),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',v.created_at),'')
|
||||
FROM pipelines p JOIN pipeline_versions v ON v.pipeline_id=p.id
|
||||
WHERE v.version=(SELECT MAX(version) FROM pipeline_versions WHERE pipeline_id=p.id)
|
||||
AND lower(p.name) LIKE ? ESCAPE '\'
|
||||
ORDER BY p.name,p.id LIMIT ?`, searchLike(terms.Text), searchRowBudget+1)
|
||||
WHERE v.version=(SELECT MAX(version) FROM pipeline_versions WHERE pipeline_id=p.id)`
|
||||
var args []any
|
||||
if terms.Browse {
|
||||
query += " ORDER BY v.created_at DESC,v.pipeline_id DESC,v.version DESC LIMIT ?"
|
||||
} else {
|
||||
query += ` AND lower(p.name) LIKE ? ESCAPE '\' ORDER BY p.name,p.id LIMIT ?`
|
||||
args = append(args, searchLike(terms.Text))
|
||||
}
|
||||
args = append(args, searchCandidateLimit(terms))
|
||||
rows, err := tx.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -244,6 +308,9 @@ ORDER BY p.name,p.id LIMIT ?`, searchLike(terms.Text), searchRowBudget+1)
|
||||
return nil, err
|
||||
}
|
||||
score, ok := searchTier(name, terms.Text)
|
||||
if terms.Browse {
|
||||
score, ok = 0, true
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
@@ -257,12 +324,20 @@ ORDER BY p.name,p.id LIMIT ?`, searchLike(terms.Text), searchRowBudget+1)
|
||||
}
|
||||
|
||||
func searchVersions(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *searchScan) ([]protocol.SearchResult, error) {
|
||||
if terms.Text == "" {
|
||||
if terms.Text == "" && !terms.Browse {
|
||||
return nil, nil
|
||||
}
|
||||
query := `SELECT v.pipeline_id,v.version,v.content_hash,COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',v.created_at),''),v.source
|
||||
FROM pipeline_versions v WHERE lower(v.source) LIKE ? ESCAPE '\'`
|
||||
args := []any{searchLike(terms.Text)}
|
||||
source := "v.source"
|
||||
if terms.Browse {
|
||||
source = "''" // Browsing needs metadata only, never retained source text.
|
||||
}
|
||||
query := `SELECT v.pipeline_id,v.version,v.content_hash,COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',v.created_at),''),` + source + `
|
||||
FROM pipeline_versions v`
|
||||
var args []any
|
||||
if !terms.Browse {
|
||||
query += ` WHERE lower(v.source) LIKE ? ESCAPE '\'`
|
||||
args = append(args, searchLike(terms.Text))
|
||||
}
|
||||
var wanted int64
|
||||
numeric := false
|
||||
if n, err := parsePositiveInt(terms.Text); err == nil {
|
||||
@@ -270,8 +345,12 @@ FROM pipeline_versions v WHERE lower(v.source) LIKE ? ESCAPE '\'`
|
||||
args = append(args, n)
|
||||
wanted, numeric = n, true
|
||||
}
|
||||
query += " ORDER BY v.pipeline_id,v.version DESC LIMIT ?"
|
||||
args = append(args, searchRowBudget+1)
|
||||
if terms.Browse {
|
||||
query += " ORDER BY v.created_at DESC,v.pipeline_id DESC,v.version DESC LIMIT ?"
|
||||
} else {
|
||||
query += " ORDER BY v.pipeline_id,v.version DESC LIMIT ?"
|
||||
}
|
||||
args = append(args, searchCandidateLimit(terms))
|
||||
rows, err := tx.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -292,7 +371,9 @@ FROM pipeline_versions v WHERE lower(v.source) LIKE ? ESCAPE '\'`
|
||||
return nil, err
|
||||
}
|
||||
score, matched := 1, "source"
|
||||
if numeric && int64(version) == wanted {
|
||||
if terms.Browse {
|
||||
score, matched = 0, ""
|
||||
} else if numeric && int64(version) == wanted {
|
||||
score, matched = 3, "version"
|
||||
} else if !searchContains(source, terms.Text) {
|
||||
continue
|
||||
@@ -313,6 +394,8 @@ func searchRuns(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *search
|
||||
var query string
|
||||
var args []any
|
||||
switch {
|
||||
case terms.Browse:
|
||||
query = base
|
||||
case terms.Status != "":
|
||||
query = base + " WHERE r.status=?"
|
||||
args = append(args, terms.Status)
|
||||
@@ -324,8 +407,12 @@ func searchRuns(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *search
|
||||
pattern := searchLike(terms.Text)
|
||||
args = append(args, pattern, pattern, pattern, pattern, terms.Text)
|
||||
}
|
||||
query += " ORDER BY r.id DESC LIMIT ?"
|
||||
args = append(args, searchRowBudget+1)
|
||||
if terms.Browse {
|
||||
query += " ORDER BY r.created_at DESC,r.id DESC LIMIT ?"
|
||||
} else {
|
||||
query += " ORDER BY r.id DESC LIMIT ?"
|
||||
}
|
||||
args = append(args, searchCandidateLimit(terms))
|
||||
rows, err := tx.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -345,7 +432,9 @@ func searchRuns(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *search
|
||||
return nil, err
|
||||
}
|
||||
score, matched := 1, "status"
|
||||
if terms.Status != "" {
|
||||
if terms.Browse {
|
||||
score, matched = 0, ""
|
||||
} else if terms.Status != "" {
|
||||
matched = "status"
|
||||
} else if terms.HasDate {
|
||||
matched = "created_at"
|
||||
@@ -377,14 +466,20 @@ func searchRuns(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *search
|
||||
}
|
||||
|
||||
func searchSteps(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *searchScan) ([]protocol.SearchResult, error) {
|
||||
if terms.Text == "" {
|
||||
if terms.Text == "" && !terms.Browse {
|
||||
return nil, nil
|
||||
}
|
||||
pattern := searchLike(terms.Text)
|
||||
rows, err := tx.QueryContext(ctx, `SELECT s.run_id,s.sequence,s.stage_key,s.outcome,s.target,s.diagnostic,`+searchStepRecency+`,`+searchStepEvidence+`
|
||||
`+searchStepJoin+`
|
||||
WHERE lower(s.stage_key) LIKE ? ESCAPE '\' OR lower(s.outcome) LIKE ? ESCAPE '\' OR lower(s.target) LIKE ? ESCAPE '\' OR lower(s.diagnostic) LIKE ? ESCAPE '\'
|
||||
ORDER BY s.run_id DESC,s.sequence DESC LIMIT ?`, pattern, pattern, pattern, pattern, searchRowBudget+1)
|
||||
query := `SELECT s.run_id,s.sequence,s.stage_key,s.outcome,s.target,s.diagnostic,` + searchStepRecency + `,` + searchStepEvidence + ` ` + searchStepJoin
|
||||
var args []any
|
||||
if terms.Browse {
|
||||
query += " ORDER BY s.started_at DESC,s.run_id DESC,s.sequence DESC LIMIT ?"
|
||||
} else {
|
||||
query += ` WHERE lower(s.stage_key) LIKE ? ESCAPE '\' OR lower(s.outcome) LIKE ? ESCAPE '\' OR lower(s.target) LIKE ? ESCAPE '\' OR lower(s.diagnostic) LIKE ? ESCAPE '\' ORDER BY s.run_id DESC,s.sequence DESC LIMIT ?`
|
||||
pattern := searchLike(terms.Text)
|
||||
args = append(args, pattern, pattern, pattern, pattern)
|
||||
}
|
||||
args = append(args, searchCandidateLimit(terms))
|
||||
rows, err := tx.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -404,7 +499,9 @@ ORDER BY s.run_id DESC,s.sequence DESC LIMIT ?`, pattern, pattern, pattern, patt
|
||||
return nil, err
|
||||
}
|
||||
score, matched := 1, "diagnostic"
|
||||
if s, ok := searchTier(stage, terms.Text); ok {
|
||||
if terms.Browse {
|
||||
score, matched = 0, ""
|
||||
} else if s, ok := searchTier(stage, terms.Text); ok {
|
||||
score, matched = s, "stage"
|
||||
} else if searchContains(outcome, terms.Text) {
|
||||
matched = "outcome"
|
||||
@@ -440,14 +537,18 @@ type searchScan struct {
|
||||
}
|
||||
|
||||
func searchFields(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *searchScan) ([]protocol.SearchResult, error) {
|
||||
if terms.Text == "" {
|
||||
if terms.Text == "" && !terms.Browse {
|
||||
return nil, nil
|
||||
}
|
||||
// One extra step is requested so a scan that hits the step bound can see that
|
||||
// more rows remain and report Truncated rather than ending silently.
|
||||
rows, err := tx.QueryContext(ctx, `SELECT s.run_id,s.sequence,s.input_json,s.output_json,s.transcript,`+searchStepEvidence+`,`+searchStepRecency+`
|
||||
`+searchStepJoin+`
|
||||
ORDER BY s.run_id DESC,s.sequence DESC LIMIT ?`, protocol.SearchMaxScanSteps+1)
|
||||
query := `SELECT s.run_id,s.sequence,s.input_json,s.output_json,s.transcript,` + searchStepEvidence + `,` + searchStepRecency + ` ` + searchStepJoin
|
||||
if terms.Browse {
|
||||
query += " ORDER BY s.started_at DESC,s.run_id DESC,s.sequence DESC LIMIT ?"
|
||||
} else {
|
||||
query += " ORDER BY s.run_id DESC,s.sequence DESC LIMIT ?"
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, query, protocol.SearchMaxScanSteps+1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -498,7 +599,7 @@ ORDER BY s.run_id DESC,s.sequence DESC LIMIT ?`, protocol.SearchMaxScanSteps+1)
|
||||
Field: &protocol.SearchField{RunID: runID, Sequence: sequence, Field: field.name, Snippet: searchSnippet(text, terms.Text)},
|
||||
})
|
||||
}
|
||||
if searchContains(transcript, terms.Text) {
|
||||
if searchContains(transcript, terms.Text) || (terms.Browse && transcript != "") {
|
||||
results = append(results, protocol.SearchResult{
|
||||
Kind: protocol.SearchKindField, Target: searchFieldTarget(runID, sequence, "transcript"),
|
||||
Score: 1, MatchedField: "transcript", Recency: recency,
|
||||
@@ -510,14 +611,23 @@ ORDER BY s.run_id DESC,s.sequence DESC LIMIT ?`, protocol.SearchMaxScanSteps+1)
|
||||
}
|
||||
|
||||
func searchArtifacts(ctx context.Context, tx *sql.Tx, terms searchTerms, scan *searchScan) ([]protocol.SearchResult, error) {
|
||||
if terms.Text == "" {
|
||||
if terms.Text == "" && !terms.Browse {
|
||||
return nil, nil
|
||||
}
|
||||
pattern := searchLike(terms.Text)
|
||||
rows, err := tx.QueryContext(ctx, `SELECT a.run_id,a.sequence,a.artifacts_json,COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),'')
|
||||
FROM step_artifacts a JOIN pipeline_runs r ON r.id=a.run_id
|
||||
WHERE lower(a.artifacts_json) LIKE ? ESCAPE '\'
|
||||
ORDER BY a.run_id DESC,a.sequence DESC LIMIT ?`, pattern, searchRowBudget+1)
|
||||
query := `SELECT a.run_id,a.sequence,a.artifacts_json,COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),'')
|
||||
FROM step_artifacts a JOIN pipeline_runs r ON r.id=a.run_id`
|
||||
var args []any
|
||||
if terms.Browse {
|
||||
query = `SELECT a.run_id,a.sequence,a.artifacts_json,COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),'')
|
||||
FROM pipeline_runs r CROSS JOIN step_artifacts a ON a.run_id=r.id
|
||||
ORDER BY r.created_at DESC,r.id DESC,a.sequence DESC LIMIT ?`
|
||||
} else {
|
||||
query += ` WHERE lower(a.artifacts_json) LIKE ? ESCAPE '\' ORDER BY a.run_id DESC,a.sequence DESC LIMIT ?`
|
||||
args = append(args, searchLike(terms.Text))
|
||||
}
|
||||
// Metadata rows can contain several artifacts or malformed retained JSON.
|
||||
args = append(args, searchRowBudget+1)
|
||||
rows, err := tx.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -544,7 +654,9 @@ ORDER BY a.run_id DESC,a.sequence DESC LIMIT ?`, pattern, searchRowBudget+1)
|
||||
}
|
||||
for _, file := range evidence.Files {
|
||||
score, matched := 1, "content_type"
|
||||
if s, ok := searchTier(file.Name, terms.Text); ok {
|
||||
if terms.Browse {
|
||||
score, matched = 0, ""
|
||||
} else if s, ok := searchTier(file.Name, terms.Text); ok {
|
||||
score, matched = s, "name"
|
||||
} else if !searchContains(file.ContentType, terms.Text) {
|
||||
continue
|
||||
@@ -565,7 +677,7 @@ ORDER BY a.run_id DESC,a.sequence DESC LIMIT ?`, pattern, searchRowBudget+1)
|
||||
func findMatch(value any, text string) (string, bool) {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
if searchContains(typed, text) {
|
||||
if searchContains(typed, text) || (text == "" && typed != "") {
|
||||
return typed, true
|
||||
}
|
||||
case map[string]any:
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func TestSearchKindOnlyCanonicalRows(t *testing.T) {
|
||||
store, _ := searchFixture(t)
|
||||
ctx := context.Background()
|
||||
for _, kind := range protocol.SearchKinds {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
for _, query := range []string{"", " \t\n "} {
|
||||
response, err := store.Search(ctx, protocol.SearchRequest{Query: query, Kind: " " + kind + " "}, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(response.Results) == 0 || response.Kind != kind || response.Query != query {
|
||||
t.Fatalf("missing kind-only results: %+v", response)
|
||||
}
|
||||
for _, hit := range response.Results {
|
||||
if hit.Kind != kind || hit.Target == "" {
|
||||
t.Fatalf("invalid canonical hit: %+v", hit)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchKindOnlyValidationAndLimits(t *testing.T) {
|
||||
store, _ := searchFixture(t)
|
||||
ctx := context.Background()
|
||||
for _, kind := range []string{"bogus", "PIPELINE", "pipeline' OR 1=1 --"} {
|
||||
if _, err := store.Search(ctx, protocol.SearchRequest{Kind: kind}, time.Now()); err == nil {
|
||||
t.Fatalf("blank text bypassed validation for %q", kind)
|
||||
}
|
||||
}
|
||||
for _, query := range []string{"", " \n\t"} {
|
||||
response, err := store.Search(ctx, protocol.SearchRequest{Query: query}, time.Now())
|
||||
if err != nil || response.Results == nil || len(response.Results) != 0 || response.Truncated {
|
||||
t.Fatalf("empty all-kind query must remain a no-op: %+v %v", response, err)
|
||||
}
|
||||
}
|
||||
for _, limit := range []int{-1, 0, 1, protocol.SearchMaxLimit + 1} {
|
||||
response, err := store.Search(ctx, protocol.SearchRequest{Kind: protocol.SearchKindField, Limit: limit}, time.Now())
|
||||
if err != nil || response.Cap != searchLimit(limit) || len(response.Results) > response.Cap {
|
||||
t.Fatalf("kind-only limit %d: %+v %v", limit, response, err)
|
||||
}
|
||||
if limit == 1 && (len(response.Results) != 1 || !response.Truncated) {
|
||||
t.Fatalf("expected bounded/truncated fields: %+v", response)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchKindOnlyRecencyThenNumericID(t *testing.T) {
|
||||
store, run := searchFixture(t)
|
||||
ctx := context.Background()
|
||||
for i := 0; i < 11; i++ {
|
||||
_, err := store.db.Exec(`INSERT INTO pipeline_runs(pipeline_id,version,request_key,workspace,allow_tools,write_paths,input_json,created_at)
|
||||
SELECT pipeline_id,version,printf('recent-%d',?),workspace,allow_tools,write_paths,input_json,'2020-01-01 00:00:00' FROM pipeline_runs WHERE id=?`, i, run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The oldest ID is newest by timestamp, followed by equal-time numeric IDs.
|
||||
if _, err := store.db.Exec("UPDATE pipeline_runs SET created_at='2030-01-01 00:00:00' WHERE id=?", run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response, err := store.Search(ctx, protocol.SearchRequest{Kind: protocol.SearchKindRun, Limit: 4}, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ids []int64
|
||||
for _, hit := range response.Results {
|
||||
ids = append(ids, hit.Run.ID)
|
||||
}
|
||||
if !reflect.DeepEqual(ids, []int64{run, run + 11, run + 10, run + 9}) || !response.Truncated {
|
||||
t.Fatalf("recency/ID ordering or limit: %+v", response)
|
||||
}
|
||||
again, err := store.Search(ctx, protocol.SearchRequest{Kind: protocol.SearchKindRun, Limit: 4}, time.Now())
|
||||
if err != nil || !reflect.DeepEqual(response, again) {
|
||||
t.Fatalf("kind-only results changed between reads: %+v %v", again, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchRecentCanonicalIndexes(t *testing.T) {
|
||||
store, _ := searchFixture(t)
|
||||
for _, tc := range []struct{ query, index string }{
|
||||
{"SELECT pipeline_id,version FROM pipeline_versions ORDER BY created_at DESC,pipeline_id DESC,version DESC LIMIT 51", "search_versions_recent"},
|
||||
{"SELECT p.id FROM pipelines p JOIN pipeline_versions v ON v.pipeline_id=p.id WHERE v.version=(SELECT MAX(version) FROM pipeline_versions WHERE pipeline_id=p.id) ORDER BY v.created_at DESC,v.pipeline_id DESC,v.version DESC LIMIT 51", "search_versions_recent"},
|
||||
{"SELECT id FROM pipeline_runs ORDER BY created_at DESC,id DESC LIMIT 51", "search_runs_recent"},
|
||||
{"SELECT run_id,sequence FROM run_steps ORDER BY started_at DESC,run_id DESC,sequence DESC LIMIT 51", "search_steps_recent"},
|
||||
{"SELECT a.run_id,a.sequence FROM pipeline_runs r CROSS JOIN step_artifacts a ON a.run_id=r.id ORDER BY r.created_at DESC,r.id DESC,a.sequence DESC LIMIT 2001", "search_runs_recent"},
|
||||
} {
|
||||
rows, err := store.db.Query("EXPLAIN QUERY PLAN " + tc.query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var plan string
|
||||
for rows.Next() {
|
||||
var id, parent, unused int
|
||||
var detail string
|
||||
if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan += detail
|
||||
}
|
||||
rows.Close()
|
||||
if !strings.Contains(plan, tc.index) || strings.Contains(plan, "USE TEMP B-TREE FOR ORDER BY") {
|
||||
t.Fatalf("recent rows must use ordered canonical index: %s", plan)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchKindOnlyMaximumResultCap(t *testing.T) {
|
||||
store, run := searchFixture(t)
|
||||
_, err := store.db.Exec(`WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<250)
|
||||
INSERT INTO pipeline_runs(pipeline_id,version,request_key,workspace,allow_tools,write_paths,input_json)
|
||||
SELECT r.pipeline_id,r.version,printf('cap-%d',n.x),r.workspace,r.allow_tools,r.write_paths,r.input_json FROM pipeline_runs r CROSS JOIN n WHERE r.id=?`, run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, limit := range []int{0, 1, 9999} {
|
||||
response, err := store.Search(context.Background(), protocol.SearchRequest{Kind: protocol.SearchKindRun, Limit: limit}, time.Now())
|
||||
if err != nil || len(response.Results) != searchLimit(limit) || !response.Truncated || response.ScannedSteps != 0 || response.ScannedBytes != 0 {
|
||||
t.Fatalf("kind-only cap %d not enforced: %+v %v", limit, response, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchKindOnlyPipelineIsLatestCanonicalVersion(t *testing.T) {
|
||||
store, _ := searchFixture(t)
|
||||
_, err := store.db.Exec(`INSERT INTO pipeline_versions(pipeline_id,version,source,content_hash,graph_json,created_at)
|
||||
SELECT pipeline_id,version+1,source,content_hash||'-next',graph_json,'2031-01-01 00:00:00' FROM pipeline_versions`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response, err := store.Search(context.Background(), protocol.SearchRequest{Kind: protocol.SearchKindPipeline}, time.Now())
|
||||
if err != nil || len(response.Results) != 1 || response.Results[0].Pipeline.Version != 2 {
|
||||
t.Fatalf("pipeline browse must return latest canonical version only: %+v %v", response, err)
|
||||
}
|
||||
versions, err := store.Search(context.Background(), protocol.SearchRequest{Kind: protocol.SearchKindVersion}, time.Now())
|
||||
if err != nil || len(versions.Results) != 2 || versions.Results[0].Version.Version != 2 {
|
||||
t.Fatalf("version browse must retain history in recent order: %+v %v", versions, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package pipeline
|
||||
|
||||
// Recent-kind browsing orders canonical table rows before applying the existing
|
||||
// row/body budgets. These covering order keys avoid sorting the whole history.
|
||||
// No duplicate search store, FTS wildcard, artifact-content scan or write during
|
||||
// Search is needed; SQLite maintains the indexes with canonical writes.
|
||||
func (s *Store) initSearch() error {
|
||||
_, err := s.db.Exec(`
|
||||
CREATE INDEX IF NOT EXISTS search_versions_recent ON pipeline_versions(created_at DESC,pipeline_id DESC,version DESC);
|
||||
CREATE INDEX IF NOT EXISTS search_runs_recent ON pipeline_runs(created_at DESC,id DESC);
|
||||
CREATE INDEX IF NOT EXISTS search_steps_recent ON run_steps(started_at DESC,run_id DESC,sequence DESC);
|
||||
`)
|
||||
return err
|
||||
}
|
||||
@@ -538,6 +538,10 @@ func TestSearchScanBoundTruncates(t *testing.T) {
|
||||
t.Fatalf("scan passed the bound and matched: %+v", hit.Field)
|
||||
}
|
||||
}
|
||||
recent, err := store.Search(ctx, protocol.SearchRequest{Kind: protocol.SearchKindField}, time.Now())
|
||||
if err != nil || !recent.Truncated || len(recent.Results) != 0 || recent.ScannedBytes > protocol.SearchMaxScanBytes {
|
||||
t.Fatalf("kind-only fields bypassed the byte budget: %+v %v", recent, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSearchScanBoundTruncatesByRowCount proves the step-count bound is
|
||||
@@ -584,6 +588,10 @@ func TestSearchScanBoundTruncatesByRowCount(t *testing.T) {
|
||||
if response.ScannedBytes > protocol.SearchMaxScanBytes {
|
||||
t.Fatalf("scanned_bytes=%d exceeds bound %d", response.ScannedBytes, protocol.SearchMaxScanBytes)
|
||||
}
|
||||
recent, err := store.Search(ctx, protocol.SearchRequest{Kind: protocol.SearchKindField}, time.Now())
|
||||
if err != nil || !recent.Truncated || recent.ScannedSteps != protocol.SearchMaxScanSteps {
|
||||
t.Fatalf("kind-only fields bypassed the step budget: %+v %v", recent, err)
|
||||
}
|
||||
}
|
||||
|
||||
func setCreated(t *testing.T, store *Store, runID int64, timestamp string) {
|
||||
|
||||
@@ -58,6 +58,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := s.initSearch(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := s.initRunNotifications(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
|
||||
@@ -61,6 +61,8 @@ const (
|
||||
// SearchRequest is the search request body. Query is the free text, a bare
|
||||
// status word or a date expression; Kind optionally narrows to one SearchKind;
|
||||
// Limit caps the returned results (0 = server default).
|
||||
// Blank Query with a valid nonempty Kind browses recent items of that kind;
|
||||
// blank Query with empty Kind remains a no-op. Invalid kinds always fail.
|
||||
type SearchRequest struct {
|
||||
Query string `json:"query"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
@@ -68,8 +70,10 @@ type SearchRequest struct {
|
||||
}
|
||||
|
||||
// SearchResponse is the ranked, bounded set of matches. Results are ordered by
|
||||
// score, then recency, then a stable kind/target tie-break. Truncated reports
|
||||
// that either the result cap or the field-scan budget dropped matches;
|
||||
// score, then recency, then a stable kind/target tie-break.
|
||||
// For kind-only browsing, order is recency descending, numeric entity/visit IDs
|
||||
// descending, then target ascending, without a text-relevance score.
|
||||
// Truncated reports that either the result cap or the field-scan budget dropped matches;
|
||||
// ScannedSteps/ScannedBytes and ScanBoundBytes make the body-scan bound
|
||||
// observable.
|
||||
type SearchResponse struct {
|
||||
|
||||
@@ -9,6 +9,10 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
"github.com/coder/websocket/wsjson"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/cli"
|
||||
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
||||
@@ -102,4 +106,42 @@ func TestCLISearchEndToEndEveryKind(t *testing.T) {
|
||||
if _, err := runCLI("search", "zephyr", "--type", "bogus"); err == nil || !strings.Contains(err.Error(), "unknown search kind") {
|
||||
t.Fatalf("invalid kind must error, got %v", err)
|
||||
}
|
||||
|
||||
// The browser sends an actual empty query, not an invented wildcard. Exercise
|
||||
// that unchanged wire shape over /connect against this throwaway store.
|
||||
wireCtx, wireCancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer wireCancel()
|
||||
conn, _, err := websocket.Dial(wireCtx, url, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.CloseNow()
|
||||
for _, kind := range append(append([]string{}, protocol.SearchKinds...), "", "bogus") {
|
||||
request := protocol.Message{Type: protocol.Search, ID: "recent-" + kind, Search: &protocol.SearchRequest{Query: "", Kind: kind, Limit: 1}}
|
||||
if err := wsjson.Write(wireCtx, conn, request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var reply protocol.Message
|
||||
for reply.ID != request.ID {
|
||||
if err := wsjson.Read(wireCtx, conn, &reply); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if kind == "bogus" {
|
||||
if !strings.Contains(reply.Error, "unknown search kind") {
|
||||
t.Fatalf("blank text must not bypass kind validation: %+v", reply)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if reply.SearchResults == nil || reply.SearchResults.Cap != 1 || reply.SearchResults.Query != "" {
|
||||
t.Fatalf("invalid kind-only reply: %+v", reply)
|
||||
}
|
||||
if kind == "" {
|
||||
if len(reply.SearchResults.Results) != 0 {
|
||||
t.Fatalf("blank unfiltered query must remain a no-op: %+v", reply)
|
||||
}
|
||||
} else if hits := reply.SearchResults.Results; len(hits) != 1 || hits[0].Kind != kind {
|
||||
t.Fatalf("missing bounded kind-only result for %q: %+v", kind, reply)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user