From 57f5219628589a3432d9eb2c5d3aad9983233395 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Wed, 4 Nov 2026 17:43:47 +0000 Subject: [PATCH] Artifacts: runner-internal turn files move to a Debugging files group Each recorded artifact now carries a server group (deliverable or debug). public/ files are deliverables; input/memory/history/output.md, session exports and private/ scratch are debugging files. The app's Artifacts tab lists deliverables and the Debug toggle reveals a separate Debugging files group (older servers fall back to the filename rule); the TUI lists deliverables first and marks debug rows; artifact search excludes debug files unless include_debug / --debug-files is set. --- .../artifacts/domain/artifact_record.dart | 23 +++++ .../artifacts/domain/artifact_selection.dart | 9 +- .../ui/artifact_document_section.dart | 11 ++- .../artifacts/ui/artifact_gallery_page.dart | 4 +- .../artifacts/ui/artifact_visit_list.dart | 2 +- .../domain/artifact_debug_group_test.dart | 58 ++++++++++++ .../ui/artifact_gallery_debug_files_test.dart | 91 +++++++++++++++++++ .../ui/artifact_gallery_documents_test.dart | 4 +- app/tool/search_selection_probe_entry.dart | 23 +++++ docs/cli.md | 5 +- docs/stage-artifacts.md | 31 ++++++- internal/cli/app.go | 4 +- internal/cli/commands.go | 4 +- internal/pipeline/runs.go | 2 +- internal/pipeline/search.go | 26 ++++-- internal/pipeline/search_debug_test.go | 85 +++++++++++++++++ internal/protocol/artifact_group.go | 62 +++++++++++++ internal/protocol/artifact_group_test.go | 49 ++++++++++ internal/protocol/runs.go | 5 + internal/protocol/search.go | 6 ++ internal/server/artifacts.go | 2 +- internal/server/artifacts_test.go | 10 ++ internal/tui/artifacts.go | 28 +++++- internal/tui/artifacts_debug_test.go | 27 ++++++ 24 files changed, 541 insertions(+), 30 deletions(-) create mode 100644 app/test/features/artifacts/domain/artifact_debug_group_test.dart create mode 100644 app/test/features/artifacts/ui/artifact_gallery_debug_files_test.dart create mode 100644 internal/pipeline/search_debug_test.go create mode 100644 internal/protocol/artifact_group.go create mode 100644 internal/protocol/artifact_group_test.go create mode 100644 internal/tui/artifacts_debug_test.go diff --git a/app/lib/features/artifacts/domain/artifact_record.dart b/app/lib/features/artifacts/domain/artifact_record.dart index 3faef2df..47a00926 100644 --- a/app/lib/features/artifacts/domain/artifact_record.dart +++ b/app/lib/features/artifacts/domain/artifact_record.dart @@ -15,6 +15,20 @@ const List systemArtifactNames = [ 'session.html', ]; +/// The server's `group` value for a runner-internal debugging file. +const String debugArtifactGroup = 'debug'; + +/// The server's `group` value for a file the stage published. +const String deliverableArtifactGroup = 'deliverable'; + +/// Whether [name] is a runner-internal debugging file by name alone. This is +/// the fallback for a server that records no `group`: the turn documents, the +/// full `history.md`, and the visit's `private/` scratch. +bool isDebugArtifactName(String name) => + isSystemArtifactName(name) || + name == 'history.md' || + name.startsWith('private/'); + /// One recorded artifact of a stage visit. /// /// The server is the authority: [name], [url], [contentType] and [sizeBytes] @@ -63,6 +77,15 @@ class ArtifactRecord { bool get isSystem => isSystemArtifactName(name); bool get isPublicName => name.startsWith('public/'); + /// Whether this is a runner-internal debugging file rather than a stage + /// deliverable. The server's recorded `group` decides; an older server that + /// records none falls back to [isDebugArtifactName]. + bool get isDebug { + final group = raw['group']; + if (group is String && group.isNotEmpty) return group == debugArtifactGroup; + return isDebugArtifactName(name); + } + static String _text(dynamic value) => value == null ? '' : '$value'; static int _intOf(dynamic value) { diff --git a/app/lib/features/artifacts/domain/artifact_selection.dart b/app/lib/features/artifacts/domain/artifact_selection.dart index 4f976473..5a6fa623 100644 --- a/app/lib/features/artifacts/domain/artifact_selection.dart +++ b/app/lib/features/artifacts/domain/artifact_selection.dart @@ -36,7 +36,7 @@ List presentedArtifacts(Iterable artifacts) { ]; } -/// A visit's stage-produced artifacts: everything except the turn-owned files +/// A visit's deliverables: everything except runner-internal debugging files /// and repaired session exports. Order is preserved. List stageArtifactsFrom( Iterable artifacts, { @@ -44,18 +44,19 @@ List stageArtifactsFrom( }) { return [ for (final artifact in presentedArtifacts(artifacts)) - if (!artifact.isSystem && !repairs.contains(artifact.name)) artifact, + if (!artifact.isDebug && !repairs.contains(artifact.name)) artifact, ]; } -/// A visit's system files and repaired exports, in recorded order. +/// A visit's debugging files (turn documents, history.md, private/ scratch) +/// and repaired exports, in recorded order. List systemArtifactsFrom( Iterable artifacts, { Set repairs = const {}, }) { return [ for (final artifact in presentedArtifacts(artifacts)) - if (artifact.isSystem || repairs.contains(artifact.name)) artifact, + if (artifact.isDebug || repairs.contains(artifact.name)) artifact, ]; } diff --git a/app/lib/features/artifacts/ui/artifact_document_section.dart b/app/lib/features/artifacts/ui/artifact_document_section.dart index fb4df48f..bb5a4631 100644 --- a/app/lib/features/artifacts/ui/artifact_document_section.dart +++ b/app/lib/features/artifacts/ui/artifact_document_section.dart @@ -8,6 +8,10 @@ import '../domain/artifact_groups.dart'; import 'artifact_record_row.dart'; import 'artifact_row_labels.dart'; +/// The gallery's label for a visit's runner-internal files (turn documents, +/// history.md, private/ scratch, session exports), shown only while Debug is on. +const String debuggingFilesTitle = 'Debugging files'; + /// One visit's turn documents, shown below its stage artifacts. /// /// A recorded document is a normal gallery row with its recorded name, MIME and @@ -21,8 +25,13 @@ class ArtifactDocumentSection extends StatelessWidget { this.selectedIdentity, this.onSelected, this.showHeader = true, + this.title = 'Turn documents', }); + /// The section label: the Session page names the turn documents; the gallery + /// names every runner-internal file it lists as debugging files. + final String title; + /// False when a collapsible title above already names the section. final bool showHeader; @@ -54,7 +63,7 @@ class ArtifactDocumentSection extends StatelessWidget { container: true, header: true, child: Text( - 'Turn documents', + title, style: cupertino ? SectionHeadingStyle.of(context).title : null, ), ), diff --git a/app/lib/features/artifacts/ui/artifact_gallery_page.dart b/app/lib/features/artifacts/ui/artifact_gallery_page.dart index 12ede23b..99b400db 100644 --- a/app/lib/features/artifacts/ui/artifact_gallery_page.dart +++ b/app/lib/features/artifacts/ui/artifact_gallery_page.dart @@ -124,7 +124,7 @@ class _ArtifactGalleryPageState extends State { for (final document in group.documents) { final record = document.record; if (record == null || record.url != url) continue; - // A turn document is listed only while the debug toggle shows them. + // A debugging file is listed only while the debug toggle shows them. _showTurnDocuments = true; _opening = (group: group, artifact: record); } @@ -556,7 +556,7 @@ class _ArtifactGalleryPageState extends State { Widget _debugToggle(BuildContext context) { final showing = _showTurnDocuments; - final label = showing ? 'Hide turn documents' : 'Show turn documents'; + final label = showing ? 'Hide debugging files' : 'Show debugging files'; // Cupertino's bundled icon font has no bug glyph; retain the universally // recognisable Material bug icon inside its native Cupertino button. final icon = showing ? Icons.bug_report : Icons.bug_report_outlined; diff --git a/app/lib/features/artifacts/ui/artifact_visit_list.dart b/app/lib/features/artifacts/ui/artifact_visit_list.dart index 2e736d7e..f9bb2199 100644 --- a/app/lib/features/artifacts/ui/artifact_visit_list.dart +++ b/app/lib/features/artifacts/ui/artifact_visit_list.dart @@ -211,7 +211,7 @@ class _ArtifactVisitListState extends State { group: group, selectedIdentity: widget.selectedIdentity, onSelected: widget.onSelected, - showHeader: false, + title: debuggingFilesTitle, ), ), ), diff --git a/app/test/features/artifacts/domain/artifact_debug_group_test.dart b/app/test/features/artifacts/domain/artifact_debug_group_test.dart new file mode 100644 index 00000000..554b8d5a --- /dev/null +++ b/app/test/features/artifacts/domain/artifact_debug_group_test.dart @@ -0,0 +1,58 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/artifacts/domain/artifact_record.dart'; +import 'package:pipelines_app/features/artifacts/domain/artifact_selection.dart'; + +import 'artifact_fixtures.dart'; + +List _names(List records) => [ + for (final record in records) record.name, +]; + +void main() { + test('the server group decides deliverable versus debugging file', () { + final records = [ + ArtifactRecord.fromJson( + artifact('notes.md', extra: const {'group': 'debug'}), + ), + ArtifactRecord.fromJson( + artifact('history.md', extra: const {'group': 'deliverable'}), + ), + ArtifactRecord.fromJson( + artifact('public/report.png', extra: const {'group': 'deliverable'}), + ), + ]; + expect(_names(stageArtifactsFrom(records)), [ + 'history.md', + 'public/report.png', + ]); + expect(_names(systemArtifactsFrom(records)), ['notes.md']); + }); + + test('an older server without groups falls back to the filename rule', () { + final records = [ + for (final name in [ + 'input.md', + 'memory.md', + 'history.md', + 'output.md', + 'session.html', + 'private/shot.png', + 'public/report.html', + 'legacy-report.txt', + ]) + ArtifactRecord.fromJson(artifact(name)), + ]; + expect(_names(stageArtifactsFrom(records)), [ + 'public/report.html', + 'legacy-report.txt', + ]); + expect(_names(systemArtifactsFrom(records)), [ + 'input.md', + 'memory.md', + 'history.md', + 'output.md', + 'session.html', + 'private/shot.png', + ]); + }); +} diff --git a/app/test/features/artifacts/ui/artifact_gallery_debug_files_test.dart b/app/test/features/artifacts/ui/artifact_gallery_debug_files_test.dart new file mode 100644 index 00000000..2cc8edc3 --- /dev/null +++ b/app/test/features/artifacts/ui/artifact_gallery_debug_files_test.dart @@ -0,0 +1,91 @@ +import 'dart:typed_data'; + +import 'package:flutter/cupertino.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/core/api_gateway.dart'; +import 'package:pipelines_app/features/artifacts/ui/artifact_gallery_page.dart'; +import 'package:pipelines_app/features/artifacts/ui/artifact_viewer_fetch.dart'; + +import '../domain/artifact_fixtures.dart'; + +enum _Platform { material, cupertino } + +JsonMap _file(String name, String group) => artifact( + name, + contentType: 'text/markdown', + sizeBytes: 10, + extra: {'group': group}, +); + +/// One turn visit as the current server records it: a public deliverable plus +/// runner-internal files, including history.md and a private capture. +List _steps() => [ + step( + stage: 'build', + sequence: 1, + visit: 1, + artifacts: [ + _file('input.md', 'debug'), + _file('memory.md', 'debug'), + _file('history.md', 'debug'), + _file('output.md', 'debug'), + _file('private/scratch.md', 'debug'), + _file('public/report.md', 'deliverable'), + ], + ), +]; + +Future _pump(WidgetTester tester, _Platform platform) async { + tester.view.physicalSize = const Size(900, 2600); + tester.view.devicePixelRatio = 1; + addTearDown(tester.view.resetPhysicalSize); + addTearDown(tester.view.resetDevicePixelRatio); + final page = ArtifactGalleryPage( + steps: _steps(), + configuredEndpoint: 'ws://127.0.0.1:8081/connect', + openExternal: (_) async => true, + fetch: (_) async => + ArtifactFetchResult(status: 200, bytes: Uint8List.fromList([35])), + markdownBuilder: (context, value) => Text(value), + ); + await tester.pumpWidget( + platform == _Platform.material + ? MaterialApp(home: page) + : CupertinoApp(home: page), + ); + await tester.pumpAndSettle(); +} + +void main() { + for (final platform in _Platform.values) { + testWidgets('${platform.name}: only deliverables show until Debugging ' + 'files is revealed', (tester) async { + await _pump(tester, platform); + expect(find.text('public/report.md'), findsOneWidget); + for (final name in [ + 'history.md', + 'memory.md', + 'input.md', + 'private/scratch.md', + ]) { + expect(find.text(name), findsNothing, reason: name); + } + expect(find.text('Debugging files'), findsNothing); + + await tester.tap(find.byKey(const Key('artifact-debug-toggle'))); + await tester.pumpAndSettle(); + expect(find.text('Debugging files'), findsOneWidget); + for (final name in [ + 'history.md', + 'memory.md', + 'input.md', + 'output.md', + 'private/scratch.md', + ]) { + expect(find.text(name), findsOneWidget, reason: name); + } + expect(find.text('public/report.md'), findsOneWidget); + }); + } +} diff --git a/app/test/features/artifacts/ui/artifact_gallery_documents_test.dart b/app/test/features/artifacts/ui/artifact_gallery_documents_test.dart index 4be4546c..177558aa 100644 --- a/app/test/features/artifacts/ui/artifact_gallery_documents_test.dart +++ b/app/test/features/artifacts/ui/artifact_gallery_documents_test.dart @@ -99,9 +99,9 @@ void main() { await _pump(tester, platform); expect(find.text('input.md'), findsNothing); expect(find.textContaining('Turn documents'), findsNothing); - expect(find.byTooltip('Show turn documents'), findsOneWidget); + expect(find.byTooltip('Show debugging files'), findsOneWidget); await _showTurnDocuments(tester); - expect(find.byTooltip('Hide turn documents'), findsOneWidget); + expect(find.byTooltip('Hide debugging files'), findsOneWidget); expect(find.text('input.md'), findsNWidgets(2)); expect(find.text('output.md'), findsNWidgets(2)); expect(find.text('session.html'), findsNWidgets(2)); diff --git a/app/tool/search_selection_probe_entry.dart b/app/tool/search_selection_probe_entry.dart index 1b20256e..55c9b249 100644 --- a/app/tool/search_selection_probe_entry.dart +++ b/app/tool/search_selection_probe_entry.dart @@ -83,6 +83,15 @@ JsonMap probeRunDetail() => { 'size_bytes': 4096, 'url': '/artifacts/7/2/release-dashboard.html', }, + if (_turnFiles) + for (final name in _turnFileNames) + { + 'name': name, + 'content_type': 'text/markdown', + 'size_bytes': 640, + 'url': '/artifacts/7/2/', + 'group': 'debug', + }, ], 'file_changes': _probeChanges([ 'lib/release/train.dart', @@ -138,6 +147,18 @@ int _fillerStepCount = 0; /// serves its bytes from the build directory (see notes/artifact-fullscreen.md). bool _htmlArtifact = false; +/// `?turn=1` adds the runner-internal turn files the server records for a model +/// visit (group `debug`) to run 7's build visit, so the Artifacts tab's hidden +/// Debugging files group can be reviewed. +bool _turnFiles = false; +const List _turnFileNames = [ + 'input.md', + 'memory.md', + 'history.md', + 'output.md', + 'private/build-log.md', +]; + List _fillerSteps() => [ for (var i = 0; i < _fillerStepCount; i++) { @@ -618,6 +639,7 @@ class _FixtureSink implements WebSocketSink { /// `?runs=200` / `?steps=60` append filler runs / run-7 steps for scroll /// performance captures; /// `?html=1` adds an HTML artifact to run 7 (`release-dashboard.html`); +/// `?turn=1` adds run 7's debug turn files (input/memory/history/output.md); /// `?q=release` types that query into Search once its field is shown; /// `?platform=cupertino` renders the Cupertino host (`material`, the /// default, renders Material as browsers do); @@ -650,6 +672,7 @@ void main() { _fillerStepCount = int.tryParse(params['steps'] ?? '') ?? 0; probeUsage = params['usage'] == '1'; _htmlArtifact = params['html'] == '1'; + _turnFiles = params['turn'] == '1'; final cupertino = params['platform'] == 'cupertino'; final route = params['route']; final opensRoute = route != null && route.startsWith('/'); diff --git a/docs/cli.md b/docs/cli.md index d641af19..c94ac3ee 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -93,12 +93,15 @@ never returned. Missing public phone configuration is labeled explicitly. ## Search -`pipelines search QUERY [--type KIND] [--limit N] [--json]` runs one read-only +`pipelines search QUERY [--type KIND] [--limit N] [--debug-files] [--json]` runs one read-only query across the whole system: pipeline names and their versions, run identity, status and dates, stage and visit identifiers, artifact names and types, and the text of stage inputs, outputs, evidence and transcripts. It never reads artifact contents, never makes a per-result follow-up request, and asks for no confirmation. +Artifact hits are deliverables only: runner-internal debugging files (see +[Stage artifacts](stage-artifacts.md#deliverables-and-debugging-files)) match +only with `--debug-files` (`include_debug` on the request). - Free text matches case-insensitively. A bare status word (`queued`, `running`, `succeeded`, `failed`, `canceled`, `interrupted`) selects those runs, and a lone diff --git a/docs/stage-artifacts.md b/docs/stage-artifacts.md index 596a0a49..5a3453f7 100644 --- a/docs/stage-artifacts.md +++ b/docs/stage-artifacts.md @@ -55,9 +55,34 @@ server-relative `url`. Public entries carry a `public/` name prefix. The same fields appear under each step's `artifacts` in `pipelines inspect RUN_ID --json` and in the TUI. -`input.md`, `memory.md`, `history.md` and `output.md` are the primary artifacts of every model -visit. `private/` files are listed without a prefix; shared files are listed as -`public/...` on every visit that can see them. +`input.md`, `memory.md`, `history.md` and `output.md` are always inventoried +first, so they are never crowded out by captures. `private/` files are listed +as `private/...`; shared files are listed as `public/...` on every visit that +can see them. + +## Deliverables and debugging files + +Every recorded artifact carries a `group` (`protocol.ClassifyArtifacts`): + +- `deliverable`: a file the stage published to `public/`. For a visit recorded + before the turn layout (no turn files), every other file is a deliverable too. +- `debug`: runner-internal files - `input.md`, `memory.md`, `history.md`, + `output.md`, `session.html` and session repairs, the visit's `private/` + scratch, and any other unprefixed file of a turn-layout visit. + +The group is derived from the recorded names on every read, so runs stored +before the field existed are classified the same way. Clients present only +deliverables by default: + +- App: the run's Artifacts tab lists deliverables; the Debug (bug) toggle + reveals a separate "Debugging files" group under each visit. The parsed + stage result stays on Summary, and the visit's Session page still shows its + turn documents. An older server records no `group`, so the app falls back to + the names above (`input.md`, `memory.md`, `history.md`, `output.md`, + `session.html`, `private/...`). +- TUI: deliverables first, debugging files after them marked `[dbg]`. +- Search: artifact hits are deliverables only; `include_debug` on the request + (`pipelines search --debug-files`) also matches debugging files. - Limits: 200 files per step, 8 MiB per file and 64 MiB total. Over-cap files are skipped and the skipped/truncated counts are recorded with the step. diff --git a/internal/cli/app.go b/internal/cli/app.go index 9cf9c90b..82f518a4 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -50,6 +50,7 @@ type options struct { keep int keepSet bool all bool + debugFiles bool json, source, noOpen, yes, allow, createOnly, force, wait bool version, expected, limit, retryVisit int timeout, waitTimeout time.Duration @@ -166,6 +167,7 @@ func (a App) flags(command string) (*flag.FlagSet, *options) { case "search": f.StringVar(&o.kind, "type", "", "Narrow to one kind: "+strings.Join(protocol.SearchKinds, ", ")) f.IntVar(&o.limit, "limit", 0, "Maximum results (default 50, max 200)") + f.BoolVar(&o.debugFiles, "debug-files", false, "Also match runner-internal debugging artifacts (turn files, private/ scratch)") case "retry": f.BoolVar(&o.yes, "yes", false, "Confirm retry without an interactive prompt") f.StringVar(&o.key, "request-key", "", "Idempotency key (default: generated unique key)") @@ -853,7 +855,7 @@ func (a App) Run(ctx context.Context, args []string) error { if strings.TrimSpace(query) == "" { return fmt.Errorf("search requires a query; use pipelines help search") } - r, err := call(protocol.Message{Type: protocol.Search, Search: &protocol.SearchRequest{Query: query, Kind: o.kind, Limit: o.limit}}) + r, err := call(protocol.Message{Type: protocol.Search, Search: &protocol.SearchRequest{Query: query, Kind: o.kind, Limit: o.limit, IncludeDebug: o.debugFiles}}) if err != nil { return err } diff --git a/internal/cli/commands.go b/internal/cli/commands.go index ab87ea9e..c661f3c3 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -11,7 +11,7 @@ var usage = map[string]string{ "validate": "validate [--json]", "submit": "submit [--name NAME] [--create-only] [--yes] [--json]", "list": "list [--json]", - "search": "search QUERY [--type KIND] [--limit N] [--json]", + "search": "search QUERY [--type KIND] [--limit N] [--debug-files] [--json]", "show": "show [--version N] [--json]", "versions": "versions [--json]", "export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]", @@ -60,7 +60,7 @@ var notes = map[string]string{ "rpc": "Accepts a JSON protocol envelope; '-' reads stdin. Every TUI server operation is available here too. Mutating commands are not dry runs. The omni-search command is `search` (fields query, optional kind and limit) and answers with ranked `search_results`.", "chat-adopt": "Adopt a session that already exists on a chat harness (an OpenCode session, or a Claude Code or Codex session by its id) so it appears in chat_sessions and can be continued from any client. Without --harness each available harness is asked in turn. The directory and title come from the harness's own record (never guessed; --title names a chat whose harness records none), and a missing id records nothing. Do not keep using the session elsewhere once it is continued here.", "retention": "Local, explicit run-history retention; never automatic and never a side effect of another command. `report` is strictly read-only and prints what a policy would remove; `apply` removes it after approval (--yes for scripted use). Policy: --older-than DURATION selects runs created before now minus that rolling UTC duration (Go durations plus d/w; not a calendar boundary), --keep N keeps the newest N runs by run id and selects older ranks (--keep 0 keeps none). Both together are the conservative intersection; at least one is required. Protected and never removed: queued/running runs, runs waiting on a person (waiting run or step), runs with an active cleanup agent, and any run referenced by a retained run's retry provenance (transitively). A run that becomes protected before deletion aborts the whole apply. Artifacts at / are removed with their run and kept with protected/retained runs; the database is VACUUMed and its WAL checkpointed, with measured reclaimed bytes. Definitions and versions are immutable and never touched. --db must name an existing database (default: the standard data directory); --artifacts defaults to artifacts/ beside --db. Works without a running server.", - "search": "One read-only query over pipelines, versions, runs, steps, field text and artifacts. QUERY is free text, a bare status word (queued, running, succeeded, failed, canceled, interrupted) selecting those runs, or a lone date expression (today, yesterday, this week [Monday 00:00 local], or YYYY-MM-DD) selecting the runs created in that window. --type is the only narrowing mechanism and takes one kind: pipeline, version, run, step, field or artifact; an unknown kind is an error. Ranking: an exact name match (3) beats a name substring (2) beats a body-text match (1); equal tiers break by recency, then a stable kind/target tie-break, and every hit reports score, matched_field and recency so a client can re-rank. Bounds: the result cap defaults to 50 and clamps to 200 (0 = default); every kind scans at most 2000 rows and field-text scanning is additionally capped at 2000 steps / 1 MiB; Truncated plus scanned_steps, scanned_bytes and scan_bound_bytes report when a bound applied. Targets are stable: pipeline:, pipeline::v, run:, run::, run:::, or an artifact URL.", + "search": "One read-only query over pipelines, versions, runs, steps, field text and artifacts. QUERY is free text, a bare status word (queued, running, succeeded, failed, canceled, interrupted) selecting those runs, or a lone date expression (today, yesterday, this week [Monday 00:00 local], or YYYY-MM-DD) selecting the runs created in that window. --type is the only narrowing mechanism and takes one kind: pipeline, version, run, step, field or artifact; an unknown kind is an error. Ranking: an exact name match (3) beats a name substring (2) beats a body-text match (1); equal tiers break by recency, then a stable kind/target tie-break, and every hit reports score, matched_field and recency so a client can re-rank. Bounds: the result cap defaults to 50 and clamps to 200 (0 = default); every kind scans at most 2000 rows and field-text scanning is additionally capped at 2000 steps / 1 MiB; Truncated plus scanned_steps, scanned_bytes and scan_bound_bytes report when a bound applied. Targets are stable: pipeline:, pipeline::v, run:, run::, run:::, or an artifact URL. Artifact hits are deliverables only; runner-internal debugging files (input.md, memory.md, history.md, output.md, session.html, private/ scratch) are matched only with --debug-files.", } var serviceHelp = map[string]string{ diff --git a/internal/pipeline/runs.go b/internal/pipeline/runs.go index c461863c..f631c5c0 100644 --- a/internal/pipeline/runs.go +++ b/internal/pipeline/runs.go @@ -214,7 +214,7 @@ strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ' rows.Close() return nil, err } - step.Artifacts = evidence.Files + step.Artifacts = protocol.ClassifyArtifacts(evidence.Files) step.ArtifactsSkipped = evidence.Skipped step.ArtifactsTruncated = evidence.Truncated } diff --git a/internal/pipeline/search.go b/internal/pipeline/search.go index 6fbecdb6..eafdd93d 100644 --- a/internal/pipeline/search.go +++ b/internal/pipeline/search.go @@ -37,6 +37,7 @@ func (s *Store) Search(ctx context.Context, req protocol.SearchRequest, now time } terms := parseSearchTerms(req.Query, now) terms.Browse = browse + terms.IncludeDebug = req.IncludeDebug terms.Limit = response.Cap // A bare status word or date selects runs; it is never treated as body text. selectorOnly := terms.Text == "" && !browse @@ -129,13 +130,15 @@ func searchLimit(limit int) int { // searchTerms is the parsed query. Text carries free text; Status and the // From/To window carry a bare status word or a date expression that selects runs. type searchTerms struct { - Browse bool - Limit int - Text string - Status string - HasDate bool - From time.Time - To time.Time + Browse bool + // IncludeDebug keeps runner-internal debugging artifacts in artifact hits. + IncludeDebug bool + Limit int + Text string + Status string + HasDate bool + From time.Time + To time.Time } // parseSearchTerms recognises a lone status word and a lone date expression and @@ -625,7 +628,12 @@ ORDER BY r.created_at DESC,r.id DESC,a.sequence DESC LIMIT ?` if err := json.Unmarshal([]byte(artifacts), &evidence); err != nil { continue } - for _, file := range evidence.Files { + // Debugging files (turn files, private/ scratch) are internal to the + // runner and stay out of results unless explicitly requested. + for _, file := range protocol.ClassifyArtifacts(evidence.Files) { + if file.IsDebug() && !terms.IncludeDebug { + continue + } score, matched := 1, "content_type" if terms.Browse { score, matched = 0, "" @@ -637,7 +645,7 @@ ORDER BY r.created_at DESC,r.id DESC,a.sequence DESC LIMIT ?` results = append(results, protocol.SearchResult{ Kind: protocol.SearchKindArtifact, Target: file.URL, Score: score, MatchedField: matched, Recency: recency, - Artifact: &protocol.SearchArtifact{RunID: runID, Sequence: sequence, Name: file.Name, ContentType: file.ContentType, SizeBytes: file.SizeBytes, URL: file.URL}, + Artifact: &protocol.SearchArtifact{RunID: runID, Sequence: sequence, Name: file.Name, ContentType: file.ContentType, SizeBytes: file.SizeBytes, URL: file.URL, Group: file.Group}, }) } } diff --git a/internal/pipeline/search_debug_test.go b/internal/pipeline/search_debug_test.go new file mode 100644 index 00000000..bdc13f68 --- /dev/null +++ b/internal/pipeline/search_debug_test.go @@ -0,0 +1,85 @@ +package pipeline + +import ( + "context" + "path/filepath" + "sort" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// Artifact search returns only deliverables unless debugging files are asked +// for, and the step read path classifies stored records that carry no group. +func TestSearchArtifactsExcludeDebugFilesByDefault(t *testing.T) { + ctx := context.Background() + store, err := Open(filepath.Join(t.TempDir(), "search-debug.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { store.Close() }) + if _, err := store.Submit(ctx, "Quartz", Template([]string{"quartz"}, true)); err != nil { + t.Fatal(err) + } + items, err := store.List(ctx) + if err != nil { + t.Fatal(err) + } + definition, err := store.Get(ctx, items[0].ID, 1) + if err != nil { + t.Fatal(err) + } + run, _, err := store.CreateRun(ctx, definition, "quartz-key", "/work/q", false, nil, map[string]any{"prompt": "task"}) + if err != nil { + t.Fatal(err) + } + if err := store.BeginStep(ctx, run, 1, "quartz", map[string]any{"prompt": "task"}); err != nil { + t.Fatal(err) + } + // Stored without a group, as a server before the field recorded them. + files := []protocol.StageArtifact{ + {Name: "input.md", ContentType: "text/markdown", URL: "/a/input.md"}, + {Name: "history.md", ContentType: "text/markdown", URL: "/a/history.md"}, + {Name: "private/qz-shot.png", ContentType: "image/png", URL: "/a/private/qz-shot.png"}, + {Name: "public/qz-report.md", ContentType: "text/markdown", URL: "/a/public/qz-report.md"}, + } + if err := store.FinishStep(ctx, run, protocol.RunStep{Sequence: 1, Status: "completed", Outcome: "success", Target: protocol.SuccessEnd, Artifacts: files}); err != nil { + t.Fatal(err) + } + names := func(req protocol.SearchRequest) []string { + t.Helper() + req.Kind = protocol.SearchKindArtifact + response, err := store.Search(ctx, req, time.Now()) + if err != nil { + t.Fatal(err) + } + var got []string + for _, hit := range response.Results { + got = append(got, hit.Artifact.Name+"="+hit.Artifact.Group) + } + sort.Strings(got) + return got + } + if got := names(protocol.SearchRequest{Query: "md"}); len(got) != 1 || got[0] != "public/qz-report.md=deliverable" { + t.Fatalf("default search = %v, want only the public deliverable", got) + } + if got := names(protocol.SearchRequest{Query: "history"}); len(got) != 0 { + t.Fatalf("history.md surfaced by default: %v", got) + } + if got := names(protocol.SearchRequest{Query: "qz", IncludeDebug: true}); len(got) != 2 || got[0] != "private/qz-shot.png=debug" { + t.Fatalf("include_debug search = %v", got) + } + + detail, err := store.GetRun(ctx, run) + if err != nil { + t.Fatal(err) + } + groups := map[string]string{} + for _, file := range detail.Steps[0].Artifacts { + groups[file.Name] = file.Group + } + if groups["history.md"] != protocol.ArtifactGroupDebug || groups["private/qz-shot.png"] != protocol.ArtifactGroupDebug || groups["public/qz-report.md"] != protocol.ArtifactGroupDeliverable { + t.Fatalf("read path groups = %v", groups) + } +} diff --git a/internal/protocol/artifact_group.go b/internal/protocol/artifact_group.go new file mode 100644 index 00000000..c0556ee2 --- /dev/null +++ b/internal/protocol/artifact_group.go @@ -0,0 +1,62 @@ +package protocol + +import "strings" + +// Artifact presentation groups. A deliverable is a file the stage published to +// the run's public/ directory (or, for a visit recorded before the turn layout, +// any file it left). A debug file is one of the runner's own turn files or the +// visit's private/ scratch: evidence for debugging a stage, not its output. +const ( + ArtifactGroupDeliverable = "deliverable" + ArtifactGroupDebug = "debug" +) + +// TurnArtifactNames are the files the server itself writes into a model visit's +// turn directory. They are always debugging files. +var TurnArtifactNames = []string{"input.md", "memory.md", "history.md", "output.md", "session.html"} + +// IsDebug reports whether the artifact is a runner-internal debugging file. +func (a StageArtifact) IsDebug() bool { return a.Group == ArtifactGroupDebug } + +func isTurnArtifactName(name string) bool { + for _, turn := range TurnArtifactNames { + if name == turn { + return true + } + } + return name == LegacySessionMarkdownArtifact || strings.HasPrefix(name, "session-repair-") +} + +// ClassifyArtifacts returns one visit's artifacts with Group set. public/ files +// are deliverables. Turn files, session exports and their repairs, and private/ +// scratch are debug files; so is every other unprefixed file of a turn-layout +// visit (one that recorded any turn file), because a stage publishes only +// through public/. A visit recorded before the turn layout keeps its other +// files as deliverables. The rule uses only the recorded names, so runs stored +// before the field existed classify identically on read. It never mutates the +// input. Pure, O(artifacts). +func ClassifyArtifacts(files []StageArtifact) []StageArtifact { + if len(files) == 0 { + return files + } + turnVisit := false + for _, file := range files { + if isTurnArtifactName(file.Name) { + turnVisit = true + break + } + } + classified := make([]StageArtifact, len(files)) + for i, file := range files { + switch { + case strings.HasPrefix(file.Name, "public/"): + file.Group = ArtifactGroupDeliverable + case turnVisit, isTurnArtifactName(file.Name), strings.HasPrefix(file.Name, "private/"): + file.Group = ArtifactGroupDebug + default: + file.Group = ArtifactGroupDeliverable + } + classified[i] = file + } + return classified +} diff --git a/internal/protocol/artifact_group_test.go b/internal/protocol/artifact_group_test.go new file mode 100644 index 00000000..16e72738 --- /dev/null +++ b/internal/protocol/artifact_group_test.go @@ -0,0 +1,49 @@ +package protocol + +import "testing" + +func groupsByName(files []StageArtifact) map[string]string { + groups := map[string]string{} + for _, file := range files { + groups[file.Name] = file.Group + } + return groups +} + +func TestClassifyArtifactsTurnVisit(t *testing.T) { + input := []StageArtifact{ + {Name: "input.md"}, {Name: "memory.md"}, {Name: "history.md"}, {Name: "output.md"}, + {Name: "session.html"}, {Name: "session-repair-1.html"}, {Name: "private/shot.png"}, + {Name: "notes.txt"}, {Name: "public/report.html"}, {Name: "public/history.md"}, + } + got := groupsByName(ClassifyArtifacts(input)) + want := map[string]string{ + "input.md": ArtifactGroupDebug, "memory.md": ArtifactGroupDebug, "history.md": ArtifactGroupDebug, + "output.md": ArtifactGroupDebug, "session.html": ArtifactGroupDebug, "session-repair-1.html": ArtifactGroupDebug, + "private/shot.png": ArtifactGroupDebug, "notes.txt": ArtifactGroupDebug, + "public/report.html": ArtifactGroupDeliverable, "public/history.md": ArtifactGroupDeliverable, + } + for name, group := range want { + if got[name] != group { + t.Errorf("%s: group %q, want %q", name, got[name], group) + } + } + for _, file := range input { + if file.Group != "" { + t.Fatalf("ClassifyArtifacts mutated its input: %+v", file) + } + } +} + +func TestClassifyArtifactsLegacyVisitKeepsFilesAsDeliverables(t *testing.T) { + got := groupsByName(ClassifyArtifacts([]StageArtifact{{Name: "report.png"}, {Name: "private/scratch.txt"}})) + if got["report.png"] != ArtifactGroupDeliverable { + t.Errorf("legacy file group %q, want deliverable", got["report.png"]) + } + if got["private/scratch.txt"] != ArtifactGroupDebug { + t.Errorf("private file group %q, want debug", got["private/scratch.txt"]) + } + if ClassifyArtifacts(nil) != nil { + t.Error("nil input must stay nil") + } +} diff --git a/internal/protocol/runs.go b/internal/protocol/runs.go index 65396543..78317c1e 100644 --- a/internal/protocol/runs.go +++ b/internal/protocol/runs.go @@ -278,6 +278,11 @@ type StageArtifact struct { SizeBytes int64 `json:"size_bytes"` ContentType string `json:"content_type"` URL string `json:"url"` + // Group is the server's presentation classification (ArtifactGroupDeliverable + // or ArtifactGroupDebug, see ClassifyArtifacts). Every listing the server + // returns sets it; an older server omits it, so clients keep a filename + // fallback. + Group string `json:"group,omitempty"` } // LegacySessionMarkdownArtifact is the retired markdown session export file name. diff --git a/internal/protocol/search.go b/internal/protocol/search.go index 9769aa47..9dae88e8 100644 --- a/internal/protocol/search.go +++ b/internal/protocol/search.go @@ -80,6 +80,10 @@ type SearchRequest struct { Query string `json:"query"` Kind string `json:"kind,omitempty"` Limit int `json:"limit,omitempty"` + // IncludeDebug also returns runner-internal debugging artifacts (turn files + // and private/ scratch, see ClassifyArtifacts). Off by default: artifact + // search surfaces only deliverables. + IncludeDebug bool `json:"include_debug,omitempty"` } // SearchResponse is the bounded set of matches. Results are ordered by recency @@ -181,6 +185,8 @@ type SearchArtifact struct { ContentType string `json:"content_type"` SizeBytes int64 `json:"size_bytes"` URL string `json:"url"` + // Group is the artifact's presentation group (see StageArtifact.Group). + Group string `json:"group,omitempty"` } // SearchChat is an operator chat session hit: its registry facts and, for a diff --git a/internal/server/artifacts.go b/internal/server/artifacts.go index 77aa014d..2f31970a 100644 --- a/internal/server/artifacts.go +++ b/internal/server/artifacts.go @@ -138,7 +138,7 @@ func (r *runner) scanArtifacts(runID int64, sequence int, directory string, publ if len(files) == 0 { files = nil } - return files, skipped, truncated + return protocol.ClassifyArtifacts(files), skipped, truncated } // artifactRel returns the slash path of directory below the artifacts root. A diff --git a/internal/server/artifacts_test.go b/internal/server/artifacts_test.go index 5d69ca06..4de81890 100644 --- a/internal/server/artifacts_test.go +++ b/internal/server/artifacts_test.go @@ -55,6 +55,13 @@ func TestSessionEvidenceIsInventoriedBeforeLargeCaptureDirectories(t *testing.T) t.Fatalf("mandatory runtime file %s hidden by user captures", name) } } + // The recorded inventory carries the server's classification: turn files and + // private captures are debugging files. + for _, file := range files { + if file.Group != protocol.ArtifactGroupDebug { + t.Fatalf("%s group %q, want debug", file.Name, file.Group) + } + } } func TestArtifactsScanRecordsSizesTypesAndSkips(t *testing.T) { @@ -396,6 +403,9 @@ func TestArtifactHTTPGetPublicSharedFile(t *testing.T) { if shared.URL == "" { t.Fatalf("public artifact missing: %+v", step.Artifacts) } + if shared.Group != protocol.ArtifactGroupDeliverable { + t.Fatalf("public artifact group %q, want deliverable", shared.Group) + } srv := httptest.NewServer(handler) defer srv.Close() resp, err := http.Get(srv.URL + shared.URL) diff --git a/internal/tui/artifacts.go b/internal/tui/artifacts.go index 833a6601..7f6de7cf 100644 --- a/internal/tui/artifacts.go +++ b/internal/tui/artifacts.go @@ -57,12 +57,34 @@ func runArtifactVisits(steps []protocol.RunStep) []artifactVisit { if step.Control != nil && step.Control.Visit > 0 { visit = step.Control.Visit } - visits = append(visits, artifactVisit{stage: stage, visit: visit, step: step, artifacts: step.PresentedArtifacts()}) + visits = append(visits, artifactVisit{stage: stage, visit: visit, step: step, artifacts: deliverablesFirst(step.PresentedArtifacts())}) } } return visits } +// deliverablesFirst orders a visit's artifacts with the stage's deliverables +// ahead of the runner-internal debugging files (marked [dbg] in each row), so +// the files a stage published lead and turn files never crowd them out. The +// order within each group is the recorded order. +func deliverablesFirst(artifacts []protocol.StageArtifact) []protocol.StageArtifact { + ordered := make([]protocol.StageArtifact, 0, len(artifacts)) + for _, artifact := range artifacts { + if !artifact.IsDebug() { + ordered = append(ordered, artifact) + } + } + for _, artifact := range artifacts { + if artifact.IsDebug() { + ordered = append(ordered, artifact) + } + } + if len(ordered) == 0 { + return nil + } + return ordered +} + // artifactRef addresses one artifact row in the flat selection order the view // uses (group order, then recorded order within the group). type artifactRef struct { @@ -170,7 +192,9 @@ func artifactRowText(artifact protocol.StageArtifact, widths artifactWidths, wid // decision depends only on width, so every row's columns stay aligned // whether or not the marker is shown. if width >= artifactCursorWidth+artifactMarkerWidth+1+artifactMinName { - if isImageContentType(artifact.ContentType) { + if artifact.IsDebug() { + prefix += "[dbg] " + } else if isImageContentType(artifact.ContentType) { prefix += "[img] " } else { prefix += strings.Repeat(" ", artifactMarkerWidth) + " " diff --git a/internal/tui/artifacts_debug_test.go b/internal/tui/artifacts_debug_test.go new file mode 100644 index 00000000..89f209b3 --- /dev/null +++ b/internal/tui/artifacts_debug_test.go @@ -0,0 +1,27 @@ +package tui + +import ( + "strings" + "testing" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func TestArtifactsDeliverablesLeadAndDebugFilesAreMarked(t *testing.T) { + artifacts := []protocol.StageArtifact{ + {Name: "input.md", ContentType: "text/markdown", Group: protocol.ArtifactGroupDebug}, + {Name: "history.md", ContentType: "text/markdown", Group: protocol.ArtifactGroupDebug}, + {Name: "public/report.png", ContentType: "image/png", Group: protocol.ArtifactGroupDeliverable}, + } + ordered := deliverablesFirst(artifacts) + if ordered[0].Name != "public/report.png" || ordered[1].Name != "input.md" || ordered[2].Name != "history.md" { + t.Fatalf("order = %+v", ordered) + } + widths := artifactWidthsFor([]artifactVisit{{artifacts: ordered}}) + if row := artifactRowText(ordered[1], widths, 100, false); !strings.Contains(row, "[dbg] input.md") { + t.Fatalf("debug row not marked: %q", row) + } + if row := artifactRowText(ordered[0], widths, 100, false); !strings.Contains(row, "[img] public/report.png") { + t.Fatalf("deliverable row: %q", row) + } +}