Artifacts: runner-internal turn files move to a Debugging files group
Each recorded artifact now carries a server group (deliverable or debug). public/ files are deliverables; input/memory/history/output.md, session exports and private/ scratch are debugging files. The app's Artifacts tab lists deliverables and the Debug toggle reveals a separate Debugging files group (older servers fall back to the filename rule); the TUI lists deliverables first and marks debug rows; artifact search excludes debug files unless include_debug / --debug-files is set.
This commit is contained in:
@@ -15,6 +15,20 @@ const List<String> systemArtifactNames = <String>[
|
||||
'session.html',
|
||||
];
|
||||
|
||||
/// The server's `group` value for a runner-internal debugging file.
|
||||
const String debugArtifactGroup = 'debug';
|
||||
|
||||
/// The server's `group` value for a file the stage published.
|
||||
const String deliverableArtifactGroup = 'deliverable';
|
||||
|
||||
/// Whether [name] is a runner-internal debugging file by name alone. This is
|
||||
/// the fallback for a server that records no `group`: the turn documents, the
|
||||
/// full `history.md`, and the visit's `private/` scratch.
|
||||
bool isDebugArtifactName(String name) =>
|
||||
isSystemArtifactName(name) ||
|
||||
name == 'history.md' ||
|
||||
name.startsWith('private/');
|
||||
|
||||
/// One recorded artifact of a stage visit.
|
||||
///
|
||||
/// The server is the authority: [name], [url], [contentType] and [sizeBytes]
|
||||
@@ -63,6 +77,15 @@ class ArtifactRecord {
|
||||
bool get isSystem => isSystemArtifactName(name);
|
||||
bool get isPublicName => name.startsWith('public/');
|
||||
|
||||
/// Whether this is a runner-internal debugging file rather than a stage
|
||||
/// deliverable. The server's recorded `group` decides; an older server that
|
||||
/// records none falls back to [isDebugArtifactName].
|
||||
bool get isDebug {
|
||||
final group = raw['group'];
|
||||
if (group is String && group.isNotEmpty) return group == debugArtifactGroup;
|
||||
return isDebugArtifactName(name);
|
||||
}
|
||||
|
||||
static String _text(dynamic value) => value == null ? '' : '$value';
|
||||
|
||||
static int _intOf(dynamic value) {
|
||||
|
||||
@@ -36,7 +36,7 @@ List<ArtifactRecord> presentedArtifacts(Iterable<ArtifactRecord> artifacts) {
|
||||
];
|
||||
}
|
||||
|
||||
/// A visit's stage-produced artifacts: everything except the turn-owned files
|
||||
/// A visit's deliverables: everything except runner-internal debugging files
|
||||
/// and repaired session exports. Order is preserved.
|
||||
List<ArtifactRecord> stageArtifactsFrom(
|
||||
Iterable<ArtifactRecord> artifacts, {
|
||||
@@ -44,18 +44,19 @@ List<ArtifactRecord> stageArtifactsFrom(
|
||||
}) {
|
||||
return <ArtifactRecord>[
|
||||
for (final artifact in presentedArtifacts(artifacts))
|
||||
if (!artifact.isSystem && !repairs.contains(artifact.name)) artifact,
|
||||
if (!artifact.isDebug && !repairs.contains(artifact.name)) artifact,
|
||||
];
|
||||
}
|
||||
|
||||
/// A visit's system files and repaired exports, in recorded order.
|
||||
/// A visit's debugging files (turn documents, history.md, private/ scratch)
|
||||
/// and repaired exports, in recorded order.
|
||||
List<ArtifactRecord> systemArtifactsFrom(
|
||||
Iterable<ArtifactRecord> artifacts, {
|
||||
Set<String> repairs = const <String>{},
|
||||
}) {
|
||||
return <ArtifactRecord>[
|
||||
for (final artifact in presentedArtifacts(artifacts))
|
||||
if (artifact.isSystem || repairs.contains(artifact.name)) artifact,
|
||||
if (artifact.isDebug || repairs.contains(artifact.name)) artifact,
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,10 @@ import '../domain/artifact_groups.dart';
|
||||
import 'artifact_record_row.dart';
|
||||
import 'artifact_row_labels.dart';
|
||||
|
||||
/// The gallery's label for a visit's runner-internal files (turn documents,
|
||||
/// history.md, private/ scratch, session exports), shown only while Debug is on.
|
||||
const String debuggingFilesTitle = 'Debugging files';
|
||||
|
||||
/// One visit's turn documents, shown below its stage artifacts.
|
||||
///
|
||||
/// A recorded document is a normal gallery row with its recorded name, MIME and
|
||||
@@ -21,8 +25,13 @@ class ArtifactDocumentSection extends StatelessWidget {
|
||||
this.selectedIdentity,
|
||||
this.onSelected,
|
||||
this.showHeader = true,
|
||||
this.title = 'Turn documents',
|
||||
});
|
||||
|
||||
/// The section label: the Session page names the turn documents; the gallery
|
||||
/// names every runner-internal file it lists as debugging files.
|
||||
final String title;
|
||||
|
||||
/// False when a collapsible title above already names the section.
|
||||
final bool showHeader;
|
||||
|
||||
@@ -54,7 +63,7 @@ class ArtifactDocumentSection extends StatelessWidget {
|
||||
container: true,
|
||||
header: true,
|
||||
child: Text(
|
||||
'Turn documents',
|
||||
title,
|
||||
style: cupertino ? SectionHeadingStyle.of(context).title : null,
|
||||
),
|
||||
),
|
||||
|
||||
@@ -124,7 +124,7 @@ class _ArtifactGalleryPageState extends State<ArtifactGalleryPage> {
|
||||
for (final document in group.documents) {
|
||||
final record = document.record;
|
||||
if (record == null || record.url != url) continue;
|
||||
// A turn document is listed only while the debug toggle shows them.
|
||||
// A debugging file is listed only while the debug toggle shows them.
|
||||
_showTurnDocuments = true;
|
||||
_opening = (group: group, artifact: record);
|
||||
}
|
||||
@@ -556,7 +556,7 @@ class _ArtifactGalleryPageState extends State<ArtifactGalleryPage> {
|
||||
|
||||
Widget _debugToggle(BuildContext context) {
|
||||
final showing = _showTurnDocuments;
|
||||
final label = showing ? 'Hide turn documents' : 'Show turn documents';
|
||||
final label = showing ? 'Hide debugging files' : 'Show debugging files';
|
||||
// Cupertino's bundled icon font has no bug glyph; retain the universally
|
||||
// recognisable Material bug icon inside its native Cupertino button.
|
||||
final icon = showing ? Icons.bug_report : Icons.bug_report_outlined;
|
||||
|
||||
@@ -211,7 +211,7 @@ class _ArtifactVisitListState extends State<ArtifactVisitList> {
|
||||
group: group,
|
||||
selectedIdentity: widget.selectedIdentity,
|
||||
onSelected: widget.onSelected,
|
||||
showHeader: false,
|
||||
title: debuggingFilesTitle,
|
||||
),
|
||||
),
|
||||
),
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:pipelines_app/features/artifacts/domain/artifact_record.dart';
|
||||
import 'package:pipelines_app/features/artifacts/domain/artifact_selection.dart';
|
||||
|
||||
import 'artifact_fixtures.dart';
|
||||
|
||||
List<String> _names(List<ArtifactRecord> records) => [
|
||||
for (final record in records) record.name,
|
||||
];
|
||||
|
||||
void main() {
|
||||
test('the server group decides deliverable versus debugging file', () {
|
||||
final records = [
|
||||
ArtifactRecord.fromJson(
|
||||
artifact('notes.md', extra: const {'group': 'debug'}),
|
||||
),
|
||||
ArtifactRecord.fromJson(
|
||||
artifact('history.md', extra: const {'group': 'deliverable'}),
|
||||
),
|
||||
ArtifactRecord.fromJson(
|
||||
artifact('public/report.png', extra: const {'group': 'deliverable'}),
|
||||
),
|
||||
];
|
||||
expect(_names(stageArtifactsFrom(records)), [
|
||||
'history.md',
|
||||
'public/report.png',
|
||||
]);
|
||||
expect(_names(systemArtifactsFrom(records)), ['notes.md']);
|
||||
});
|
||||
|
||||
test('an older server without groups falls back to the filename rule', () {
|
||||
final records = [
|
||||
for (final name in [
|
||||
'input.md',
|
||||
'memory.md',
|
||||
'history.md',
|
||||
'output.md',
|
||||
'session.html',
|
||||
'private/shot.png',
|
||||
'public/report.html',
|
||||
'legacy-report.txt',
|
||||
])
|
||||
ArtifactRecord.fromJson(artifact(name)),
|
||||
];
|
||||
expect(_names(stageArtifactsFrom(records)), [
|
||||
'public/report.html',
|
||||
'legacy-report.txt',
|
||||
]);
|
||||
expect(_names(systemArtifactsFrom(records)), [
|
||||
'input.md',
|
||||
'memory.md',
|
||||
'history.md',
|
||||
'output.md',
|
||||
'session.html',
|
||||
'private/shot.png',
|
||||
]);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:flutter/cupertino.dart';
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:pipelines_app/core/api_gateway.dart';
|
||||
import 'package:pipelines_app/features/artifacts/ui/artifact_gallery_page.dart';
|
||||
import 'package:pipelines_app/features/artifacts/ui/artifact_viewer_fetch.dart';
|
||||
|
||||
import '../domain/artifact_fixtures.dart';
|
||||
|
||||
enum _Platform { material, cupertino }
|
||||
|
||||
JsonMap _file(String name, String group) => artifact(
|
||||
name,
|
||||
contentType: 'text/markdown',
|
||||
sizeBytes: 10,
|
||||
extra: <String, dynamic>{'group': group},
|
||||
);
|
||||
|
||||
/// One turn visit as the current server records it: a public deliverable plus
|
||||
/// runner-internal files, including history.md and a private capture.
|
||||
List<JsonMap> _steps() => <JsonMap>[
|
||||
step(
|
||||
stage: 'build',
|
||||
sequence: 1,
|
||||
visit: 1,
|
||||
artifacts: [
|
||||
_file('input.md', 'debug'),
|
||||
_file('memory.md', 'debug'),
|
||||
_file('history.md', 'debug'),
|
||||
_file('output.md', 'debug'),
|
||||
_file('private/scratch.md', 'debug'),
|
||||
_file('public/report.md', 'deliverable'),
|
||||
],
|
||||
),
|
||||
];
|
||||
|
||||
Future<void> _pump(WidgetTester tester, _Platform platform) async {
|
||||
tester.view.physicalSize = const Size(900, 2600);
|
||||
tester.view.devicePixelRatio = 1;
|
||||
addTearDown(tester.view.resetPhysicalSize);
|
||||
addTearDown(tester.view.resetDevicePixelRatio);
|
||||
final page = ArtifactGalleryPage(
|
||||
steps: _steps(),
|
||||
configuredEndpoint: 'ws://127.0.0.1:8081/connect',
|
||||
openExternal: (_) async => true,
|
||||
fetch: (_) async =>
|
||||
ArtifactFetchResult(status: 200, bytes: Uint8List.fromList([35])),
|
||||
markdownBuilder: (context, value) => Text(value),
|
||||
);
|
||||
await tester.pumpWidget(
|
||||
platform == _Platform.material
|
||||
? MaterialApp(home: page)
|
||||
: CupertinoApp(home: page),
|
||||
);
|
||||
await tester.pumpAndSettle();
|
||||
}
|
||||
|
||||
void main() {
|
||||
for (final platform in _Platform.values) {
|
||||
testWidgets('${platform.name}: only deliverables show until Debugging '
|
||||
'files is revealed', (tester) async {
|
||||
await _pump(tester, platform);
|
||||
expect(find.text('public/report.md'), findsOneWidget);
|
||||
for (final name in [
|
||||
'history.md',
|
||||
'memory.md',
|
||||
'input.md',
|
||||
'private/scratch.md',
|
||||
]) {
|
||||
expect(find.text(name), findsNothing, reason: name);
|
||||
}
|
||||
expect(find.text('Debugging files'), findsNothing);
|
||||
|
||||
await tester.tap(find.byKey(const Key('artifact-debug-toggle')));
|
||||
await tester.pumpAndSettle();
|
||||
expect(find.text('Debugging files'), findsOneWidget);
|
||||
for (final name in [
|
||||
'history.md',
|
||||
'memory.md',
|
||||
'input.md',
|
||||
'output.md',
|
||||
'private/scratch.md',
|
||||
]) {
|
||||
expect(find.text(name), findsOneWidget, reason: name);
|
||||
}
|
||||
expect(find.text('public/report.md'), findsOneWidget);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -99,9 +99,9 @@ void main() {
|
||||
await _pump(tester, platform);
|
||||
expect(find.text('input.md'), findsNothing);
|
||||
expect(find.textContaining('Turn documents'), findsNothing);
|
||||
expect(find.byTooltip('Show turn documents'), findsOneWidget);
|
||||
expect(find.byTooltip('Show debugging files'), findsOneWidget);
|
||||
await _showTurnDocuments(tester);
|
||||
expect(find.byTooltip('Hide turn documents'), findsOneWidget);
|
||||
expect(find.byTooltip('Hide debugging files'), findsOneWidget);
|
||||
expect(find.text('input.md'), findsNWidgets(2));
|
||||
expect(find.text('output.md'), findsNWidgets(2));
|
||||
expect(find.text('session.html'), findsNWidgets(2));
|
||||
|
||||
@@ -83,6 +83,15 @@ JsonMap probeRunDetail() => {
|
||||
'size_bytes': 4096,
|
||||
'url': '/artifacts/7/2/release-dashboard.html',
|
||||
},
|
||||
if (_turnFiles)
|
||||
for (final name in _turnFileNames)
|
||||
{
|
||||
'name': name,
|
||||
'content_type': 'text/markdown',
|
||||
'size_bytes': 640,
|
||||
'url': '/artifacts/7/2/',
|
||||
'group': 'debug',
|
||||
},
|
||||
],
|
||||
'file_changes': _probeChanges([
|
||||
'lib/release/train.dart',
|
||||
@@ -138,6 +147,18 @@ int _fillerStepCount = 0;
|
||||
/// serves its bytes from the build directory (see notes/artifact-fullscreen.md).
|
||||
bool _htmlArtifact = false;
|
||||
|
||||
/// `?turn=1` adds the runner-internal turn files the server records for a model
|
||||
/// visit (group `debug`) to run 7's build visit, so the Artifacts tab's hidden
|
||||
/// Debugging files group can be reviewed.
|
||||
bool _turnFiles = false;
|
||||
const List<String> _turnFileNames = [
|
||||
'input.md',
|
||||
'memory.md',
|
||||
'history.md',
|
||||
'output.md',
|
||||
'private/build-log.md',
|
||||
];
|
||||
|
||||
List<JsonMap> _fillerSteps() => [
|
||||
for (var i = 0; i < _fillerStepCount; i++)
|
||||
{
|
||||
@@ -618,6 +639,7 @@ class _FixtureSink implements WebSocketSink {
|
||||
/// `?runs=200` / `?steps=60` append filler runs / run-7 steps for scroll
|
||||
/// performance captures;
|
||||
/// `?html=1` adds an HTML artifact to run 7 (`release-dashboard.html`);
|
||||
/// `?turn=1` adds run 7's debug turn files (input/memory/history/output.md);
|
||||
/// `?q=release` types that query into Search once its field is shown;
|
||||
/// `?platform=cupertino` renders the Cupertino host (`material`, the
|
||||
/// default, renders Material as browsers do);
|
||||
@@ -650,6 +672,7 @@ void main() {
|
||||
_fillerStepCount = int.tryParse(params['steps'] ?? '') ?? 0;
|
||||
probeUsage = params['usage'] == '1';
|
||||
_htmlArtifact = params['html'] == '1';
|
||||
_turnFiles = params['turn'] == '1';
|
||||
final cupertino = params['platform'] == 'cupertino';
|
||||
final route = params['route'];
|
||||
final opensRoute = route != null && route.startsWith('/');
|
||||
|
||||
+4
-1
@@ -93,12 +93,15 @@ never returned. Missing public phone configuration is labeled explicitly.
|
||||
|
||||
## Search
|
||||
|
||||
`pipelines search QUERY [--type KIND] [--limit N] [--json]` runs one read-only
|
||||
`pipelines search QUERY [--type KIND] [--limit N] [--debug-files] [--json]` runs one read-only
|
||||
query across the whole system: pipeline names and their versions, run identity,
|
||||
status and dates, stage and visit identifiers, artifact names and types, and the
|
||||
text of stage inputs, outputs, evidence and transcripts. It never reads
|
||||
artifact contents, never makes a per-result follow-up request, and asks for no
|
||||
confirmation.
|
||||
Artifact hits are deliverables only: runner-internal debugging files (see
|
||||
[Stage artifacts](stage-artifacts.md#deliverables-and-debugging-files)) match
|
||||
only with `--debug-files` (`include_debug` on the request).
|
||||
|
||||
- Free text matches case-insensitively. A bare status word (`queued`, `running`,
|
||||
`succeeded`, `failed`, `canceled`, `interrupted`) selects those runs, and a lone
|
||||
|
||||
+28
-3
@@ -55,9 +55,34 @@ server-relative `url`. Public entries carry a `public/` name prefix. The same
|
||||
fields appear under each step's `artifacts` in `pipelines inspect RUN_ID --json`
|
||||
and in the TUI.
|
||||
|
||||
`input.md`, `memory.md`, `history.md` and `output.md` are the primary artifacts of every model
|
||||
visit. `private/` files are listed without a prefix; shared files are listed as
|
||||
`public/...` on every visit that can see them.
|
||||
`input.md`, `memory.md`, `history.md` and `output.md` are always inventoried
|
||||
first, so they are never crowded out by captures. `private/` files are listed
|
||||
as `private/...`; shared files are listed as `public/...` on every visit that
|
||||
can see them.
|
||||
|
||||
## Deliverables and debugging files
|
||||
|
||||
Every recorded artifact carries a `group` (`protocol.ClassifyArtifacts`):
|
||||
|
||||
- `deliverable`: a file the stage published to `public/`. For a visit recorded
|
||||
before the turn layout (no turn files), every other file is a deliverable too.
|
||||
- `debug`: runner-internal files - `input.md`, `memory.md`, `history.md`,
|
||||
`output.md`, `session.html` and session repairs, the visit's `private/`
|
||||
scratch, and any other unprefixed file of a turn-layout visit.
|
||||
|
||||
The group is derived from the recorded names on every read, so runs stored
|
||||
before the field existed are classified the same way. Clients present only
|
||||
deliverables by default:
|
||||
|
||||
- App: the run's Artifacts tab lists deliverables; the Debug (bug) toggle
|
||||
reveals a separate "Debugging files" group under each visit. The parsed
|
||||
stage result stays on Summary, and the visit's Session page still shows its
|
||||
turn documents. An older server records no `group`, so the app falls back to
|
||||
the names above (`input.md`, `memory.md`, `history.md`, `output.md`,
|
||||
`session.html`, `private/...`).
|
||||
- TUI: deliverables first, debugging files after them marked `[dbg]`.
|
||||
- Search: artifact hits are deliverables only; `include_debug` on the request
|
||||
(`pipelines search --debug-files`) also matches debugging files.
|
||||
|
||||
- Limits: 200 files per step, 8 MiB per file and 64 MiB total. Over-cap files are
|
||||
skipped and the skipped/truncated counts are recorded with the step.
|
||||
|
||||
+3
-1
@@ -50,6 +50,7 @@ type options struct {
|
||||
keep int
|
||||
keepSet bool
|
||||
all bool
|
||||
debugFiles bool
|
||||
json, source, noOpen, yes, allow, createOnly, force, wait bool
|
||||
version, expected, limit, retryVisit int
|
||||
timeout, waitTimeout time.Duration
|
||||
@@ -166,6 +167,7 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
|
||||
case "search":
|
||||
f.StringVar(&o.kind, "type", "", "Narrow to one kind: "+strings.Join(protocol.SearchKinds, ", "))
|
||||
f.IntVar(&o.limit, "limit", 0, "Maximum results (default 50, max 200)")
|
||||
f.BoolVar(&o.debugFiles, "debug-files", false, "Also match runner-internal debugging artifacts (turn files, private/ scratch)")
|
||||
case "retry":
|
||||
f.BoolVar(&o.yes, "yes", false, "Confirm retry without an interactive prompt")
|
||||
f.StringVar(&o.key, "request-key", "", "Idempotency key (default: generated unique key)")
|
||||
@@ -853,7 +855,7 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
if strings.TrimSpace(query) == "" {
|
||||
return fmt.Errorf("search requires a query; use pipelines help search")
|
||||
}
|
||||
r, err := call(protocol.Message{Type: protocol.Search, Search: &protocol.SearchRequest{Query: query, Kind: o.kind, Limit: o.limit}})
|
||||
r, err := call(protocol.Message{Type: protocol.Search, Search: &protocol.SearchRequest{Query: query, Kind: o.kind, Limit: o.limit, IncludeDebug: o.debugFiles}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ var usage = map[string]string{
|
||||
"validate": "validate <draft.md> [--json]",
|
||||
"submit": "submit <draft.md> [--name NAME] [--create-only] [--yes] [--json]",
|
||||
"list": "list [--json]",
|
||||
"search": "search QUERY [--type KIND] [--limit N] [--json]",
|
||||
"search": "search QUERY [--type KIND] [--limit N] [--debug-files] [--json]",
|
||||
"show": "show <name|id> [--version N] [--json]",
|
||||
"versions": "versions <name|id> [--json]",
|
||||
"export": "export [name|id ...] [--output FILE.zip] [--all] [--version N]",
|
||||
@@ -60,7 +60,7 @@ var notes = map[string]string{
|
||||
"rpc": "Accepts a JSON protocol envelope; '-' reads stdin. Every TUI server operation is available here too. Mutating commands are not dry runs. The omni-search command is `search` (fields query, optional kind and limit) and answers with ranked `search_results`.",
|
||||
"chat-adopt": "Adopt a session that already exists on a chat harness (an OpenCode session, or a Claude Code or Codex session by its id) so it appears in chat_sessions and can be continued from any client. Without --harness each available harness is asked in turn. The directory and title come from the harness's own record (never guessed; --title names a chat whose harness records none), and a missing id records nothing. Do not keep using the session elsewhere once it is continued here.",
|
||||
"retention": "Local, explicit run-history retention; never automatic and never a side effect of another command. `report` is strictly read-only and prints what a policy would remove; `apply` removes it after approval (--yes for scripted use). Policy: --older-than DURATION selects runs created before now minus that rolling UTC duration (Go durations plus d/w; not a calendar boundary), --keep N keeps the newest N runs by run id and selects older ranks (--keep 0 keeps none). Both together are the conservative intersection; at least one is required. Protected and never removed: queued/running runs, runs waiting on a person (waiting run or step), runs with an active cleanup agent, and any run referenced by a retained run's retry provenance (transitively). A run that becomes protected before deletion aborts the whole apply. Artifacts at <root>/<run> are removed with their run and kept with protected/retained runs; the database is VACUUMed and its WAL checkpointed, with measured reclaimed bytes. Definitions and versions are immutable and never touched. --db must name an existing database (default: the standard data directory); --artifacts defaults to artifacts/ beside --db. Works without a running server.",
|
||||
"search": "One read-only query over pipelines, versions, runs, steps, field text and artifacts. QUERY is free text, a bare status word (queued, running, succeeded, failed, canceled, interrupted) selecting those runs, or a lone date expression (today, yesterday, this week [Monday 00:00 local], or YYYY-MM-DD) selecting the runs created in that window. --type is the only narrowing mechanism and takes one kind: pipeline, version, run, step, field or artifact; an unknown kind is an error. Ranking: an exact name match (3) beats a name substring (2) beats a body-text match (1); equal tiers break by recency, then a stable kind/target tie-break, and every hit reports score, matched_field and recency so a client can re-rank. Bounds: the result cap defaults to 50 and clamps to 200 (0 = default); every kind scans at most 2000 rows and field-text scanning is additionally capped at 2000 steps / 1 MiB; Truncated plus scanned_steps, scanned_bytes and scan_bound_bytes report when a bound applied. Targets are stable: pipeline:<id>, pipeline:<id>:v<version>, run:<id>, run:<id>:<sequence>, run:<id>:<sequence>:<field>, or an artifact URL.",
|
||||
"search": "One read-only query over pipelines, versions, runs, steps, field text and artifacts. QUERY is free text, a bare status word (queued, running, succeeded, failed, canceled, interrupted) selecting those runs, or a lone date expression (today, yesterday, this week [Monday 00:00 local], or YYYY-MM-DD) selecting the runs created in that window. --type is the only narrowing mechanism and takes one kind: pipeline, version, run, step, field or artifact; an unknown kind is an error. Ranking: an exact name match (3) beats a name substring (2) beats a body-text match (1); equal tiers break by recency, then a stable kind/target tie-break, and every hit reports score, matched_field and recency so a client can re-rank. Bounds: the result cap defaults to 50 and clamps to 200 (0 = default); every kind scans at most 2000 rows and field-text scanning is additionally capped at 2000 steps / 1 MiB; Truncated plus scanned_steps, scanned_bytes and scan_bound_bytes report when a bound applied. Targets are stable: pipeline:<id>, pipeline:<id>:v<version>, run:<id>, run:<id>:<sequence>, run:<id>:<sequence>:<field>, or an artifact URL. Artifact hits are deliverables only; runner-internal debugging files (input.md, memory.md, history.md, output.md, session.html, private/ scratch) are matched only with --debug-files.",
|
||||
}
|
||||
|
||||
var serviceHelp = map[string]string{
|
||||
|
||||
@@ -214,7 +214,7 @@ strftime('%Y-%m-%dT%H:%M:%SZ',started_at),COALESCE(strftime('%Y-%m-%dT%H:%M:%SZ'
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
step.Artifacts = evidence.Files
|
||||
step.Artifacts = protocol.ClassifyArtifacts(evidence.Files)
|
||||
step.ArtifactsSkipped = evidence.Skipped
|
||||
step.ArtifactsTruncated = evidence.Truncated
|
||||
}
|
||||
|
||||
@@ -37,6 +37,7 @@ func (s *Store) Search(ctx context.Context, req protocol.SearchRequest, now time
|
||||
}
|
||||
terms := parseSearchTerms(req.Query, now)
|
||||
terms.Browse = browse
|
||||
terms.IncludeDebug = req.IncludeDebug
|
||||
terms.Limit = response.Cap
|
||||
// A bare status word or date selects runs; it is never treated as body text.
|
||||
selectorOnly := terms.Text == "" && !browse
|
||||
@@ -129,13 +130,15 @@ func searchLimit(limit int) int {
|
||||
// searchTerms is the parsed query. Text carries free text; Status and the
|
||||
// From/To window carry a bare status word or a date expression that selects runs.
|
||||
type searchTerms struct {
|
||||
Browse bool
|
||||
Limit int
|
||||
Text string
|
||||
Status string
|
||||
HasDate bool
|
||||
From time.Time
|
||||
To time.Time
|
||||
Browse bool
|
||||
// IncludeDebug keeps runner-internal debugging artifacts in artifact hits.
|
||||
IncludeDebug bool
|
||||
Limit int
|
||||
Text string
|
||||
Status string
|
||||
HasDate bool
|
||||
From time.Time
|
||||
To time.Time
|
||||
}
|
||||
|
||||
// parseSearchTerms recognises a lone status word and a lone date expression and
|
||||
@@ -625,7 +628,12 @@ ORDER BY r.created_at DESC,r.id DESC,a.sequence DESC LIMIT ?`
|
||||
if err := json.Unmarshal([]byte(artifacts), &evidence); err != nil {
|
||||
continue
|
||||
}
|
||||
for _, file := range evidence.Files {
|
||||
// Debugging files (turn files, private/ scratch) are internal to the
|
||||
// runner and stay out of results unless explicitly requested.
|
||||
for _, file := range protocol.ClassifyArtifacts(evidence.Files) {
|
||||
if file.IsDebug() && !terms.IncludeDebug {
|
||||
continue
|
||||
}
|
||||
score, matched := 1, "content_type"
|
||||
if terms.Browse {
|
||||
score, matched = 0, ""
|
||||
@@ -637,7 +645,7 @@ ORDER BY r.created_at DESC,r.id DESC,a.sequence DESC LIMIT ?`
|
||||
results = append(results, protocol.SearchResult{
|
||||
Kind: protocol.SearchKindArtifact, Target: file.URL,
|
||||
Score: score, MatchedField: matched, Recency: recency,
|
||||
Artifact: &protocol.SearchArtifact{RunID: runID, Sequence: sequence, Name: file.Name, ContentType: file.ContentType, SizeBytes: file.SizeBytes, URL: file.URL},
|
||||
Artifact: &protocol.SearchArtifact{RunID: runID, Sequence: sequence, Name: file.Name, ContentType: file.ContentType, SizeBytes: file.SizeBytes, URL: file.URL, Group: file.Group},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// Artifact search returns only deliverables unless debugging files are asked
|
||||
// for, and the step read path classifies stored records that carry no group.
|
||||
func TestSearchArtifactsExcludeDebugFilesByDefault(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store, err := Open(filepath.Join(t.TempDir(), "search-debug.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { store.Close() })
|
||||
if _, err := store.Submit(ctx, "Quartz", Template([]string{"quartz"}, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
items, err := store.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, err := store.Get(ctx, items[0].ID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run, _, err := store.CreateRun(ctx, definition, "quartz-key", "/work/q", false, nil, map[string]any{"prompt": "task"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.BeginStep(ctx, run, 1, "quartz", map[string]any{"prompt": "task"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Stored without a group, as a server before the field recorded them.
|
||||
files := []protocol.StageArtifact{
|
||||
{Name: "input.md", ContentType: "text/markdown", URL: "/a/input.md"},
|
||||
{Name: "history.md", ContentType: "text/markdown", URL: "/a/history.md"},
|
||||
{Name: "private/qz-shot.png", ContentType: "image/png", URL: "/a/private/qz-shot.png"},
|
||||
{Name: "public/qz-report.md", ContentType: "text/markdown", URL: "/a/public/qz-report.md"},
|
||||
}
|
||||
if err := store.FinishStep(ctx, run, protocol.RunStep{Sequence: 1, Status: "completed", Outcome: "success", Target: protocol.SuccessEnd, Artifacts: files}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names := func(req protocol.SearchRequest) []string {
|
||||
t.Helper()
|
||||
req.Kind = protocol.SearchKindArtifact
|
||||
response, err := store.Search(ctx, req, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []string
|
||||
for _, hit := range response.Results {
|
||||
got = append(got, hit.Artifact.Name+"="+hit.Artifact.Group)
|
||||
}
|
||||
sort.Strings(got)
|
||||
return got
|
||||
}
|
||||
if got := names(protocol.SearchRequest{Query: "md"}); len(got) != 1 || got[0] != "public/qz-report.md=deliverable" {
|
||||
t.Fatalf("default search = %v, want only the public deliverable", got)
|
||||
}
|
||||
if got := names(protocol.SearchRequest{Query: "history"}); len(got) != 0 {
|
||||
t.Fatalf("history.md surfaced by default: %v", got)
|
||||
}
|
||||
if got := names(protocol.SearchRequest{Query: "qz", IncludeDebug: true}); len(got) != 2 || got[0] != "private/qz-shot.png=debug" {
|
||||
t.Fatalf("include_debug search = %v", got)
|
||||
}
|
||||
|
||||
detail, err := store.GetRun(ctx, run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
groups := map[string]string{}
|
||||
for _, file := range detail.Steps[0].Artifacts {
|
||||
groups[file.Name] = file.Group
|
||||
}
|
||||
if groups["history.md"] != protocol.ArtifactGroupDebug || groups["private/qz-shot.png"] != protocol.ArtifactGroupDebug || groups["public/qz-report.md"] != protocol.ArtifactGroupDeliverable {
|
||||
t.Fatalf("read path groups = %v", groups)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package protocol
|
||||
|
||||
import "strings"
|
||||
|
||||
// Artifact presentation groups. A deliverable is a file the stage published to
|
||||
// the run's public/ directory (or, for a visit recorded before the turn layout,
|
||||
// any file it left). A debug file is one of the runner's own turn files or the
|
||||
// visit's private/ scratch: evidence for debugging a stage, not its output.
|
||||
const (
|
||||
ArtifactGroupDeliverable = "deliverable"
|
||||
ArtifactGroupDebug = "debug"
|
||||
)
|
||||
|
||||
// TurnArtifactNames are the files the server itself writes into a model visit's
|
||||
// turn directory. They are always debugging files.
|
||||
var TurnArtifactNames = []string{"input.md", "memory.md", "history.md", "output.md", "session.html"}
|
||||
|
||||
// IsDebug reports whether the artifact is a runner-internal debugging file.
|
||||
func (a StageArtifact) IsDebug() bool { return a.Group == ArtifactGroupDebug }
|
||||
|
||||
func isTurnArtifactName(name string) bool {
|
||||
for _, turn := range TurnArtifactNames {
|
||||
if name == turn {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return name == LegacySessionMarkdownArtifact || strings.HasPrefix(name, "session-repair-")
|
||||
}
|
||||
|
||||
// ClassifyArtifacts returns one visit's artifacts with Group set. public/ files
|
||||
// are deliverables. Turn files, session exports and their repairs, and private/
|
||||
// scratch are debug files; so is every other unprefixed file of a turn-layout
|
||||
// visit (one that recorded any turn file), because a stage publishes only
|
||||
// through public/. A visit recorded before the turn layout keeps its other
|
||||
// files as deliverables. The rule uses only the recorded names, so runs stored
|
||||
// before the field existed classify identically on read. It never mutates the
|
||||
// input. Pure, O(artifacts).
|
||||
func ClassifyArtifacts(files []StageArtifact) []StageArtifact {
|
||||
if len(files) == 0 {
|
||||
return files
|
||||
}
|
||||
turnVisit := false
|
||||
for _, file := range files {
|
||||
if isTurnArtifactName(file.Name) {
|
||||
turnVisit = true
|
||||
break
|
||||
}
|
||||
}
|
||||
classified := make([]StageArtifact, len(files))
|
||||
for i, file := range files {
|
||||
switch {
|
||||
case strings.HasPrefix(file.Name, "public/"):
|
||||
file.Group = ArtifactGroupDeliverable
|
||||
case turnVisit, isTurnArtifactName(file.Name), strings.HasPrefix(file.Name, "private/"):
|
||||
file.Group = ArtifactGroupDebug
|
||||
default:
|
||||
file.Group = ArtifactGroupDeliverable
|
||||
}
|
||||
classified[i] = file
|
||||
}
|
||||
return classified
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package protocol
|
||||
|
||||
import "testing"
|
||||
|
||||
func groupsByName(files []StageArtifact) map[string]string {
|
||||
groups := map[string]string{}
|
||||
for _, file := range files {
|
||||
groups[file.Name] = file.Group
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func TestClassifyArtifactsTurnVisit(t *testing.T) {
|
||||
input := []StageArtifact{
|
||||
{Name: "input.md"}, {Name: "memory.md"}, {Name: "history.md"}, {Name: "output.md"},
|
||||
{Name: "session.html"}, {Name: "session-repair-1.html"}, {Name: "private/shot.png"},
|
||||
{Name: "notes.txt"}, {Name: "public/report.html"}, {Name: "public/history.md"},
|
||||
}
|
||||
got := groupsByName(ClassifyArtifacts(input))
|
||||
want := map[string]string{
|
||||
"input.md": ArtifactGroupDebug, "memory.md": ArtifactGroupDebug, "history.md": ArtifactGroupDebug,
|
||||
"output.md": ArtifactGroupDebug, "session.html": ArtifactGroupDebug, "session-repair-1.html": ArtifactGroupDebug,
|
||||
"private/shot.png": ArtifactGroupDebug, "notes.txt": ArtifactGroupDebug,
|
||||
"public/report.html": ArtifactGroupDeliverable, "public/history.md": ArtifactGroupDeliverable,
|
||||
}
|
||||
for name, group := range want {
|
||||
if got[name] != group {
|
||||
t.Errorf("%s: group %q, want %q", name, got[name], group)
|
||||
}
|
||||
}
|
||||
for _, file := range input {
|
||||
if file.Group != "" {
|
||||
t.Fatalf("ClassifyArtifacts mutated its input: %+v", file)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyArtifactsLegacyVisitKeepsFilesAsDeliverables(t *testing.T) {
|
||||
got := groupsByName(ClassifyArtifacts([]StageArtifact{{Name: "report.png"}, {Name: "private/scratch.txt"}}))
|
||||
if got["report.png"] != ArtifactGroupDeliverable {
|
||||
t.Errorf("legacy file group %q, want deliverable", got["report.png"])
|
||||
}
|
||||
if got["private/scratch.txt"] != ArtifactGroupDebug {
|
||||
t.Errorf("private file group %q, want debug", got["private/scratch.txt"])
|
||||
}
|
||||
if ClassifyArtifacts(nil) != nil {
|
||||
t.Error("nil input must stay nil")
|
||||
}
|
||||
}
|
||||
@@ -278,6 +278,11 @@ type StageArtifact struct {
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
ContentType string `json:"content_type"`
|
||||
URL string `json:"url"`
|
||||
// Group is the server's presentation classification (ArtifactGroupDeliverable
|
||||
// or ArtifactGroupDebug, see ClassifyArtifacts). Every listing the server
|
||||
// returns sets it; an older server omits it, so clients keep a filename
|
||||
// fallback.
|
||||
Group string `json:"group,omitempty"`
|
||||
}
|
||||
|
||||
// LegacySessionMarkdownArtifact is the retired markdown session export file name.
|
||||
|
||||
@@ -80,6 +80,10 @@ type SearchRequest struct {
|
||||
Query string `json:"query"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
Limit int `json:"limit,omitempty"`
|
||||
// IncludeDebug also returns runner-internal debugging artifacts (turn files
|
||||
// and private/ scratch, see ClassifyArtifacts). Off by default: artifact
|
||||
// search surfaces only deliverables.
|
||||
IncludeDebug bool `json:"include_debug,omitempty"`
|
||||
}
|
||||
|
||||
// SearchResponse is the bounded set of matches. Results are ordered by recency
|
||||
@@ -181,6 +185,8 @@ type SearchArtifact struct {
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
URL string `json:"url"`
|
||||
// Group is the artifact's presentation group (see StageArtifact.Group).
|
||||
Group string `json:"group,omitempty"`
|
||||
}
|
||||
|
||||
// SearchChat is an operator chat session hit: its registry facts and, for a
|
||||
|
||||
@@ -138,7 +138,7 @@ func (r *runner) scanArtifacts(runID int64, sequence int, directory string, publ
|
||||
if len(files) == 0 {
|
||||
files = nil
|
||||
}
|
||||
return files, skipped, truncated
|
||||
return protocol.ClassifyArtifacts(files), skipped, truncated
|
||||
}
|
||||
|
||||
// artifactRel returns the slash path of directory below the artifacts root. A
|
||||
|
||||
@@ -55,6 +55,13 @@ func TestSessionEvidenceIsInventoriedBeforeLargeCaptureDirectories(t *testing.T)
|
||||
t.Fatalf("mandatory runtime file %s hidden by user captures", name)
|
||||
}
|
||||
}
|
||||
// The recorded inventory carries the server's classification: turn files and
|
||||
// private captures are debugging files.
|
||||
for _, file := range files {
|
||||
if file.Group != protocol.ArtifactGroupDebug {
|
||||
t.Fatalf("%s group %q, want debug", file.Name, file.Group)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestArtifactsScanRecordsSizesTypesAndSkips(t *testing.T) {
|
||||
@@ -396,6 +403,9 @@ func TestArtifactHTTPGetPublicSharedFile(t *testing.T) {
|
||||
if shared.URL == "" {
|
||||
t.Fatalf("public artifact missing: %+v", step.Artifacts)
|
||||
}
|
||||
if shared.Group != protocol.ArtifactGroupDeliverable {
|
||||
t.Fatalf("public artifact group %q, want deliverable", shared.Group)
|
||||
}
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
resp, err := http.Get(srv.URL + shared.URL)
|
||||
|
||||
@@ -57,12 +57,34 @@ func runArtifactVisits(steps []protocol.RunStep) []artifactVisit {
|
||||
if step.Control != nil && step.Control.Visit > 0 {
|
||||
visit = step.Control.Visit
|
||||
}
|
||||
visits = append(visits, artifactVisit{stage: stage, visit: visit, step: step, artifacts: step.PresentedArtifacts()})
|
||||
visits = append(visits, artifactVisit{stage: stage, visit: visit, step: step, artifacts: deliverablesFirst(step.PresentedArtifacts())})
|
||||
}
|
||||
}
|
||||
return visits
|
||||
}
|
||||
|
||||
// deliverablesFirst orders a visit's artifacts with the stage's deliverables
|
||||
// ahead of the runner-internal debugging files (marked [dbg] in each row), so
|
||||
// the files a stage published lead and turn files never crowd them out. The
|
||||
// order within each group is the recorded order.
|
||||
func deliverablesFirst(artifacts []protocol.StageArtifact) []protocol.StageArtifact {
|
||||
ordered := make([]protocol.StageArtifact, 0, len(artifacts))
|
||||
for _, artifact := range artifacts {
|
||||
if !artifact.IsDebug() {
|
||||
ordered = append(ordered, artifact)
|
||||
}
|
||||
}
|
||||
for _, artifact := range artifacts {
|
||||
if artifact.IsDebug() {
|
||||
ordered = append(ordered, artifact)
|
||||
}
|
||||
}
|
||||
if len(ordered) == 0 {
|
||||
return nil
|
||||
}
|
||||
return ordered
|
||||
}
|
||||
|
||||
// artifactRef addresses one artifact row in the flat selection order the view
|
||||
// uses (group order, then recorded order within the group).
|
||||
type artifactRef struct {
|
||||
@@ -170,7 +192,9 @@ func artifactRowText(artifact protocol.StageArtifact, widths artifactWidths, wid
|
||||
// decision depends only on width, so every row's columns stay aligned
|
||||
// whether or not the marker is shown.
|
||||
if width >= artifactCursorWidth+artifactMarkerWidth+1+artifactMinName {
|
||||
if isImageContentType(artifact.ContentType) {
|
||||
if artifact.IsDebug() {
|
||||
prefix += "[dbg] "
|
||||
} else if isImageContentType(artifact.ContentType) {
|
||||
prefix += "[img] "
|
||||
} else {
|
||||
prefix += strings.Repeat(" ", artifactMarkerWidth) + " "
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package tui
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func TestArtifactsDeliverablesLeadAndDebugFilesAreMarked(t *testing.T) {
|
||||
artifacts := []protocol.StageArtifact{
|
||||
{Name: "input.md", ContentType: "text/markdown", Group: protocol.ArtifactGroupDebug},
|
||||
{Name: "history.md", ContentType: "text/markdown", Group: protocol.ArtifactGroupDebug},
|
||||
{Name: "public/report.png", ContentType: "image/png", Group: protocol.ArtifactGroupDeliverable},
|
||||
}
|
||||
ordered := deliverablesFirst(artifacts)
|
||||
if ordered[0].Name != "public/report.png" || ordered[1].Name != "input.md" || ordered[2].Name != "history.md" {
|
||||
t.Fatalf("order = %+v", ordered)
|
||||
}
|
||||
widths := artifactWidthsFor([]artifactVisit{{artifacts: ordered}})
|
||||
if row := artifactRowText(ordered[1], widths, 100, false); !strings.Contains(row, "[dbg] input.md") {
|
||||
t.Fatalf("debug row not marked: %q", row)
|
||||
}
|
||||
if row := artifactRowText(ordered[0], widths, 100, false); !strings.Contains(row, "[img] public/report.png") {
|
||||
t.Fatalf("deliverable row: %q", row)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user