From 06bf580154a3b6f21790cbbf492c681c1ceca64b Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Wed, 4 Nov 2026 23:25:55 +0000 Subject: [PATCH] Archive pipelines: hide unused ones while keeping their history archived_at on pipelines (migrated in place). archive_pipeline and unarchive_pipeline by id or name with an expected-version guard; list_pipelines hides archived unless include_archived. Archived pipelines refuse runs, retries, new versions and name reuse, and cannot be archived while an enabled schedule points at them. CLI archive/unarchive and list --archived/--all, docs and skill note. --- docs/cli.md | 42 ++++- .../author/skill/references/operations.md | 3 + internal/cli/app.go | 28 +++- internal/cli/archive.go | 52 ++++++ internal/cli/commands.go | 7 +- internal/cli/help.go | 4 +- internal/cli/transfer.go | 3 +- internal/pipeline/archive.go | 140 ++++++++++++++++ internal/pipeline/archive_test.go | 117 +++++++++++++ internal/pipeline/saved_schedules.go | 9 + internal/pipeline/store.go | 31 +++- internal/protocol/message.go | 12 ++ internal/server/archive.go | 23 +++ internal/server/archive_test.go | 157 ++++++++++++++++++ internal/server/runner.go | 3 + internal/server/websocket.go | 13 +- 16 files changed, 630 insertions(+), 14 deletions(-) create mode 100644 internal/cli/archive.go create mode 100644 internal/pipeline/archive.go create mode 100644 internal/pipeline/archive_test.go create mode 100644 internal/server/archive.go create mode 100644 internal/server/archive_test.go diff --git a/docs/cli.md b/docs/cli.md index 3c97a604..39c7e0a6 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -316,6 +316,7 @@ Submitting activates included schedules, so review them before publishing. | Server/ntfy status popup | `status --json` | | Usage statistics/charts | `stats --json` | | Run-history retention | `retention report --keep N --json`, `retention apply --keep N --yes` | +| Retire an unused pipeline | `archive NAME --yes`, `unarchive NAME`, `list --archived` | | Legacy operational diagnostics | managed `logs`, `doctor` | The TUI does not spawn CLI processes for each request. Both front ends share @@ -327,7 +328,8 @@ exposes the full protocol as an escape hatch, so no server command is TUI-only: printf '%s' '{"type":"get_pipeline","pipeline_id":1,"version":1}' | pipelines rpc - ``` -Existing protocol commands: `ping`, `list_pipelines`, `get_pipeline`, +Existing protocol commands: `ping`, `list_pipelines` (optional `include_archived`), +`get_pipeline`, `archive_pipeline`, `unarchive_pipeline`, `list_templates`, `validate`, `submit`, `run_pipeline`, `list_runs`, `get_run`, `cancel_run`, `retry_run`, `decide_run`, `set_run_title`, `get_run_evidence`, `get_status`, `search`. `search` takes a `search` request object (`query`, with an @@ -369,6 +371,44 @@ whose scope covers the runs receive the `runs_removed` push. `pipelines runs rm have real effects; they are not previews. TUI-only animation, scrolling, and transient client logs are presentation, not separate server capabilities. +## Archiving pipelines + +There is no pipeline deletion: an unused pipeline is archived instead. + +```sh +pipelines archive "old flow" --yes # hide it; history is kept +pipelines list --archived # only archived ones (--all lists both) +pipelines unarchive "old flow" +``` + +Archiving sets `archived_at` on the pipeline and nothing else. Its versions, +runs, artifacts, `inspect`, `runs --pipeline` and search history keep working and +keep showing its name. While archived: + +- `list_pipelines` omits it unless the request sets `include_archived`; then the + summary carries `archived_at` (RFC 3339). The TUI and the app's main list hide it; + the app's Archived filter shows and unarchives it. +- `run_pipeline` and `retry_run` are refused: `pipeline "X" is archived; unarchive + it first`. +- Its name stays reserved: `new`, `submit` (with or without `--create-only`) and + import are refused with the same message rather than creating a second pipeline. +- Archiving is refused while an enabled schedule points at it, naming each + schedule; pause or repoint them first. Enabling a schedule for an archived + pipeline is refused too. Refusing is deliberate rather than pausing schedules + implicitly: the operator sees exactly which timers stop, and unarchiving never + needs to guess which schedules to turn back on. + +| Command | Payload | Reply | +| --- | --- | --- | +| `archive_pipeline` | `pipeline_id` or `name`, optional `expected_version` | `pipeline_archived` with `detail` | +| `unarchive_pipeline` | `pipeline_id` or `name`, optional `expected_version` | `pipeline_unarchived` with `detail` | + +`name` matches exactly first and then as a numeric ID, as in the CLI. A positive +`expected_version` refuses the change when the latest version differs (the CLI +sends the version it resolved). Repeating a request already in effect succeeds +without change. Like schedule saves, these are lifecycle controls and are not +gated by authoring being disabled. + ## Saved run schedules Schedules are independent pipeline/version + run-information + timer records, diff --git a/internal/author/skill/references/operations.md b/internal/author/skill/references/operations.md index a4c584d1..24329324 100644 --- a/internal/author/skill/references/operations.md +++ b/internal/author/skill/references/operations.md @@ -15,6 +15,9 @@ parallel/distributed task engine or a guarantee that model reports are correct. an editable prefilled name or customized first. Editing a bundled repository template changes no installed or published definition; only explicit authoring then submission creates a new immutable version. +- Retire an unused pipeline with `pipelines archive NAME --yes` (no deletion exists): + it leaves lists and cannot run, its history stays, its name stays reserved, and + enabled schedules must be paused first; `unarchive` and `list --archived` undo/show it. - A run pins its exact definition version, initial input and resolved directory. The client invocation directory is the default, but files/tools execute on the server. Docker needs that directory mounted at the same absolute path. A remote diff --git a/internal/cli/app.go b/internal/cli/app.go index 7cd88615..72349b9e 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -51,6 +51,7 @@ type options struct { keepSet bool all bool json, source, noOpen, yes, allow, createOnly, force, wait bool + archived bool version, expected, limit, retryVisit int timeout, waitTimeout time.Duration stageModels stageModelFlags @@ -137,6 +138,11 @@ func (a App) flags(command string) (*flag.FlagSet, *options) { f.BoolVar(&o.yes, "yes", false, "Confirm publication (schedules may activate)") case "template": f.BoolVar(&o.source, "source", false, "Print original Markdown instead of metadata") + case "list": + f.BoolVar(&o.archived, "archived", false, "List only archived pipelines") + f.BoolVar(&o.all, "all", false, "List active and archived pipelines (archived ones are marked)") + case "archive": + f.BoolVar(&o.yes, "yes", false, "Confirm archiving without an interactive prompt") case "submit": f.BoolVar(&o.yes, "yes", false, "Confirm publication without an interactive prompt") f.StringVar(&o.name, "name", "", "Name for an external Markdown file (managed drafts already know their name)") @@ -511,8 +517,10 @@ func (a App) Run(ctx context.Context, args []string) error { defer cancel() return c.Call(requestCtx, request) } + // Resolution includes archived pipelines so history, show and unarchive + // still address them; the server refuses runs and publications for them. resolve := func(value string) (protocol.PipelineSummary, error) { - r, err := call(protocol.Message{Type: protocol.List}) + r, err := call(protocol.Message{Type: protocol.List, IncludeArchived: true}) if err != nil { return protocol.PipelineSummary{}, err } @@ -755,6 +763,8 @@ func (a App) Run(ctx context.Context, args []string) error { } } return nil + case "archive", "unarchive": + return a.archive(command, pos, o, call, resolve) case "list", "templates", "runs", "ping", "connection": if command == "runs" && len(pos) > 0 && pos[0] == "rm" { return a.removeRuns(o, pos[1:], call) @@ -763,10 +773,13 @@ func (a App) Run(ctx context.Context, args []string) error { return fmt.Errorf("%s takes no positional arguments", command) } typeName := map[string]string{"list": protocol.List, "templates": protocol.ListTemplates, "runs": protocol.ListRuns, "ping": protocol.Ping, "connection": protocol.List}[command] - r, err := call(protocol.Message{Type: typeName}) + r, err := call(protocol.Message{Type: typeName, IncludeArchived: command == "list" && (o.all || o.archived)}) if err != nil { return err } + if command == "list" && o.archived && !o.all { + r.Pipelines = archivedOnly(r.Pipelines) + } if command == "runs" && o.name != "" { p, err := resolve(o.name) if err != nil { @@ -786,7 +799,11 @@ func (a App) Run(ctx context.Context, args []string) error { switch command { case "list": for _, p := range r.Pipelines { - fmt.Fprintf(a.Out, "%d\t%s\tv%d\t%d stages\n", p.ID, p.Name, p.Version, p.Stages) + note := "" + if p.ArchivedAt != "" { + note = "\tarchived " + p.ArchivedAt + } + fmt.Fprintf(a.Out, "%d\t%s\tv%d\t%d stages%s\n", p.ID, p.Name, p.Version, p.Stages, note) } case "templates": for _, t := range r.Templates { @@ -1195,11 +1212,14 @@ func (a App) author(ctx context.Context, command string, pos []string, o *option if (o.name != "" && strings.TrimSpace(o.name) == "") || len(o.name) > 200 || strings.ContainsAny(o.name, "\r\n\x00") { return fmt.Errorf("name must be 1–200 bytes without line breaks or NUL") } - r, err := call(protocol.Message{Type: protocol.List}) + r, err := call(protocol.Message{Type: protocol.List, IncludeArchived: true}) if err != nil { return err } for _, p := range r.Pipelines { + if p.Name == o.name && p.ArchivedAt != "" { + return &pipeline.ArchivedError{Name: p.Name} + } if p.Name == o.name { return fmt.Errorf("pipeline already exists; use edit or choose another name") } diff --git a/internal/cli/archive.go b/internal/cli/archive.go new file mode 100644 index 00000000..9a699109 --- /dev/null +++ b/internal/cli/archive.go @@ -0,0 +1,52 @@ +package cli + +import ( + "fmt" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// archive implements `pipelines archive` and `pipelines unarchive`. The +// server keeps every version and run; the latest version is the guard. +func (a App) archive(command string, pos []string, o *options, call callFunc, resolve resolveFunc) error { + if len(pos) != 1 { + return fmt.Errorf("%s requires one pipeline name or ID; use pipelines help %s", command, command) + } + p, err := resolve(pos[0]) + if err != nil { + return err + } + request := protocol.Message{Type: protocol.UnarchivePipeline, PipelineID: p.ID, ExpectedVersion: p.Version} + if command == "archive" { + request.Type = protocol.ArchivePipeline + if err := a.confirmAction("Archive "+p.Name+"? It leaves the pipeline list and cannot start runs; its versions, runs and artifacts are kept and it can be unarchived.", o.yes); err != nil { + return err + } + } + r, err := call(request) + if err != nil { + return err + } + if r.Detail == nil { + return fmt.Errorf("server did not return the pipeline; update pipelines-server") + } + if o.json { + return a.output(r.Detail.PipelineSummary) + } + if command == "archive" { + fmt.Fprintf(a.Out, "Archived %s (history kept; pipelines unarchive %d restores it)\n", r.Detail.Name, r.Detail.ID) + } else { + fmt.Fprintf(a.Out, "Unarchived %s\n", r.Detail.Name) + } + return nil +} + +func archivedOnly(items []protocol.PipelineSummary) []protocol.PipelineSummary { + kept := []protocol.PipelineSummary{} + for _, p := range items { + if p.ArchivedAt != "" { + kept = append(kept, p) + } + } + return kept +} diff --git a/internal/cli/commands.go b/internal/cli/commands.go index b9136240..25a8b0b9 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -10,7 +10,9 @@ var usage = map[string]string{ "draft-delete": "draft-delete [--json]", "validate": "validate [--json]", "submit": "submit [--name NAME] [--create-only] [--yes] [--json]", - "list": "list [--json]", + "list": "list [--archived | --all] [--json]", + "archive": "archive [--yes] [--json]", + "unarchive": "unarchive [--json]", "search": "search QUERY [--type KIND] [--limit N] [--json]", "show": "show [--version N] [--json]", "versions": "versions [--json]", @@ -52,6 +54,9 @@ var notes = map[string]string{ "submit": "Confirm publication at the prompt, or use --yes for explicit automation; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.", "run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. --model PROVIDER/MODEL overrides the model for every stage of that run; --stage-model STAGE=PROVIDER/MODEL overrides one stage and is repeatable, with a repeated key last-wins. Precedence is per-stage > whole-run > the model pinned in the pipeline version; overrides apply to that run only and never change the pinned definition. An unknown stage key or a malformed provider/model is refused before the run starts, naming the valid stage keys. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).", "retry": "Resume a failed stage by default, restoring the completed prefix handoff; use --mode=restart to start at entry. --stage selects a recorded stage (use --visit for repeated stages). --input-file accepts a bounded JSON object, merged with replacement at the selected stage; '-' reads stdin. Original history and pinned identity are retained; --version selects an explicit published version of the same pipeline only. --wait-timeout leaves an active retry for later read-only inspection or reattachment with the original --request-key, rather than starting a duplicate retry.", + "list": "Lists active pipelines. Archived pipelines are hidden; --archived lists only them and --all lists both, marking archived ones with their archive time.", + "archive": "Retire a pipeline from lists after confirmation (--yes for scripts). Nothing is deleted: versions, runs, artifacts and search history stay and keep its name. An archived pipeline cannot start runs and its name stays reserved (new/submit say so). Archiving is refused while an enabled schedule points at it, naming the schedule; pause or repoint it first. The latest version is sent as a guard, so a publication in between is refused.", + "unarchive": "Return an archived pipeline to lists; it can run and take new versions again. Schedules stay paused until you enable them.", "runs": "Lists runs newest first; --json includes each run's size_bytes (bytes on disk: artifacts, session exports, server-created Pi session directories and an estimate of its database rows; 0 until first measured). `runs rm ` permanently deletes finished runs (succeeded, failed, canceled, interrupted) with every database row and server-owned file for them, after confirmation (--yes for scripts). Active runs, runs an unfinished retry depends on and unknown ids are refused individually with a reason, and the command then exits 1. User workspaces are never touched. Database pages are reused by SQLite; `retention apply` additionally VACUUMs to shrink the file.", "title": "Rename a run's operator-facing title after the fact without touching its input, steps or execution. The title is collapsed to a single bounded line and recorded as explicitly set. `pipelines runs` and `pipelines inspect` then show it.", "wait": "Block until each named run reaches a terminal status - succeeded, failed, canceled or interrupted, and also through a run waiting on a person - then print one line per run with its id, status and outcome, its stage and how long it took, and exit 0 even for a run that finished unsuccessfully. Several ids wait together in one command. The exit is non-zero only for a client-side failure: an unknown run id, a connection that could not be established, or --timeout expiring while a run is still active; a timeout says so plainly, never cancels, retries or otherwise touches a run, and is never reported as its outcome. Read-only: it needs no workspace, is not refused by the workspace guard and writes nothing. --json prints the final run envelope instead of the line (one envelope for one id, an array for several).", diff --git a/internal/cli/help.go b/internal/cli/help.go index 8376d36e..3a6a4fc5 100644 --- a/internal/cli/help.go +++ b/internal/cli/help.go @@ -25,7 +25,7 @@ EXECUTE rpc Send any protocol command (automation escape hatch) BROWSE - list List registered pipelines + list List active pipelines; --archived/--all show archived ones search QUERY Search pipelines, versions, runs, steps, field text, artifacts show Inspect graph, settings, source and history versions List immutable versions @@ -42,6 +42,8 @@ AUTHORING — only when the user explicitly asks to create or change a definitio import [FILE.zip] Select/import definitions; explicitly resolve conflicts MAINTAIN + archive Hide an unused pipeline; versions and runs are kept + unarchive Return an archived pipeline to lists and runs retention report Show what a run-history policy would remove (read-only) retention apply Remove it after approval; artifacts go with their run diff --git a/internal/cli/transfer.go b/internal/cli/transfer.go index ff26d759..fc8db8cf 100644 --- a/internal/cli/transfer.go +++ b/internal/cli/transfer.go @@ -139,7 +139,8 @@ func (a App) importBundle(ctx context.Context, o *options, pos []string, call ca entries = keep } // All selected sources/names are checked before any publication. - existing, err := call(protocol.Message{Type: protocol.List}) + // Archived names stay reserved, so they count as existing here. + existing, err := call(protocol.Message{Type: protocol.List, IncludeArchived: true}) if err != nil { return err } diff --git a/internal/pipeline/archive.go b/internal/pipeline/archive.go new file mode 100644 index 00000000..a907341e --- /dev/null +++ b/internal/pipeline/archive.go @@ -0,0 +1,140 @@ +package pipeline + +import ( + "context" + "database/sql" + "fmt" + "strconv" + "strings" + "time" +) + +// Archiving retires a pipeline from everyday lists without deleting anything: +// its versions, runs, artifacts and search history stay as they are, and its +// name stays reserved. Only the pipelines row carries the archived_at stamp. + +func (s *Store) initArchive() error { + return s.addColumnIfMissing("pipelines", "archived_at", "TEXT") +} + +// ArchivedError reports that a name or ID belongs to an archived pipeline. +type ArchivedError struct{ Name string } + +func (e *ArchivedError) Error() string { + return fmt.Sprintf("pipeline %q is archived; unarchive it first (pipelines unarchive %s)", e.Name, quoteArg(e.Name)) +} + +func quoteArg(name string) string { + if strings.ContainsAny(name, " \t'\"\\$`") { + return "'" + strings.ReplaceAll(name, "'", `'\''`) + "'" + } + return name +} + +// ResolvePipelineID finds a pipeline by exact name first, then by numeric ID, +// archived or not, matching the CLI's name-over-ID rule. +func (s *Store) ResolvePipelineID(ctx context.Context, id int64, name string) (int64, error) { + name = strings.TrimSpace(name) + if name != "" { + var found int64 + err := s.db.QueryRowContext(ctx, `SELECT id FROM pipelines WHERE name=?`, name).Scan(&found) + if err == nil { + return found, nil + } + if err != sql.ErrNoRows { + return 0, err + } + if id == 0 { + id, _ = strconv.ParseInt(name, 10, 64) + } + if id <= 0 { + return 0, fmt.Errorf("pipeline %q not found", name) + } + } + if id <= 0 { + return 0, fmt.Errorf("a pipeline name or ID is required") + } + var found int64 + err := s.db.QueryRowContext(ctx, `SELECT id FROM pipelines WHERE id=?`, id).Scan(&found) + if err == sql.ErrNoRows { + return 0, fmt.Errorf("pipeline not found") + } + return found, err +} + +// SetArchived archives or unarchives one pipeline. A positive expectedVersion +// refuses the change when a newer version was published meanwhile. Archiving +// is refused while an enabled schedule still points at the pipeline, naming +// the schedules, so nothing silently stops firing; repeating a request that is +// already in effect is a no-op. +func (s *Store) SetArchived(ctx context.Context, id int64, archived bool, expectedVersion int, now time.Time) error { + if expectedVersion < 0 { + return fmt.Errorf("expected version must be nonnegative") + } + s.mu.Lock() + defer s.mu.Unlock() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + var name string + var stamp sql.NullString + var latest int + err = tx.QueryRowContext(ctx, `SELECT p.name,p.archived_at,COALESCE((SELECT MAX(version) FROM pipeline_versions WHERE pipeline_id=p.id),0) FROM pipelines p WHERE p.id=?`, id).Scan(&name, &stamp, &latest) + if err == sql.ErrNoRows { + return fmt.Errorf("pipeline not found") + } + if err != nil { + return err + } + if expectedVersion > 0 && latest != expectedVersion { + return fmt.Errorf("pipeline %q changed: expected latest version %d, found %d; reload first", name, expectedVersion, latest) + } + if stamp.Valid == archived { + return tx.Commit() + } + if !archived { + if _, err = tx.ExecContext(ctx, `UPDATE pipelines SET archived_at=NULL WHERE id=?`, id); err != nil { + return err + } + return tx.Commit() + } + rows, err := tx.QueryContext(ctx, savedScheduleSelect+` WHERE pipeline_id=? AND deleted=0 AND enabled=1 ORDER BY id`, id) + if err != nil { + return err + } + var enabled []string + for rows.Next() { + item, err := scanSavedSchedule(rows) + if err != nil { + rows.Close() + return err + } + enabled = append(enabled, fmt.Sprintf("%q (#%d)", item.Name, item.ID)) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + if len(enabled) > 0 { + return fmt.Errorf("pipeline %q has enabled schedule(s) %s; pause or repoint them before archiving", name, strings.Join(enabled, ", ")) + } + if _, err = tx.ExecContext(ctx, `UPDATE pipelines SET archived_at=? WHERE id=?`, now.UTC().Format(time.RFC3339), id); err != nil { + return err + } + return tx.Commit() +} + +// archivedName reports whether the pipeline row is archived inside a +// transaction, so a submission or schedule edit cannot race an archive. +func archivedName(ctx context.Context, tx *sql.Tx, query string, arg any) (string, bool, error) { + var name string + var stamp sql.NullString + err := tx.QueryRowContext(ctx, `SELECT name,archived_at FROM pipelines WHERE `+query, arg).Scan(&name, &stamp) + if err == sql.ErrNoRows { + return "", false, nil + } + return name, stamp.Valid, err +} diff --git a/internal/pipeline/archive_test.go b/internal/pipeline/archive_test.go new file mode 100644 index 00000000..6021ecf8 --- /dev/null +++ b/internal/pipeline/archive_test.go @@ -0,0 +1,117 @@ +package pipeline + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "strings" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" + _ "modernc.org/sqlite" +) + +func TestArchiveHidesReservesAndGuardsSchedules(t *testing.T) { + ctx := context.Background() + s, err := Open(filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if _, err := s.Submit(ctx, "old", Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + if _, err := s.Submit(ctx, "keep", Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + id, err := s.ResolvePipelineID(ctx, 0, "old") + if err != nil { + t.Fatal(err) + } + if byID, err := s.ResolvePipelineID(ctx, 0, "1"); err != nil || byID != id { + t.Fatal("numeric name did not resolve as an ID", byID, err) + } + spec := protocol.SavedSchedule{Name: "nightly", PipelineID: id, Version: 1, Enabled: true, Workspace: t.TempDir(), Input: map[string]any{"prompt": "go"}, Rule: protocol.Schedule{Every: "1h"}} + saved, err := s.SaveSchedule(ctx, spec, time.Now()) + if err != nil { + t.Fatal(err) + } + err = s.SetArchived(ctx, id, true, 0, time.Now()) + if err == nil || !strings.Contains(err.Error(), `"nightly"`) || !strings.Contains(err.Error(), "pause") { + t.Fatal("archive with an enabled schedule was not refused by name", err) + } + paused := *saved + paused.Enabled = false + if _, err = s.SaveSchedule(ctx, paused, time.Now()); err != nil { + t.Fatal(err) + } + if err = s.SetArchived(ctx, id, true, 7, time.Now()); err == nil || !strings.Contains(err.Error(), "expected latest version 7") { + t.Fatal("stale expected version accepted", err) + } + if err = s.SetArchived(ctx, id, true, 1, time.Now()); err != nil { + t.Fatal(err) + } + if err = s.SetArchived(ctx, id, true, 0, time.Now()); err != nil { + t.Fatal("repeated archive must be a no-op", err) + } + active, _ := s.List(ctx) + if len(active) != 1 || active[0].Name != "keep" { + t.Fatal("archived pipeline listed by default", active) + } + all, _ := s.ListPipelines(ctx, true) + if len(all) != 2 || all[1].Name != "old" || all[1].ArchivedAt == "" || all[0].ArchivedAt != "" { + t.Fatal("include_archived did not mark the archived pipeline", all) + } + detail, err := s.Get(ctx, id, 1) + if err != nil || detail.ArchivedAt == "" || detail.Source == "" { + t.Fatal("archived detail lost history", detail, err) + } + var archived *ArchivedError + if _, err = s.Submit(ctx, "old", Template([]string{"one", "two"}, true), true); !errors.As(err, &archived) || !strings.Contains(err.Error(), "unarchive") { + t.Fatal("create-only reuse of an archived name not refused", err) + } + if _, err = s.Submit(ctx, "old", Template([]string{"one", "two"}, true)); !errors.As(err, &archived) { + t.Fatal("new version of an archived pipeline accepted", err) + } + enabled := paused + enabled.Revision++ + enabled.Enabled = true + if _, err = s.SaveSchedule(ctx, enabled, time.Now()); !errors.As(err, &archived) { + t.Fatal("enabled schedule saved for an archived pipeline", err) + } + if err = s.SetArchived(ctx, id, false, 1, time.Now()); err != nil { + t.Fatal(err) + } + if active, _ = s.List(ctx); len(active) != 2 { + t.Fatal("unarchived pipeline still hidden", active) + } + if v, err := s.Submit(ctx, "old", Template([]string{"one", "two"}, true)); err != nil || v != 2 { + t.Fatal("unarchived pipeline cannot publish", v, err) + } +} + +func TestArchiveColumnMigratesExistingDatabase(t *testing.T) { + path := filepath.Join(t.TempDir(), "legacy.db") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + if _, err = db.Exec(`CREATE TABLE pipelines (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP); +INSERT INTO pipelines(name) VALUES ('legacy');`); err != nil { + t.Fatal(err) + } + db.Close() + for range 2 { + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + var stamp sql.NullString + if err = s.db.QueryRow(`SELECT archived_at FROM pipelines WHERE name='legacy'`).Scan(&stamp); err != nil || stamp.Valid { + t.Fatal("legacy pipeline not migrated as active", stamp, err) + } + s.Close() + } +} diff --git a/internal/pipeline/saved_schedules.go b/internal/pipeline/saved_schedules.go index 624acb9c..bc893740 100644 --- a/internal/pipeline/saved_schedules.go +++ b/internal/pipeline/saved_schedules.go @@ -267,6 +267,15 @@ func (s *Store) SaveSchedule(ctx context.Context, spec protocol.SavedSchedule, n return nil, err } defer tx.Rollback() + // An archived pipeline cannot gain an enabled schedule; a paused one may + // still be saved so the operator can repoint or tidy it. + if spec.Enabled { + if name, archived, err := archivedName(ctx, tx, "id=?", spec.PipelineID); err != nil { + return nil, err + } else if archived { + return nil, &ArchivedError{Name: name} + } + } if spec.ID == 0 && requestKey != "" { var id int64 var oldHash string diff --git a/internal/pipeline/store.go b/internal/pipeline/store.go index d3cb9aff..ef17889b 100644 --- a/internal/pipeline/store.go +++ b/internal/pipeline/store.go @@ -53,6 +53,10 @@ CREATE TABLE IF NOT EXISTS submissions (id INTEGER PRIMARY KEY, name TEXT NOT NU return nil, err } s := &Store{db: db, dir: filepath.Dir(path), path: path, done: make(chan struct{})} + if err := s.initArchive(); err != nil { + db.Close() + return nil, err + } if err := s.initRuns(); err != nil { db.Close() return nil, err @@ -161,8 +165,15 @@ func OpenMaintenance(path string) (*Store, error) { // adjacent server state such as stage artifacts. func (s *Store) Directory() string { return s.dir } +// List returns the active (unarchived) pipelines. func (s *Store) List(ctx context.Context) ([]protocol.PipelineSummary, error) { - rows, err := s.db.QueryContext(ctx, `SELECT p.id, p.name, v.version, + return s.ListPipelines(ctx, false) +} + +// ListPipelines returns the latest version of every pipeline; archived ones +// are included, marked by ArchivedAt, only when includeArchived is set. +func (s *Store) ListPipelines(ctx context.Context, includeArchived bool) ([]protocol.PipelineSummary, error) { + rows, err := s.db.QueryContext(ctx, `SELECT p.id, p.name, v.version, COALESCE(p.archived_at, ''), json_array_length(v.graph_json, '$.stages'), strftime('%Y-%m-%dT%H:%M:%SZ', p.created_at), strftime('%Y-%m-%dT%H:%M:%SZ', v.created_at), v.graph_json, @@ -170,7 +181,8 @@ v.graph_json, (SELECT COUNT(*) FROM saved_schedules WHERE pipeline_id=p.id AND deleted=0 AND enabled=1) FROM pipelines p JOIN pipeline_versions v ON v.pipeline_id=p.id WHERE v.version=(SELECT MAX(version) FROM pipeline_versions WHERE pipeline_id=p.id) -ORDER BY p.name, p.id`) +AND (? OR p.archived_at IS NULL) +ORDER BY p.name, p.id`, includeArchived) if err != nil { return nil, err } @@ -179,7 +191,7 @@ ORDER BY p.name, p.id`) for rows.Next() { var item protocol.PipelineSummary var graphJSON string - if err := rows.Scan(&item.ID, &item.Name, &item.Version, &item.Stages, &item.CreatedAt, &item.UpdatedAt, &graphJSON, &item.ScheduleCount, &item.EnabledScheduleCount); err != nil { + if err := rows.Scan(&item.ID, &item.Name, &item.Version, &item.ArchivedAt, &item.Stages, &item.CreatedAt, &item.UpdatedAt, &graphJSON, &item.ScheduleCount, &item.EnabledScheduleCount); err != nil { return nil, err } var shape protocol.Graph @@ -219,10 +231,10 @@ func (s *Store) Get(ctx context.Context, id int64, version int) (*protocol.Pipel defer tx.Rollback() detail := &protocol.PipelineDetail{} err = tx.QueryRowContext(ctx, `SELECT p.id,p.name,strftime('%Y-%m-%dT%H:%M:%SZ',p.created_at), -v.version,strftime('%Y-%m-%dT%H:%M:%SZ',v.created_at) +v.version,strftime('%Y-%m-%dT%H:%M:%SZ',v.created_at),COALESCE(p.archived_at,'') FROM pipelines p JOIN pipeline_versions v ON v.pipeline_id=p.id WHERE p.id=? ORDER BY v.version DESC LIMIT 1`, id). - Scan(&detail.ID, &detail.Name, &detail.CreatedAt, &detail.LatestVersion, &detail.UpdatedAt) + Scan(&detail.ID, &detail.Name, &detail.CreatedAt, &detail.LatestVersion, &detail.UpdatedAt, &detail.ArchivedAt) if err == sql.ErrNoRows { return nil, fmt.Errorf("pipeline not found") } @@ -310,6 +322,15 @@ func (s *Store) SubmitChecked(ctx context.Context, name, source string, createOn var version int var versionID any hash := fmt.Sprintf("%x", sha256.Sum256([]byte(source))) + if validation == nil { + // An archived name stays reserved: neither a fresh pipeline nor a new + // version may take it until the operator unarchives it. + if _, archived, err := archivedName(ctx, tx, "name=?", name); err != nil { + return 0, err + } else if archived { + validation = &ArchivedError{Name: name} + } + } if validation == nil { _, err = tx.ExecContext(ctx, "INSERT INTO pipelines(name) VALUES (?) ON CONFLICT(name) DO NOTHING", name) if err != nil { diff --git a/internal/protocol/message.go b/internal/protocol/message.go index e8353824..8b8e0489 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -16,6 +16,13 @@ const ( Templates = "templates" Validate = "validate" Validated = "validated" + // Archive and unarchive name the pipeline by PipelineID or Name; an + // optional ExpectedVersion guards against a concurrent publication. Both + // reply with the pipeline detail (ArchivedAt set while archived). + ArchivePipeline = "archive_pipeline" + PipelineArchived = "pipeline_archived" + UnarchivePipeline = "unarchive_pipeline" + PipelineUnarchived = "pipeline_unarchived" ) type Stage struct { @@ -103,6 +110,10 @@ type PipelineSummary struct { Stages int `json:"stages"` CreatedAt string `json:"created_at"` UpdatedAt string `json:"updated_at"` + // ArchivedAt is set (RFC 3339) while the pipeline is archived: it is + // hidden from list_pipelines unless include_archived is set, cannot start + // runs and keeps its name reserved. History is untouched. + ArchivedAt string `json:"archived_at,omitempty"` // Shape is the current version's parsed graph, included so a list can draw // a miniature without one detail request per row. Detail responses keep the // full Graph field; this one is deliberately optional and additive. @@ -131,6 +142,7 @@ type Message struct { ExpectedVersion int `json:"expected_version,omitempty"` Templates []Template `json:"templates,omitempty"` CreateOnly bool `json:"create_only,omitempty"` + IncludeArchived bool `json:"include_archived,omitempty"` NotificationEndpoints []NotificationEndpoint `json:"notification_endpoints,omitempty"` Type string `json:"type"` ID string `json:"id,omitempty"` diff --git a/internal/server/archive.go b/internal/server/archive.go new file mode 100644 index 00000000..f2cdbe1c --- /dev/null +++ b/internal/server/archive.go @@ -0,0 +1,23 @@ +package server + +import ( + "context" + "time" + + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +// archivePipeline serves archive_pipeline and unarchive_pipeline. Like saving +// a schedule it is a lifecycle change, not authoring: no definition or version +// is written, so it is not gated by AllowAuthoring. +func archivePipeline(ctx context.Context, store *pipeline.Store, request protocol.Message) (*protocol.PipelineDetail, error) { + id, err := store.ResolvePipelineID(ctx, request.PipelineID, request.Name) + if err != nil { + return nil, err + } + if err := store.SetArchived(ctx, id, request.Type == protocol.ArchivePipeline, request.ExpectedVersion, time.Now()); err != nil { + return nil, err + } + return store.Get(ctx, id, 0) +} diff --git a/internal/server/archive_test.go b/internal/server/archive_test.go new file mode 100644 index 00000000..82a775a8 --- /dev/null +++ b/internal/server/archive_test.go @@ -0,0 +1,157 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/cli" + "git.a-wan.net/alan/pipelines/internal/pipeline" + "git.a-wan.net/alan/pipelines/internal/protocol" + "git.a-wan.net/alan/pipelines/internal/stage" +) + +func TestArchivePipelineOverCLIKeepsHistoryAndRefusesRuns(t *testing.T) { + t.Setenv("PIPELINES_HARNESS_CONFIG", filepath.Join(t.TempDir(), "harnesses.json")) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "archive.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err = store.Submit(ctx, "old flow", pipeline.Template([]string{"worker"}, true)); err != nil { + t.Fatal(err) + } + if _, err = store.Submit(ctx, "current", pipeline.Template([]string{"worker"}, true)); err != nil { + t.Fatal(err) + } + handler := Handler(ctx, store) + r := handler.(*runHandler).runner + // The first run fails so a retry of it can be attempted after archiving. + var failing atomic.Bool + failing.Store(true) + r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { + if failing.Load() { + return stage.Result{}, fmt.Errorf("stage failed") + } + return stage.Result{Outcome: "success", Output: stage.Values{"summary": "done"}}, nil + } + srv := httptest.NewServer(handler) + defer srv.Close() + defer func() { cancel(); WaitRuns(handler) }() + url := "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect" + run := func(args ...string) (string, error) { + var out bytes.Buffer + app := cli.App{Out: &out, Err: &out, In: strings.NewReader("")} + err := app.Run(ctx, append([]string{"--url", url}, args...)) + return out.String(), err + } + workspace := t.TempDir() + output, err := run("run", "old flow", "before archive", "--yes", "--json", "--workspace", workspace) + if err != nil { + t.Fatal(output, err) + } + var started protocol.Message + if err = json.Unmarshal([]byte(output), &started); err != nil || started.RunDetail == nil { + t.Fatal(output, err) + } + runID := started.RunDetail.ID + if detail := waitTerminal(t, r, runID); detail.Status != "failed" { + t.Fatal(detail.Status) + } + failing.Store(false) + items, _ := store.List(ctx) + schedule := protocol.SavedSchedule{Name: "nightly", PipelineID: items[1].ID, Version: 1, Enabled: true, Workspace: workspace, Input: map[string]any{"prompt": "go"}, Rule: protocol.Schedule{Every: "24h"}} + if items[1].Name != "old flow" { + t.Fatal(items) + } + saved, err := store.SaveSchedule(ctx, schedule, time.Now()) + if err != nil { + t.Fatal(err) + } + if output, err = run("archive", "old flow", "--yes"); err == nil || !strings.Contains(err.Error(), `"nightly"`) { + t.Fatal("archive with an enabled schedule not refused by name", output, err) + } + saved.Enabled = false + if _, err = store.SaveSchedule(ctx, *saved, time.Now()); err != nil { + t.Fatal(err) + } + if output, err = run("archive", "old flow", "--yes"); err != nil || !strings.Contains(output, "Archived old flow") { + t.Fatal(output, err) + } + if output, _ = run("list"); strings.Contains(output, "old flow") || !strings.Contains(output, "current") { + t.Fatal("default list shows archived pipeline", output) + } + if output, _ = run("list", "--archived"); !strings.Contains(output, "old flow\tv1\t1 stages\tarchived ") || strings.Contains(output, "current") { + t.Fatal("list --archived", output) + } + if output, _ = run("list", "--all"); !strings.Contains(output, "old flow") || !strings.Contains(output, "current") { + t.Fatal("list --all", output) + } + if output, err = run("run", "old flow", "after archive", "--yes", "--json", "--workspace", workspace); err == nil || !strings.Contains(err.Error(), "is archived; unarchive it first") { + t.Fatal("archived pipeline started a run", output, err) + } + if output, err = run("retry", fmt.Sprint(runID), "--yes", "--mode", "restart", "--json"); err == nil || !strings.Contains(err.Error(), "archived") { + t.Fatal("archived pipeline retried a run", output, err) + } + // History stays: the run, its inspection, filtering by name and search. + if output, err = run("inspect", fmt.Sprint(runID), "--json"); err != nil || !strings.Contains(output, `"old flow"`) { + t.Fatal("archived run inspection", output, err) + } + if output, err = run("runs", "--pipeline", "old flow"); err != nil || !strings.Contains(output, "old flow") { + t.Fatal("archived run history", output, err) + } + if output, err = run("show", "old flow", "--json"); err != nil || !strings.Contains(output, "archived_at") { + t.Fatal("archived show", output, err) + } + if output, err = run("search", "old flow", "--type", "run"); err != nil || !strings.Contains(output, fmt.Sprintf("run:%d", runID)) { + t.Fatal("archived run search", output, err) + } + if output, err = run("new", "old flow", "--no-open", "--yes"); err == nil || !strings.Contains(err.Error(), "archived") { + t.Fatal("archived name reused by new", output, err) + } + if output, err = run("unarchive", "old flow"); err != nil || !strings.Contains(output, "Unarchived old flow") { + t.Fatal(output, err) + } + if output, err = run("run", "old flow", "after unarchive", "--yes", "--json", "--workspace", workspace); err != nil { + t.Fatal("unarchived pipeline cannot run", output, err) + } +} + +func TestArchiveProtocolByNameReportsDetail(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + store, err := pipeline.Open(filepath.Join(t.TempDir(), "archive-rpc.db")) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if _, err = store.Submit(ctx, "retire", pipeline.Template([]string{"one"}, true)); err != nil { + t.Fatal(err) + } + handler := HandlerWithOptions(ctx, Options{}, store) + defer func() { cancel(); WaitRuns(handler) }() + srv := httptest.NewServer(handler) + defer srv.Close() + var out bytes.Buffer + app := cli.App{Out: &out, Err: &out, In: strings.NewReader(`{"type":"archive_pipeline","name":"retire","expected_version":1}`)} + if err = app.Run(ctx, []string{"--url", "ws" + strings.TrimPrefix(srv.URL, "http") + "/connect", "rpc", "-"}); err != nil { + t.Fatal(out.String(), err) + } + var reply protocol.Message + if err = json.Unmarshal(out.Bytes(), &reply); err != nil || reply.Type != protocol.PipelineArchived || reply.Detail == nil || reply.Detail.ArchivedAt == "" { + t.Fatal("archive reply", out.String(), err) + } + all, _ := store.ListPipelines(ctx, true) + if len(all) != 1 || all[0].ArchivedAt == "" { + t.Fatal(all) + } +} diff --git a/internal/server/runner.go b/internal/server/runner.go index 6c4dd2bb..a9339077 100644 --- a/internal/server/runner.go +++ b/internal/server/runner.go @@ -185,6 +185,9 @@ func (r *runner) startWithSchedule(request protocol.Message, origin *pipeline.Sc if err != nil { return nil, err } + if d.ArchivedAt != "" { + return nil, &pipeline.ArchivedError{Name: d.Name} + } if _, err := pipeline.Parse(pipeline.ExecutionSource(d.Source)); err != nil { return nil, err } diff --git a/internal/server/websocket.go b/internal/server/websocket.go index 319e6b6f..68c680dd 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -350,13 +350,24 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin } else if request.Type == protocol.List && len(stores) > 0 { response.Type = protocol.PipelineList response.DefaultModel, response.DefaultHarness = starterDefaults() - items, err := stores[0].List(ctx) + items, err := stores[0].ListPipelines(ctx, request.IncludeArchived) if err != nil { response.Type = protocol.Error response.Error = err.Error() } else { response.Pipelines = items } + } else if (request.Type == protocol.ArchivePipeline || request.Type == protocol.UnarchivePipeline) && len(stores) > 0 { + detail, err := archivePipeline(ctx, stores[0], request) + response.Detail = detail + response.Type = protocol.PipelineArchived + if request.Type == protocol.UnarchivePipeline { + response.Type = protocol.PipelineUnarchived + } + if err != nil { + response.Type = protocol.Error + response.Error = err.Error() + } } else if request.Type == protocol.Search { if len(stores) == 0 { response.Type = protocol.Error