764 lines
28 KiB
Go
764 lines
28 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// RetentionPolicy selects run history an operator has asked to remove. At least
|
|
// one bound must be set:
|
|
//
|
|
// - OlderThan selects runs created strictly before now-OlderThan (a rolling
|
|
// UTC duration, never a calendar-day boundary). Zero means no age bound.
|
|
// - Keep retains the newest Keep runs by run id and selects every older rank
|
|
// (N+1 and beyond). -1 means no count bound; 0 is the explicit "keep none".
|
|
//
|
|
// When both are set the selection is their conservative intersection: a run is
|
|
// selected only when it is both older than the age bound and outside the newest
|
|
// Keep. Selecting a run is not the same as removing it: a protected run is
|
|
// never in the removal set.
|
|
type RetentionPolicy struct {
|
|
OlderThan time.Duration `json:"older_than"`
|
|
Keep int `json:"keep"`
|
|
}
|
|
|
|
// Validate reports whether a policy is well-formed: at least one bound is set,
|
|
// the age is not negative and the count is not below the -1 "unbounded" sentinel.
|
|
func (p RetentionPolicy) Validate() error {
|
|
if p.OlderThan < 0 {
|
|
return fmt.Errorf("retention age must not be negative")
|
|
}
|
|
if p.Keep < -1 {
|
|
return fmt.Errorf("retention count must not be negative")
|
|
}
|
|
if p.OlderThan == 0 && p.Keep < 0 {
|
|
return fmt.Errorf("retention policy requires an age bound, a count bound, or both")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// RetentionArtifacts is the stat-only size of one run's artifact tree.
|
|
type RetentionArtifacts struct {
|
|
Files int64 `json:"files"`
|
|
Bytes int64 `json:"bytes"`
|
|
Partial bool `json:"partial,omitempty"`
|
|
}
|
|
|
|
// RetentionRun is one run selected by the policy for removal, with the measured
|
|
// size of the artifacts that would be removed with it.
|
|
type RetentionRun struct {
|
|
ID int64 `json:"id"`
|
|
Status string `json:"status"`
|
|
CreatedAt string `json:"created_at"`
|
|
Artifacts RetentionArtifacts `json:"artifacts"`
|
|
}
|
|
|
|
// ProtectedRun is a run the policy selected but that must not be removed,
|
|
// with the reason it is protected.
|
|
type ProtectedRun struct {
|
|
ID int64 `json:"id"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
|
|
// RetentionReport is the read-only description of what a policy would remove.
|
|
// WouldRemove and Protected partition the policy-selected runs; RetainedRuns
|
|
// counts the runs the policy never selected.
|
|
type RetentionReport struct {
|
|
Now string `json:"now"`
|
|
Policy RetentionPolicy `json:"policy"`
|
|
Database string `json:"database"`
|
|
ArtifactsRoot string `json:"artifacts_root"`
|
|
TotalRuns int `json:"total_runs"`
|
|
RetainedRuns int `json:"retained_runs"`
|
|
WouldRemove []RetentionRun `json:"would_remove"`
|
|
Protected []ProtectedRun `json:"protected"`
|
|
ArtifactFiles int64 `json:"artifact_files"`
|
|
ArtifactBytes int64 `json:"artifact_bytes"`
|
|
ArtifactsPartial bool `json:"artifacts_partial,omitempty"`
|
|
ArtifactWalkErrors []string `json:"artifact_walk_errors,omitempty"`
|
|
DatabaseBytes int64 `json:"database_bytes"`
|
|
WALBytes int64 `json:"wal_bytes"`
|
|
DatabaseFreeBytes int64 `json:"database_free_bytes"`
|
|
Pipelines int `json:"pipelines"`
|
|
PipelineVersions int `json:"pipeline_versions"`
|
|
ReclaimableBytes int64 `json:"reclaimable_bytes"`
|
|
// Applied is always false for a report; an apply sets it true once it has
|
|
// actually removed run history.
|
|
Applied bool `json:"applied"`
|
|
}
|
|
|
|
// RetentionApplyResult is the report plus what an apply actually removed and
|
|
// the measured database space it reclaimed. It embeds the pre-removal report so
|
|
// refusal and vacuum errors still carry the selection and reasons.
|
|
type RetentionApplyResult struct {
|
|
RetentionReport
|
|
RemovedRuns []int64 `json:"removed_runs"`
|
|
RemovedRunCount int `json:"removed_run_count"`
|
|
RemovedArtifactFiles int64 `json:"removed_artifact_files"`
|
|
RemovedArtifactBytes int64 `json:"removed_artifact_bytes"`
|
|
Vacuumed bool `json:"vacuumed"`
|
|
VacuumError string `json:"vacuum_error,omitempty"`
|
|
DatabaseBytesAfter int64 `json:"database_bytes_after"`
|
|
DatabaseFreeBytesAfter int64 `json:"database_free_bytes_after"`
|
|
WALBytesAfter int64 `json:"wal_bytes_after"`
|
|
DatabaseBytesReclaimed int64 `json:"database_bytes_reclaimed"`
|
|
SpaceReclaimedBytes int64 `json:"space_reclaimed_bytes"`
|
|
StagingPath string `json:"staging_path,omitempty"`
|
|
StagingRemainingFiles int64 `json:"staging_remaining_files,omitempty"`
|
|
StagingRemainingBytes int64 `json:"staging_remaining_bytes,omitempty"`
|
|
}
|
|
|
|
// retentionQuerier is the read surface shared by *sql.DB, *sql.Tx and *sql.Conn,
|
|
// so the selection runs identically against the report connection and inside
|
|
// the apply write transaction without a second query projection.
|
|
type retentionQuerier interface {
|
|
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
|
|
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
|
|
}
|
|
|
|
// retentionRunInfo is one pipeline_runs row plus the step aggregates protection
|
|
// needs, read in the same statement so the report never issues a per-run query.
|
|
type retentionRunInfo struct {
|
|
id int64
|
|
status string
|
|
createdAt time.Time
|
|
waitingSteps int
|
|
activeSteps int
|
|
}
|
|
|
|
// protectionReasons lists why a run must not be removed for a reason intrinsic
|
|
// to itself: it is active, waiting on a person, or still running a cleanup agent.
|
|
func (r retentionRunInfo) protectionReasons() []string {
|
|
var reasons []string
|
|
switch r.status {
|
|
case protocol.RunStatusQueued:
|
|
reasons = append(reasons, "run is queued")
|
|
case protocol.RunStatusRunning:
|
|
reasons = append(reasons, "run is running")
|
|
case protocol.RunStatusWaiting:
|
|
reasons = append(reasons, "run is waiting on a person")
|
|
}
|
|
if r.waitingSteps > 0 {
|
|
reasons = append(reasons, "a step is waiting on a person")
|
|
}
|
|
if r.activeSteps > 0 {
|
|
reasons = append(reasons, "cleanup agent still active (agent_active=1)")
|
|
}
|
|
return reasons
|
|
}
|
|
|
|
// retentionSelection is the raw run rows and retry edges the policy is applied
|
|
// to. It is deliberately independent of the removal decision so the same
|
|
// selection can be recomputed inside the apply transaction.
|
|
type retentionSelection struct {
|
|
runs []retentionRunInfo // ordered by run id DESC
|
|
edges [][2]int64 // [retry run id, original run id]
|
|
}
|
|
|
|
// selectRetentionRuns reads every run's status, creation time and step
|
|
// protection aggregates in one statement (plus one retry-edge statement), so the
|
|
// report is a bounded number of round trips rather than one query per run.
|
|
func selectRetentionRuns(ctx context.Context, q retentionQuerier, policy RetentionPolicy, now time.Time) (retentionSelection, error) {
|
|
var sel retentionSelection
|
|
rows, err := q.QueryContext(ctx, `SELECT r.id,r.status,strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at),
|
|
COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.status=?),0),
|
|
COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_active=1),0) `+
|
|
runFrom+` ORDER BY r.id DESC`, protocol.StepStatusWaiting)
|
|
if err != nil {
|
|
return sel, err
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
var run retentionRunInfo
|
|
var created string
|
|
if err := rows.Scan(&run.id, &run.status, &created, &run.waitingSteps, &run.activeSteps); err != nil {
|
|
return sel, err
|
|
}
|
|
run.createdAt = parseRetentionTime(created)
|
|
sel.runs = append(sel.runs, run)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return sel, err
|
|
}
|
|
edges, err := q.QueryContext(ctx, `SELECT run_id,original_run_id FROM run_retries`)
|
|
if err != nil {
|
|
// A database predating retry provenance has no run_retries table; that is
|
|
// an empty edge set, not a report failure.
|
|
if !strings.Contains(err.Error(), "no such table") {
|
|
return sel, err
|
|
}
|
|
return sel, nil
|
|
}
|
|
defer edges.Close()
|
|
for edges.Next() {
|
|
var retry, original int64
|
|
if err := edges.Scan(&retry, &original); err != nil {
|
|
return sel, err
|
|
}
|
|
sel.edges = append(sel.edges, [2]int64{retry, original})
|
|
}
|
|
return sel, edges.Err()
|
|
}
|
|
|
|
// parseRetentionTime reads the ISO timestamp the selection quotes, accepting the
|
|
// stored space-separated UTC form as a fallback. Values without a zone are UTC.
|
|
func parseRetentionTime(value string) time.Time {
|
|
for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05Z", "2006-01-02 15:04:05"} {
|
|
if parsed, err := time.Parse(layout, value); err == nil {
|
|
return parsed.UTC()
|
|
}
|
|
}
|
|
return time.Time{}
|
|
}
|
|
|
|
// policySelects reports whether the run at the given newest-first rank is chosen
|
|
// by the policy. Rank 0 is the newest run.
|
|
func policySelects(policy RetentionPolicy, info retentionRunInfo, rank int, now time.Time) bool {
|
|
if policy.OlderThan > 0 && !info.createdAt.Before(now.Add(-policy.OlderThan)) {
|
|
return false
|
|
}
|
|
if policy.Keep >= 0 && rank < policy.Keep {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// retentionDecision is the policy selection split into runs that would be
|
|
// removed and runs that are protected, with each protected run's reason.
|
|
type retentionDecision struct {
|
|
wouldRemove []retentionRunInfo
|
|
protected []ProtectedRun
|
|
}
|
|
|
|
// decide applies the policy and then protection. Protection is intrinsic state
|
|
// (active/waiting/agent) plus retry provenance propagated to a fixpoint: a run
|
|
// that remains and references a selected run protects it; if both a retry and
|
|
// its original are selected, no retained link remains and both may be removed.
|
|
func (s retentionSelection) decide(policy RetentionPolicy, now time.Time) retentionDecision {
|
|
selected := map[int64]bool{}
|
|
for rank, run := range s.runs {
|
|
if policySelects(policy, run, rank, now) {
|
|
selected[run.id] = true
|
|
}
|
|
}
|
|
reasons := map[int64][]string{}
|
|
remaining := map[int64]bool{}
|
|
for _, run := range s.runs {
|
|
if !selected[run.id] {
|
|
remaining[run.id] = true
|
|
continue
|
|
}
|
|
if found := run.protectionReasons(); len(found) > 0 {
|
|
reasons[run.id] = append(reasons[run.id], found...)
|
|
remaining[run.id] = true
|
|
}
|
|
}
|
|
for changed := true; changed; {
|
|
changed = false
|
|
for _, edge := range s.edges {
|
|
retry, original := edge[0], edge[1]
|
|
if !remaining[retry] || !selected[original] {
|
|
continue
|
|
}
|
|
if _, done := reasons[original]; done {
|
|
continue
|
|
}
|
|
reasons[original] = append(reasons[original], fmt.Sprintf("referenced by retained retry run #%d", retry))
|
|
remaining[original] = true
|
|
changed = true
|
|
}
|
|
}
|
|
decision := retentionDecision{}
|
|
for _, run := range s.runs {
|
|
if !selected[run.id] {
|
|
continue
|
|
}
|
|
if found, ok := reasons[run.id]; ok {
|
|
decision.protected = append(decision.protected, ProtectedRun{ID: run.id, Reason: strings.Join(found, "; ")})
|
|
continue
|
|
}
|
|
decision.wouldRemove = append(decision.wouldRemove, run)
|
|
}
|
|
return decision
|
|
}
|
|
|
|
// RetentionReport describes what a policy would remove without changing
|
|
// anything: no table write, VACUUM, checkpoint or temp table runs. It uses one
|
|
// run query, one retry-edge query, one definition/version count query and the
|
|
// page/freelist PRAGMAs plus file stats, so it is a bounded read-only pass.
|
|
func (s *Store) RetentionReport(ctx context.Context, policy RetentionPolicy, artifactsRoot string, now time.Time) (*RetentionReport, error) {
|
|
if err := policy.Validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
now = now.UTC()
|
|
selection, err := selectRetentionRuns(ctx, s.db, policy, now)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
decision := selection.decide(policy, now)
|
|
report := &RetentionReport{
|
|
Now: now.Format(time.RFC3339),
|
|
Policy: policy,
|
|
Database: s.path,
|
|
ArtifactsRoot: artifactsRoot,
|
|
TotalRuns: len(selection.runs),
|
|
WouldRemove: []RetentionRun{},
|
|
Protected: decision.protected,
|
|
}
|
|
if report.Protected == nil {
|
|
report.Protected = []ProtectedRun{}
|
|
}
|
|
partial := false
|
|
for _, run := range decision.wouldRemove {
|
|
files, bytes, errs := retentionArtifactsSize(artifactsRoot, run.id)
|
|
entry := RetentionRun{
|
|
ID: run.id,
|
|
Status: run.status,
|
|
CreatedAt: run.createdAt.Format(time.RFC3339),
|
|
Artifacts: RetentionArtifacts{Files: files, Bytes: bytes, Partial: len(errs) > 0},
|
|
}
|
|
if len(errs) > 0 {
|
|
partial = true
|
|
report.ArtifactWalkErrors = append(report.ArtifactWalkErrors, errs...)
|
|
}
|
|
report.ArtifactFiles += files
|
|
report.ArtifactBytes += bytes
|
|
report.WouldRemove = append(report.WouldRemove, entry)
|
|
}
|
|
report.ArtifactsPartial = partial
|
|
report.RetainedRuns = report.TotalRuns - len(report.WouldRemove) - len(report.Protected)
|
|
if report.DatabaseBytes, report.WALBytes, report.DatabaseFreeBytes, err = s.retentionDBSize(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := s.db.QueryRowContext(ctx, `SELECT (SELECT COUNT(*) FROM pipelines),(SELECT COUNT(*) FROM pipeline_versions)`).
|
|
Scan(&report.Pipelines, &report.PipelineVersions); err != nil {
|
|
return nil, err
|
|
}
|
|
report.ReclaimableBytes = report.ArtifactBytes + report.DatabaseFreeBytes
|
|
return report, nil
|
|
}
|
|
|
|
// retentionArtifactsSize stat-walks one run's artifact directory, counting every
|
|
// regular file including files the server's inventory caps did not record. It
|
|
// skips symlinks and non-regular entries, never reads file contents, and treats
|
|
// a missing directory as zero bytes. A symlink or file at the run path owns no
|
|
// tree under the root, so it is not followed. Walk errors are returned so the
|
|
// report can mark the size partial rather than fail.
|
|
func retentionArtifactsSize(root string, id int64) (int64, int64, []string) {
|
|
if root == "" {
|
|
return 0, 0, nil
|
|
}
|
|
dir := filepath.Join(root, strconv.FormatInt(id, 10))
|
|
info, err := os.Lstat(dir)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return 0, 0, nil
|
|
}
|
|
return 0, 0, []string{fmt.Sprintf("run %d: %v", id, err)}
|
|
}
|
|
// A symlink or file at the run path owns no tree under the root, so it is
|
|
// not walked; only a real run directory contributes artifact bytes.
|
|
if !info.IsDir() {
|
|
return 0, 0, nil
|
|
}
|
|
files, bytes, errs := retentionWalkSize(dir)
|
|
for i, walkErr := range errs {
|
|
errs[i] = fmt.Sprintf("run %d: %s", id, walkErr)
|
|
}
|
|
return files, bytes, errs
|
|
}
|
|
|
|
// retentionWalkSize stat-walks a directory counting every regular file and its
|
|
// bytes without reading any file contents. It skips directories, symlinks and
|
|
// other non-regular entries, accumulates only what it could stat, and returns
|
|
// every walk error so a caller can report a partial size rather than fail. It is
|
|
// the one walk shared by artifact sizing and the staging-cleanup report.
|
|
func retentionWalkSize(dir string) (int64, int64, []string) {
|
|
var files, bytes int64
|
|
var errs []string
|
|
walkErr := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
errs = append(errs, fmt.Sprintf("%s: %v", path, err))
|
|
return nil
|
|
}
|
|
if entry.IsDir() || !entry.Type().IsRegular() {
|
|
return nil
|
|
}
|
|
stat, statErr := entry.Info()
|
|
if statErr != nil {
|
|
errs = append(errs, fmt.Sprintf("%s: %v", path, statErr))
|
|
return nil
|
|
}
|
|
files++
|
|
bytes += stat.Size()
|
|
return nil
|
|
})
|
|
if walkErr != nil {
|
|
errs = append(errs, fmt.Sprintf("%s: %v", dir, walkErr))
|
|
}
|
|
return files, bytes, errs
|
|
}
|
|
|
|
// ApplyRetention removes exactly the runs the policy selected and is not
|
|
// protecting, together with their artifacts, then reclaims database space. It
|
|
// is the explicit operator command: nothing here runs automatically. A run that
|
|
// becomes protected between the report and the delete aborts the whole apply
|
|
// with the run ids and reasons, so a removal never half-completes.
|
|
func (s *Store) ApplyRetention(ctx context.Context, policy RetentionPolicy, artifactsRoot string, now time.Time) (*RetentionApplyResult, error) {
|
|
if err := policy.Validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
now = now.UTC()
|
|
report, err := s.RetentionReport(ctx, policy, artifactsRoot, now)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
result := &RetentionApplyResult{RetentionReport: *report, RemovedRuns: []int64{}}
|
|
ids := make([]int64, 0, len(report.WouldRemove))
|
|
for _, run := range report.WouldRemove {
|
|
ids = append(ids, run.ID)
|
|
}
|
|
if len(ids) > 0 {
|
|
if err := s.removeRetentionRuns(ctx, policy, artifactsRoot, now, ids, result); err != nil {
|
|
return result, err
|
|
}
|
|
}
|
|
// Apply is the explicit reclaim command: even with nothing to remove it may
|
|
// compact pages freed earlier. VACUUM failure is reported, never hidden.
|
|
beforeBytes := report.DatabaseBytes
|
|
if err := s.vacuumAndCheckpoint(ctx); err != nil {
|
|
result.VacuumError = err.Error()
|
|
// The run history and artifacts were already removed by
|
|
// removeRetentionRuns; report only the artifact space actually
|
|
// reclaimed and no database pages.
|
|
result.SpaceReclaimedBytes = result.RemovedArtifactBytes
|
|
return result, fmt.Errorf("reclaim database space: %w", err)
|
|
}
|
|
result.Vacuumed = true
|
|
afterBytes, afterWAL, afterFree, err := s.retentionDBSize(ctx)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
result.DatabaseBytesAfter = afterBytes
|
|
result.DatabaseFreeBytesAfter = afterFree
|
|
result.WALBytesAfter = afterWAL
|
|
if reclaimed := beforeBytes - afterBytes; reclaimed > 0 {
|
|
result.DatabaseBytesReclaimed = reclaimed
|
|
}
|
|
result.SpaceReclaimedBytes = result.RemovedArtifactBytes + result.DatabaseBytesReclaimed
|
|
result.Applied = result.RemovedRunCount > 0
|
|
return result, nil
|
|
}
|
|
|
|
// removeRetentionRuns performs the staged, single-transaction removal of the
|
|
// selected runs: re-check protection under an immediate write lock, rename the
|
|
// run artifact directories into one staging directory, delete the rows, commit,
|
|
// then delete the staged files. Every failure path restores what it can and
|
|
// leaves nothing half-removed.
|
|
func (s *Store) removeRetentionRuns(ctx context.Context, policy RetentionPolicy, artifactsRoot string, now time.Time, ids []int64, result *RetentionApplyResult) error {
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("begin retention transaction: %w", err)
|
|
}
|
|
committed := false
|
|
defer func() {
|
|
if !committed {
|
|
_ = tx.Rollback()
|
|
}
|
|
}()
|
|
|
|
// Re-check protection inside the write transaction: a run that gained a
|
|
// retry reference, a waiting state or agent activity since the report must
|
|
// abort the whole apply.
|
|
selection, err := selectRetentionRuns(ctx, tx, policy, now)
|
|
if err != nil {
|
|
return fmt.Errorf("re-check retention selection: %w", err)
|
|
}
|
|
decision := selection.decide(policy, now)
|
|
stillRemove := map[int64]bool{}
|
|
for _, run := range decision.wouldRemove {
|
|
stillRemove[run.id] = true
|
|
}
|
|
became := []string{}
|
|
for _, id := range ids {
|
|
if stillRemove[id] {
|
|
continue
|
|
}
|
|
reason := "run is no longer selected for removal"
|
|
for _, protected := range decision.protected {
|
|
if protected.ID == id {
|
|
reason = protected.Reason
|
|
}
|
|
}
|
|
became = append(became, fmt.Sprintf("run #%d: %s", id, reason))
|
|
}
|
|
if len(became) > 0 {
|
|
return fmt.Errorf("retention aborted: selected run(s) became protected before deletion; nothing was removed: %s", strings.Join(became, "; "))
|
|
}
|
|
|
|
staging := ""
|
|
staged := []string{}
|
|
if artifactsRoot != "" {
|
|
// Stage only inside an artifacts root that already exists: creating the
|
|
// root here would leave an empty artifacts directory behind after an
|
|
// apply that had no artifacts to remove, and with no root there is no
|
|
// <root>/<id> directory to stage anyway. os.Stat (not Lstat) follows a
|
|
// symlinked root so staging happens through the same directory the
|
|
// report sized; skipping a symlinked root would claim its bytes as
|
|
// removed while leaving every artifact in place.
|
|
if info, err := os.Stat(artifactsRoot); err != nil {
|
|
if !os.IsNotExist(err) {
|
|
return fmt.Errorf("inspect artifacts root %s: %w", artifactsRoot, err)
|
|
}
|
|
} else if info.IsDir() {
|
|
staging = filepath.Join(artifactsRoot, fmt.Sprintf(".retention-staging-%d-%d", now.Unix(), os.Getpid()))
|
|
if err := os.MkdirAll(staging, 0700); err != nil {
|
|
return fmt.Errorf("create retention staging directory: %w", err)
|
|
}
|
|
}
|
|
}
|
|
// unstage restores every renamed run directory and removes the staging
|
|
// directory, returning any restore/cleanup error so the rollback path cannot
|
|
// silently leave directories behind while only the original failure is named.
|
|
unstage := func() error {
|
|
var problems []string
|
|
for _, original := range staged {
|
|
if err := os.Rename(filepath.Join(staging, filepath.Base(original)), original); err != nil {
|
|
problems = append(problems, fmt.Sprintf("restore %s: %v", original, err))
|
|
}
|
|
}
|
|
if staging != "" {
|
|
if err := os.RemoveAll(staging); err != nil {
|
|
problems = append(problems, fmt.Sprintf("remove staging directory %s: %v", staging, err))
|
|
}
|
|
}
|
|
if len(problems) > 0 {
|
|
return fmt.Errorf("un-stage artifacts: %s", strings.Join(problems, "; "))
|
|
}
|
|
return nil
|
|
}
|
|
for _, id := range ids {
|
|
if staging == "" {
|
|
break
|
|
}
|
|
original := filepath.Join(artifactsRoot, strconv.FormatInt(id, 10))
|
|
info, lerr := os.Lstat(original)
|
|
if lerr != nil {
|
|
if os.IsNotExist(lerr) {
|
|
continue
|
|
}
|
|
return errors.Join(fmt.Errorf("inspect artifacts for run %d: %w", id, lerr), unstage())
|
|
}
|
|
// Only the exact numeric run entry is staged. A regular file at that path
|
|
// is refused rather than deleted; a symlink is renamed (removing the link
|
|
// only), never its target.
|
|
if !info.IsDir() && info.Mode()&os.ModeSymlink == 0 {
|
|
return errors.Join(fmt.Errorf("refusing to remove run %d: %s is a regular file, not a run artifact directory", id, original), unstage())
|
|
}
|
|
if err := os.Rename(original, filepath.Join(staging, strconv.FormatInt(id, 10))); err != nil {
|
|
return errors.Join(fmt.Errorf("stage artifacts for run %d: %w", id, err), unstage())
|
|
}
|
|
staged = append(staged, original)
|
|
}
|
|
|
|
if err := deleteRetentionRows(ctx, tx, ids); err != nil {
|
|
return errors.Join(fmt.Errorf("delete run history: %w", err), unstage())
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return errors.Join(fmt.Errorf("commit retention removal: %w", err), unstage())
|
|
}
|
|
committed = true
|
|
|
|
// The rows are committed: these runs are gone whatever happens to the
|
|
// staged files, so record exactly what was removed before any cleanup
|
|
// error can return, and never let a cleanup failure be reported as zero
|
|
// runs removed or zero reclaimed space. SpaceReclaimedBytes is set here so
|
|
// a later VACUUM failure still reports the artifacts actually removed.
|
|
result.RemovedRuns = append(result.RemovedRuns, ids...)
|
|
sort.Slice(result.RemovedRuns, func(i, j int) bool { return result.RemovedRuns[i] < result.RemovedRuns[j] })
|
|
result.RemovedRunCount = len(ids)
|
|
for _, run := range result.WouldRemove {
|
|
result.RemovedArtifactFiles += run.Artifacts.Files
|
|
result.RemovedArtifactBytes += run.Artifacts.Bytes
|
|
}
|
|
result.SpaceReclaimedBytes = result.RemovedArtifactBytes
|
|
result.Applied = result.RemovedRunCount > 0
|
|
|
|
if staging != "" {
|
|
if err := os.RemoveAll(staging); err != nil {
|
|
remainingFiles, remainingBytes, _ := retentionWalkSize(staging)
|
|
result.StagingPath = staging
|
|
result.StagingRemainingFiles = remainingFiles
|
|
result.StagingRemainingBytes = remainingBytes
|
|
// Files still in staging were staged out of the run tree but not
|
|
// deleted, so they are not reclaimed space.
|
|
if reclaimed := result.RemovedArtifactBytes - remainingBytes; reclaimed > 0 {
|
|
result.SpaceReclaimedBytes = reclaimed
|
|
} else {
|
|
result.SpaceReclaimedBytes = 0
|
|
}
|
|
return fmt.Errorf("run history removed but staged artifact cleanup failed at %s (%d files, %d bytes remain and are not counted as reclaimed): %w", staging, remainingFiles, remainingBytes, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// deleteRetentionRows removes every table row scoped to the selected runs, in
|
|
// foreign-key order, batched at 500 ids per statement in the caller's single
|
|
// transaction: step-scoped evidence first, then run_steps, then retry edges,
|
|
// then the runs.
|
|
func deleteRetentionRows(ctx context.Context, tx *sql.Tx, ids []int64) error {
|
|
for _, table := range []struct{ name, column string }{
|
|
{"step_notifications", "run_id"},
|
|
{"run_notifications", "run_id"},
|
|
{"step_file_changes", "run_id"},
|
|
{"step_control", "run_id"},
|
|
{"step_artifacts", "run_id"},
|
|
{"step_verdicts", "run_id"},
|
|
{"step_session_exports", "run_id"},
|
|
{"step_usage", "run_id"},
|
|
{"run_session_owners", "run_id"},
|
|
{"run_steps", "run_id"},
|
|
} {
|
|
if err := deleteRetentionBatch(ctx, tx, table.name, table.column, ids); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// Retry provenance is removed when either end is selected, so a removed
|
|
// original leaves no dangling edge and a removed retry leaves none either.
|
|
for start := 0; start < len(ids); start += retentionDeleteBatch {
|
|
batch := ids[start:min(start+retentionDeleteBatch, len(ids))]
|
|
placeholders, args := retentionBindings(batch, 2)
|
|
if _, err := tx.ExecContext(ctx, "DELETE FROM run_retries WHERE run_id IN ("+placeholders+") OR original_run_id IN ("+placeholders+")", args...); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return deleteRetentionBatch(ctx, tx, "pipeline_runs", "id", ids)
|
|
}
|
|
|
|
// retentionDeleteBatch bounds each IN clause so a large removal does not build
|
|
// an unbounded statement.
|
|
const retentionDeleteBatch = 500
|
|
|
|
// retentionBindings builds an "?,?,..." placeholder list for len(ids) values and
|
|
// returns args as copies repeats of ids, so a statement with several IN clauses
|
|
// over the same id set shares one construction instead of duplicating it.
|
|
func retentionBindings(ids []int64, copies int) (string, []any) {
|
|
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",")
|
|
args := make([]any, 0, len(ids)*copies)
|
|
for i := 0; i < copies; i++ {
|
|
for _, id := range ids {
|
|
args = append(args, id)
|
|
}
|
|
}
|
|
return placeholders, args
|
|
}
|
|
|
|
func deleteRetentionBatch(ctx context.Context, tx *sql.Tx, table, column string, ids []int64) error {
|
|
for start := 0; start < len(ids); start += retentionDeleteBatch {
|
|
batch := ids[start:min(start+retentionDeleteBatch, len(ids))]
|
|
placeholders, args := retentionBindings(batch, 1)
|
|
if _, err := tx.ExecContext(ctx, "DELETE FROM "+table+" WHERE "+column+" IN ("+placeholders+")", args...); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// retentionDBSize measures the database file, its -wal sidecar and the free
|
|
// pages SQLite reports, in bytes. It is a measurement, never an estimate.
|
|
func (s *Store) retentionDBSize(ctx context.Context) (int64, int64, int64, error) {
|
|
var free int64
|
|
var pageSize, freelist int64
|
|
if err := s.db.QueryRowContext(ctx, "PRAGMA page_size").Scan(&pageSize); err != nil {
|
|
return 0, 0, 0, err
|
|
}
|
|
if err := s.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&freelist); err != nil {
|
|
return 0, 0, 0, err
|
|
}
|
|
free = pageSize * freelist
|
|
bytes, err := fileSize(s.path)
|
|
if err != nil {
|
|
return 0, 0, 0, err
|
|
}
|
|
wal, err := fileSize(s.path + "-wal")
|
|
if err != nil {
|
|
return 0, 0, 0, err
|
|
}
|
|
return bytes, wal, free, nil
|
|
}
|
|
|
|
func fileSize(path string) (int64, error) {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return 0, nil
|
|
}
|
|
return 0, err
|
|
}
|
|
if !info.Mode().IsRegular() {
|
|
return 0, nil
|
|
}
|
|
return info.Size(), nil
|
|
}
|
|
|
|
// vacuumAndCheckpoint reclaims freed pages and truncates the write-ahead log.
|
|
// SQLITE_BUSY is retried a bounded number of times; any other failure, or a
|
|
// still-busy database, is returned as an honest error rather than a hang.
|
|
func (s *Store) vacuumAndCheckpoint(ctx context.Context) error {
|
|
if err := s.retryBusy(ctx, func() error {
|
|
_, err := s.db.ExecContext(ctx, "VACUUM")
|
|
return err
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
return s.retryBusy(ctx, func() error {
|
|
var busy, logFrames, checkpointed int
|
|
if err := s.db.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logFrames, &checkpointed); err != nil {
|
|
return err
|
|
}
|
|
if busy != 0 {
|
|
return fmt.Errorf("wal checkpoint busy")
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// retentionBusyAttempts bounds VACUUM/checkpoint retries; a locked database is
|
|
// reported, never retried forever.
|
|
const retentionBusyAttempts = 3
|
|
|
|
func (s *Store) retryBusy(ctx context.Context, action func() error) error {
|
|
var err error
|
|
for attempt := 0; attempt < retentionBusyAttempts; attempt++ {
|
|
if err = action(); err == nil || !isBusyError(err) {
|
|
return err
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(time.Duration(attempt+1) * 200 * time.Millisecond):
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
|
|
func isBusyError(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
message := strings.ToLower(err.Error())
|
|
return strings.Contains(message, "busy") || strings.Contains(message, "locked")
|
|
}
|