package pipeline import ( "context" "database/sql" "errors" "fmt" "io/fs" "os" "path/filepath" "sort" "strconv" "strings" "time" "git.a-wan.net/alan/pipelines/internal/protocol" ) // RetentionPolicy selects run history an operator has asked to remove. At least // one bound must be set: // // - OlderThan selects runs created strictly before now-OlderThan (a rolling // UTC duration, never a calendar-day boundary). Zero means no age bound. // - Keep retains the newest Keep runs by run id and selects every older rank // (N+1 and beyond). -1 means no count bound; 0 is the explicit "keep none". // // When both are set the selection is their conservative intersection: a run is // selected only when it is both older than the age bound and outside the newest // Keep. Selecting a run is not the same as removing it: a protected run is // never in the removal set. type RetentionPolicy struct { OlderThan time.Duration `json:"older_than"` Keep int `json:"keep"` } // Validate reports whether a policy is well-formed: at least one bound is set, // the age is not negative and the count is not below the -1 "unbounded" sentinel. func (p RetentionPolicy) Validate() error { if p.OlderThan < 0 { return fmt.Errorf("retention age must not be negative") } if p.Keep < -1 { return fmt.Errorf("retention count must not be negative") } if p.OlderThan == 0 && p.Keep < 0 { return fmt.Errorf("retention policy requires an age bound, a count bound, or both") } return nil } // RetentionArtifacts is the stat-only size of one run's artifact tree. type RetentionArtifacts struct { Files int64 `json:"files"` Bytes int64 `json:"bytes"` Partial bool `json:"partial,omitempty"` } // RetentionRun is one run selected by the policy for removal, with the measured // size of the artifacts that would be removed with it. type RetentionRun struct { ID int64 `json:"id"` Status string `json:"status"` CreatedAt string `json:"created_at"` Artifacts RetentionArtifacts `json:"artifacts"` } // ProtectedRun is a run the policy selected but that must not be removed, // with the reason it is protected. type ProtectedRun struct { ID int64 `json:"id"` Reason string `json:"reason"` } // RetentionReport is the read-only description of what a policy would remove. // WouldRemove and Protected partition the policy-selected runs; RetainedRuns // counts the runs the policy never selected. type RetentionReport struct { Now string `json:"now"` Policy RetentionPolicy `json:"policy"` Database string `json:"database"` ArtifactsRoot string `json:"artifacts_root"` TotalRuns int `json:"total_runs"` RetainedRuns int `json:"retained_runs"` WouldRemove []RetentionRun `json:"would_remove"` Protected []ProtectedRun `json:"protected"` ArtifactFiles int64 `json:"artifact_files"` ArtifactBytes int64 `json:"artifact_bytes"` ArtifactsPartial bool `json:"artifacts_partial,omitempty"` ArtifactWalkErrors []string `json:"artifact_walk_errors,omitempty"` DatabaseBytes int64 `json:"database_bytes"` WALBytes int64 `json:"wal_bytes"` DatabaseFreeBytes int64 `json:"database_free_bytes"` Pipelines int `json:"pipelines"` PipelineVersions int `json:"pipeline_versions"` ReclaimableBytes int64 `json:"reclaimable_bytes"` // Applied is always false for a report; an apply sets it true once it has // actually removed run history. Applied bool `json:"applied"` } // RetentionApplyResult is the report plus what an apply actually removed and // the measured database space it reclaimed. It embeds the pre-removal report so // refusal and vacuum errors still carry the selection and reasons. type RetentionApplyResult struct { RetentionReport RemovedRuns []int64 `json:"removed_runs"` RemovedRunCount int `json:"removed_run_count"` RemovedArtifactFiles int64 `json:"removed_artifact_files"` RemovedArtifactBytes int64 `json:"removed_artifact_bytes"` Vacuumed bool `json:"vacuumed"` VacuumError string `json:"vacuum_error,omitempty"` DatabaseBytesAfter int64 `json:"database_bytes_after"` DatabaseFreeBytesAfter int64 `json:"database_free_bytes_after"` WALBytesAfter int64 `json:"wal_bytes_after"` DatabaseBytesReclaimed int64 `json:"database_bytes_reclaimed"` SpaceReclaimedBytes int64 `json:"space_reclaimed_bytes"` StagingPath string `json:"staging_path,omitempty"` StagingRemainingFiles int64 `json:"staging_remaining_files,omitempty"` StagingRemainingBytes int64 `json:"staging_remaining_bytes,omitempty"` } // retentionQuerier is the read surface shared by *sql.DB, *sql.Tx and *sql.Conn, // so the selection runs identically against the report connection and inside // the apply write transaction without a second query projection. type retentionQuerier interface { QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row } // retentionRunInfo is one pipeline_runs row plus the step aggregates protection // needs, read in the same statement so the report never issues a per-run query. type retentionRunInfo struct { id int64 status string createdAt time.Time waitingSteps int activeSteps int } // protectionReasons lists why a run must not be removed for a reason intrinsic // to itself: it is active, waiting on a person, or still running a cleanup agent. func (r retentionRunInfo) protectionReasons() []string { var reasons []string switch r.status { case protocol.RunStatusQueued: reasons = append(reasons, "run is queued") case protocol.RunStatusRunning: reasons = append(reasons, "run is running") case protocol.RunStatusWaiting: reasons = append(reasons, "run is waiting on a person") } if r.waitingSteps > 0 { reasons = append(reasons, "a step is waiting on a person") } if r.activeSteps > 0 { reasons = append(reasons, "cleanup agent still active (agent_active=1)") } return reasons } // retentionSelection is the raw run rows and retry edges the policy is applied // to. It is deliberately independent of the removal decision so the same // selection can be recomputed inside the apply transaction. type retentionSelection struct { runs []retentionRunInfo // ordered by run id DESC edges [][2]int64 // [retry run id, original run id] } // selectRetentionRuns reads every run's status, creation time and step // protection aggregates in one statement (plus one retry-edge statement), so the // report is a bounded number of round trips rather than one query per run. func selectRetentionRuns(ctx context.Context, q retentionQuerier, policy RetentionPolicy, now time.Time) (retentionSelection, error) { var sel retentionSelection rows, err := q.QueryContext(ctx, `SELECT r.id,r.status,strftime('%Y-%m-%dT%H:%M:%SZ',r.created_at), COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.status=?),0), COALESCE((SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND s.agent_active=1),0) `+ runFrom+` ORDER BY r.id DESC`, protocol.StepStatusWaiting) if err != nil { return sel, err } defer rows.Close() for rows.Next() { var run retentionRunInfo var created string if err := rows.Scan(&run.id, &run.status, &created, &run.waitingSteps, &run.activeSteps); err != nil { return sel, err } run.createdAt = parseRetentionTime(created) sel.runs = append(sel.runs, run) } if err := rows.Err(); err != nil { return sel, err } edges, err := q.QueryContext(ctx, `SELECT run_id,original_run_id FROM run_retries`) if err != nil { // A database predating retry provenance has no run_retries table; that is // an empty edge set, not a report failure. if !strings.Contains(err.Error(), "no such table") { return sel, err } return sel, nil } defer edges.Close() for edges.Next() { var retry, original int64 if err := edges.Scan(&retry, &original); err != nil { return sel, err } sel.edges = append(sel.edges, [2]int64{retry, original}) } return sel, edges.Err() } // parseRetentionTime reads the ISO timestamp the selection quotes, accepting the // stored space-separated UTC form as a fallback. Values without a zone are UTC. func parseRetentionTime(value string) time.Time { for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05Z", "2006-01-02 15:04:05"} { if parsed, err := time.Parse(layout, value); err == nil { return parsed.UTC() } } return time.Time{} } // policySelects reports whether the run at the given newest-first rank is chosen // by the policy. Rank 0 is the newest run. func policySelects(policy RetentionPolicy, info retentionRunInfo, rank int, now time.Time) bool { if policy.OlderThan > 0 && !info.createdAt.Before(now.Add(-policy.OlderThan)) { return false } if policy.Keep >= 0 && rank < policy.Keep { return false } return true } // retentionDecision is the policy selection split into runs that would be // removed and runs that are protected, with each protected run's reason. type retentionDecision struct { wouldRemove []retentionRunInfo protected []ProtectedRun } // decide applies the policy and then protection. Protection is intrinsic state // (active/waiting/agent) plus retry provenance propagated to a fixpoint: a run // that remains and references a selected run protects it; if both a retry and // its original are selected, no retained link remains and both may be removed. func (s retentionSelection) decide(policy RetentionPolicy, now time.Time) retentionDecision { selected := map[int64]bool{} for rank, run := range s.runs { if policySelects(policy, run, rank, now) { selected[run.id] = true } } reasons := map[int64][]string{} remaining := map[int64]bool{} for _, run := range s.runs { if !selected[run.id] { remaining[run.id] = true continue } if found := run.protectionReasons(); len(found) > 0 { reasons[run.id] = append(reasons[run.id], found...) remaining[run.id] = true } } for changed := true; changed; { changed = false for _, edge := range s.edges { retry, original := edge[0], edge[1] if !remaining[retry] || !selected[original] { continue } if _, done := reasons[original]; done { continue } reasons[original] = append(reasons[original], fmt.Sprintf("referenced by retained retry run #%d", retry)) remaining[original] = true changed = true } } decision := retentionDecision{} for _, run := range s.runs { if !selected[run.id] { continue } if found, ok := reasons[run.id]; ok { decision.protected = append(decision.protected, ProtectedRun{ID: run.id, Reason: strings.Join(found, "; ")}) continue } decision.wouldRemove = append(decision.wouldRemove, run) } return decision } // RetentionReport describes what a policy would remove without changing // anything: no table write, VACUUM, checkpoint or temp table runs. It uses one // run query, one retry-edge query, one definition/version count query and the // page/freelist PRAGMAs plus file stats, so it is a bounded read-only pass. func (s *Store) RetentionReport(ctx context.Context, policy RetentionPolicy, artifactsRoot string, now time.Time) (*RetentionReport, error) { if err := policy.Validate(); err != nil { return nil, err } now = now.UTC() selection, err := selectRetentionRuns(ctx, s.db, policy, now) if err != nil { return nil, err } decision := selection.decide(policy, now) report := &RetentionReport{ Now: now.Format(time.RFC3339), Policy: policy, Database: s.path, ArtifactsRoot: artifactsRoot, TotalRuns: len(selection.runs), WouldRemove: []RetentionRun{}, Protected: decision.protected, } if report.Protected == nil { report.Protected = []ProtectedRun{} } partial := false for _, run := range decision.wouldRemove { files, bytes, errs := retentionArtifactsSize(artifactsRoot, run.id) entry := RetentionRun{ ID: run.id, Status: run.status, CreatedAt: run.createdAt.Format(time.RFC3339), Artifacts: RetentionArtifacts{Files: files, Bytes: bytes, Partial: len(errs) > 0}, } if len(errs) > 0 { partial = true report.ArtifactWalkErrors = append(report.ArtifactWalkErrors, errs...) } report.ArtifactFiles += files report.ArtifactBytes += bytes report.WouldRemove = append(report.WouldRemove, entry) } report.ArtifactsPartial = partial report.RetainedRuns = report.TotalRuns - len(report.WouldRemove) - len(report.Protected) if report.DatabaseBytes, report.WALBytes, report.DatabaseFreeBytes, err = s.retentionDBSize(ctx); err != nil { return nil, err } if err := s.db.QueryRowContext(ctx, `SELECT (SELECT COUNT(*) FROM pipelines),(SELECT COUNT(*) FROM pipeline_versions)`). Scan(&report.Pipelines, &report.PipelineVersions); err != nil { return nil, err } report.ReclaimableBytes = report.ArtifactBytes + report.DatabaseFreeBytes return report, nil } // retentionArtifactsSize stat-walks one run's artifact directory, counting every // regular file including files the server's inventory caps did not record. It // skips symlinks and non-regular entries, never reads file contents, and treats // a missing directory as zero bytes. A symlink or file at the run path owns no // tree under the root, so it is not followed. Walk errors are returned so the // report can mark the size partial rather than fail. func retentionArtifactsSize(root string, id int64) (int64, int64, []string) { if root == "" { return 0, 0, nil } dir := filepath.Join(root, strconv.FormatInt(id, 10)) info, err := os.Lstat(dir) if err != nil { if os.IsNotExist(err) { return 0, 0, nil } return 0, 0, []string{fmt.Sprintf("run %d: %v", id, err)} } // A symlink or file at the run path owns no tree under the root, so it is // not walked; only a real run directory contributes artifact bytes. if !info.IsDir() { return 0, 0, nil } files, bytes, errs := retentionWalkSize(dir) for i, walkErr := range errs { errs[i] = fmt.Sprintf("run %d: %s", id, walkErr) } return files, bytes, errs } // retentionWalkSize stat-walks a directory counting every regular file and its // bytes without reading any file contents. It skips directories, symlinks and // other non-regular entries, accumulates only what it could stat, and returns // every walk error so a caller can report a partial size rather than fail. It is // the one walk shared by artifact sizing and the staging-cleanup report. func retentionWalkSize(dir string) (int64, int64, []string) { var files, bytes int64 var errs []string walkErr := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error { if err != nil { errs = append(errs, fmt.Sprintf("%s: %v", path, err)) return nil } if entry.IsDir() || !entry.Type().IsRegular() { return nil } stat, statErr := entry.Info() if statErr != nil { errs = append(errs, fmt.Sprintf("%s: %v", path, statErr)) return nil } files++ bytes += stat.Size() return nil }) if walkErr != nil { errs = append(errs, fmt.Sprintf("%s: %v", dir, walkErr)) } return files, bytes, errs } // ApplyRetention removes exactly the runs the policy selected and is not // protecting, together with their artifacts, then reclaims database space. It // is the explicit operator command: nothing here runs automatically. A run that // becomes protected between the report and the delete aborts the whole apply // with the run ids and reasons, so a removal never half-completes. func (s *Store) ApplyRetention(ctx context.Context, policy RetentionPolicy, artifactsRoot string, now time.Time) (*RetentionApplyResult, error) { if err := policy.Validate(); err != nil { return nil, err } now = now.UTC() report, err := s.RetentionReport(ctx, policy, artifactsRoot, now) if err != nil { return nil, err } result := &RetentionApplyResult{RetentionReport: *report, RemovedRuns: []int64{}} ids := make([]int64, 0, len(report.WouldRemove)) for _, run := range report.WouldRemove { ids = append(ids, run.ID) } if len(ids) > 0 { if err := s.removeRetentionRuns(ctx, policy, artifactsRoot, now, ids, result); err != nil { return result, err } } // Apply is the explicit reclaim command: even with nothing to remove it may // compact pages freed earlier. VACUUM failure is reported, never hidden. beforeBytes := report.DatabaseBytes if err := s.vacuumAndCheckpoint(ctx); err != nil { result.VacuumError = err.Error() // The run history and artifacts were already removed by // removeRetentionRuns; report only the artifact space actually // reclaimed and no database pages. result.SpaceReclaimedBytes = result.RemovedArtifactBytes return result, fmt.Errorf("reclaim database space: %w", err) } result.Vacuumed = true afterBytes, afterWAL, afterFree, err := s.retentionDBSize(ctx) if err != nil { return result, err } result.DatabaseBytesAfter = afterBytes result.DatabaseFreeBytesAfter = afterFree result.WALBytesAfter = afterWAL if reclaimed := beforeBytes - afterBytes; reclaimed > 0 { result.DatabaseBytesReclaimed = reclaimed } result.SpaceReclaimedBytes = result.RemovedArtifactBytes + result.DatabaseBytesReclaimed result.Applied = result.RemovedRunCount > 0 return result, nil } // removeRetentionRuns performs the staged, single-transaction removal of the // selected runs: re-check protection under an immediate write lock, rename the // run artifact directories into one staging directory, delete the rows, commit, // then delete the staged files. Every failure path restores what it can and // leaves nothing half-removed. func (s *Store) removeRetentionRuns(ctx context.Context, policy RetentionPolicy, artifactsRoot string, now time.Time, ids []int64, result *RetentionApplyResult) error { tx, err := s.db.BeginTx(ctx, nil) if err != nil { return fmt.Errorf("begin retention transaction: %w", err) } committed := false defer func() { if !committed { _ = tx.Rollback() } }() // Re-check protection inside the write transaction: a run that gained a // retry reference, a waiting state or agent activity since the report must // abort the whole apply. selection, err := selectRetentionRuns(ctx, tx, policy, now) if err != nil { return fmt.Errorf("re-check retention selection: %w", err) } decision := selection.decide(policy, now) stillRemove := map[int64]bool{} for _, run := range decision.wouldRemove { stillRemove[run.id] = true } became := []string{} for _, id := range ids { if stillRemove[id] { continue } reason := "run is no longer selected for removal" for _, protected := range decision.protected { if protected.ID == id { reason = protected.Reason } } became = append(became, fmt.Sprintf("run #%d: %s", id, reason)) } if len(became) > 0 { return fmt.Errorf("retention aborted: selected run(s) became protected before deletion; nothing was removed: %s", strings.Join(became, "; ")) } staging := "" staged := []string{} if artifactsRoot != "" { // Stage only inside an artifacts root that already exists: creating the // root here would leave an empty artifacts directory behind after an // apply that had no artifacts to remove, and with no root there is no // / directory to stage anyway. os.Stat (not Lstat) follows a // symlinked root so staging happens through the same directory the // report sized; skipping a symlinked root would claim its bytes as // removed while leaving every artifact in place. if info, err := os.Stat(artifactsRoot); err != nil { if !os.IsNotExist(err) { return fmt.Errorf("inspect artifacts root %s: %w", artifactsRoot, err) } } else if info.IsDir() { staging = filepath.Join(artifactsRoot, fmt.Sprintf(".retention-staging-%d-%d", now.Unix(), os.Getpid())) if err := os.MkdirAll(staging, 0700); err != nil { return fmt.Errorf("create retention staging directory: %w", err) } } } // unstage restores every renamed run directory and removes the staging // directory, returning any restore/cleanup error so the rollback path cannot // silently leave directories behind while only the original failure is named. unstage := func() error { var problems []string for _, original := range staged { if err := os.Rename(filepath.Join(staging, filepath.Base(original)), original); err != nil { problems = append(problems, fmt.Sprintf("restore %s: %v", original, err)) } } if staging != "" { if err := os.RemoveAll(staging); err != nil { problems = append(problems, fmt.Sprintf("remove staging directory %s: %v", staging, err)) } } if len(problems) > 0 { return fmt.Errorf("un-stage artifacts: %s", strings.Join(problems, "; ")) } return nil } for _, id := range ids { if staging == "" { break } original := filepath.Join(artifactsRoot, strconv.FormatInt(id, 10)) info, lerr := os.Lstat(original) if lerr != nil { if os.IsNotExist(lerr) { continue } return errors.Join(fmt.Errorf("inspect artifacts for run %d: %w", id, lerr), unstage()) } // Only the exact numeric run entry is staged. A regular file at that path // is refused rather than deleted; a symlink is renamed (removing the link // only), never its target. if !info.IsDir() && info.Mode()&os.ModeSymlink == 0 { return errors.Join(fmt.Errorf("refusing to remove run %d: %s is a regular file, not a run artifact directory", id, original), unstage()) } if err := os.Rename(original, filepath.Join(staging, strconv.FormatInt(id, 10))); err != nil { return errors.Join(fmt.Errorf("stage artifacts for run %d: %w", id, err), unstage()) } staged = append(staged, original) } if err := deleteRetentionRows(ctx, tx, ids); err != nil { return errors.Join(fmt.Errorf("delete run history: %w", err), unstage()) } if err := tx.Commit(); err != nil { return errors.Join(fmt.Errorf("commit retention removal: %w", err), unstage()) } committed = true // The rows are committed: these runs are gone whatever happens to the // staged files, so record exactly what was removed before any cleanup // error can return, and never let a cleanup failure be reported as zero // runs removed or zero reclaimed space. SpaceReclaimedBytes is set here so // a later VACUUM failure still reports the artifacts actually removed. result.RemovedRuns = append(result.RemovedRuns, ids...) sort.Slice(result.RemovedRuns, func(i, j int) bool { return result.RemovedRuns[i] < result.RemovedRuns[j] }) result.RemovedRunCount = len(ids) for _, run := range result.WouldRemove { result.RemovedArtifactFiles += run.Artifacts.Files result.RemovedArtifactBytes += run.Artifacts.Bytes } result.SpaceReclaimedBytes = result.RemovedArtifactBytes result.Applied = result.RemovedRunCount > 0 if staging != "" { if err := os.RemoveAll(staging); err != nil { remainingFiles, remainingBytes, _ := retentionWalkSize(staging) result.StagingPath = staging result.StagingRemainingFiles = remainingFiles result.StagingRemainingBytes = remainingBytes // Files still in staging were staged out of the run tree but not // deleted, so they are not reclaimed space. if reclaimed := result.RemovedArtifactBytes - remainingBytes; reclaimed > 0 { result.SpaceReclaimedBytes = reclaimed } else { result.SpaceReclaimedBytes = 0 } return fmt.Errorf("run history removed but staged artifact cleanup failed at %s (%d files, %d bytes remain and are not counted as reclaimed): %w", staging, remainingFiles, remainingBytes, err) } } return nil } // deleteRetentionRows removes every table row scoped to the selected runs, in // foreign-key order, batched at 500 ids per statement in the caller's single // transaction: step-scoped evidence first, then run_steps, then retry edges, // then the runs. func deleteRetentionRows(ctx context.Context, tx *sql.Tx, ids []int64) error { for _, table := range []struct{ name, column string }{ {"step_notifications", "run_id"}, {"run_notifications", "run_id"}, {"step_file_changes", "run_id"}, {"step_control", "run_id"}, {"step_artifacts", "run_id"}, {"step_verdicts", "run_id"}, {"step_session_exports", "run_id"}, {"step_usage", "run_id"}, {"run_session_owners", "run_id"}, {"run_steps", "run_id"}, } { if err := deleteRetentionBatch(ctx, tx, table.name, table.column, ids); err != nil { return err } } // Retry provenance is removed when either end is selected, so a removed // original leaves no dangling edge and a removed retry leaves none either. for start := 0; start < len(ids); start += retentionDeleteBatch { batch := ids[start:min(start+retentionDeleteBatch, len(ids))] placeholders, args := retentionBindings(batch, 2) if _, err := tx.ExecContext(ctx, "DELETE FROM run_retries WHERE run_id IN ("+placeholders+") OR original_run_id IN ("+placeholders+")", args...); err != nil { return err } } return deleteRetentionBatch(ctx, tx, "pipeline_runs", "id", ids) } // retentionDeleteBatch bounds each IN clause so a large removal does not build // an unbounded statement. const retentionDeleteBatch = 500 // retentionBindings builds an "?,?,..." placeholder list for len(ids) values and // returns args as copies repeats of ids, so a statement with several IN clauses // over the same id set shares one construction instead of duplicating it. func retentionBindings(ids []int64, copies int) (string, []any) { placeholders := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",") args := make([]any, 0, len(ids)*copies) for i := 0; i < copies; i++ { for _, id := range ids { args = append(args, id) } } return placeholders, args } func deleteRetentionBatch(ctx context.Context, tx *sql.Tx, table, column string, ids []int64) error { for start := 0; start < len(ids); start += retentionDeleteBatch { batch := ids[start:min(start+retentionDeleteBatch, len(ids))] placeholders, args := retentionBindings(batch, 1) if _, err := tx.ExecContext(ctx, "DELETE FROM "+table+" WHERE "+column+" IN ("+placeholders+")", args...); err != nil { return err } } return nil } // retentionDBSize measures the database file, its -wal sidecar and the free // pages SQLite reports, in bytes. It is a measurement, never an estimate. func (s *Store) retentionDBSize(ctx context.Context) (int64, int64, int64, error) { var free int64 var pageSize, freelist int64 if err := s.db.QueryRowContext(ctx, "PRAGMA page_size").Scan(&pageSize); err != nil { return 0, 0, 0, err } if err := s.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&freelist); err != nil { return 0, 0, 0, err } free = pageSize * freelist bytes, err := fileSize(s.path) if err != nil { return 0, 0, 0, err } wal, err := fileSize(s.path + "-wal") if err != nil { return 0, 0, 0, err } return bytes, wal, free, nil } func fileSize(path string) (int64, error) { info, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { return 0, nil } return 0, err } if !info.Mode().IsRegular() { return 0, nil } return info.Size(), nil } // vacuumAndCheckpoint reclaims freed pages and truncates the write-ahead log. // SQLITE_BUSY is retried a bounded number of times; any other failure, or a // still-busy database, is returned as an honest error rather than a hang. func (s *Store) vacuumAndCheckpoint(ctx context.Context) error { if err := s.retryBusy(ctx, func() error { _, err := s.db.ExecContext(ctx, "VACUUM") return err }); err != nil { return err } return s.retryBusy(ctx, func() error { var busy, logFrames, checkpointed int if err := s.db.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logFrames, &checkpointed); err != nil { return err } if busy != 0 { return fmt.Errorf("wal checkpoint busy") } return nil }) } // retentionBusyAttempts bounds VACUUM/checkpoint retries; a locked database is // reported, never retried forever. const retentionBusyAttempts = 3 func (s *Store) retryBusy(ctx context.Context, action func() error) error { var err error for attempt := 0; attempt < retentionBusyAttempts; attempt++ { if err = action(); err == nil || !isBusyError(err) { return err } select { case <-ctx.Done(): return ctx.Err() case <-time.After(time.Duration(attempt+1) * 200 * time.Millisecond): } } return err } func isBusyError(err error) bool { if err == nil { return false } message := strings.ToLower(err.Error()) return strings.Contains(message, "busy") || strings.Contains(message, "locked") }