package pipeline import ( "context" "database/sql" "errors" "fmt" "os" "path/filepath" "strconv" "strings" "time" "git.a-wan.net/alan/pipelines/internal/protocol" ) // MaxDeleteRuns bounds one delete_runs request. const MaxDeleteRuns = 1000 // terminalRunStatuses are the only run statuses delete_runs removes. Anything // else (queued, running, waiting, or a status this build does not know) is // refused. var terminalRunStatuses = map[string]bool{"succeeded": true, "failed": true, "canceled": true, "interrupted": true} // DeleteRunsResult is what DeleteRuns removed. Workspaces maps each deleted run // to its workspace so the server can scope the runs_removed push. type DeleteRunsResult struct { protocol.RunsDeletedResult Workspaces map[int64]string } // DeleteRuns removes each requested terminal run and everything the server // stores for it: every database row scoped to it (see deleteRunRows), its // artifact tree /, and Pi session directories the server // created for its stages. User workspaces and repositories are never touched. // // Eligibility is checked inside the write transaction. busy, when non-nil, lets // the caller veto a run it knows to be executing; its non-empty reason refuses // the run. Unknown ids, invalid ids and duplicates are refused individually; an // I/O or database failure aborts the whole request with nothing removed (the // artifact trees are staged by rename and restored on failure). func (s *Store) DeleteRuns(ctx context.Context, ids []int64, artifactsRoot string, busy func(int64) string) (*DeleteRunsResult, error) { if len(ids) == 0 { return nil, fmt.Errorf("run_ids is required") } if len(ids) > MaxDeleteRuns { return nil, fmt.Errorf("at most %d runs can be deleted at once", MaxDeleteRuns) } result := &DeleteRunsResult{ RunsDeletedResult: protocol.RunsDeletedResult{Deleted: []int64{}, Refused: []protocol.RunDeleteRefusal{}}, Workspaces: map[int64]string{}, } refuse := func(id int64, reason string) { result.Refused = append(result.Refused, protocol.RunDeleteRefusal{RunID: id, Reason: reason}) } seen := map[int64]bool{} var candidates []int64 for _, id := range ids { switch { case id <= 0: refuse(id, "invalid run id") case seen[id]: // A repeated id is the same request; report it once. default: seen[id] = true candidates = append(candidates, id) } } tx, err := s.db.BeginTx(ctx, nil) if err != nil { return nil, err } committed := false defer func() { if !committed { _ = tx.Rollback() } }() var deletable []int64 sizes := map[int64]int64{} for _, id := range candidates { reason, workspace, err := deleteRefusal(ctx, tx, id) if err != nil { return nil, err } if reason == "" && busy != nil { reason = busy(id) } if reason != "" { refuse(id, reason) continue } size, err := computeRunSize(ctx, tx, artifactsRoot, id) if err != nil { return nil, err } sizes[id] = size result.Workspaces[id] = workspace deletable = append(deletable, id) } if len(deletable) == 0 { return result, nil } piDirs, err := deletablePiDirs(ctx, tx, deletable) if err != nil { return nil, err } stage, err := stageRunArtifacts(artifactsRoot, deletable) if err != nil { return nil, err } if err := deleteRunRows(ctx, tx, deletable); err != nil { return nil, errors.Join(fmt.Errorf("delete run history: %w", err), stage.restore()) } if err := tx.Commit(); err != nil { return nil, errors.Join(fmt.Errorf("commit run deletion: %w", err), stage.restore()) } committed = true // The rows are gone; file cleanup failures cannot resurrect the runs, so // they are reported only in the freed byte count, never as an error. for _, id := range deletable { result.Deleted = append(result.Deleted, id) result.FreedBytes += sizes[id] } if stage.dir != "" { if err := removeRunTree(ctx, stage.dir); err != nil { remaining := result.FreedBytes if ctx.Err() == nil { _, remaining, _ = retentionWalkSize(stage.dir) } result.FreedBytes = max(result.FreedBytes-remaining, 0) result.RemainingFileBytes += remaining } } for _, dir := range piDirs { if err := removeRunTree(ctx, dir); err != nil { remaining := result.FreedBytes if ctx.Err() == nil { _, remaining, _ = retentionWalkSize(dir) } result.FreedBytes = max(result.FreedBytes-remaining, 0) result.RemainingFileBytes += remaining } } return result, nil } // deleteRefusal returns why a run must not be deleted ("" when it may be), and // its workspace. It refuses unknown runs, every non-terminal status, runs with // unconfirmed cleanup, active or waiting visits, a running session export, and // runs an unfinished retry still depends on. func deleteRefusal(ctx context.Context, q sizeQuerier, id int64) (string, string, error) { var status, workspace string var cleanupUnconfirmed bool var activeSteps, runningExports, dependents int err := q.QueryRowContext(ctx, `SELECT r.status,r.workspace,r.cleanup_unconfirmed, (SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND (s.status IN ('running','waiting') OR s.agent_active<>0)), (SELECT COUNT(*) FROM session_export_jobs j WHERE j.run_id=r.id AND j.status='running'), (SELECT COUNT(*) FROM pipeline_runs d WHERE d.id<>r.id AND d.status NOT IN ('succeeded','failed','canceled','interrupted') AND (d.id IN (SELECT run_id FROM run_retries WHERE original_run_id=r.id) OR d.id IN (SELECT run_id FROM retry_checkpoints WHERE source_run_id=r.id))) FROM pipeline_runs r WHERE r.id=?`, id).Scan(&status, &workspace, &cleanupUnconfirmed, &activeSteps, &runningExports, &dependents) if err == sql.ErrNoRows { return "run not found", "", nil } if err != nil { return "", "", err } switch { case !terminalRunStatuses[status]: return fmt.Sprintf("run is %s; cancel it or wait for it to finish", status), workspace, nil case cleanupUnconfirmed: return "run cleanup is not confirmed yet", workspace, nil case activeSteps > 0: return "a stage of this run is still active or waiting", workspace, nil case runningExports > 0: return "a session export for this run is in progress", workspace, nil case dependents > 0: return "an unfinished retry depends on this run", workspace, nil } return "", workspace, nil } // deletablePiDirs lists the owned Pi session directories of the runs being // deleted that no surviving run also references. func deletablePiDirs(ctx context.Context, q sizeQuerier, ids []int64) ([]string, error) { deleting := map[int64]bool{} for _, id := range ids { deleting[id] = true } var dirs []string for _, id := range ids { found, err := runPiSessionDirs(ctx, q, id) if err != nil { return nil, err } for _, dir := range found { rows, err := q.QueryContext(ctx, `SELECT DISTINCT run_id FROM run_session_owners WHERE reference=?`, dir) if err != nil { return nil, err } shared := false for rows.Next() { var owner int64 if err := rows.Scan(&owner); err != nil { rows.Close() return nil, err } if !deleting[owner] { shared = true } } err = rows.Err() rows.Close() if err != nil { return nil, err } if !shared { dirs = append(dirs, dir) } } } return dirs, nil } // stagedArtifacts is a set of run artifact trees renamed into one staging // directory under the artifacts root, so the database removal and the file // removal commit or roll back together. type stagedArtifacts struct { dir string staged []string } // stageRunArtifacts renames / for each run into a fresh staging // directory. Only the exact numeric entry is staged; a symlink is moved (the // link, never its target) and a regular file there aborts with an error. func stageRunArtifacts(root string, ids []int64) (*stagedArtifacts, error) { stage := &stagedArtifacts{} if root == "" { return stage, nil } info, err := os.Stat(root) if os.IsNotExist(err) { return stage, nil } if err != nil { return nil, fmt.Errorf("inspect artifacts root %s: %w", root, err) } if !info.IsDir() { return stage, nil } for _, id := range ids { original := filepath.Join(root, strconv.FormatInt(id, 10)) entry, err := os.Lstat(original) if os.IsNotExist(err) { continue } if err != nil { return nil, errors.Join(fmt.Errorf("inspect artifacts for run %d: %w", id, err), stage.restore()) } if !entry.IsDir() && entry.Mode()&os.ModeSymlink == 0 { return nil, errors.Join(fmt.Errorf("refusing to remove run %d: %s is a regular file, not a run artifact directory", id, original), stage.restore()) } if stage.dir == "" { stage.dir = filepath.Join(root, fmt.Sprintf(".delete-staging-%d-%d", time.Now().UnixNano(), os.Getpid())) if err := os.Mkdir(stage.dir, 0o700); err != nil { stage.dir = "" return nil, fmt.Errorf("create delete staging directory: %w", err) } } if err := os.Rename(original, filepath.Join(stage.dir, filepath.Base(original))); err != nil { return nil, errors.Join(fmt.Errorf("stage artifacts for run %d: %w", id, err), stage.restore()) } stage.staged = append(stage.staged, original) } return stage, nil } // restore moves every staged tree back and removes the staging directory. func (s *stagedArtifacts) restore() error { var problems []string for _, original := range s.staged { if err := os.Rename(filepath.Join(s.dir, filepath.Base(original)), original); err != nil { problems = append(problems, fmt.Sprintf("restore %s: %v", original, err)) } } s.staged = nil if s.dir != "" { if err := os.Remove(s.dir); err != nil && !os.IsNotExist(err) { problems = append(problems, fmt.Sprintf("remove staging directory %s: %v", s.dir, err)) } } if len(problems) > 0 { return fmt.Errorf("restore artifacts: %s", strings.Join(problems, "; ")) } return nil }