Files

301 lines
9.6 KiB
Go

package pipeline
import (
"context"
"database/sql"
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// MaxDeleteRuns bounds one delete_runs request.
const MaxDeleteRuns = 1000
// terminalRunStatuses are the only run statuses delete_runs removes. Anything
// else (queued, running, waiting, or a status this build does not know) is
// refused.
var terminalRunStatuses = map[string]bool{"succeeded": true, "failed": true, "canceled": true, "interrupted": true}
// DeleteRunsResult is what DeleteRuns removed. Workspaces maps each deleted run
// to its workspace so the server can scope the runs_removed push.
type DeleteRunsResult struct {
protocol.RunsDeletedResult
Workspaces map[int64]string
}
// DeleteRuns removes each requested terminal run and everything the server
// stores for it: every database row scoped to it (see deleteRunRows), its
// artifact tree <artifactsRoot>/<id>, and Pi session directories the server
// created for its stages. User workspaces and repositories are never touched.
//
// Eligibility is checked inside the write transaction. busy, when non-nil, lets
// the caller veto a run it knows to be executing; its non-empty reason refuses
// the run. Unknown ids, invalid ids and duplicates are refused individually; an
// I/O or database failure aborts the whole request with nothing removed (the
// artifact trees are staged by rename and restored on failure).
func (s *Store) DeleteRuns(ctx context.Context, ids []int64, artifactsRoot string, busy func(int64) string) (*DeleteRunsResult, error) {
if len(ids) == 0 {
return nil, fmt.Errorf("run_ids is required")
}
if len(ids) > MaxDeleteRuns {
return nil, fmt.Errorf("at most %d runs can be deleted at once", MaxDeleteRuns)
}
result := &DeleteRunsResult{
RunsDeletedResult: protocol.RunsDeletedResult{Deleted: []int64{}, Refused: []protocol.RunDeleteRefusal{}},
Workspaces: map[int64]string{},
}
refuse := func(id int64, reason string) {
result.Refused = append(result.Refused, protocol.RunDeleteRefusal{RunID: id, Reason: reason})
}
seen := map[int64]bool{}
var candidates []int64
for _, id := range ids {
switch {
case id <= 0:
refuse(id, "invalid run id")
case seen[id]:
// A repeated id is the same request; report it once.
default:
seen[id] = true
candidates = append(candidates, id)
}
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return nil, err
}
committed := false
defer func() {
if !committed {
_ = tx.Rollback()
}
}()
var deletable []int64
sizes := map[int64]int64{}
for _, id := range candidates {
reason, workspace, err := deleteRefusal(ctx, tx, id)
if err != nil {
return nil, err
}
if reason == "" && busy != nil {
reason = busy(id)
}
if reason != "" {
refuse(id, reason)
continue
}
size, err := computeRunSize(ctx, tx, artifactsRoot, id)
if err != nil {
return nil, err
}
sizes[id] = size
result.Workspaces[id] = workspace
deletable = append(deletable, id)
}
if len(deletable) == 0 {
return result, nil
}
piDirs, err := deletablePiDirs(ctx, tx, deletable)
if err != nil {
return nil, err
}
stage, err := stageRunArtifacts(artifactsRoot, deletable)
if err != nil {
return nil, err
}
if err := deleteRunRows(ctx, tx, deletable); err != nil {
return nil, errors.Join(fmt.Errorf("delete run history: %w", err), stage.restore())
}
if err := tx.Commit(); err != nil {
return nil, errors.Join(fmt.Errorf("commit run deletion: %w", err), stage.restore())
}
committed = true
// The rows are gone; file cleanup failures cannot resurrect the runs, so
// they are reported only in the freed byte count, never as an error.
for _, id := range deletable {
result.Deleted = append(result.Deleted, id)
result.FreedBytes += sizes[id]
}
if stage.dir != "" {
if err := removeRunTree(ctx, stage.dir); err != nil {
remaining := result.FreedBytes
if ctx.Err() == nil {
_, remaining, _ = retentionWalkSize(stage.dir)
}
result.FreedBytes = max(result.FreedBytes-remaining, 0)
result.RemainingFileBytes += remaining
}
}
for _, dir := range piDirs {
if err := removeRunTree(ctx, dir); err != nil {
remaining := result.FreedBytes
if ctx.Err() == nil {
_, remaining, _ = retentionWalkSize(dir)
}
result.FreedBytes = max(result.FreedBytes-remaining, 0)
result.RemainingFileBytes += remaining
}
}
return result, nil
}
// deleteRefusal returns why a run must not be deleted ("" when it may be), and
// its workspace. It refuses unknown runs, every non-terminal status, runs with
// unconfirmed cleanup, active or waiting visits, a running session export, and
// runs an unfinished retry still depends on.
func deleteRefusal(ctx context.Context, q sizeQuerier, id int64) (string, string, error) {
var status, workspace string
var cleanupUnconfirmed bool
var activeSteps, runningExports, dependents int
err := q.QueryRowContext(ctx, `SELECT r.status,r.workspace,r.cleanup_unconfirmed,
(SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND (s.status IN ('running','waiting') OR s.agent_active<>0)),
(SELECT COUNT(*) FROM session_export_jobs j WHERE j.run_id=r.id AND j.status='running'),
(SELECT COUNT(*) FROM pipeline_runs d WHERE d.id<>r.id AND d.status NOT IN ('succeeded','failed','canceled','interrupted')
AND (d.id IN (SELECT run_id FROM run_retries WHERE original_run_id=r.id) OR d.id IN (SELECT run_id FROM retry_checkpoints WHERE source_run_id=r.id)))
FROM pipeline_runs r WHERE r.id=?`, id).Scan(&status, &workspace, &cleanupUnconfirmed, &activeSteps, &runningExports, &dependents)
if err == sql.ErrNoRows {
return "run not found", "", nil
}
if err != nil {
return "", "", err
}
switch {
case !terminalRunStatuses[status]:
return fmt.Sprintf("run is %s; cancel it or wait for it to finish", status), workspace, nil
case cleanupUnconfirmed:
return "run cleanup is not confirmed yet", workspace, nil
case activeSteps > 0:
return "a stage of this run is still active or waiting", workspace, nil
case runningExports > 0:
return "a session export for this run is in progress", workspace, nil
case dependents > 0:
return "an unfinished retry depends on this run", workspace, nil
}
return "", workspace, nil
}
// deletablePiDirs lists the owned Pi session directories of the runs being
// deleted that no surviving run also references.
func deletablePiDirs(ctx context.Context, q sizeQuerier, ids []int64) ([]string, error) {
deleting := map[int64]bool{}
for _, id := range ids {
deleting[id] = true
}
var dirs []string
for _, id := range ids {
found, err := runPiSessionDirs(ctx, q, id)
if err != nil {
return nil, err
}
for _, dir := range found {
rows, err := q.QueryContext(ctx, `SELECT DISTINCT run_id FROM run_session_owners WHERE reference=?`, dir)
if err != nil {
return nil, err
}
shared := false
for rows.Next() {
var owner int64
if err := rows.Scan(&owner); err != nil {
rows.Close()
return nil, err
}
if !deleting[owner] {
shared = true
}
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
if !shared {
dirs = append(dirs, dir)
}
}
}
return dirs, nil
}
// stagedArtifacts is a set of run artifact trees renamed into one staging
// directory under the artifacts root, so the database removal and the file
// removal commit or roll back together.
type stagedArtifacts struct {
dir string
staged []string
}
// stageRunArtifacts renames <root>/<id> for each run into a fresh staging
// directory. Only the exact numeric entry is staged; a symlink is moved (the
// link, never its target) and a regular file there aborts with an error.
func stageRunArtifacts(root string, ids []int64) (*stagedArtifacts, error) {
stage := &stagedArtifacts{}
if root == "" {
return stage, nil
}
info, err := os.Stat(root)
if os.IsNotExist(err) {
return stage, nil
}
if err != nil {
return nil, fmt.Errorf("inspect artifacts root %s: %w", root, err)
}
if !info.IsDir() {
return stage, nil
}
for _, id := range ids {
original := filepath.Join(root, strconv.FormatInt(id, 10))
entry, err := os.Lstat(original)
if os.IsNotExist(err) {
continue
}
if err != nil {
return nil, errors.Join(fmt.Errorf("inspect artifacts for run %d: %w", id, err), stage.restore())
}
if !entry.IsDir() && entry.Mode()&os.ModeSymlink == 0 {
return nil, errors.Join(fmt.Errorf("refusing to remove run %d: %s is a regular file, not a run artifact directory", id, original), stage.restore())
}
if stage.dir == "" {
stage.dir = filepath.Join(root, fmt.Sprintf(".delete-staging-%d-%d", time.Now().UnixNano(), os.Getpid()))
if err := os.Mkdir(stage.dir, 0o700); err != nil {
stage.dir = ""
return nil, fmt.Errorf("create delete staging directory: %w", err)
}
}
if err := os.Rename(original, filepath.Join(stage.dir, filepath.Base(original))); err != nil {
return nil, errors.Join(fmt.Errorf("stage artifacts for run %d: %w", id, err), stage.restore())
}
stage.staged = append(stage.staged, original)
}
return stage, nil
}
// restore moves every staged tree back and removes the staging directory.
func (s *stagedArtifacts) restore() error {
var problems []string
for _, original := range s.staged {
if err := os.Rename(filepath.Join(s.dir, filepath.Base(original)), original); err != nil {
problems = append(problems, fmt.Sprintf("restore %s: %v", original, err))
}
}
s.staged = nil
if s.dir != "" {
if err := os.Remove(s.dir); err != nil && !os.IsNotExist(err) {
problems = append(problems, fmt.Sprintf("remove staging directory %s: %v", s.dir, err))
}
}
if len(problems) > 0 {
return fmt.Errorf("restore artifacts: %s", strings.Join(problems, "; "))
}
return nil
}