301 lines
9.6 KiB
Go
301 lines
9.6 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// MaxDeleteRuns bounds one delete_runs request.
|
|
const MaxDeleteRuns = 1000
|
|
|
|
// terminalRunStatuses are the only run statuses delete_runs removes. Anything
|
|
// else (queued, running, waiting, or a status this build does not know) is
|
|
// refused.
|
|
var terminalRunStatuses = map[string]bool{"succeeded": true, "failed": true, "canceled": true, "interrupted": true}
|
|
|
|
// DeleteRunsResult is what DeleteRuns removed. Workspaces maps each deleted run
|
|
// to its workspace so the server can scope the runs_removed push.
|
|
type DeleteRunsResult struct {
|
|
protocol.RunsDeletedResult
|
|
Workspaces map[int64]string
|
|
}
|
|
|
|
// DeleteRuns removes each requested terminal run and everything the server
|
|
// stores for it: every database row scoped to it (see deleteRunRows), its
|
|
// artifact tree <artifactsRoot>/<id>, and Pi session directories the server
|
|
// created for its stages. User workspaces and repositories are never touched.
|
|
//
|
|
// Eligibility is checked inside the write transaction. busy, when non-nil, lets
|
|
// the caller veto a run it knows to be executing; its non-empty reason refuses
|
|
// the run. Unknown ids, invalid ids and duplicates are refused individually; an
|
|
// I/O or database failure aborts the whole request with nothing removed (the
|
|
// artifact trees are staged by rename and restored on failure).
|
|
func (s *Store) DeleteRuns(ctx context.Context, ids []int64, artifactsRoot string, busy func(int64) string) (*DeleteRunsResult, error) {
|
|
if len(ids) == 0 {
|
|
return nil, fmt.Errorf("run_ids is required")
|
|
}
|
|
if len(ids) > MaxDeleteRuns {
|
|
return nil, fmt.Errorf("at most %d runs can be deleted at once", MaxDeleteRuns)
|
|
}
|
|
result := &DeleteRunsResult{
|
|
RunsDeletedResult: protocol.RunsDeletedResult{Deleted: []int64{}, Refused: []protocol.RunDeleteRefusal{}},
|
|
Workspaces: map[int64]string{},
|
|
}
|
|
refuse := func(id int64, reason string) {
|
|
result.Refused = append(result.Refused, protocol.RunDeleteRefusal{RunID: id, Reason: reason})
|
|
}
|
|
seen := map[int64]bool{}
|
|
var candidates []int64
|
|
for _, id := range ids {
|
|
switch {
|
|
case id <= 0:
|
|
refuse(id, "invalid run id")
|
|
case seen[id]:
|
|
// A repeated id is the same request; report it once.
|
|
default:
|
|
seen[id] = true
|
|
candidates = append(candidates, id)
|
|
}
|
|
}
|
|
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
committed := false
|
|
defer func() {
|
|
if !committed {
|
|
_ = tx.Rollback()
|
|
}
|
|
}()
|
|
|
|
var deletable []int64
|
|
sizes := map[int64]int64{}
|
|
for _, id := range candidates {
|
|
reason, workspace, err := deleteRefusal(ctx, tx, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if reason == "" && busy != nil {
|
|
reason = busy(id)
|
|
}
|
|
if reason != "" {
|
|
refuse(id, reason)
|
|
continue
|
|
}
|
|
size, err := computeRunSize(ctx, tx, artifactsRoot, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sizes[id] = size
|
|
result.Workspaces[id] = workspace
|
|
deletable = append(deletable, id)
|
|
}
|
|
if len(deletable) == 0 {
|
|
return result, nil
|
|
}
|
|
|
|
piDirs, err := deletablePiDirs(ctx, tx, deletable)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stage, err := stageRunArtifacts(artifactsRoot, deletable)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := deleteRunRows(ctx, tx, deletable); err != nil {
|
|
return nil, errors.Join(fmt.Errorf("delete run history: %w", err), stage.restore())
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return nil, errors.Join(fmt.Errorf("commit run deletion: %w", err), stage.restore())
|
|
}
|
|
committed = true
|
|
|
|
// The rows are gone; file cleanup failures cannot resurrect the runs, so
|
|
// they are reported only in the freed byte count, never as an error.
|
|
for _, id := range deletable {
|
|
result.Deleted = append(result.Deleted, id)
|
|
result.FreedBytes += sizes[id]
|
|
}
|
|
if stage.dir != "" {
|
|
if err := removeRunTree(ctx, stage.dir); err != nil {
|
|
remaining := result.FreedBytes
|
|
if ctx.Err() == nil {
|
|
_, remaining, _ = retentionWalkSize(stage.dir)
|
|
}
|
|
result.FreedBytes = max(result.FreedBytes-remaining, 0)
|
|
result.RemainingFileBytes += remaining
|
|
}
|
|
}
|
|
for _, dir := range piDirs {
|
|
if err := removeRunTree(ctx, dir); err != nil {
|
|
remaining := result.FreedBytes
|
|
if ctx.Err() == nil {
|
|
_, remaining, _ = retentionWalkSize(dir)
|
|
}
|
|
result.FreedBytes = max(result.FreedBytes-remaining, 0)
|
|
result.RemainingFileBytes += remaining
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
// deleteRefusal returns why a run must not be deleted ("" when it may be), and
|
|
// its workspace. It refuses unknown runs, every non-terminal status, runs with
|
|
// unconfirmed cleanup, active or waiting visits, a running session export, and
|
|
// runs an unfinished retry still depends on.
|
|
func deleteRefusal(ctx context.Context, q sizeQuerier, id int64) (string, string, error) {
|
|
var status, workspace string
|
|
var cleanupUnconfirmed bool
|
|
var activeSteps, runningExports, dependents int
|
|
err := q.QueryRowContext(ctx, `SELECT r.status,r.workspace,r.cleanup_unconfirmed,
|
|
(SELECT COUNT(*) FROM run_steps s WHERE s.run_id=r.id AND (s.status IN ('running','waiting') OR s.agent_active<>0)),
|
|
(SELECT COUNT(*) FROM session_export_jobs j WHERE j.run_id=r.id AND j.status='running'),
|
|
(SELECT COUNT(*) FROM pipeline_runs d WHERE d.id<>r.id AND d.status NOT IN ('succeeded','failed','canceled','interrupted')
|
|
AND (d.id IN (SELECT run_id FROM run_retries WHERE original_run_id=r.id) OR d.id IN (SELECT run_id FROM retry_checkpoints WHERE source_run_id=r.id)))
|
|
FROM pipeline_runs r WHERE r.id=?`, id).Scan(&status, &workspace, &cleanupUnconfirmed, &activeSteps, &runningExports, &dependents)
|
|
if err == sql.ErrNoRows {
|
|
return "run not found", "", nil
|
|
}
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
switch {
|
|
case !terminalRunStatuses[status]:
|
|
return fmt.Sprintf("run is %s; cancel it or wait for it to finish", status), workspace, nil
|
|
case cleanupUnconfirmed:
|
|
return "run cleanup is not confirmed yet", workspace, nil
|
|
case activeSteps > 0:
|
|
return "a stage of this run is still active or waiting", workspace, nil
|
|
case runningExports > 0:
|
|
return "a session export for this run is in progress", workspace, nil
|
|
case dependents > 0:
|
|
return "an unfinished retry depends on this run", workspace, nil
|
|
}
|
|
return "", workspace, nil
|
|
}
|
|
|
|
// deletablePiDirs lists the owned Pi session directories of the runs being
|
|
// deleted that no surviving run also references.
|
|
func deletablePiDirs(ctx context.Context, q sizeQuerier, ids []int64) ([]string, error) {
|
|
deleting := map[int64]bool{}
|
|
for _, id := range ids {
|
|
deleting[id] = true
|
|
}
|
|
var dirs []string
|
|
for _, id := range ids {
|
|
found, err := runPiSessionDirs(ctx, q, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, dir := range found {
|
|
rows, err := q.QueryContext(ctx, `SELECT DISTINCT run_id FROM run_session_owners WHERE reference=?`, dir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
shared := false
|
|
for rows.Next() {
|
|
var owner int64
|
|
if err := rows.Scan(&owner); err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
if !deleting[owner] {
|
|
shared = true
|
|
}
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !shared {
|
|
dirs = append(dirs, dir)
|
|
}
|
|
}
|
|
}
|
|
return dirs, nil
|
|
}
|
|
|
|
// stagedArtifacts is a set of run artifact trees renamed into one staging
|
|
// directory under the artifacts root, so the database removal and the file
|
|
// removal commit or roll back together.
|
|
type stagedArtifacts struct {
|
|
dir string
|
|
staged []string
|
|
}
|
|
|
|
// stageRunArtifacts renames <root>/<id> for each run into a fresh staging
|
|
// directory. Only the exact numeric entry is staged; a symlink is moved (the
|
|
// link, never its target) and a regular file there aborts with an error.
|
|
func stageRunArtifacts(root string, ids []int64) (*stagedArtifacts, error) {
|
|
stage := &stagedArtifacts{}
|
|
if root == "" {
|
|
return stage, nil
|
|
}
|
|
info, err := os.Stat(root)
|
|
if os.IsNotExist(err) {
|
|
return stage, nil
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("inspect artifacts root %s: %w", root, err)
|
|
}
|
|
if !info.IsDir() {
|
|
return stage, nil
|
|
}
|
|
for _, id := range ids {
|
|
original := filepath.Join(root, strconv.FormatInt(id, 10))
|
|
entry, err := os.Lstat(original)
|
|
if os.IsNotExist(err) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return nil, errors.Join(fmt.Errorf("inspect artifacts for run %d: %w", id, err), stage.restore())
|
|
}
|
|
if !entry.IsDir() && entry.Mode()&os.ModeSymlink == 0 {
|
|
return nil, errors.Join(fmt.Errorf("refusing to remove run %d: %s is a regular file, not a run artifact directory", id, original), stage.restore())
|
|
}
|
|
if stage.dir == "" {
|
|
stage.dir = filepath.Join(root, fmt.Sprintf(".delete-staging-%d-%d", time.Now().UnixNano(), os.Getpid()))
|
|
if err := os.Mkdir(stage.dir, 0o700); err != nil {
|
|
stage.dir = ""
|
|
return nil, fmt.Errorf("create delete staging directory: %w", err)
|
|
}
|
|
}
|
|
if err := os.Rename(original, filepath.Join(stage.dir, filepath.Base(original))); err != nil {
|
|
return nil, errors.Join(fmt.Errorf("stage artifacts for run %d: %w", id, err), stage.restore())
|
|
}
|
|
stage.staged = append(stage.staged, original)
|
|
}
|
|
return stage, nil
|
|
}
|
|
|
|
// restore moves every staged tree back and removes the staging directory.
|
|
func (s *stagedArtifacts) restore() error {
|
|
var problems []string
|
|
for _, original := range s.staged {
|
|
if err := os.Rename(filepath.Join(s.dir, filepath.Base(original)), original); err != nil {
|
|
problems = append(problems, fmt.Sprintf("restore %s: %v", original, err))
|
|
}
|
|
}
|
|
s.staged = nil
|
|
if s.dir != "" {
|
|
if err := os.Remove(s.dir); err != nil && !os.IsNotExist(err) {
|
|
problems = append(problems, fmt.Sprintf("remove staging directory %s: %v", s.dir, err))
|
|
}
|
|
}
|
|
if len(problems) > 0 {
|
|
return fmt.Errorf("restore artifacts: %s", strings.Join(problems, "; "))
|
|
}
|
|
return nil
|
|
}
|