152 lines
4.3 KiB
Go
152 lines
4.3 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// listRunsSizeBudget bounds how many stale run sizes one list_runs request
|
|
// measures synchronously; the rest are left to the background backfill so a
|
|
// first list over hundreds of unmeasured runs stays fast.
|
|
const listRunsSizeBudget = 25
|
|
|
|
// runSizeInterval is how often the background sizer re-checks for stale sizes
|
|
// when nothing kicks it.
|
|
const runSizeInterval = time.Minute
|
|
|
|
// runSizeBatch is how many stale runs one backfill step measures.
|
|
const runSizeBatch = 50
|
|
|
|
// kickSizes wakes the background sizer without blocking.
|
|
func (r *runner) kickSizes() {
|
|
if r.sizeKick == nil {
|
|
return
|
|
}
|
|
select {
|
|
case r.sizeKick <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// sizeLoop is the background run-size backfill. It starts with a pass (so an
|
|
// upgraded database is measured without blocking startup), then runs on every
|
|
// kick or interval until the runner's context or store closes. It never pushes
|
|
// frames: sizes reach clients on list_runs, subscribe snapshots and terminal
|
|
// frames.
|
|
func (r *runner) sizeLoop() {
|
|
defer r.schedulerWG.Done()
|
|
ticker := time.NewTicker(runSizeInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
for {
|
|
measured, err := r.store.RefreshRunSizes(r.ctx, r.artifactsRoot, runSizeBatch)
|
|
if err != nil {
|
|
if r.ctx.Err() == nil {
|
|
log.Printf("refresh run sizes: %v", err)
|
|
}
|
|
break
|
|
}
|
|
// A short batch means the backlog is drained; runs that changed while
|
|
// measured stay stale until the next kick or tick, so an active run
|
|
// never spins this loop.
|
|
if len(measured) < runSizeBatch {
|
|
break
|
|
}
|
|
select {
|
|
case <-r.ctx.Done():
|
|
return
|
|
case <-r.store.Done():
|
|
return
|
|
default:
|
|
}
|
|
}
|
|
select {
|
|
case <-r.ctx.Done():
|
|
return
|
|
case <-r.store.Done():
|
|
return
|
|
case <-ticker.C:
|
|
case <-r.sizeKick:
|
|
}
|
|
}
|
|
}
|
|
|
|
// refreshListSizes measures a bounded number of stale sizes before list_runs
|
|
// reads them and kicks the background sizer for any remainder.
|
|
func (r *runner) refreshListSizes(ctx context.Context) {
|
|
if _, err := r.store.RefreshRunSizes(ctx, r.artifactsRoot, listRunsSizeBudget); err != nil {
|
|
log.Printf("refresh run sizes: %v", err)
|
|
}
|
|
if stale, err := r.store.StaleRunSizes(ctx); err == nil && stale > 0 {
|
|
r.kickSizes()
|
|
}
|
|
}
|
|
|
|
// finishedSize measures a finishing run's size so its terminal frame carries
|
|
// it ("computed when a run finishes"). It is one directory walk and one
|
|
// aggregate query; 0 on any error.
|
|
func (r *runner) finishedSize(id int64) int64 {
|
|
ctx, cancel := context.WithTimeout(r.ctx, 10*time.Second)
|
|
defer cancel()
|
|
size, err := r.store.RefreshRunSize(ctx, r.artifactsRoot, id)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
return size
|
|
}
|
|
|
|
// deleteRuns removes terminal runs and everything the server stores for them,
|
|
// then tells every subscribed client which runs are gone. It holds r.mu so a run
|
|
// cannot become active between the executing check and the delete.
|
|
func (r *runner) deleteRuns(ids []int64) (*protocol.RunsDeletedResult, error) {
|
|
return r.deleteRunsContext(r.ctx, ids, false)
|
|
}
|
|
|
|
// Storage maintenance uses its own deadline and never queues behind execution.
|
|
func (r *runner) deleteRunsContext(ctx context.Context, ids []int64, try bool) (*protocol.RunsDeletedResult, error) {
|
|
if try {
|
|
if !r.mu.TryLock() {
|
|
return nil, fmt.Errorf("run maintenance is busy; retry shortly")
|
|
}
|
|
} else {
|
|
r.mu.Lock()
|
|
}
|
|
result, err := r.store.DeleteRuns(ctx, ids, r.artifactsRoot, func(id int64) string {
|
|
if _, ok := r.active[id]; ok {
|
|
return "run is executing"
|
|
}
|
|
return ""
|
|
})
|
|
r.mu.Unlock()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, id := range result.Deleted {
|
|
r.identMu.Lock()
|
|
delete(r.idents, id)
|
|
r.identMu.Unlock()
|
|
r.attentionMu.Lock()
|
|
delete(r.attentionSent, id)
|
|
r.attentionMu.Unlock()
|
|
r.notifyMu.Lock()
|
|
delete(r.notifyModes, id)
|
|
r.notifyMu.Unlock()
|
|
// A deleted run's notifications are gone on every device.
|
|
if r.notes != nil {
|
|
c, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
r.notes.clearFor(c, pipeline.AppNotificationFilter{RunID: id})
|
|
cancel()
|
|
}
|
|
}
|
|
if len(result.Deleted) > 0 {
|
|
log.Printf("deleted runs %v (refused %d, freed %d bytes)", result.Deleted, len(result.Refused), result.FreedBytes)
|
|
r.events.removeRuns(result.Deleted, result.Workspaces)
|
|
}
|
|
return &result.RunsDeletedResult, nil
|
|
}
|