d1846441cc
Record every notification published to ntfy (run outbox, stage notify hooks, chat question/approval/reply) in a persisted app_notifications outbox and stream it to subscribe_notifications clients with an after cursor; clear_notifications and ack_attention clear across devices, as do runs leaving waiting, deleted runs and answered chat questions. Add the notify.ntfy_enabled setting (default on) to stop only the ntfy publish.
140 lines
4.0 KiB
Go
140 lines
4.0 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"time"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// listRunsSizeBudget bounds how many stale run sizes one list_runs request
|
|
// measures synchronously; the rest are left to the background backfill so a
|
|
// first list over hundreds of unmeasured runs stays fast.
|
|
const listRunsSizeBudget = 25
|
|
|
|
// runSizeInterval is how often the background sizer re-checks for stale sizes
|
|
// when nothing kicks it.
|
|
const runSizeInterval = time.Minute
|
|
|
|
// runSizeBatch is how many stale runs one backfill step measures.
|
|
const runSizeBatch = 50
|
|
|
|
// kickSizes wakes the background sizer without blocking.
|
|
func (r *runner) kickSizes() {
|
|
if r.sizeKick == nil {
|
|
return
|
|
}
|
|
select {
|
|
case r.sizeKick <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// sizeLoop is the background run-size backfill. It starts with a pass (so an
|
|
// upgraded database is measured without blocking startup), then runs on every
|
|
// kick or interval until the runner's context or store closes. It never pushes
|
|
// frames: sizes reach clients on list_runs, subscribe snapshots and terminal
|
|
// frames.
|
|
func (r *runner) sizeLoop() {
|
|
defer r.schedulerWG.Done()
|
|
ticker := time.NewTicker(runSizeInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
for {
|
|
measured, err := r.store.RefreshRunSizes(r.ctx, r.artifactsRoot, runSizeBatch)
|
|
if err != nil {
|
|
if r.ctx.Err() == nil {
|
|
log.Printf("refresh run sizes: %v", err)
|
|
}
|
|
break
|
|
}
|
|
// A short batch means the backlog is drained; runs that changed while
|
|
// measured stay stale until the next kick or tick, so an active run
|
|
// never spins this loop.
|
|
if len(measured) < runSizeBatch {
|
|
break
|
|
}
|
|
select {
|
|
case <-r.ctx.Done():
|
|
return
|
|
case <-r.store.Done():
|
|
return
|
|
default:
|
|
}
|
|
}
|
|
select {
|
|
case <-r.ctx.Done():
|
|
return
|
|
case <-r.store.Done():
|
|
return
|
|
case <-ticker.C:
|
|
case <-r.sizeKick:
|
|
}
|
|
}
|
|
}
|
|
|
|
// refreshListSizes measures a bounded number of stale sizes before list_runs
|
|
// reads them and kicks the background sizer for any remainder.
|
|
func (r *runner) refreshListSizes(ctx context.Context) {
|
|
if _, err := r.store.RefreshRunSizes(ctx, r.artifactsRoot, listRunsSizeBudget); err != nil {
|
|
log.Printf("refresh run sizes: %v", err)
|
|
}
|
|
if stale, err := r.store.StaleRunSizes(ctx); err == nil && stale > 0 {
|
|
r.kickSizes()
|
|
}
|
|
}
|
|
|
|
// finishedSize measures a finishing run's size so its terminal frame carries
|
|
// it ("computed when a run finishes"). It is one directory walk and one
|
|
// aggregate query; 0 on any error.
|
|
func (r *runner) finishedSize(id int64) int64 {
|
|
ctx, cancel := context.WithTimeout(r.ctx, 10*time.Second)
|
|
defer cancel()
|
|
size, err := r.store.RefreshRunSize(ctx, r.artifactsRoot, id)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
return size
|
|
}
|
|
|
|
// deleteRuns removes terminal runs and everything the server stores for them,
|
|
// then tells every subscribed client which runs are gone. It holds r.mu so a run
|
|
// cannot become active between the executing check and the delete.
|
|
func (r *runner) deleteRuns(ids []int64) (*protocol.RunsDeletedResult, error) {
|
|
r.mu.Lock()
|
|
result, err := r.store.DeleteRuns(r.ctx, ids, r.artifactsRoot, func(id int64) string {
|
|
if _, ok := r.active[id]; ok {
|
|
return "run is executing"
|
|
}
|
|
return ""
|
|
})
|
|
r.mu.Unlock()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, id := range result.Deleted {
|
|
r.identMu.Lock()
|
|
delete(r.idents, id)
|
|
r.identMu.Unlock()
|
|
r.attentionMu.Lock()
|
|
delete(r.attentionSent, id)
|
|
r.attentionMu.Unlock()
|
|
r.notifyMu.Lock()
|
|
delete(r.notifyModes, id)
|
|
r.notifyMu.Unlock()
|
|
// A deleted run's notifications are gone on every device.
|
|
if r.notes != nil {
|
|
c, cancel := context.WithTimeout(r.ctx, 5*time.Second)
|
|
r.notes.clearFor(c, pipeline.AppNotificationFilter{RunID: id})
|
|
cancel()
|
|
}
|
|
}
|
|
if len(result.Deleted) > 0 {
|
|
log.Printf("deleted runs %v (refused %d, freed %d bytes)", result.Deleted, len(result.Refused), result.FreedBytes)
|
|
r.events.removeRuns(result.Deleted, result.Workspaces)
|
|
}
|
|
return &result.RunsDeletedResult, nil
|
|
}
|