Add server stats websocket reporting

This commit is contained in:
2026-11-07 01:35:39 +00:00
parent 81955059de
commit 95849f636a
10 changed files with 516 additions and 2 deletions
+6
View File
@@ -0,0 +1,6 @@
# Server stats
Send a `server_stats` websocket request to receive a `server_stats_result` with
host, CPU, memory, storage, Pipelines data, process, and best-effort GPU health.
Unsupported or unavailable sections include an error or reason rather than
failing the complete response.
+8
View File
@@ -15,6 +15,7 @@ require (
github.com/muesli/termenv v0.16.0
github.com/pmezard/go-difflib v1.0.0
github.com/sahilm/fuzzy v0.1.1
github.com/shirou/gopsutil/v4 v4.26.10
github.com/yuin/goldmark v1.7.8
golang.org/x/sys v0.48.0
golang.org/x/term v0.46.0
@@ -31,10 +32,13 @@ require (
github.com/charmbracelet/x/term v0.2.1 // indirect
github.com/dlclark/regexp2 v1.11.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/ebitengine/purego v0.11.2 // indirect
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
@@ -43,10 +47,14 @@ require (
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/reflow v0.3.0 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/tklauser/go-sysconf v0.4.0 // indirect
github.com/tklauser/numcpus v0.12.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/yuin/goldmark-emoji v1.0.5 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
golang.org/x/mod v0.41.0 // indirect
golang.org/x/net v0.59.0 // indirect
golang.org/x/sync v0.23.0 // indirect
+26 -2
View File
@@ -40,10 +40,15 @@ github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxK
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/ebitengine/purego v0.11.2 h1:WWiTx8rD8zYdOwSj8O/NiNbsOvSAgJMTlyV1XlbUSeU=
github.com/ebitengine/purego v0.11.2/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -60,6 +65,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
@@ -83,6 +90,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU=
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -91,6 +100,14 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA=
github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
github.com/shirou/gopsutil/v4 v4.26.10 h1:MJN6euesJx53MVEjscYeyRSRcrdIcy3sz5k9nosjldg=
github.com/shirou/gopsutil/v4 v4.26.10/go.mod h1:XpkIrcUMQhwte/dGxu57KtnMMFsbKelI7PP23zweKBk=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
@@ -98,6 +115,10 @@ github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic=
github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
github.com/yuin/goldmark-emoji v1.0.5 h1:EMVWyCGPlXJfUXBXpuMu+ii3TIaxbVBnEX9uaDC4cIk=
github.com/yuin/goldmark-emoji v1.0.5/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E=
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
@@ -106,6 +127,8 @@ golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
@@ -115,6 +138,7 @@ golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg=
modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE=
+1
View File
@@ -282,6 +282,7 @@ type Message struct {
// whole set) and the attention_acks_changed push (the delta). See
// attention_acks.go.
AttentionAcks *AttentionAcks `json:"attention_acks,omitempty"`
ServerStats *ServerStats `json:"server_stats,omitempty"`
// App notification stream (see notifications.go): After is the
// subscribe_notifications cursor, Notifications the subscribe reply,
// Notification one push, and IDs the clear_notifications request, the
+90
View File
@@ -0,0 +1,90 @@
package protocol
const (
ServerStatsRequest = "server_stats"
ServerStatsResult = "server_stats_result"
)
type ServerStats struct {
SampledAt string `json:"sampled_at"`
Host ServerHostStats `json:"host"`
CPU ServerCPUStats `json:"cpu"`
Memory ServerMemoryStats `json:"memory"`
Storage []ServerDiskStats `json:"storage"`
Data ServerDataStats `json:"data"`
Process ServerProcessStats `json:"process"`
GPU ServerGPUStats `json:"gpu"`
}
type ServerHostStats struct {
Hostname string `json:"hostname"`
OS string `json:"os"`
Platform string `json:"platform"`
PlatformVersion string `json:"platform_version"`
Kernel string `json:"kernel"`
Arch string `json:"arch"`
UptimeSeconds uint64 `json:"uptime_seconds"`
Load1 *float64 `json:"load1,omitempty"`
Load5 *float64 `json:"load5,omitempty"`
Load15 *float64 `json:"load15,omitempty"`
Error string `json:"error,omitempty"`
}
type ServerCPUStats struct {
Model string `json:"model"`
LogicalCores int `json:"logical_cores"`
PhysicalCores int `json:"physical_cores"`
UsagePercent float64 `json:"usage_percent"`
PerCorePercent []float64 `json:"per_core_percent,omitempty"`
Error string `json:"error,omitempty"`
}
type ServerMemoryStats struct {
TotalBytes uint64 `json:"total_bytes"`
UsedBytes uint64 `json:"used_bytes"`
AvailableBytes uint64 `json:"available_bytes"`
UsedPercent float64 `json:"used_percent"`
SwapTotalBytes uint64 `json:"swap_total_bytes"`
SwapUsedBytes uint64 `json:"swap_used_bytes"`
Error string `json:"error,omitempty"`
}
type ServerDiskStats struct {
Mount string `json:"mount"`
Device string `json:"device"`
FSType string `json:"fs_type"`
TotalBytes uint64 `json:"total_bytes"`
UsedBytes uint64 `json:"used_bytes"`
FreeBytes uint64 `json:"free_bytes"`
UsedPercent float64 `json:"used_percent"`
Error string `json:"error,omitempty"`
}
type ServerDataStats struct {
Dir string `json:"dir"`
DBBytes uint64 `json:"db_bytes"`
ArtifactsBytes uint64 `json:"artifacts_bytes"`
TotalBytes uint64 `json:"total_bytes"`
Error string `json:"error,omitempty"`
}
type ServerProcessStats struct {
PID int32 `json:"pid"`
StartedAt string `json:"started_at"`
RSSBytes uint64 `json:"rss_bytes,omitempty"`
CPUPercent float64 `json:"cpu_percent,omitempty"`
Goroutines int `json:"goroutines,omitempty"`
HeapBytes uint64 `json:"heap_bytes,omitempty"`
Threads int `json:"threads,omitempty"`
GoVersion string `json:"go_version,omitempty"`
Build string `json:"build,omitempty"`
Error string `json:"error,omitempty"`
}
type ServerGPUStats struct {
Available bool `json:"available"`
Reason string `json:"reason,omitempty"`
Devices []ServerGPUDevice `json:"devices,omitempty"`
Error string `json:"error,omitempty"`
}
type ServerGPUDevice struct {
Name string `json:"name"`
Vendor string `json:"vendor,omitempty"`
UtilizationPercent *float64 `json:"utilization_percent,omitempty"`
MemoryTotalBytes *uint64 `json:"memory_total_bytes,omitempty"`
MemoryUsedBytes *uint64 `json:"memory_used_bytes,omitempty"`
TemperatureC *float64 `json:"temperature_c,omitempty"`
}
+1
View File
@@ -22,6 +22,7 @@ go test ./... # ~1 min for everything; `gofmt -l .` must print
ownership and settlement. This is where a run's state changes.
- `internal/server/runevents.go` — the push hub: subscriptions scoped per workspace, per-run frames,
coalescing. Live clients fold these instead of polling.
- `internal/server/server_stats.go` and `server_stats_gpu.go` — bounded host, process, storage and GPU health sampling.
- `internal/server/chat_*.go` — the chat broker: `chat_sessions`, claims, the stream, permissions,
session registry.
- `internal/stage/` — one client per harness and stage type: `opencode.go` (native sessions),
+236
View File
@@ -0,0 +1,236 @@
package server
import (
"context"
"os"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/shirou/gopsutil/v4/cpu"
"github.com/shirou/gopsutil/v4/disk"
"github.com/shirou/gopsutil/v4/host"
"github.com/shirou/gopsutil/v4/load"
"github.com/shirou/gopsutil/v4/mem"
"github.com/shirou/gopsutil/v4/process"
"git.a-wan.net/alan/pipelines/internal/buildinfo"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
type ServerStatsSampler interface {
Snapshot(context.Context) protocol.ServerStats
}
type systemStatsSampler struct {
dataDir, dbPath, artifactsRoot string
started time.Time
build buildinfo.Info
}
func handleServerStats(ctx context.Context, request protocol.Message, runs *runner, injected ServerStatsSampler, started time.Time, build buildinfo.Info, write func(protocol.Message) error) {
var result protocol.ServerStats
if injected != nil {
result = injected.Snapshot(ctx)
} else if runs == nil {
result.Data.Error = "server store unavailable"
} else {
stop, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
result = (&systemStatsSampler{dataDir: runs.store.Directory(), dbPath: runs.store.DatabasePath(), artifactsRoot: runs.artifactsRoot, started: started, build: build}).Snapshot(stop)
}
if result.SampledAt == "" {
result.SampledAt = time.Now().UTC().Format(time.RFC3339)
}
_ = write(protocol.Message{Type: protocol.ServerStatsResult, ID: request.ID, ServerStats: &result})
}
func (s *systemStatsSampler) Snapshot(ctx context.Context) protocol.ServerStats {
r := protocol.ServerStats{SampledAt: time.Now().UTC().Format(time.RFC3339), Storage: []protocol.ServerDiskStats{}}
s.host(ctx, &r.Host)
s.cpu(ctx, &r.CPU)
s.memory(ctx, &r.Memory)
s.storage(ctx, &r.Storage)
r.Data = protocol.ServerDataStats{Dir: s.dataDir}
var dataErr error
r.Data.DBBytes, dataErr = boundedSize(ctx, s.dataDir, func(p string) bool {
b := filepath.Base(p)
n := filepath.Base(s.dbPath)
return b == n || strings.HasPrefix(b, n+"-")
})
var err error
if r.Data.ArtifactsBytes, err = boundedSize(ctx, s.artifactsRoot, nil); dataErr == nil {
dataErr = err
}
if r.Data.TotalBytes, err = boundedSize(ctx, s.dataDir, nil); dataErr == nil {
dataErr = err
}
if dataErr != nil {
r.Data.Error = dataErr.Error()
}
s.process(ctx, &r.Process)
r.GPU = sampleGPU(ctx)
return r
}
func (s *systemStatsSampler) host(ctx context.Context, r *protocol.ServerHostStats) {
r.OS = runtime.GOOS
r.Arch = runtime.GOARCH
if v, e := host.InfoWithContext(ctx); e == nil {
r.Hostname = v.Hostname
r.Platform = v.Platform
r.PlatformVersion = v.PlatformVersion
r.Kernel = v.KernelVersion
r.UptimeSeconds = v.Uptime
} else {
r.Error = e.Error()
}
if v, e := load.AvgWithContext(ctx); e == nil {
r.Load1 = &v.Load1
r.Load5 = &v.Load5
r.Load15 = &v.Load15
}
}
func (s *systemStatsSampler) cpu(ctx context.Context, r *protocol.ServerCPUStats) {
if v, e := cpu.InfoWithContext(ctx); e == nil && len(v) > 0 {
r.Model = v[0].ModelName
}
r.LogicalCores, _ = cpu.CountsWithContext(ctx, false)
r.PhysicalCores, _ = cpu.CountsWithContext(ctx, true)
if v, e := cpu.PercentWithContext(ctx, 500*time.Millisecond, true); e == nil {
r.PerCorePercent = v
for _, n := range v {
r.UsagePercent += n
}
if len(v) > 0 {
r.UsagePercent /= float64(len(v))
}
} else {
r.Error = e.Error()
}
}
func (s *systemStatsSampler) memory(ctx context.Context, r *protocol.ServerMemoryStats) {
v, e := mem.VirtualMemoryWithContext(ctx)
if e != nil {
r.Error = e.Error()
return
}
r.TotalBytes = v.Total
r.UsedBytes = v.Used
r.AvailableBytes = v.Available
r.UsedPercent = v.UsedPercent
if v, e := mem.SwapMemoryWithContext(ctx); e == nil {
r.SwapTotalBytes = v.Total
r.SwapUsedBytes = v.Used
}
}
func (s *systemStatsSampler) storage(ctx context.Context, out *[]protocol.ServerDiskStats) {
ps, e := disk.PartitionsWithContext(ctx, false)
if e != nil {
*out = append(*out, protocol.ServerDiskStats{Error: e.Error()})
return
}
for _, p := range ps {
if pseudoFS(p.Fstype) || remoteFS(p.Fstype, p.Device) || p.Mountpoint == "" {
continue
}
v, e := disk.UsageWithContext(ctx, p.Mountpoint)
if e != nil {
*out = append(*out, protocol.ServerDiskStats{Mount: p.Mountpoint, Device: p.Device, FSType: p.Fstype, Error: e.Error()})
continue
}
if v.Total < 100<<20 && p.Mountpoint != "/" {
continue
}
*out = append(*out, protocol.ServerDiskStats{Mount: p.Mountpoint, Device: p.Device, FSType: p.Fstype, TotalBytes: v.Total, UsedBytes: v.Used, FreeBytes: v.Free, UsedPercent: v.UsedPercent})
}
}
func (s *systemStatsSampler) process(ctx context.Context, r *protocol.ServerProcessStats) {
p, e := process.NewProcess(int32(os.Getpid()))
if e != nil {
r.Error = e.Error()
return
}
r.PID = p.Pid
r.StartedAt = s.started.UTC().Format(time.RFC3339)
r.GoVersion = runtime.Version()
r.Build = s.build.Version
r.Goroutines = runtime.NumGoroutine()
var m runtime.MemStats
runtime.ReadMemStats(&m)
r.HeapBytes = m.HeapAlloc
if v, e := p.MemoryInfoWithContext(ctx); e == nil {
r.RSSBytes = v.RSS
}
if v, e := p.CPUPercentWithContext(ctx); e == nil {
r.CPUPercent = v
}
if v, e := p.NumThreadsWithContext(ctx); e == nil {
r.Threads = int(v)
}
}
func pseudoFS(v string) bool {
v = strings.ToLower(v)
for _, x := range []string{"proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "cgroup", "overlay", "squashfs", "autofs", "nsfs", "ramfs"} {
if strings.Contains(v, x) {
return true
}
}
return false
}
func remoteFS(fsType, device string) bool {
v := strings.ToLower(fsType)
for _, x := range []string{"nfs", "nfs4", "cifs", "smb", "afp", "sshfs", "fuse.sshfs", "ceph", "gluster", "9p", "davfs", "fuse.rclone"} {
if strings.Contains(v, x) {
return true
}
}
// Linux reports some network mounts with an unhelpful type. These device
// forms are unambiguously remote without excluding local block devices.
return strings.HasPrefix(device, "//") || strings.Contains(device, ":/")
}
const maxSizeWalkEntries = 100_000
func boundedSize(ctx context.Context, root string, match func(string) bool) (uint64, error) {
var n uint64
var entries int
if root == "" {
return 0, nil
}
var walkErr error
err := filepath.WalkDir(root, func(path string, d os.DirEntry, e error) error {
if err := ctx.Err(); err != nil {
return err
}
if e != nil {
if walkErr == nil {
walkErr = e
}
return nil
}
if d == nil {
return nil
}
entries++
if entries > maxSizeWalkEntries {
return filepath.SkipAll
}
if d.Type()&os.ModeSymlink != 0 {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if !d.IsDir() && (match == nil || match(path)) {
if i, e := d.Info(); e == nil && i.Size() > 0 {
n += uint64(i.Size())
}
}
return nil
})
if err != nil {
return n, err
}
return n, walkErr
}
+87
View File
@@ -0,0 +1,87 @@
package server
import (
"context"
"encoding/json"
"os/exec"
"runtime"
"strconv"
"strings"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func sampleGPU(ctx context.Context) protocol.ServerGPUStats {
if runtime.GOOS == "linux" {
return sampleNVIDIA(ctx)
}
if runtime.GOOS == "darwin" {
return sampleMacGPU(ctx)
}
return protocol.ServerGPUStats{Reason: "GPU detection is unsupported on this platform"}
}
func sampleNVIDIA(parent context.Context) protocol.ServerGPUStats {
ctx, cancel := context.WithTimeout(parent, 2*time.Second)
defer cancel()
out, e := exec.CommandContext(ctx, "nvidia-smi", "--query-gpu=name,utilization.gpu,memory.total,memory.used,temperature.gpu", "--format=csv,noheader,nounits").Output()
if e != nil {
return protocol.ServerGPUStats{Reason: "nvidia-smi unavailable"}
}
devices := parseNVIDIA(string(out))
if len(devices) == 0 {
return protocol.ServerGPUStats{Reason: "nvidia-smi returned no usable GPUs"}
}
return protocol.ServerGPUStats{Available: true, Devices: devices}
}
func parseNVIDIA(text string) []protocol.ServerGPUDevice {
var out []protocol.ServerGPUDevice
for _, line := range strings.Split(text, "\n") {
p := strings.Split(line, ",")
if len(p) != 5 {
continue
}
d := protocol.ServerGPUDevice{Name: strings.TrimSpace(p[0]), Vendor: "NVIDIA"}
if d.Name == "" {
continue
}
if v, e := strconv.ParseFloat(strings.TrimSpace(p[1]), 64); e == nil {
d.UtilizationPercent = &v
}
if v, e := strconv.ParseUint(strings.TrimSpace(p[2]), 10, 64); e == nil {
v *= 1024 * 1024
d.MemoryTotalBytes = &v
}
if v, e := strconv.ParseUint(strings.TrimSpace(p[3]), 10, 64); e == nil {
v *= 1024 * 1024
d.MemoryUsedBytes = &v
}
if v, e := strconv.ParseFloat(strings.TrimSpace(p[4]), 64); e == nil {
d.TemperatureC = &v
}
out = append(out, d)
}
return out
}
func sampleMacGPU(parent context.Context) protocol.ServerGPUStats {
ctx, cancel := context.WithTimeout(parent, 3*time.Second)
defer cancel()
out, e := exec.CommandContext(ctx, "system_profiler", "SPDisplaysDataType", "-json").Output()
if e != nil {
return protocol.ServerGPUStats{Reason: "system_profiler unavailable"}
}
var v struct {
Items []struct {
Chipset string `json:"sppci_model"`
Vendor string `json:"spdisplays_vendor"`
} `json:"SPDisplaysDataType"`
}
if json.Unmarshal(out, &v) != nil || len(v.Items) == 0 {
return protocol.ServerGPUStats{Reason: "no display GPU found"}
}
r := protocol.ServerGPUStats{Available: true}
for _, i := range v.Items {
r.Devices = append(r.Devices, protocol.ServerGPUDevice{Name: i.Chipset, Vendor: i.Vendor})
}
return r
}
+55
View File
@@ -0,0 +1,55 @@
package server
import (
"context"
"os"
"path/filepath"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/buildinfo"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
type fakeStatsSampler struct{ snapshot protocol.ServerStats }
func (f fakeStatsSampler) Snapshot(context.Context) protocol.ServerStats { return f.snapshot }
func TestHandleServerStatsUsesSamplerAndEchoesID(t *testing.T) {
want := protocol.ServerStats{SampledAt: "2026-01-02T03:04:05Z"}
var got protocol.Message
handleServerStats(context.Background(), protocol.Message{Type: protocol.ServerStatsRequest, ID: "abc"}, nil, fakeStatsSampler{snapshot: want}, time.Time{}, buildinfo.Info{}, func(m protocol.Message) error { got = m; return nil })
if got.Type != protocol.ServerStatsResult || got.ID != "abc" || got.ServerStats == nil || got.ServerStats.SampledAt != want.SampledAt {
t.Fatalf("unexpected reply: %+v", got)
}
}
func TestParseNVIDIA(t *testing.T) {
devices := parseNVIDIA("GeForce RTX, 42, 8192, 1024, 55\nmalformed")
if len(devices) != 1 || devices[0].Name != "GeForce RTX" || devices[0].UtilizationPercent == nil || *devices[0].UtilizationPercent != 42 {
t.Fatalf("unexpected device: %+v", devices)
}
if devices[0].MemoryTotalBytes == nil || *devices[0].MemoryTotalBytes != uint64(8192)*1024*1024 {
t.Fatalf("memory total=%v, want %d", devices[0].MemoryTotalBytes, uint64(8192)*1024*1024)
}
if devices[0].MemoryUsedBytes == nil || *devices[0].MemoryUsedBytes != uint64(1024)*1024*1024 {
t.Fatalf("memory used=%v, want %d", devices[0].MemoryUsedBytes, uint64(1024)*1024*1024)
}
}
func TestBoundedSizeSkipsSymlinks(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "a"), []byte("1234"), 0600); err != nil {
t.Fatal(err)
}
if err := os.Symlink(filepath.Join(dir, "a"), filepath.Join(dir, "link")); err != nil {
t.Fatal(err)
}
got, err := boundedSize(context.Background(), dir, nil)
if err != nil {
t.Fatal(err)
}
if got != 4 {
t.Fatalf("size=%d, want 4", got)
}
}
+6
View File
@@ -21,6 +21,8 @@ import (
// Options configures the HTTP handler.
type Options struct {
// ServerStatsSampler overrides host sampling for deterministic tests.
ServerStatsSampler ServerStatsSampler
// AllowAuthoring permits publishing immutable definitions. Execution servers
// default to run-only: a CLI --yes flag or raw RPC cannot grant this server
// capability. Enable deliberately on a separate trusted authoring instance.
@@ -277,6 +279,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
if handleDirectoryListing(request, write) {
continue
}
if request.Type == protocol.ServerStatsRequest {
go handleServerStats(connCtx, request, runs, options.ServerStatsSampler, started, build, write)
continue
}
if handleNotificationSettings(ctx, request, settingsStore, write) {
continue
}