From 95849f636a11664713668e67292c5039e0f2e501 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sat, 7 Nov 2026 01:35:39 +0000 Subject: [PATCH] Add server stats websocket reporting --- docs/notes/server-stats.md | 6 + go.mod | 8 + go.sum | 28 +++- internal/protocol/message.go | 1 + internal/protocol/server_stats.go | 90 ++++++++++ internal/server/AGENTS.md | 1 + internal/server/server_stats.go | 236 +++++++++++++++++++++++++++ internal/server/server_stats_gpu.go | 87 ++++++++++ internal/server/server_stats_test.go | 55 +++++++ internal/server/websocket.go | 6 + 10 files changed, 516 insertions(+), 2 deletions(-) create mode 100644 docs/notes/server-stats.md create mode 100644 internal/protocol/server_stats.go create mode 100644 internal/server/server_stats.go create mode 100644 internal/server/server_stats_gpu.go create mode 100644 internal/server/server_stats_test.go diff --git a/docs/notes/server-stats.md b/docs/notes/server-stats.md new file mode 100644 index 00000000..ca3004a9 --- /dev/null +++ b/docs/notes/server-stats.md @@ -0,0 +1,6 @@ +# Server stats + +Send a `server_stats` websocket request to receive a `server_stats_result` with +host, CPU, memory, storage, Pipelines data, process, and best-effort GPU health. +Unsupported or unavailable sections include an error or reason rather than +failing the complete response. diff --git a/go.mod b/go.mod index 701a06ba..3a59f6d3 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/muesli/termenv v0.16.0 github.com/pmezard/go-difflib v1.0.0 github.com/sahilm/fuzzy v0.1.1 + github.com/shirou/gopsutil/v4 v4.26.10 github.com/yuin/goldmark v1.7.8 golang.org/x/sys v0.48.0 golang.org/x/term v0.46.0 @@ -31,10 +32,13 @@ require ( github.com/charmbracelet/x/term v0.2.1 // indirect github.com/dlclark/regexp2 v1.11.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.11.2 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -43,10 +47,14 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yuin/goldmark-emoji v1.0.5 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/mod v0.41.0 // indirect golang.org/x/net v0.59.0 // indirect golang.org/x/sync v0.23.0 // indirect diff --git a/go.sum b/go.sum index baa77c26..45486442 100644 --- a/go.sum +++ b/go.sum @@ -40,10 +40,15 @@ github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxK github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.11.2 h1:WWiTx8rD8zYdOwSj8O/NiNbsOvSAgJMTlyV1XlbUSeU= +github.com/ebitengine/purego v0.11.2/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -60,6 +65,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= @@ -83,6 +90,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU= +github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -91,6 +100,14 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA= github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= +github.com/shirou/gopsutil/v4 v4.26.10 h1:MJN6euesJx53MVEjscYeyRSRcrdIcy3sz5k9nosjldg= +github.com/shirou/gopsutil/v4 v4.26.10/go.mod h1:XpkIrcUMQhwte/dGxu57KtnMMFsbKelI7PP23zweKBk= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= @@ -98,6 +115,10 @@ github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic= github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= github.com/yuin/goldmark-emoji v1.0.5 h1:EMVWyCGPlXJfUXBXpuMu+ii3TIaxbVBnEX9uaDC4cIk= github.com/yuin/goldmark-emoji v1.0.5/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E= golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= @@ -106,6 +127,8 @@ golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= @@ -115,6 +138,7 @@ golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg= modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE= diff --git a/internal/protocol/message.go b/internal/protocol/message.go index 557cde70..1d7306da 100644 --- a/internal/protocol/message.go +++ b/internal/protocol/message.go @@ -282,6 +282,7 @@ type Message struct { // whole set) and the attention_acks_changed push (the delta). See // attention_acks.go. AttentionAcks *AttentionAcks `json:"attention_acks,omitempty"` + ServerStats *ServerStats `json:"server_stats,omitempty"` // App notification stream (see notifications.go): After is the // subscribe_notifications cursor, Notifications the subscribe reply, // Notification one push, and IDs the clear_notifications request, the diff --git a/internal/protocol/server_stats.go b/internal/protocol/server_stats.go new file mode 100644 index 00000000..2888e03f --- /dev/null +++ b/internal/protocol/server_stats.go @@ -0,0 +1,90 @@ +package protocol + +const ( + ServerStatsRequest = "server_stats" + ServerStatsResult = "server_stats_result" +) + +type ServerStats struct { + SampledAt string `json:"sampled_at"` + Host ServerHostStats `json:"host"` + CPU ServerCPUStats `json:"cpu"` + Memory ServerMemoryStats `json:"memory"` + Storage []ServerDiskStats `json:"storage"` + Data ServerDataStats `json:"data"` + Process ServerProcessStats `json:"process"` + GPU ServerGPUStats `json:"gpu"` +} +type ServerHostStats struct { + Hostname string `json:"hostname"` + OS string `json:"os"` + Platform string `json:"platform"` + PlatformVersion string `json:"platform_version"` + Kernel string `json:"kernel"` + Arch string `json:"arch"` + UptimeSeconds uint64 `json:"uptime_seconds"` + Load1 *float64 `json:"load1,omitempty"` + Load5 *float64 `json:"load5,omitempty"` + Load15 *float64 `json:"load15,omitempty"` + Error string `json:"error,omitempty"` +} +type ServerCPUStats struct { + Model string `json:"model"` + LogicalCores int `json:"logical_cores"` + PhysicalCores int `json:"physical_cores"` + UsagePercent float64 `json:"usage_percent"` + PerCorePercent []float64 `json:"per_core_percent,omitempty"` + Error string `json:"error,omitempty"` +} +type ServerMemoryStats struct { + TotalBytes uint64 `json:"total_bytes"` + UsedBytes uint64 `json:"used_bytes"` + AvailableBytes uint64 `json:"available_bytes"` + UsedPercent float64 `json:"used_percent"` + SwapTotalBytes uint64 `json:"swap_total_bytes"` + SwapUsedBytes uint64 `json:"swap_used_bytes"` + Error string `json:"error,omitempty"` +} +type ServerDiskStats struct { + Mount string `json:"mount"` + Device string `json:"device"` + FSType string `json:"fs_type"` + TotalBytes uint64 `json:"total_bytes"` + UsedBytes uint64 `json:"used_bytes"` + FreeBytes uint64 `json:"free_bytes"` + UsedPercent float64 `json:"used_percent"` + Error string `json:"error,omitempty"` +} +type ServerDataStats struct { + Dir string `json:"dir"` + DBBytes uint64 `json:"db_bytes"` + ArtifactsBytes uint64 `json:"artifacts_bytes"` + TotalBytes uint64 `json:"total_bytes"` + Error string `json:"error,omitempty"` +} +type ServerProcessStats struct { + PID int32 `json:"pid"` + StartedAt string `json:"started_at"` + RSSBytes uint64 `json:"rss_bytes,omitempty"` + CPUPercent float64 `json:"cpu_percent,omitempty"` + Goroutines int `json:"goroutines,omitempty"` + HeapBytes uint64 `json:"heap_bytes,omitempty"` + Threads int `json:"threads,omitempty"` + GoVersion string `json:"go_version,omitempty"` + Build string `json:"build,omitempty"` + Error string `json:"error,omitempty"` +} +type ServerGPUStats struct { + Available bool `json:"available"` + Reason string `json:"reason,omitempty"` + Devices []ServerGPUDevice `json:"devices,omitempty"` + Error string `json:"error,omitempty"` +} +type ServerGPUDevice struct { + Name string `json:"name"` + Vendor string `json:"vendor,omitempty"` + UtilizationPercent *float64 `json:"utilization_percent,omitempty"` + MemoryTotalBytes *uint64 `json:"memory_total_bytes,omitempty"` + MemoryUsedBytes *uint64 `json:"memory_used_bytes,omitempty"` + TemperatureC *float64 `json:"temperature_c,omitempty"` +} diff --git a/internal/server/AGENTS.md b/internal/server/AGENTS.md index 58f1d490..a4acb44a 100644 --- a/internal/server/AGENTS.md +++ b/internal/server/AGENTS.md @@ -22,6 +22,7 @@ go test ./... # ~1 min for everything; `gofmt -l .` must print ownership and settlement. This is where a run's state changes. - `internal/server/runevents.go` — the push hub: subscriptions scoped per workspace, per-run frames, coalescing. Live clients fold these instead of polling. +- `internal/server/server_stats.go` and `server_stats_gpu.go` — bounded host, process, storage and GPU health sampling. - `internal/server/chat_*.go` — the chat broker: `chat_sessions`, claims, the stream, permissions, session registry. - `internal/stage/` — one client per harness and stage type: `opencode.go` (native sessions), diff --git a/internal/server/server_stats.go b/internal/server/server_stats.go new file mode 100644 index 00000000..936e9373 --- /dev/null +++ b/internal/server/server_stats.go @@ -0,0 +1,236 @@ +package server + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/shirou/gopsutil/v4/cpu" + "github.com/shirou/gopsutil/v4/disk" + "github.com/shirou/gopsutil/v4/host" + "github.com/shirou/gopsutil/v4/load" + "github.com/shirou/gopsutil/v4/mem" + "github.com/shirou/gopsutil/v4/process" + + "git.a-wan.net/alan/pipelines/internal/buildinfo" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +type ServerStatsSampler interface { + Snapshot(context.Context) protocol.ServerStats +} +type systemStatsSampler struct { + dataDir, dbPath, artifactsRoot string + started time.Time + build buildinfo.Info +} + +func handleServerStats(ctx context.Context, request protocol.Message, runs *runner, injected ServerStatsSampler, started time.Time, build buildinfo.Info, write func(protocol.Message) error) { + var result protocol.ServerStats + if injected != nil { + result = injected.Snapshot(ctx) + } else if runs == nil { + result.Data.Error = "server store unavailable" + } else { + stop, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + result = (&systemStatsSampler{dataDir: runs.store.Directory(), dbPath: runs.store.DatabasePath(), artifactsRoot: runs.artifactsRoot, started: started, build: build}).Snapshot(stop) + } + if result.SampledAt == "" { + result.SampledAt = time.Now().UTC().Format(time.RFC3339) + } + _ = write(protocol.Message{Type: protocol.ServerStatsResult, ID: request.ID, ServerStats: &result}) +} + +func (s *systemStatsSampler) Snapshot(ctx context.Context) protocol.ServerStats { + r := protocol.ServerStats{SampledAt: time.Now().UTC().Format(time.RFC3339), Storage: []protocol.ServerDiskStats{}} + s.host(ctx, &r.Host) + s.cpu(ctx, &r.CPU) + s.memory(ctx, &r.Memory) + s.storage(ctx, &r.Storage) + r.Data = protocol.ServerDataStats{Dir: s.dataDir} + var dataErr error + r.Data.DBBytes, dataErr = boundedSize(ctx, s.dataDir, func(p string) bool { + b := filepath.Base(p) + n := filepath.Base(s.dbPath) + return b == n || strings.HasPrefix(b, n+"-") + }) + var err error + if r.Data.ArtifactsBytes, err = boundedSize(ctx, s.artifactsRoot, nil); dataErr == nil { + dataErr = err + } + if r.Data.TotalBytes, err = boundedSize(ctx, s.dataDir, nil); dataErr == nil { + dataErr = err + } + if dataErr != nil { + r.Data.Error = dataErr.Error() + } + s.process(ctx, &r.Process) + r.GPU = sampleGPU(ctx) + return r +} +func (s *systemStatsSampler) host(ctx context.Context, r *protocol.ServerHostStats) { + r.OS = runtime.GOOS + r.Arch = runtime.GOARCH + if v, e := host.InfoWithContext(ctx); e == nil { + r.Hostname = v.Hostname + r.Platform = v.Platform + r.PlatformVersion = v.PlatformVersion + r.Kernel = v.KernelVersion + r.UptimeSeconds = v.Uptime + } else { + r.Error = e.Error() + } + if v, e := load.AvgWithContext(ctx); e == nil { + r.Load1 = &v.Load1 + r.Load5 = &v.Load5 + r.Load15 = &v.Load15 + } +} +func (s *systemStatsSampler) cpu(ctx context.Context, r *protocol.ServerCPUStats) { + if v, e := cpu.InfoWithContext(ctx); e == nil && len(v) > 0 { + r.Model = v[0].ModelName + } + r.LogicalCores, _ = cpu.CountsWithContext(ctx, false) + r.PhysicalCores, _ = cpu.CountsWithContext(ctx, true) + if v, e := cpu.PercentWithContext(ctx, 500*time.Millisecond, true); e == nil { + r.PerCorePercent = v + for _, n := range v { + r.UsagePercent += n + } + if len(v) > 0 { + r.UsagePercent /= float64(len(v)) + } + } else { + r.Error = e.Error() + } +} +func (s *systemStatsSampler) memory(ctx context.Context, r *protocol.ServerMemoryStats) { + v, e := mem.VirtualMemoryWithContext(ctx) + if e != nil { + r.Error = e.Error() + return + } + r.TotalBytes = v.Total + r.UsedBytes = v.Used + r.AvailableBytes = v.Available + r.UsedPercent = v.UsedPercent + if v, e := mem.SwapMemoryWithContext(ctx); e == nil { + r.SwapTotalBytes = v.Total + r.SwapUsedBytes = v.Used + } +} +func (s *systemStatsSampler) storage(ctx context.Context, out *[]protocol.ServerDiskStats) { + ps, e := disk.PartitionsWithContext(ctx, false) + if e != nil { + *out = append(*out, protocol.ServerDiskStats{Error: e.Error()}) + return + } + for _, p := range ps { + if pseudoFS(p.Fstype) || remoteFS(p.Fstype, p.Device) || p.Mountpoint == "" { + continue + } + v, e := disk.UsageWithContext(ctx, p.Mountpoint) + if e != nil { + *out = append(*out, protocol.ServerDiskStats{Mount: p.Mountpoint, Device: p.Device, FSType: p.Fstype, Error: e.Error()}) + continue + } + if v.Total < 100<<20 && p.Mountpoint != "/" { + continue + } + *out = append(*out, protocol.ServerDiskStats{Mount: p.Mountpoint, Device: p.Device, FSType: p.Fstype, TotalBytes: v.Total, UsedBytes: v.Used, FreeBytes: v.Free, UsedPercent: v.UsedPercent}) + } +} +func (s *systemStatsSampler) process(ctx context.Context, r *protocol.ServerProcessStats) { + p, e := process.NewProcess(int32(os.Getpid())) + if e != nil { + r.Error = e.Error() + return + } + r.PID = p.Pid + r.StartedAt = s.started.UTC().Format(time.RFC3339) + r.GoVersion = runtime.Version() + r.Build = s.build.Version + r.Goroutines = runtime.NumGoroutine() + var m runtime.MemStats + runtime.ReadMemStats(&m) + r.HeapBytes = m.HeapAlloc + if v, e := p.MemoryInfoWithContext(ctx); e == nil { + r.RSSBytes = v.RSS + } + if v, e := p.CPUPercentWithContext(ctx); e == nil { + r.CPUPercent = v + } + if v, e := p.NumThreadsWithContext(ctx); e == nil { + r.Threads = int(v) + } +} +func pseudoFS(v string) bool { + v = strings.ToLower(v) + for _, x := range []string{"proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "cgroup", "overlay", "squashfs", "autofs", "nsfs", "ramfs"} { + if strings.Contains(v, x) { + return true + } + } + return false +} + +func remoteFS(fsType, device string) bool { + v := strings.ToLower(fsType) + for _, x := range []string{"nfs", "nfs4", "cifs", "smb", "afp", "sshfs", "fuse.sshfs", "ceph", "gluster", "9p", "davfs", "fuse.rclone"} { + if strings.Contains(v, x) { + return true + } + } + // Linux reports some network mounts with an unhelpful type. These device + // forms are unambiguously remote without excluding local block devices. + return strings.HasPrefix(device, "//") || strings.Contains(device, ":/") +} + +const maxSizeWalkEntries = 100_000 + +func boundedSize(ctx context.Context, root string, match func(string) bool) (uint64, error) { + var n uint64 + var entries int + if root == "" { + return 0, nil + } + var walkErr error + err := filepath.WalkDir(root, func(path string, d os.DirEntry, e error) error { + if err := ctx.Err(); err != nil { + return err + } + if e != nil { + if walkErr == nil { + walkErr = e + } + return nil + } + if d == nil { + return nil + } + entries++ + if entries > maxSizeWalkEntries { + return filepath.SkipAll + } + if d.Type()&os.ModeSymlink != 0 { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if !d.IsDir() && (match == nil || match(path)) { + if i, e := d.Info(); e == nil && i.Size() > 0 { + n += uint64(i.Size()) + } + } + return nil + }) + if err != nil { + return n, err + } + return n, walkErr +} diff --git a/internal/server/server_stats_gpu.go b/internal/server/server_stats_gpu.go new file mode 100644 index 00000000..049c290b --- /dev/null +++ b/internal/server/server_stats_gpu.go @@ -0,0 +1,87 @@ +package server + +import ( + "context" + "encoding/json" + "os/exec" + "runtime" + "strconv" + "strings" + "time" + + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +func sampleGPU(ctx context.Context) protocol.ServerGPUStats { + if runtime.GOOS == "linux" { + return sampleNVIDIA(ctx) + } + if runtime.GOOS == "darwin" { + return sampleMacGPU(ctx) + } + return protocol.ServerGPUStats{Reason: "GPU detection is unsupported on this platform"} +} +func sampleNVIDIA(parent context.Context) protocol.ServerGPUStats { + ctx, cancel := context.WithTimeout(parent, 2*time.Second) + defer cancel() + out, e := exec.CommandContext(ctx, "nvidia-smi", "--query-gpu=name,utilization.gpu,memory.total,memory.used,temperature.gpu", "--format=csv,noheader,nounits").Output() + if e != nil { + return protocol.ServerGPUStats{Reason: "nvidia-smi unavailable"} + } + devices := parseNVIDIA(string(out)) + if len(devices) == 0 { + return protocol.ServerGPUStats{Reason: "nvidia-smi returned no usable GPUs"} + } + return protocol.ServerGPUStats{Available: true, Devices: devices} +} +func parseNVIDIA(text string) []protocol.ServerGPUDevice { + var out []protocol.ServerGPUDevice + for _, line := range strings.Split(text, "\n") { + p := strings.Split(line, ",") + if len(p) != 5 { + continue + } + d := protocol.ServerGPUDevice{Name: strings.TrimSpace(p[0]), Vendor: "NVIDIA"} + if d.Name == "" { + continue + } + if v, e := strconv.ParseFloat(strings.TrimSpace(p[1]), 64); e == nil { + d.UtilizationPercent = &v + } + if v, e := strconv.ParseUint(strings.TrimSpace(p[2]), 10, 64); e == nil { + v *= 1024 * 1024 + d.MemoryTotalBytes = &v + } + if v, e := strconv.ParseUint(strings.TrimSpace(p[3]), 10, 64); e == nil { + v *= 1024 * 1024 + d.MemoryUsedBytes = &v + } + if v, e := strconv.ParseFloat(strings.TrimSpace(p[4]), 64); e == nil { + d.TemperatureC = &v + } + out = append(out, d) + } + return out +} +func sampleMacGPU(parent context.Context) protocol.ServerGPUStats { + ctx, cancel := context.WithTimeout(parent, 3*time.Second) + defer cancel() + out, e := exec.CommandContext(ctx, "system_profiler", "SPDisplaysDataType", "-json").Output() + if e != nil { + return protocol.ServerGPUStats{Reason: "system_profiler unavailable"} + } + var v struct { + Items []struct { + Chipset string `json:"sppci_model"` + Vendor string `json:"spdisplays_vendor"` + } `json:"SPDisplaysDataType"` + } + if json.Unmarshal(out, &v) != nil || len(v.Items) == 0 { + return protocol.ServerGPUStats{Reason: "no display GPU found"} + } + r := protocol.ServerGPUStats{Available: true} + for _, i := range v.Items { + r.Devices = append(r.Devices, protocol.ServerGPUDevice{Name: i.Chipset, Vendor: i.Vendor}) + } + return r +} diff --git a/internal/server/server_stats_test.go b/internal/server/server_stats_test.go new file mode 100644 index 00000000..7cf8bd94 --- /dev/null +++ b/internal/server/server_stats_test.go @@ -0,0 +1,55 @@ +package server + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "git.a-wan.net/alan/pipelines/internal/buildinfo" + "git.a-wan.net/alan/pipelines/internal/protocol" +) + +type fakeStatsSampler struct{ snapshot protocol.ServerStats } + +func (f fakeStatsSampler) Snapshot(context.Context) protocol.ServerStats { return f.snapshot } + +func TestHandleServerStatsUsesSamplerAndEchoesID(t *testing.T) { + want := protocol.ServerStats{SampledAt: "2026-01-02T03:04:05Z"} + var got protocol.Message + handleServerStats(context.Background(), protocol.Message{Type: protocol.ServerStatsRequest, ID: "abc"}, nil, fakeStatsSampler{snapshot: want}, time.Time{}, buildinfo.Info{}, func(m protocol.Message) error { got = m; return nil }) + if got.Type != protocol.ServerStatsResult || got.ID != "abc" || got.ServerStats == nil || got.ServerStats.SampledAt != want.SampledAt { + t.Fatalf("unexpected reply: %+v", got) + } +} + +func TestParseNVIDIA(t *testing.T) { + devices := parseNVIDIA("GeForce RTX, 42, 8192, 1024, 55\nmalformed") + if len(devices) != 1 || devices[0].Name != "GeForce RTX" || devices[0].UtilizationPercent == nil || *devices[0].UtilizationPercent != 42 { + t.Fatalf("unexpected device: %+v", devices) + } + if devices[0].MemoryTotalBytes == nil || *devices[0].MemoryTotalBytes != uint64(8192)*1024*1024 { + t.Fatalf("memory total=%v, want %d", devices[0].MemoryTotalBytes, uint64(8192)*1024*1024) + } + if devices[0].MemoryUsedBytes == nil || *devices[0].MemoryUsedBytes != uint64(1024)*1024*1024 { + t.Fatalf("memory used=%v, want %d", devices[0].MemoryUsedBytes, uint64(1024)*1024*1024) + } +} + +func TestBoundedSizeSkipsSymlinks(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "a"), []byte("1234"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(dir, "a"), filepath.Join(dir, "link")); err != nil { + t.Fatal(err) + } + got, err := boundedSize(context.Background(), dir, nil) + if err != nil { + t.Fatal(err) + } + if got != 4 { + t.Fatalf("size=%d, want 4", got) + } +} diff --git a/internal/server/websocket.go b/internal/server/websocket.go index efece1a5..5bcb6d52 100644 --- a/internal/server/websocket.go +++ b/internal/server/websocket.go @@ -21,6 +21,8 @@ import ( // Options configures the HTTP handler. type Options struct { + // ServerStatsSampler overrides host sampling for deterministic tests. + ServerStatsSampler ServerStatsSampler // AllowAuthoring permits publishing immutable definitions. Execution servers // default to run-only: a CLI --yes flag or raw RPC cannot grant this server // capability. Enable deliberately on a separate trusted authoring instance. @@ -277,6 +279,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin if handleDirectoryListing(request, write) { continue } + if request.Type == protocol.ServerStatsRequest { + go handleServerStats(connCtx, request, runs, options.ServerStatsSampler, started, build, write) + continue + } if handleNotificationSettings(ctx, request, settingsStore, write) { continue }