Add server stats websocket reporting
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
# Server stats
|
||||
|
||||
Send a `server_stats` websocket request to receive a `server_stats_result` with
|
||||
host, CPU, memory, storage, Pipelines data, process, and best-effort GPU health.
|
||||
Unsupported or unavailable sections include an error or reason rather than
|
||||
failing the complete response.
|
||||
@@ -15,6 +15,7 @@ require (
|
||||
github.com/muesli/termenv v0.16.0
|
||||
github.com/pmezard/go-difflib v1.0.0
|
||||
github.com/sahilm/fuzzy v0.1.1
|
||||
github.com/shirou/gopsutil/v4 v4.26.10
|
||||
github.com/yuin/goldmark v1.7.8
|
||||
golang.org/x/sys v0.48.0
|
||||
golang.org/x/term v0.46.0
|
||||
@@ -31,10 +32,13 @@ require (
|
||||
github.com/charmbracelet/x/term v0.2.1 // indirect
|
||||
github.com/dlclark/regexp2 v1.11.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.11.2 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
@@ -43,10 +47,14 @@ require (
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/reflow v0.3.0 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
||||
github.com/tklauser/numcpus v0.12.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/yuin/goldmark-emoji v1.0.5 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
golang.org/x/mod v0.41.0 // indirect
|
||||
golang.org/x/net v0.59.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
|
||||
@@ -40,10 +40,15 @@ github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxK
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/ebitengine/purego v0.11.2 h1:WWiTx8rD8zYdOwSj8O/NiNbsOvSAgJMTlyV1XlbUSeU=
|
||||
github.com/ebitengine/purego v0.11.2/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
@@ -60,6 +65,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
@@ -83,6 +90,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU=
|
||||
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
@@ -91,6 +100,14 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA=
|
||||
github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y=
|
||||
github.com/shirou/gopsutil/v4 v4.26.10 h1:MJN6euesJx53MVEjscYeyRSRcrdIcy3sz5k9nosjldg=
|
||||
github.com/shirou/gopsutil/v4 v4.26.10/go.mod h1:XpkIrcUMQhwte/dGxu57KtnMMFsbKelI7PP23zweKBk=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
|
||||
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
|
||||
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
|
||||
github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
|
||||
@@ -98,6 +115,10 @@ github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic=
|
||||
github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
|
||||
github.com/yuin/goldmark-emoji v1.0.5 h1:EMVWyCGPlXJfUXBXpuMu+ii3TIaxbVBnEX9uaDC4cIk=
|
||||
github.com/yuin/goldmark-emoji v1.0.5/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E=
|
||||
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
|
||||
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
|
||||
@@ -106,6 +127,8 @@ golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
||||
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
@@ -115,6 +138,7 @@ golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
|
||||
golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg=
|
||||
modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE=
|
||||
|
||||
@@ -282,6 +282,7 @@ type Message struct {
|
||||
// whole set) and the attention_acks_changed push (the delta). See
|
||||
// attention_acks.go.
|
||||
AttentionAcks *AttentionAcks `json:"attention_acks,omitempty"`
|
||||
ServerStats *ServerStats `json:"server_stats,omitempty"`
|
||||
// App notification stream (see notifications.go): After is the
|
||||
// subscribe_notifications cursor, Notifications the subscribe reply,
|
||||
// Notification one push, and IDs the clear_notifications request, the
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package protocol
|
||||
|
||||
const (
|
||||
ServerStatsRequest = "server_stats"
|
||||
ServerStatsResult = "server_stats_result"
|
||||
)
|
||||
|
||||
type ServerStats struct {
|
||||
SampledAt string `json:"sampled_at"`
|
||||
Host ServerHostStats `json:"host"`
|
||||
CPU ServerCPUStats `json:"cpu"`
|
||||
Memory ServerMemoryStats `json:"memory"`
|
||||
Storage []ServerDiskStats `json:"storage"`
|
||||
Data ServerDataStats `json:"data"`
|
||||
Process ServerProcessStats `json:"process"`
|
||||
GPU ServerGPUStats `json:"gpu"`
|
||||
}
|
||||
type ServerHostStats struct {
|
||||
Hostname string `json:"hostname"`
|
||||
OS string `json:"os"`
|
||||
Platform string `json:"platform"`
|
||||
PlatformVersion string `json:"platform_version"`
|
||||
Kernel string `json:"kernel"`
|
||||
Arch string `json:"arch"`
|
||||
UptimeSeconds uint64 `json:"uptime_seconds"`
|
||||
Load1 *float64 `json:"load1,omitempty"`
|
||||
Load5 *float64 `json:"load5,omitempty"`
|
||||
Load15 *float64 `json:"load15,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
type ServerCPUStats struct {
|
||||
Model string `json:"model"`
|
||||
LogicalCores int `json:"logical_cores"`
|
||||
PhysicalCores int `json:"physical_cores"`
|
||||
UsagePercent float64 `json:"usage_percent"`
|
||||
PerCorePercent []float64 `json:"per_core_percent,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
type ServerMemoryStats struct {
|
||||
TotalBytes uint64 `json:"total_bytes"`
|
||||
UsedBytes uint64 `json:"used_bytes"`
|
||||
AvailableBytes uint64 `json:"available_bytes"`
|
||||
UsedPercent float64 `json:"used_percent"`
|
||||
SwapTotalBytes uint64 `json:"swap_total_bytes"`
|
||||
SwapUsedBytes uint64 `json:"swap_used_bytes"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
type ServerDiskStats struct {
|
||||
Mount string `json:"mount"`
|
||||
Device string `json:"device"`
|
||||
FSType string `json:"fs_type"`
|
||||
TotalBytes uint64 `json:"total_bytes"`
|
||||
UsedBytes uint64 `json:"used_bytes"`
|
||||
FreeBytes uint64 `json:"free_bytes"`
|
||||
UsedPercent float64 `json:"used_percent"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
type ServerDataStats struct {
|
||||
Dir string `json:"dir"`
|
||||
DBBytes uint64 `json:"db_bytes"`
|
||||
ArtifactsBytes uint64 `json:"artifacts_bytes"`
|
||||
TotalBytes uint64 `json:"total_bytes"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
type ServerProcessStats struct {
|
||||
PID int32 `json:"pid"`
|
||||
StartedAt string `json:"started_at"`
|
||||
RSSBytes uint64 `json:"rss_bytes,omitempty"`
|
||||
CPUPercent float64 `json:"cpu_percent,omitempty"`
|
||||
Goroutines int `json:"goroutines,omitempty"`
|
||||
HeapBytes uint64 `json:"heap_bytes,omitempty"`
|
||||
Threads int `json:"threads,omitempty"`
|
||||
GoVersion string `json:"go_version,omitempty"`
|
||||
Build string `json:"build,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
type ServerGPUStats struct {
|
||||
Available bool `json:"available"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Devices []ServerGPUDevice `json:"devices,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
type ServerGPUDevice struct {
|
||||
Name string `json:"name"`
|
||||
Vendor string `json:"vendor,omitempty"`
|
||||
UtilizationPercent *float64 `json:"utilization_percent,omitempty"`
|
||||
MemoryTotalBytes *uint64 `json:"memory_total_bytes,omitempty"`
|
||||
MemoryUsedBytes *uint64 `json:"memory_used_bytes,omitempty"`
|
||||
TemperatureC *float64 `json:"temperature_c,omitempty"`
|
||||
}
|
||||
@@ -22,6 +22,7 @@ go test ./... # ~1 min for everything; `gofmt -l .` must print
|
||||
ownership and settlement. This is where a run's state changes.
|
||||
- `internal/server/runevents.go` — the push hub: subscriptions scoped per workspace, per-run frames,
|
||||
coalescing. Live clients fold these instead of polling.
|
||||
- `internal/server/server_stats.go` and `server_stats_gpu.go` — bounded host, process, storage and GPU health sampling.
|
||||
- `internal/server/chat_*.go` — the chat broker: `chat_sessions`, claims, the stream, permissions,
|
||||
session registry.
|
||||
- `internal/stage/` — one client per harness and stage type: `opencode.go` (native sessions),
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/shirou/gopsutil/v4/cpu"
|
||||
"github.com/shirou/gopsutil/v4/disk"
|
||||
"github.com/shirou/gopsutil/v4/host"
|
||||
"github.com/shirou/gopsutil/v4/load"
|
||||
"github.com/shirou/gopsutil/v4/mem"
|
||||
"github.com/shirou/gopsutil/v4/process"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/buildinfo"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
type ServerStatsSampler interface {
|
||||
Snapshot(context.Context) protocol.ServerStats
|
||||
}
|
||||
type systemStatsSampler struct {
|
||||
dataDir, dbPath, artifactsRoot string
|
||||
started time.Time
|
||||
build buildinfo.Info
|
||||
}
|
||||
|
||||
func handleServerStats(ctx context.Context, request protocol.Message, runs *runner, injected ServerStatsSampler, started time.Time, build buildinfo.Info, write func(protocol.Message) error) {
|
||||
var result protocol.ServerStats
|
||||
if injected != nil {
|
||||
result = injected.Snapshot(ctx)
|
||||
} else if runs == nil {
|
||||
result.Data.Error = "server store unavailable"
|
||||
} else {
|
||||
stop, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
result = (&systemStatsSampler{dataDir: runs.store.Directory(), dbPath: runs.store.DatabasePath(), artifactsRoot: runs.artifactsRoot, started: started, build: build}).Snapshot(stop)
|
||||
}
|
||||
if result.SampledAt == "" {
|
||||
result.SampledAt = time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
_ = write(protocol.Message{Type: protocol.ServerStatsResult, ID: request.ID, ServerStats: &result})
|
||||
}
|
||||
|
||||
func (s *systemStatsSampler) Snapshot(ctx context.Context) protocol.ServerStats {
|
||||
r := protocol.ServerStats{SampledAt: time.Now().UTC().Format(time.RFC3339), Storage: []protocol.ServerDiskStats{}}
|
||||
s.host(ctx, &r.Host)
|
||||
s.cpu(ctx, &r.CPU)
|
||||
s.memory(ctx, &r.Memory)
|
||||
s.storage(ctx, &r.Storage)
|
||||
r.Data = protocol.ServerDataStats{Dir: s.dataDir}
|
||||
var dataErr error
|
||||
r.Data.DBBytes, dataErr = boundedSize(ctx, s.dataDir, func(p string) bool {
|
||||
b := filepath.Base(p)
|
||||
n := filepath.Base(s.dbPath)
|
||||
return b == n || strings.HasPrefix(b, n+"-")
|
||||
})
|
||||
var err error
|
||||
if r.Data.ArtifactsBytes, err = boundedSize(ctx, s.artifactsRoot, nil); dataErr == nil {
|
||||
dataErr = err
|
||||
}
|
||||
if r.Data.TotalBytes, err = boundedSize(ctx, s.dataDir, nil); dataErr == nil {
|
||||
dataErr = err
|
||||
}
|
||||
if dataErr != nil {
|
||||
r.Data.Error = dataErr.Error()
|
||||
}
|
||||
s.process(ctx, &r.Process)
|
||||
r.GPU = sampleGPU(ctx)
|
||||
return r
|
||||
}
|
||||
func (s *systemStatsSampler) host(ctx context.Context, r *protocol.ServerHostStats) {
|
||||
r.OS = runtime.GOOS
|
||||
r.Arch = runtime.GOARCH
|
||||
if v, e := host.InfoWithContext(ctx); e == nil {
|
||||
r.Hostname = v.Hostname
|
||||
r.Platform = v.Platform
|
||||
r.PlatformVersion = v.PlatformVersion
|
||||
r.Kernel = v.KernelVersion
|
||||
r.UptimeSeconds = v.Uptime
|
||||
} else {
|
||||
r.Error = e.Error()
|
||||
}
|
||||
if v, e := load.AvgWithContext(ctx); e == nil {
|
||||
r.Load1 = &v.Load1
|
||||
r.Load5 = &v.Load5
|
||||
r.Load15 = &v.Load15
|
||||
}
|
||||
}
|
||||
func (s *systemStatsSampler) cpu(ctx context.Context, r *protocol.ServerCPUStats) {
|
||||
if v, e := cpu.InfoWithContext(ctx); e == nil && len(v) > 0 {
|
||||
r.Model = v[0].ModelName
|
||||
}
|
||||
r.LogicalCores, _ = cpu.CountsWithContext(ctx, false)
|
||||
r.PhysicalCores, _ = cpu.CountsWithContext(ctx, true)
|
||||
if v, e := cpu.PercentWithContext(ctx, 500*time.Millisecond, true); e == nil {
|
||||
r.PerCorePercent = v
|
||||
for _, n := range v {
|
||||
r.UsagePercent += n
|
||||
}
|
||||
if len(v) > 0 {
|
||||
r.UsagePercent /= float64(len(v))
|
||||
}
|
||||
} else {
|
||||
r.Error = e.Error()
|
||||
}
|
||||
}
|
||||
func (s *systemStatsSampler) memory(ctx context.Context, r *protocol.ServerMemoryStats) {
|
||||
v, e := mem.VirtualMemoryWithContext(ctx)
|
||||
if e != nil {
|
||||
r.Error = e.Error()
|
||||
return
|
||||
}
|
||||
r.TotalBytes = v.Total
|
||||
r.UsedBytes = v.Used
|
||||
r.AvailableBytes = v.Available
|
||||
r.UsedPercent = v.UsedPercent
|
||||
if v, e := mem.SwapMemoryWithContext(ctx); e == nil {
|
||||
r.SwapTotalBytes = v.Total
|
||||
r.SwapUsedBytes = v.Used
|
||||
}
|
||||
}
|
||||
func (s *systemStatsSampler) storage(ctx context.Context, out *[]protocol.ServerDiskStats) {
|
||||
ps, e := disk.PartitionsWithContext(ctx, false)
|
||||
if e != nil {
|
||||
*out = append(*out, protocol.ServerDiskStats{Error: e.Error()})
|
||||
return
|
||||
}
|
||||
for _, p := range ps {
|
||||
if pseudoFS(p.Fstype) || remoteFS(p.Fstype, p.Device) || p.Mountpoint == "" {
|
||||
continue
|
||||
}
|
||||
v, e := disk.UsageWithContext(ctx, p.Mountpoint)
|
||||
if e != nil {
|
||||
*out = append(*out, protocol.ServerDiskStats{Mount: p.Mountpoint, Device: p.Device, FSType: p.Fstype, Error: e.Error()})
|
||||
continue
|
||||
}
|
||||
if v.Total < 100<<20 && p.Mountpoint != "/" {
|
||||
continue
|
||||
}
|
||||
*out = append(*out, protocol.ServerDiskStats{Mount: p.Mountpoint, Device: p.Device, FSType: p.Fstype, TotalBytes: v.Total, UsedBytes: v.Used, FreeBytes: v.Free, UsedPercent: v.UsedPercent})
|
||||
}
|
||||
}
|
||||
func (s *systemStatsSampler) process(ctx context.Context, r *protocol.ServerProcessStats) {
|
||||
p, e := process.NewProcess(int32(os.Getpid()))
|
||||
if e != nil {
|
||||
r.Error = e.Error()
|
||||
return
|
||||
}
|
||||
r.PID = p.Pid
|
||||
r.StartedAt = s.started.UTC().Format(time.RFC3339)
|
||||
r.GoVersion = runtime.Version()
|
||||
r.Build = s.build.Version
|
||||
r.Goroutines = runtime.NumGoroutine()
|
||||
var m runtime.MemStats
|
||||
runtime.ReadMemStats(&m)
|
||||
r.HeapBytes = m.HeapAlloc
|
||||
if v, e := p.MemoryInfoWithContext(ctx); e == nil {
|
||||
r.RSSBytes = v.RSS
|
||||
}
|
||||
if v, e := p.CPUPercentWithContext(ctx); e == nil {
|
||||
r.CPUPercent = v
|
||||
}
|
||||
if v, e := p.NumThreadsWithContext(ctx); e == nil {
|
||||
r.Threads = int(v)
|
||||
}
|
||||
}
|
||||
func pseudoFS(v string) bool {
|
||||
v = strings.ToLower(v)
|
||||
for _, x := range []string{"proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "cgroup", "overlay", "squashfs", "autofs", "nsfs", "ramfs"} {
|
||||
if strings.Contains(v, x) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func remoteFS(fsType, device string) bool {
|
||||
v := strings.ToLower(fsType)
|
||||
for _, x := range []string{"nfs", "nfs4", "cifs", "smb", "afp", "sshfs", "fuse.sshfs", "ceph", "gluster", "9p", "davfs", "fuse.rclone"} {
|
||||
if strings.Contains(v, x) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// Linux reports some network mounts with an unhelpful type. These device
|
||||
// forms are unambiguously remote without excluding local block devices.
|
||||
return strings.HasPrefix(device, "//") || strings.Contains(device, ":/")
|
||||
}
|
||||
|
||||
const maxSizeWalkEntries = 100_000
|
||||
|
||||
func boundedSize(ctx context.Context, root string, match func(string) bool) (uint64, error) {
|
||||
var n uint64
|
||||
var entries int
|
||||
if root == "" {
|
||||
return 0, nil
|
||||
}
|
||||
var walkErr error
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, e error) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if e != nil {
|
||||
if walkErr == nil {
|
||||
walkErr = e
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
entries++
|
||||
if entries > maxSizeWalkEntries {
|
||||
return filepath.SkipAll
|
||||
}
|
||||
if d.Type()&os.ModeSymlink != 0 {
|
||||
if d.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !d.IsDir() && (match == nil || match(path)) {
|
||||
if i, e := d.Info(); e == nil && i.Size() > 0 {
|
||||
n += uint64(i.Size())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
return n, walkErr
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
func sampleGPU(ctx context.Context) protocol.ServerGPUStats {
|
||||
if runtime.GOOS == "linux" {
|
||||
return sampleNVIDIA(ctx)
|
||||
}
|
||||
if runtime.GOOS == "darwin" {
|
||||
return sampleMacGPU(ctx)
|
||||
}
|
||||
return protocol.ServerGPUStats{Reason: "GPU detection is unsupported on this platform"}
|
||||
}
|
||||
func sampleNVIDIA(parent context.Context) protocol.ServerGPUStats {
|
||||
ctx, cancel := context.WithTimeout(parent, 2*time.Second)
|
||||
defer cancel()
|
||||
out, e := exec.CommandContext(ctx, "nvidia-smi", "--query-gpu=name,utilization.gpu,memory.total,memory.used,temperature.gpu", "--format=csv,noheader,nounits").Output()
|
||||
if e != nil {
|
||||
return protocol.ServerGPUStats{Reason: "nvidia-smi unavailable"}
|
||||
}
|
||||
devices := parseNVIDIA(string(out))
|
||||
if len(devices) == 0 {
|
||||
return protocol.ServerGPUStats{Reason: "nvidia-smi returned no usable GPUs"}
|
||||
}
|
||||
return protocol.ServerGPUStats{Available: true, Devices: devices}
|
||||
}
|
||||
func parseNVIDIA(text string) []protocol.ServerGPUDevice {
|
||||
var out []protocol.ServerGPUDevice
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
p := strings.Split(line, ",")
|
||||
if len(p) != 5 {
|
||||
continue
|
||||
}
|
||||
d := protocol.ServerGPUDevice{Name: strings.TrimSpace(p[0]), Vendor: "NVIDIA"}
|
||||
if d.Name == "" {
|
||||
continue
|
||||
}
|
||||
if v, e := strconv.ParseFloat(strings.TrimSpace(p[1]), 64); e == nil {
|
||||
d.UtilizationPercent = &v
|
||||
}
|
||||
if v, e := strconv.ParseUint(strings.TrimSpace(p[2]), 10, 64); e == nil {
|
||||
v *= 1024 * 1024
|
||||
d.MemoryTotalBytes = &v
|
||||
}
|
||||
if v, e := strconv.ParseUint(strings.TrimSpace(p[3]), 10, 64); e == nil {
|
||||
v *= 1024 * 1024
|
||||
d.MemoryUsedBytes = &v
|
||||
}
|
||||
if v, e := strconv.ParseFloat(strings.TrimSpace(p[4]), 64); e == nil {
|
||||
d.TemperatureC = &v
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out
|
||||
}
|
||||
func sampleMacGPU(parent context.Context) protocol.ServerGPUStats {
|
||||
ctx, cancel := context.WithTimeout(parent, 3*time.Second)
|
||||
defer cancel()
|
||||
out, e := exec.CommandContext(ctx, "system_profiler", "SPDisplaysDataType", "-json").Output()
|
||||
if e != nil {
|
||||
return protocol.ServerGPUStats{Reason: "system_profiler unavailable"}
|
||||
}
|
||||
var v struct {
|
||||
Items []struct {
|
||||
Chipset string `json:"sppci_model"`
|
||||
Vendor string `json:"spdisplays_vendor"`
|
||||
} `json:"SPDisplaysDataType"`
|
||||
}
|
||||
if json.Unmarshal(out, &v) != nil || len(v.Items) == 0 {
|
||||
return protocol.ServerGPUStats{Reason: "no display GPU found"}
|
||||
}
|
||||
r := protocol.ServerGPUStats{Available: true}
|
||||
for _, i := range v.Items {
|
||||
r.Devices = append(r.Devices, protocol.ServerGPUDevice{Name: i.Chipset, Vendor: i.Vendor})
|
||||
}
|
||||
return r
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/buildinfo"
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
type fakeStatsSampler struct{ snapshot protocol.ServerStats }
|
||||
|
||||
func (f fakeStatsSampler) Snapshot(context.Context) protocol.ServerStats { return f.snapshot }
|
||||
|
||||
func TestHandleServerStatsUsesSamplerAndEchoesID(t *testing.T) {
|
||||
want := protocol.ServerStats{SampledAt: "2026-01-02T03:04:05Z"}
|
||||
var got protocol.Message
|
||||
handleServerStats(context.Background(), protocol.Message{Type: protocol.ServerStatsRequest, ID: "abc"}, nil, fakeStatsSampler{snapshot: want}, time.Time{}, buildinfo.Info{}, func(m protocol.Message) error { got = m; return nil })
|
||||
if got.Type != protocol.ServerStatsResult || got.ID != "abc" || got.ServerStats == nil || got.ServerStats.SampledAt != want.SampledAt {
|
||||
t.Fatalf("unexpected reply: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNVIDIA(t *testing.T) {
|
||||
devices := parseNVIDIA("GeForce RTX, 42, 8192, 1024, 55\nmalformed")
|
||||
if len(devices) != 1 || devices[0].Name != "GeForce RTX" || devices[0].UtilizationPercent == nil || *devices[0].UtilizationPercent != 42 {
|
||||
t.Fatalf("unexpected device: %+v", devices)
|
||||
}
|
||||
if devices[0].MemoryTotalBytes == nil || *devices[0].MemoryTotalBytes != uint64(8192)*1024*1024 {
|
||||
t.Fatalf("memory total=%v, want %d", devices[0].MemoryTotalBytes, uint64(8192)*1024*1024)
|
||||
}
|
||||
if devices[0].MemoryUsedBytes == nil || *devices[0].MemoryUsedBytes != uint64(1024)*1024*1024 {
|
||||
t.Fatalf("memory used=%v, want %d", devices[0].MemoryUsedBytes, uint64(1024)*1024*1024)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedSizeSkipsSymlinks(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "a"), []byte("1234"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(filepath.Join(dir, "a"), filepath.Join(dir, "link")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := boundedSize(context.Background(), dir, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != 4 {
|
||||
t.Fatalf("size=%d, want 4", got)
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,8 @@ import (
|
||||
|
||||
// Options configures the HTTP handler.
|
||||
type Options struct {
|
||||
// ServerStatsSampler overrides host sampling for deterministic tests.
|
||||
ServerStatsSampler ServerStatsSampler
|
||||
// AllowAuthoring permits publishing immutable definitions. Execution servers
|
||||
// default to run-only: a CLI --yes flag or raw RPC cannot grant this server
|
||||
// capability. Enable deliberately on a separate trusted authoring instance.
|
||||
@@ -277,6 +279,10 @@ func HandlerWithOptions(ctx context.Context, options Options, stores ...*pipelin
|
||||
if handleDirectoryListing(request, write) {
|
||||
continue
|
||||
}
|
||||
if request.Type == protocol.ServerStatsRequest {
|
||||
go handleServerStats(connCtx, request, runs, options.ServerStatsSampler, started, build, write)
|
||||
continue
|
||||
}
|
||||
if handleNotificationSettings(ctx, request, settingsStore, write) {
|
||||
continue
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user