241 lines
6.9 KiB
Go
241 lines
6.9 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/shirou/gopsutil/v4/cpu"
|
|
"github.com/shirou/gopsutil/v4/disk"
|
|
"github.com/shirou/gopsutil/v4/host"
|
|
"github.com/shirou/gopsutil/v4/load"
|
|
"github.com/shirou/gopsutil/v4/mem"
|
|
"github.com/shirou/gopsutil/v4/process"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/buildinfo"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
type ServerStatsSampler interface {
|
|
Snapshot(context.Context) protocol.ServerStats
|
|
}
|
|
type systemStatsSampler struct {
|
|
dataDir, dbPath, artifactsRoot string
|
|
started time.Time
|
|
build buildinfo.Info
|
|
}
|
|
|
|
func handleServerStats(ctx context.Context, request protocol.Message, runs *runner, injected ServerStatsSampler, started time.Time, build buildinfo.Info, write func(protocol.Message) error) {
|
|
var result protocol.ServerStats
|
|
if injected != nil {
|
|
result = injected.Snapshot(ctx)
|
|
} else if runs == nil {
|
|
result.Data.Error = "server store unavailable"
|
|
} else {
|
|
stop, cancel := context.WithTimeout(ctx, 3*time.Second)
|
|
defer cancel()
|
|
result = (&systemStatsSampler{dataDir: runs.store.Directory(), dbPath: runs.store.DatabasePath(), artifactsRoot: runs.artifactsRoot, started: started, build: build}).Snapshot(stop)
|
|
}
|
|
if result.SampledAt == "" {
|
|
result.SampledAt = time.Now().UTC().Format(time.RFC3339)
|
|
}
|
|
_ = write(protocol.Message{Type: protocol.ServerStatsResult, ID: request.ID, ServerStats: &result})
|
|
}
|
|
|
|
func (s *systemStatsSampler) Snapshot(ctx context.Context) protocol.ServerStats {
|
|
r := protocol.ServerStats{SampledAt: time.Now().UTC().Format(time.RFC3339), Storage: []protocol.ServerDiskStats{}}
|
|
s.host(ctx, &r.Host)
|
|
s.cpu(ctx, &r.CPU)
|
|
s.memory(ctx, &r.Memory)
|
|
s.storage(ctx, &r.Storage)
|
|
r.Data = protocol.ServerDataStats{Dir: s.dataDir}
|
|
var dataErr error
|
|
r.Data.DBBytes, dataErr = boundedSize(ctx, s.dataDir, func(p string) bool {
|
|
b := filepath.Base(p)
|
|
n := filepath.Base(s.dbPath)
|
|
return b == n || strings.HasPrefix(b, n+"-")
|
|
})
|
|
var err error
|
|
if r.Data.ArtifactsBytes, err = boundedSize(ctx, s.artifactsRoot, nil); dataErr == nil {
|
|
dataErr = err
|
|
}
|
|
if r.Data.TotalBytes, err = boundedSize(ctx, s.dataDir, nil); dataErr == nil {
|
|
dataErr = err
|
|
}
|
|
if dataErr != nil {
|
|
r.Data.Error = dataErr.Error()
|
|
}
|
|
s.process(ctx, &r.Process)
|
|
r.GPU = sampleGPU(ctx)
|
|
return r
|
|
}
|
|
func (s *systemStatsSampler) host(ctx context.Context, r *protocol.ServerHostStats) {
|
|
r.OS = runtime.GOOS
|
|
r.Arch = runtime.GOARCH
|
|
if v, e := host.InfoWithContext(ctx); e == nil {
|
|
r.Hostname = v.Hostname
|
|
r.Platform = v.Platform
|
|
r.PlatformVersion = v.PlatformVersion
|
|
r.Kernel = v.KernelVersion
|
|
r.UptimeSeconds = v.Uptime
|
|
} else {
|
|
r.Error = e.Error()
|
|
}
|
|
if v, e := load.AvgWithContext(ctx); e == nil {
|
|
r.Load1 = &v.Load1
|
|
r.Load5 = &v.Load5
|
|
r.Load15 = &v.Load15
|
|
}
|
|
}
|
|
func (s *systemStatsSampler) cpu(ctx context.Context, r *protocol.ServerCPUStats) {
|
|
if v, e := cpu.InfoWithContext(ctx); e == nil && len(v) > 0 {
|
|
r.Model = v[0].ModelName
|
|
}
|
|
r.LogicalCores, _ = cpu.CountsWithContext(ctx, true)
|
|
r.PhysicalCores, _ = cpu.CountsWithContext(ctx, false)
|
|
if v, e := cpu.PercentWithContext(ctx, 500*time.Millisecond, true); e == nil {
|
|
r.PerCorePercent = v
|
|
for _, n := range v {
|
|
r.UsagePercent += n
|
|
}
|
|
if len(v) > 0 {
|
|
r.UsagePercent /= float64(len(v))
|
|
}
|
|
} else {
|
|
r.Error = e.Error()
|
|
}
|
|
}
|
|
func (s *systemStatsSampler) memory(ctx context.Context, r *protocol.ServerMemoryStats) {
|
|
v, e := mem.VirtualMemoryWithContext(ctx)
|
|
if e != nil {
|
|
r.Error = e.Error()
|
|
return
|
|
}
|
|
r.TotalBytes = v.Total
|
|
r.UsedBytes = v.Used
|
|
r.AvailableBytes = v.Available
|
|
r.UsedPercent = v.UsedPercent
|
|
if v, e := mem.SwapMemoryWithContext(ctx); e == nil {
|
|
r.SwapTotalBytes = v.Total
|
|
r.SwapUsedBytes = v.Used
|
|
}
|
|
}
|
|
func (s *systemStatsSampler) storage(ctx context.Context, out *[]protocol.ServerDiskStats) {
|
|
ps, e := disk.PartitionsWithContext(ctx, false)
|
|
if e != nil {
|
|
*out = append(*out, protocol.ServerDiskStats{Error: e.Error()})
|
|
return
|
|
}
|
|
for _, p := range ps {
|
|
if pseudoFS(p.Fstype) || remoteFS(p.Fstype, p.Device) || p.Mountpoint == "" {
|
|
continue
|
|
}
|
|
v, e := disk.UsageWithContext(ctx, p.Mountpoint)
|
|
if e != nil {
|
|
*out = append(*out, protocol.ServerDiskStats{Mount: p.Mountpoint, Device: p.Device, FSType: p.Fstype, Error: e.Error()})
|
|
continue
|
|
}
|
|
if v.Total < 100<<20 && p.Mountpoint != "/" {
|
|
continue
|
|
}
|
|
*out = append(*out, protocol.ServerDiskStats{Mount: p.Mountpoint, Device: p.Device, FSType: p.Fstype, TotalBytes: v.Total, UsedBytes: v.Used, FreeBytes: v.Free, UsedPercent: v.UsedPercent})
|
|
}
|
|
}
|
|
func (s *systemStatsSampler) process(ctx context.Context, r *protocol.ServerProcessStats) {
|
|
p, e := process.NewProcess(int32(os.Getpid()))
|
|
if e != nil {
|
|
r.Error = e.Error()
|
|
return
|
|
}
|
|
r.PID = p.Pid
|
|
r.StartedAt = s.started.UTC().Format(time.RFC3339)
|
|
r.GoVersion = runtime.Version()
|
|
r.Build = s.build.Version
|
|
r.Goroutines = runtime.NumGoroutine()
|
|
var m runtime.MemStats
|
|
runtime.ReadMemStats(&m)
|
|
r.HeapBytes = m.HeapAlloc
|
|
if v, e := p.MemoryInfoWithContext(ctx); e == nil {
|
|
r.RSSBytes = v.RSS
|
|
}
|
|
if v, e := p.CPUPercentWithContext(ctx); e == nil {
|
|
r.CPUPercent = v
|
|
}
|
|
if v, e := p.NumThreadsWithContext(ctx); e == nil {
|
|
r.Threads = int(v)
|
|
}
|
|
}
|
|
func pseudoFS(v string) bool {
|
|
v = strings.ToLower(v)
|
|
for _, x := range []string{"proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "cgroup", "overlay", "squashfs", "autofs", "nsfs", "ramfs"} {
|
|
if strings.Contains(v, x) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func remoteFS(fsType, device string) bool {
|
|
v := strings.ToLower(fsType)
|
|
for _, x := range []string{"nfs", "nfs4", "cifs", "smb", "afp", "sshfs", "fuse.sshfs", "ceph", "gluster", "9p", "davfs", "fuse.rclone"} {
|
|
if strings.Contains(v, x) {
|
|
return true
|
|
}
|
|
}
|
|
// Linux reports some network mounts with an unhelpful type. These device
|
|
// forms are unambiguously remote without excluding local block devices.
|
|
return strings.HasPrefix(device, "//") || strings.Contains(device, ":/")
|
|
}
|
|
|
|
const maxSizeWalkEntries = 100_000
|
|
|
|
func boundedSize(ctx context.Context, root string, match func(string) bool) (uint64, error) {
|
|
var n uint64
|
|
var entries int
|
|
if root == "" {
|
|
return 0, nil
|
|
}
|
|
var walkErr error
|
|
err := filepath.WalkDir(root, func(path string, d os.DirEntry, e error) error {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
if e != nil {
|
|
if walkErr == nil {
|
|
walkErr = e
|
|
}
|
|
return nil
|
|
}
|
|
if d == nil {
|
|
return nil
|
|
}
|
|
entries++
|
|
if entries > maxSizeWalkEntries {
|
|
walkErr = errors.New("directory size is partial: entry limit reached")
|
|
return filepath.SkipAll
|
|
}
|
|
if d.Type()&os.ModeSymlink != 0 {
|
|
if d.IsDir() {
|
|
return filepath.SkipDir
|
|
}
|
|
return nil
|
|
}
|
|
if !d.IsDir() && (match == nil || match(path)) {
|
|
if i, e := d.Info(); e == nil && i.Size() > 0 {
|
|
n += uint64(i.Size())
|
|
} else if e != nil && walkErr == nil {
|
|
walkErr = e
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return n, err
|
|
}
|
|
return n, walkErr
|
|
}
|