Files

241 lines
6.9 KiB
Go

package server
import (
"context"
"errors"
"os"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/shirou/gopsutil/v4/cpu"
"github.com/shirou/gopsutil/v4/disk"
"github.com/shirou/gopsutil/v4/host"
"github.com/shirou/gopsutil/v4/load"
"github.com/shirou/gopsutil/v4/mem"
"github.com/shirou/gopsutil/v4/process"
"git.a-wan.net/alan/pipelines/internal/buildinfo"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
type ServerStatsSampler interface {
Snapshot(context.Context) protocol.ServerStats
}
type systemStatsSampler struct {
dataDir, dbPath, artifactsRoot string
started time.Time
build buildinfo.Info
}
func handleServerStats(ctx context.Context, request protocol.Message, runs *runner, injected ServerStatsSampler, started time.Time, build buildinfo.Info, write func(protocol.Message) error) {
var result protocol.ServerStats
if injected != nil {
result = injected.Snapshot(ctx)
} else if runs == nil {
result.Data.Error = "server store unavailable"
} else {
stop, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
result = (&systemStatsSampler{dataDir: runs.store.Directory(), dbPath: runs.store.DatabasePath(), artifactsRoot: runs.artifactsRoot, started: started, build: build}).Snapshot(stop)
}
if result.SampledAt == "" {
result.SampledAt = time.Now().UTC().Format(time.RFC3339)
}
_ = write(protocol.Message{Type: protocol.ServerStatsResult, ID: request.ID, ServerStats: &result})
}
func (s *systemStatsSampler) Snapshot(ctx context.Context) protocol.ServerStats {
r := protocol.ServerStats{SampledAt: time.Now().UTC().Format(time.RFC3339), Storage: []protocol.ServerDiskStats{}}
s.host(ctx, &r.Host)
s.cpu(ctx, &r.CPU)
s.memory(ctx, &r.Memory)
s.storage(ctx, &r.Storage)
r.Data = protocol.ServerDataStats{Dir: s.dataDir}
var dataErr error
r.Data.DBBytes, dataErr = boundedSize(ctx, s.dataDir, func(p string) bool {
b := filepath.Base(p)
n := filepath.Base(s.dbPath)
return b == n || strings.HasPrefix(b, n+"-")
})
var err error
if r.Data.ArtifactsBytes, err = boundedSize(ctx, s.artifactsRoot, nil); dataErr == nil {
dataErr = err
}
if r.Data.TotalBytes, err = boundedSize(ctx, s.dataDir, nil); dataErr == nil {
dataErr = err
}
if dataErr != nil {
r.Data.Error = dataErr.Error()
}
s.process(ctx, &r.Process)
r.GPU = sampleGPU(ctx)
return r
}
func (s *systemStatsSampler) host(ctx context.Context, r *protocol.ServerHostStats) {
r.OS = runtime.GOOS
r.Arch = runtime.GOARCH
if v, e := host.InfoWithContext(ctx); e == nil {
r.Hostname = v.Hostname
r.Platform = v.Platform
r.PlatformVersion = v.PlatformVersion
r.Kernel = v.KernelVersion
r.UptimeSeconds = v.Uptime
} else {
r.Error = e.Error()
}
if v, e := load.AvgWithContext(ctx); e == nil {
r.Load1 = &v.Load1
r.Load5 = &v.Load5
r.Load15 = &v.Load15
}
}
func (s *systemStatsSampler) cpu(ctx context.Context, r *protocol.ServerCPUStats) {
if v, e := cpu.InfoWithContext(ctx); e == nil && len(v) > 0 {
r.Model = v[0].ModelName
}
r.LogicalCores, _ = cpu.CountsWithContext(ctx, true)
r.PhysicalCores, _ = cpu.CountsWithContext(ctx, false)
if v, e := cpu.PercentWithContext(ctx, 500*time.Millisecond, true); e == nil {
r.PerCorePercent = v
for _, n := range v {
r.UsagePercent += n
}
if len(v) > 0 {
r.UsagePercent /= float64(len(v))
}
} else {
r.Error = e.Error()
}
}
func (s *systemStatsSampler) memory(ctx context.Context, r *protocol.ServerMemoryStats) {
v, e := mem.VirtualMemoryWithContext(ctx)
if e != nil {
r.Error = e.Error()
return
}
r.TotalBytes = v.Total
r.UsedBytes = v.Used
r.AvailableBytes = v.Available
r.UsedPercent = v.UsedPercent
if v, e := mem.SwapMemoryWithContext(ctx); e == nil {
r.SwapTotalBytes = v.Total
r.SwapUsedBytes = v.Used
}
}
func (s *systemStatsSampler) storage(ctx context.Context, out *[]protocol.ServerDiskStats) {
ps, e := disk.PartitionsWithContext(ctx, false)
if e != nil {
*out = append(*out, protocol.ServerDiskStats{Error: e.Error()})
return
}
for _, p := range ps {
if pseudoFS(p.Fstype) || remoteFS(p.Fstype, p.Device) || p.Mountpoint == "" {
continue
}
v, e := disk.UsageWithContext(ctx, p.Mountpoint)
if e != nil {
*out = append(*out, protocol.ServerDiskStats{Mount: p.Mountpoint, Device: p.Device, FSType: p.Fstype, Error: e.Error()})
continue
}
if v.Total < 100<<20 && p.Mountpoint != "/" {
continue
}
*out = append(*out, protocol.ServerDiskStats{Mount: p.Mountpoint, Device: p.Device, FSType: p.Fstype, TotalBytes: v.Total, UsedBytes: v.Used, FreeBytes: v.Free, UsedPercent: v.UsedPercent})
}
}
func (s *systemStatsSampler) process(ctx context.Context, r *protocol.ServerProcessStats) {
p, e := process.NewProcess(int32(os.Getpid()))
if e != nil {
r.Error = e.Error()
return
}
r.PID = p.Pid
r.StartedAt = s.started.UTC().Format(time.RFC3339)
r.GoVersion = runtime.Version()
r.Build = s.build.Version
r.Goroutines = runtime.NumGoroutine()
var m runtime.MemStats
runtime.ReadMemStats(&m)
r.HeapBytes = m.HeapAlloc
if v, e := p.MemoryInfoWithContext(ctx); e == nil {
r.RSSBytes = v.RSS
}
if v, e := p.CPUPercentWithContext(ctx); e == nil {
r.CPUPercent = v
}
if v, e := p.NumThreadsWithContext(ctx); e == nil {
r.Threads = int(v)
}
}
func pseudoFS(v string) bool {
v = strings.ToLower(v)
for _, x := range []string{"proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "cgroup", "overlay", "squashfs", "autofs", "nsfs", "ramfs"} {
if strings.Contains(v, x) {
return true
}
}
return false
}
func remoteFS(fsType, device string) bool {
v := strings.ToLower(fsType)
for _, x := range []string{"nfs", "nfs4", "cifs", "smb", "afp", "sshfs", "fuse.sshfs", "ceph", "gluster", "9p", "davfs", "fuse.rclone"} {
if strings.Contains(v, x) {
return true
}
}
// Linux reports some network mounts with an unhelpful type. These device
// forms are unambiguously remote without excluding local block devices.
return strings.HasPrefix(device, "//") || strings.Contains(device, ":/")
}
const maxSizeWalkEntries = 100_000
func boundedSize(ctx context.Context, root string, match func(string) bool) (uint64, error) {
var n uint64
var entries int
if root == "" {
return 0, nil
}
var walkErr error
err := filepath.WalkDir(root, func(path string, d os.DirEntry, e error) error {
if err := ctx.Err(); err != nil {
return err
}
if e != nil {
if walkErr == nil {
walkErr = e
}
return nil
}
if d == nil {
return nil
}
entries++
if entries > maxSizeWalkEntries {
walkErr = errors.New("directory size is partial: entry limit reached")
return filepath.SkipAll
}
if d.Type()&os.ModeSymlink != 0 {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if !d.IsDir() && (match == nil || match(path)) {
if i, e := d.Info(); e == nil && i.Size() > 0 {
n += uint64(i.Size())
} else if e != nil && walkErr == nil {
walkErr = e
}
}
return nil
})
if err != nil {
return n, err
}
return n, walkErr
}