package server import ( "context" "errors" "os" "path/filepath" "runtime" "strings" "time" "github.com/shirou/gopsutil/v4/cpu" "github.com/shirou/gopsutil/v4/disk" "github.com/shirou/gopsutil/v4/host" "github.com/shirou/gopsutil/v4/load" "github.com/shirou/gopsutil/v4/mem" "github.com/shirou/gopsutil/v4/process" "git.a-wan.net/alan/pipelines/internal/buildinfo" "git.a-wan.net/alan/pipelines/internal/protocol" ) type ServerStatsSampler interface { Snapshot(context.Context) protocol.ServerStats } type systemStatsSampler struct { dataDir, dbPath, artifactsRoot string started time.Time build buildinfo.Info } func handleServerStats(ctx context.Context, request protocol.Message, runs *runner, injected ServerStatsSampler, started time.Time, build buildinfo.Info, write func(protocol.Message) error) { var result protocol.ServerStats if injected != nil { result = injected.Snapshot(ctx) } else if runs == nil { result.Data.Error = "server store unavailable" } else { stop, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() result = (&systemStatsSampler{dataDir: runs.store.Directory(), dbPath: runs.store.DatabasePath(), artifactsRoot: runs.artifactsRoot, started: started, build: build}).Snapshot(stop) } if result.SampledAt == "" { result.SampledAt = time.Now().UTC().Format(time.RFC3339) } _ = write(protocol.Message{Type: protocol.ServerStatsResult, ID: request.ID, ServerStats: &result}) } func (s *systemStatsSampler) Snapshot(ctx context.Context) protocol.ServerStats { r := protocol.ServerStats{SampledAt: time.Now().UTC().Format(time.RFC3339), Storage: []protocol.ServerDiskStats{}} s.host(ctx, &r.Host) s.cpu(ctx, &r.CPU) s.memory(ctx, &r.Memory) s.storage(ctx, &r.Storage) r.Data = protocol.ServerDataStats{Dir: s.dataDir} var dataErr error r.Data.DBBytes, dataErr = boundedSize(ctx, s.dataDir, func(p string) bool { b := filepath.Base(p) n := filepath.Base(s.dbPath) return b == n || strings.HasPrefix(b, n+"-") }) var err error if r.Data.ArtifactsBytes, err = boundedSize(ctx, s.artifactsRoot, nil); dataErr == nil { dataErr = err } if r.Data.TotalBytes, err = boundedSize(ctx, s.dataDir, nil); dataErr == nil { dataErr = err } if dataErr != nil { r.Data.Error = dataErr.Error() } s.process(ctx, &r.Process) r.GPU = sampleGPU(ctx) return r } func (s *systemStatsSampler) host(ctx context.Context, r *protocol.ServerHostStats) { r.OS = runtime.GOOS r.Arch = runtime.GOARCH if v, e := host.InfoWithContext(ctx); e == nil { r.Hostname = v.Hostname r.Platform = v.Platform r.PlatformVersion = v.PlatformVersion r.Kernel = v.KernelVersion r.UptimeSeconds = v.Uptime } else { r.Error = e.Error() } if v, e := load.AvgWithContext(ctx); e == nil { r.Load1 = &v.Load1 r.Load5 = &v.Load5 r.Load15 = &v.Load15 } } func (s *systemStatsSampler) cpu(ctx context.Context, r *protocol.ServerCPUStats) { if v, e := cpu.InfoWithContext(ctx); e == nil && len(v) > 0 { r.Model = v[0].ModelName } r.LogicalCores, _ = cpu.CountsWithContext(ctx, true) r.PhysicalCores, _ = cpu.CountsWithContext(ctx, false) if v, e := cpu.PercentWithContext(ctx, 500*time.Millisecond, true); e == nil { r.PerCorePercent = v for _, n := range v { r.UsagePercent += n } if len(v) > 0 { r.UsagePercent /= float64(len(v)) } } else { r.Error = e.Error() } } func (s *systemStatsSampler) memory(ctx context.Context, r *protocol.ServerMemoryStats) { v, e := mem.VirtualMemoryWithContext(ctx) if e != nil { r.Error = e.Error() return } r.TotalBytes = v.Total r.UsedBytes = v.Used r.AvailableBytes = v.Available r.UsedPercent = v.UsedPercent if v, e := mem.SwapMemoryWithContext(ctx); e == nil { r.SwapTotalBytes = v.Total r.SwapUsedBytes = v.Used } } func (s *systemStatsSampler) storage(ctx context.Context, out *[]protocol.ServerDiskStats) { ps, e := disk.PartitionsWithContext(ctx, false) if e != nil { *out = append(*out, protocol.ServerDiskStats{Error: e.Error()}) return } for _, p := range ps { if pseudoFS(p.Fstype) || remoteFS(p.Fstype, p.Device) || p.Mountpoint == "" { continue } v, e := disk.UsageWithContext(ctx, p.Mountpoint) if e != nil { *out = append(*out, protocol.ServerDiskStats{Mount: p.Mountpoint, Device: p.Device, FSType: p.Fstype, Error: e.Error()}) continue } if v.Total < 100<<20 && p.Mountpoint != "/" { continue } *out = append(*out, protocol.ServerDiskStats{Mount: p.Mountpoint, Device: p.Device, FSType: p.Fstype, TotalBytes: v.Total, UsedBytes: v.Used, FreeBytes: v.Free, UsedPercent: v.UsedPercent}) } } func (s *systemStatsSampler) process(ctx context.Context, r *protocol.ServerProcessStats) { p, e := process.NewProcess(int32(os.Getpid())) if e != nil { r.Error = e.Error() return } r.PID = p.Pid r.StartedAt = s.started.UTC().Format(time.RFC3339) r.GoVersion = runtime.Version() r.Build = s.build.Version r.Goroutines = runtime.NumGoroutine() var m runtime.MemStats runtime.ReadMemStats(&m) r.HeapBytes = m.HeapAlloc if v, e := p.MemoryInfoWithContext(ctx); e == nil { r.RSSBytes = v.RSS } if v, e := p.CPUPercentWithContext(ctx); e == nil { r.CPUPercent = v } if v, e := p.NumThreadsWithContext(ctx); e == nil { r.Threads = int(v) } } func pseudoFS(v string) bool { v = strings.ToLower(v) for _, x := range []string{"proc", "sysfs", "devtmpfs", "devpts", "tmpfs", "cgroup", "overlay", "squashfs", "autofs", "nsfs", "ramfs"} { if strings.Contains(v, x) { return true } } return false } func remoteFS(fsType, device string) bool { v := strings.ToLower(fsType) for _, x := range []string{"nfs", "nfs4", "cifs", "smb", "afp", "sshfs", "fuse.sshfs", "ceph", "gluster", "9p", "davfs", "fuse.rclone"} { if strings.Contains(v, x) { return true } } // Linux reports some network mounts with an unhelpful type. These device // forms are unambiguously remote without excluding local block devices. return strings.HasPrefix(device, "//") || strings.Contains(device, ":/") } const maxSizeWalkEntries = 100_000 func boundedSize(ctx context.Context, root string, match func(string) bool) (uint64, error) { var n uint64 var entries int if root == "" { return 0, nil } var walkErr error err := filepath.WalkDir(root, func(path string, d os.DirEntry, e error) error { if err := ctx.Err(); err != nil { return err } if e != nil { if walkErr == nil { walkErr = e } return nil } if d == nil { return nil } entries++ if entries > maxSizeWalkEntries { walkErr = errors.New("directory size is partial: entry limit reached") return filepath.SkipAll } if d.Type()&os.ModeSymlink != 0 { if d.IsDir() { return filepath.SkipDir } return nil } if !d.IsDir() && (match == nil || match(path)) { if i, e := d.Info(); e == nil && i.Size() > 0 { n += uint64(i.Size()) } else if e != nil && walkErr == nil { walkErr = e } } return nil }) if err != nil { return n, err } return n, walkErr }