[ws-daemon] Plugin for PSI metrics

Author: Furisto

components/common-go/cgroups/cgroup.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/containerd/cgroups"
 	v2 "github.com/containerd/cgroups/v2"
-	"github.com/gitpod-io/gitpod/common-go/log"
 )
 
 const DefaultMountPoint = "/sys/fs/cgroup"
@@ -105,7 +104,6 @@ func ReadPSIValue(path string) (PSI, error) {
 			return PSI{}, fmt.Errorf("could not find total stalled time")
 		}
 
-		log.Infof("total is %v", line[i+6:])
 		total, err := strconv.ParseUint(line[i+6:], 10, 64)
 		if err != nil {
 			return PSI{}, fmt.Errorf("could not parse total stalled time: %w", err)

components/ws-daemon/pkg/cgroup/cgroup.go

@@ -87,7 +87,7 @@ func (host *PluginHost) WorkspaceAdded(ctx context.Context, ws *dispatch.Workspa
 	opts := &PluginOptions{
 		BasePath:   host.CGroupBasePath,
 		CgroupPath: cgroupPath,
-		InstanceID: ws.InstanceID,
+		InstanceId: ws.InstanceID,
 	}
 
 	for _, plg := range host.Plugins {
@@ -128,5 +128,5 @@ const (
 type PluginOptions struct {
 	BasePath   string
 	CgroupPath string
-	InstanceID string
+	InstanceId string
 }

components/ws-daemon/pkg/cgroup/plugin_psi.go

@@ -0,0 +1,98 @@
+// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License-AGPL.txt in the project root for license information.
+
+package cgroup
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"time"
+
+	cgroups "github.com/gitpod-io/gitpod/common-go/cgroups/v2"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type PSIMetrics struct {
+	cpu      *prometheus.GaugeVec
+	memory   *prometheus.GaugeVec
+	io       *prometheus.GaugeVec
+	nodeName string
+}
+
+func NewPSIMetrics(prom prometheus.Registerer) *PSIMetrics {
+	p := &PSIMetrics{
+		cpu: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "workspace_cpu_psi_total_seconds",
+			Help: "Total time spent under cpu pressure in microseconds",
+		}, []string{"node", "workspace", "kind"}),
+
+		memory: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "workspace_memory_psi_total_seconds",
+			Help: "Total time spent under memory pressure in microseconds",
+		}, []string{"node", "workspace", "kind"}),
+
+		io: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "workspace_io_psi_total_seconds",
+			Help: "Total time spent under io pressure in microseconds",
+		}, []string{"node", "workspace", "kind"}),
+
+		nodeName: os.Getenv("NODENAME"),
+	}
+
+	prom.MustRegister(
+		p.cpu,
+		p.memory,
+		p.io,
+	)
+
+	return p
+}
+
+func (p *PSIMetrics) Name() string  { return "psi-metrics" }
+func (p *PSIMetrics) Type() Version { return Version2 }
+
+func (p *PSIMetrics) Apply(ctx context.Context, opts *PluginOptions) error {
+	fullPath := filepath.Join(opts.BasePath, opts.CgroupPath)
+	if _, err := os.Stat(fullPath); err != nil {
+		return err
+	}
+
+	cpu := cgroups.NewCpuController(fullPath)
+	memory := cgroups.NewMemoryController(fullPath)
+	io := cgroups.NewIOController(fullPath)
+
+	go func() {
+		ticker := time.NewTicker(10 * time.Second)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ticker.C:
+				p.scrape(cpu, memory, io, opts.InstanceId)
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return nil
+}
+
+func (p *PSIMetrics) scrape(cpu *cgroups.Cpu, memory *cgroups.Memory, io *cgroups.IO, instanceID string) {
+	if psi, err := cpu.PSI(); err == nil {
+		p.cpu.WithLabelValues(p.nodeName, instanceID, "some").Set(float64(psi.Some))
+		p.cpu.WithLabelValues(p.nodeName, instanceID, "full").Set(float64(psi.Full))
+	}
+
+	if psi, err := memory.PSI(); err == nil {
+		p.memory.WithLabelValues(p.nodeName, instanceID, "some").Set(float64(psi.Some))
+		p.memory.WithLabelValues(p.nodeName, instanceID, "full").Set(float64(psi.Full))
+	}
+
+	if psi, err := io.PSI(); err == nil {
+		p.io.WithLabelValues(p.nodeName, instanceID, "some").Set(float64(psi.Some))
+		p.io.WithLabelValues(p.nodeName, instanceID, "full").Set(float64(psi.Full))
+	}
+}

components/ws-daemon/pkg/daemon/daemon.go

@@ -87,6 +87,7 @@ func NewDaemon(config Config, reg prometheus.Registerer) (*Daemon, error) {
 			},
 		},
 		procV2Plugin,
+		cgroup.NewPSIMetrics(reg),
 	)
 	if err != nil {
 		return nil, err