Ensure cgroup v2 cpu controller is enabled

Author: Furisto

components/common-go/cgroups/cgroup.go

@@ -0,0 +1,111 @@
+package cgroups
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const DefaultCGroupMount = "/sys/fs/cgroup"
+
+type CgroupSetup int
+
+const (
+	Unknown CgroupSetup = iota
+	Legacy
+	Unified
+)
+
+func (s CgroupSetup) String() string {
+	return [...]string{"Legacy", "Unified"}[s]
+}
+
+func GetCgroupSetup() (CgroupSetup, error) {
+	controllers := filepath.Join(DefaultCGroupMount, "cgroup.controllers")
+	_, err := os.Stat(controllers)
+
+	if os.IsNotExist(err) {
+		return Legacy, nil
+	}
+
+	if err == nil {
+		return Unified, nil
+	}
+
+	return Unknown, err
+}
+
+func IsUnifiedCgroupSetup() (bool, error) {
+	setup, err := GetCgroupSetup()
+	if err != nil {
+		return false, err
+	}
+
+	return setup == Unified, nil
+}
+
+func IsLegacyCgroupSetup() (bool, error) {
+	setup, err := GetCgroupSetup()
+	if err != nil {
+		return false, err
+	}
+
+	return setup == Legacy, nil
+}
+
+func EnsureCpuControllerEnabled(basePath, cgroupPath string) error {
+	targetPath := filepath.Join(basePath, cgroupPath)
+	if enabled, err := isCpuControllerEnabled(targetPath); err != nil || enabled {
+		return err
+	}
+
+	err := writeCpuController(basePath)
+	if err != nil {
+		return err
+	}
+
+	levelPath := basePath
+	cgroupPath = strings.TrimPrefix(cgroupPath, "/")
+	levels := strings.Split(cgroupPath, string(os.PathSeparator))
+	for _, l := range levels[:len(levels)-1] {
+		levelPath = filepath.Join(levelPath, l)
+		err = writeCpuController(levelPath)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func isCpuControllerEnabled(path string) (bool, error) {
+	controllerFile := filepath.Join(path, "cgroup.controllers")
+	controllers, err := os.ReadFile(controllerFile)
+	if err != nil {
+		return false, err
+	}
+
+	for _, ctrl := range strings.Fields(string(controllers)) {
+		if ctrl == "cpu" {
+			// controller is already activated
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func writeCpuController(path string) error {
+	f, err := os.OpenFile(filepath.Join(path, "cgroup.subtree_control"), os.O_WRONLY, 0)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = f.Write([]byte("+cpu"))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

components/common-go/cgroups/cgroups_test.go

@@ -0,0 +1,85 @@
+package cgroups
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+var cgroupPath = []string{"kubepods", "burstable", "pods234sdf", "234as8df34"}
+
+func createHierarchy(t *testing.T, cpuEnabled bool) (string, string) {
+	testRoot := t.TempDir()
+	if err := os.WriteFile(filepath.Join(testRoot, "cgroup.controllers"), []byte(""), 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := os.WriteFile(filepath.Join(testRoot, "cgroup.subtree_control"), []byte(""), 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	testCgroup := ""
+	for i, level := range cgroupPath {
+		testCgroup = filepath.Join(testCgroup, level)
+		fullPath := filepath.Join(testRoot, testCgroup)
+		if err := os.Mkdir(fullPath, 0o755); err != nil {
+			t.Fatal(err)
+		}
+
+		ctrlFile, err := os.Create(filepath.Join(fullPath, "cgroup.controllers"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer ctrlFile.Close()
+
+		if cpuEnabled {
+			if _, err := ctrlFile.WriteString("cpu"); err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		subTreeFile, err := os.Create(filepath.Join(fullPath, "cgroup.subtree_control"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer subTreeFile.Close()
+
+		if cpuEnabled && i < len(cgroupPath)-1 {
+			if _, err := subTreeFile.WriteString("cpu"); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	return testRoot, testCgroup
+}
+
+func TestEnableController(t *testing.T) {
+	root, cgroup := createHierarchy(t, false)
+	if err := EnsureCpuControllerEnabled(root, cgroup); err != nil {
+		t.Fatal(err)
+	}
+
+	levelPath := root
+	for _, level := range cgroupPath {
+		verifyCpuControllerToggled(t, levelPath, true)
+		levelPath = filepath.Join(levelPath, level)
+	}
+
+	verifyCpuControllerToggled(t, levelPath, false)
+}
+
+func verifyCpuControllerToggled(t *testing.T, path string, enabled bool) {
+	t.Helper()
+
+	content, err := os.ReadFile(filepath.Join(path, "cgroup.subtree_control"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if enabled && string(content) != "+cpu" {
+		t.Fatalf("%s should have enabled cpu controller", path)
+	} else if !enabled && string(content) == "+cpu" {
+		t.Fatalf("%s should not have enabled cpu controller", path)
+	}
+}

components/ws-daemon/pkg/cpulimit/dispatch.go

@@ -6,19 +6,16 @@ package cpulimit
 
 import (
 	"context"
-	"os"
 	"path/filepath"
-	"strings"
 	"sync"
 	"time"
 
-	"github.com/opencontainers/runc/libcontainer/cgroups/fs2"
-	"github.com/opencontainers/runc/libcontainer/configs"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/xerrors"
 	"k8s.io/apimachinery/pkg/api/resource"
 
+	"github.com/gitpod-io/gitpod/common-go/cgroups"
 	wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
 	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/common-go/util"
@@ -225,38 +222,19 @@ func (d *DispatchListener) WorkspaceUpdated(ctx context.Context, ws *dispatch.Wo
 }
 
 func newCFSController(basePath, cgroupPath string) (CFSController, error) {
-	controllers := filepath.Join(basePath, "cgroup.controllers")
-	_, err := os.Stat(controllers)
-
-	if os.IsNotExist(err) {
-		return CgroupV1CFSController(filepath.Join(basePath, "cpu", cgroupPath)), nil
+	unified, err := cgroups.IsUnifiedCgroupSetup()
+	if err != nil {
+		return nil, xerrors.Errorf("could not determine cgroup setup: %w", err)
 	}
 
-	if err == nil {
+	if unified {
 		fullPath := filepath.Join(basePath, cgroupPath)
-		if err := ensureControllerEnabled(fullPath, "cpu"); err != nil {
+		if err := cgroups.EnsureCpuControllerEnabled(basePath, cgroupPath); err != nil {
 			return nil, err
 		}
 
 		return CgroupV2CFSController(fullPath), nil
+	} else {
+		return CgroupV1CFSController(filepath.Join(basePath, "cpu", cgroupPath)), nil
 	}
-
-	return nil, err
-}
-
-func ensureControllerEnabled(targetPath, controller string) error {
-	controllerFile := filepath.Join(targetPath, "cgroup.controllers")
-	controllers, err := os.ReadFile(controllerFile)
-	if err != nil {
-		return err
-	}
-
-	for _, ctrl := range strings.Fields(string(controllers)) {
-		if ctrl == controller {
-			// controller is already activated
-			return nil
-		}
-	}
-
-	return fs2.CreateCgroupPath(targetPath, &configs.Cgroup{})
 }