[user-namespaces] Support FUSE FS shift (#3384)

[user-namespaces] Support FUSE FS shift

Author: aledbf

chart/templates/ws-daemon-configmap.yaml

@@ -30,6 +30,8 @@ daemon:
     backup:
       timeout: "5m"
       attempts: 3
+    userNamespaces:
+      fsShift: {{ $comp.userNamespaces.fsShift | default "fuse" }}
     fullWorkspaceBackup:
       workdir: "/mnt/node0/gitpod-{{ .Release.Namespace }}"
     initializer:

chart/values.yaml

@@ -414,6 +414,11 @@ components:
       # - /run/containerd/io.containerd.runtime.v1.linux/moby
       # - /run/containerd/io.containerd.runtime.v2.task/k8s.io
     userNamespaces:
+      # Valid values for fsShift are:
+      #    fuse: uses fuse-overlayfs
+      #    shiftfs: uses shiftfs. Depending on the underlying OS/distribution you
+      #             might want to enable the shiftfsModuleLoader.
+      fsShift: fuse
       shiftfsModuleLoader:
         enabled: false
         imageName: "shiftfs-module-loader"

components/supervisor/BUILD.yaml

@@ -26,6 +26,7 @@ packages:
       - :app
       - components/supervisor/frontend:app
       - components/workspacekit:app
+      - components/workspacekit:fuse-overlayfs
     argdeps:
       - imageRepoBase
     config:

components/supervisor/leeway.Dockerfile

@@ -13,5 +13,6 @@ WORKDIR "/.supervisor"
 COPY components-supervisor--app/supervisor /.supervisor/supervisor
 COPY supervisor-config.json /.supervisor/supervisor-config.json
 COPY components-workspacekit--app/workspacekit /.supervisor/workspacekit
+COPY components-workspacekit--fuse-overlayfs/fuse-overlayfs /.supervisor/fuse-overlayfs
 
 ENTRYPOINT ["/.supervisor/supervisor"]

components/workspacekit/BUILD.yaml

@@ -15,12 +15,11 @@ packages:
     config:
       packaging: app
       buildCommand: ["go", "build", "-ldflags", "-w -extldflags \"-static\""]
-  - name: libseccomp
+  - name: fuse-overlayfs
     type: generic
     config:
       commands:
-        - ["sh", "-c", "curl -L https://github.com/seccomp/libseccomp/releases/download/v2.5.1/libseccomp-2.5.1.tar.gz | tar xz"]
-        - ["sh", "-c", "cd libseccomp-2.5.1 && ./configure --prefix=$PWD/../lib && make && make install"]
+        - ["sh", "-c", "curl -o fuse-overlayfs -L https://github.com/containers/fuse-overlayfs/releases/download/v1.4.0/fuse-overlayfs-x86_64 && chmod +x fuse-overlayfs"]
   - name: lib
     type: go
     srcs:

components/workspacekit/cmd/rings.go

@@ -13,6 +13,7 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"syscall"
 	"time"
 
@@ -28,6 +29,7 @@ import (
 	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/workspacekit/pkg/lift"
 	"github.com/gitpod-io/gitpod/workspacekit/pkg/seccomp"
+	"github.com/gitpod-io/gitpod/ws-daemon/api"
 	daemonapi "github.com/gitpod-io/gitpod/ws-daemon/api"
 )
 
@@ -70,7 +72,7 @@ var ring0Cmd = &cobra.Command{
 		}
 		defer conn.Close()
 
-		_, err = client.PrepareForUserNS(ctx, &daemonapi.PrepareForUserNSRequest{})
+		prep, err := client.PrepareForUserNS(ctx, &daemonapi.PrepareForUserNSRequest{})
 		if err != nil {
 			log.WithError(err).Fatal("cannot prepare for user namespaces")
 			return
@@ -95,7 +97,7 @@ var ring0Cmd = &cobra.Command{
 		cmd.Stdin = os.Stdin
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
-		cmd.Env = os.Environ()
+		cmd.Env = append(os.Environ(), "WORKSPACEKIT_FSSHIFT="+prep.FsShift.String())
 
 		if err := cmd.Start(); err != nil {
 			log.WithError(err).Error("failed to start ring0")
@@ -183,11 +185,11 @@ var ring1Cmd = &cobra.Command{
 		}
 		defer conn.Close()
 
+		mapping := []*daemonapi.WriteIDMappingRequest_Mapping{
+			{ContainerId: 0, HostId: 33333, Size: 1},
+			{ContainerId: 1, HostId: 100000, Size: 65534},
+		}
 		if !ring1Opts.MappingEstablished {
-			mapping := []*daemonapi.WriteIDMappingRequest_Mapping{
-				{ContainerId: 0, HostId: 33333, Size: 1},
-				{ContainerId: 1, HostId: 100000, Size: 65534},
-			}
 			_, err = client.WriteIDMapping(ctx, &daemonapi.WriteIDMappingRequest{Pid: int64(os.Getpid()), Gid: false, Mapping: mapping})
 			if err != nil {
 				log.WithError(err).Error("cannot establish UID mapping")
@@ -223,23 +225,46 @@ var ring1Cmd = &cobra.Command{
 			log.WithError(err).Fatal("cannot create tempdir")
 		}
 
-		mnts := []struct {
+		var fsshift api.FSShiftMethod
+		if v, ok := api.FSShiftMethod_value[os.Getenv("WORKSPACEKIT_FSSHIFT")]; !ok {
+			log.WithField("fsshift", os.Getenv("WORKSPACEKIT_FSSHIFT")).Fatal("unknown FS shift method")
+		} else {
+			fsshift = api.FSShiftMethod(v)
+		}
+
+		type mnte struct {
 			Target string
 			Source string
 			FSType string
 			Flags  uintptr
-		}{
-			// TODO(cw): pull mark mount location from config
-			{Target: "/", Source: "/.workspace/mark", FSType: "shiftfs"},
-			{Target: "/sys", Flags: unix.MS_BIND | unix.MS_REC},
-			{Target: "/dev", Flags: unix.MS_BIND | unix.MS_REC},
-			// TODO(cw): only mount /workspace if it's in the mount table, i.e. this isn't an FWB workspace
-			{Target: "/workspace", Flags: unix.MS_BIND | unix.MS_REC},
-			{Target: "/etc/hosts", Flags: unix.MS_BIND | unix.MS_REC},
-			{Target: "/etc/hostname", Flags: unix.MS_BIND | unix.MS_REC},
-			{Target: "/etc/resolv.conf", Flags: unix.MS_BIND | unix.MS_REC},
-			{Target: "/tmp", Source: "tmpfs", FSType: "tmpfs"},
 		}
+
+		var mnts []mnte
+
+		switch fsshift {
+		case api.FSShiftMethod_FUSE:
+			mnts = append(mnts,
+				mnte{Target: "/", Source: "/.workspace/mark", Flags: unix.MS_BIND | unix.MS_REC},
+			)
+		case api.FSShiftMethod_SHIFTFS:
+			mnts = append(mnts,
+				mnte{Target: "/", Source: "/.workspace/mark", FSType: "shiftfs"},
+			)
+		default:
+			log.WithField("fsshift", fsshift).Fatal("unknown FS shift method")
+		}
+
+		mnts = append(mnts,
+			mnte{Target: "/sys", Flags: unix.MS_BIND | unix.MS_REC},
+			mnte{Target: "/dev", Flags: unix.MS_BIND | unix.MS_REC},
+			// TODO(cw): only mount /workspace if it's in the mount table, i.e. this isn't an FWB workspace
+			mnte{Target: "/workspace", Flags: unix.MS_BIND | unix.MS_REC},
+			mnte{Target: "/etc/hosts", Flags: unix.MS_BIND | unix.MS_REC},
+			mnte{Target: "/etc/hostname", Flags: unix.MS_BIND | unix.MS_REC},
+			mnte{Target: "/etc/resolv.conf", Flags: unix.MS_BIND | unix.MS_REC},
+			mnte{Target: "/tmp", Source: "tmpfs", FSType: "tmpfs"},
+		)
+
 		for _, m := range mnts {
 			dst := filepath.Join(ring2Root, m.Target)
 			_ = os.MkdirAll(dst, 0644)
@@ -265,6 +290,14 @@ var ring1Cmd = &cobra.Command{
 			}
 		}
 
+		env := make([]string, 0, len(os.Environ()))
+		for _, e := range os.Environ() {
+			if strings.HasPrefix(e, "WORKSPACEKIT_") {
+				continue
+			}
+			env = append(env, e)
+		}
+
 		socketFN := filepath.Join(os.TempDir(), fmt.Sprintf("workspacekit-ring1-%d.unix", time.Now().UnixNano()))
 		skt, err := net.Listen("unix", socketFN)
 		if err != nil {
@@ -283,7 +316,7 @@ var ring1Cmd = &cobra.Command{
 		cmd.Stdin = os.Stdin
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
-		cmd.Env = os.Environ()
+		cmd.Env = env
 		if err := cmd.Start(); err != nil {
 			log.WithError(err).Error("failed to start the child process")
 			failed = true
@@ -358,8 +391,9 @@ var ring1Cmd = &cobra.Command{
 
 		log.Info("signaling to child process")
 		_, err = msgutil.MarshalToWriter(ring2Conn, ringSyncMsg{
-			Stage:  1,
-			Rootfs: ring2Root,
+			Stage:   1,
+			Rootfs:  ring2Root,
+			FSShift: fsshift,
 		})
 		if err != nil {
 			log.WithError(err).Error("cannot send ring sync msg to ring2")
@@ -504,7 +538,7 @@ var ring2Cmd = &cobra.Command{
 			return
 		}
 
-		err = pivotRoot(msg.Rootfs)
+		err = pivotRoot(msg.Rootfs, msg.FSShift)
 		if err != nil {
 			log.WithError(err).Error("cannot pivot root")
 			failed = true
@@ -562,13 +596,27 @@ var ring2Cmd = &cobra.Command{
 // filesystem, and everything else is cleaned up.
 //
 // copied from runc: https://github.com/opencontainers/runc/blob/cf6c074115d00c932ef01dedb3e13ba8b8f964c3/libcontainer/rootfs_linux.go#L760
-func pivotRoot(rootfs string) error {
+func pivotRoot(rootfs string, fsshift api.FSShiftMethod) error {
 	// While the documentation may claim otherwise, pivot_root(".", ".") is
 	// actually valid. What this results in is / being the new root but
 	// /proc/self/cwd being the old root. Since we can play around with the cwd
 	// with pivot_root this allows us to pivot without creating directories in
 	// the rootfs. Shout-outs to the LXC developers for giving us this idea.
 
+	if fsshift == api.FSShiftMethod_FUSE {
+		err := unix.Chroot(rootfs)
+		if err != nil {
+			return fmt.Errorf("cannot chroot: %v", err)
+		}
+
+		err = unix.Chdir("/")
+		if err != nil {
+			return fmt.Errorf("cannot chdir to new root :%v", err)
+		}
+
+		return nil
+	}
+
 	oldroot, err := unix.Open("/", unix.O_DIRECTORY|unix.O_RDONLY, 0)
 	if err != nil {
 		return err
@@ -616,6 +664,7 @@ func pivotRoot(rootfs string) error {
 	if err := unix.Chdir("/"); err != nil {
 		return fmt.Errorf("chdir / %s", err)
 	}
+
 	return nil
 }
 
@@ -631,8 +680,9 @@ func sleepForDebugging() {
 }
 
 type ringSyncMsg struct {
-	Stage  int    `json:"stage"`
-	Rootfs string `json:"rootfs"`
+	Stage   int               `json:"stage"`
+	Rootfs  string            `json:"rootfs"`
+	FSShift api.FSShiftMethod `json:"fsshift"`
 }
 
 // ConnectToInWorkspaceDaemonService attempts to connect to the InWorkspaceService offered by the ws-daemon.

components/workspacekit/go.mod

@@ -5,6 +5,7 @@ go 1.16
 replace github.com/seccomp/libseccomp-golang => github.com/kinvolk/libseccomp-golang v0.9.2-0.20201113182948-883917843313
 
 require (
+	github.com/containerd/containerd v1.4.1
 	github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
 	github.com/gitpod-io/gitpod/ws-daemon/api v0.0.0-00010101000000-000000000000
 	github.com/moby/sys/mountinfo v0.4.0