[loadgen] Support workspace classes

dev/loadgen/README.md

@@ -2,4 +2,60 @@
 
 A load generator (framework) for Gitpod.
 
-**Note:** this is a development tool only - there's no support for this.
+**Note:** this is a development tool only - there's no support for this.
+
+You can find a short explanation of this tool in this [loom video](https://www.loom.com/share/6487e3403c0746cc97bb3f766e15fab6).
+
+## How to run a benchmark
+- Ensure your kubeconfig has the configuration for the cluster you want to benchmark. You can use kubecdl to update your kubeconfig with the cluster information
+`kubecdl -p workspace-clusters [cluster-name]`
+- Fetch the TLS config from ws-manager
+`gpctl clusters get-tls-config`
+- Port-forward ws-manager
+`kubectl port-forward [ws-manager-pod] 12001:8080`
+- Now you can start the benchmark with loadgen. If you want to keep the workspaces around after testing, add --interactive. Loadgen will then ask you before taking any destructive action.
+`loadgen benchmark [config-file] --host localhost:12001 --tls ./wsman-tls --interactive`
+
+In order to configure the benchmark, you can use the configuration file
+
+| Parameter  | Description |
+| ------------- | ------------- |
+| workspaces  | The number of workspaces that will be started during the benchmark  |
+| ideImage  | The image that will be used for the IDE |
+| waitForRunning | How long to wait for workspaces to enter running state |
+| waitForStopping | How long to wait until all workspaces are stopped |
+| successRate | Percentage of started workspaces that should enter running state to count as a successful run
+| environment | Global environment variables that will be set for all repositories |
+| workspaceClass | The workspace class to use for workspaces. This setting can be overriden for individual repositories.
+| repos | The repositories that will be used to create workspaces |
+| repo.cloneURL | The url of the repository |
+| repo.cloneTarget | The branch to clone from |
+| repo.score | The score decides how often a repository is used for the creation of a workspace. |
+| repo.workspaceImage | The docker image that will be used for the workspace |
+| repo.environment | Environment variables that will only be set for this repository |
+| repo.workspaceClass | The workspace class to use for the workspace that will be created for this repository |
+
+After the benchmark has completed, you will find a benchmark-result.json file in your working directory, that contains information about every started workspace.
+
+```
+[
+  {
+    "WorkspaceName": "moccasin-lynx-aqjtmmi4",
+    "InstanceId": "d25c1a63-0319-4ecc-881d-68804a0d1e4a",
+    "Phase": 4,
+    "Class": "default",
+    "NodeName": "workspace-ws-ephemeral-fo3-pool-wtcj",
+    "Pod": "ws-d25c1a63-0319-4ecc-881d-68804a0d1e4a",
+    "Context": "https://github.com/gitpod-io/template-python-flask"
+  },
+  {
+    "WorkspaceName": "black-wren-lqa4698w",
+    "InstanceId": "95f24d47-8c0c-4249-be24-fcc5d4d7b6fb",
+    "Phase": 4,
+    "Class": "default",
+    "NodeName": "workspace-ws-ephemeral-fo3-pool-wtcj",
+    "Pod": "ws-95f24d47-8c0c-4249-be24-fcc5d4d7b6fb",
+    "Context": "https://github.com/gitpod-io/template-python-django"
+  },
+  ...
+```

dev/loadgen/cmd/benchmark.go

@@ -78,6 +78,7 @@ var benchmarkCommand = &cobra.Command{
 				WorkspaceImage:    "will-be-overriden",
 				WorkspaceLocation: "workspace-stress",
 				Envvars:           scenario.Environment,
+				Class:             scenario.WorkspaceClass,
 			},
 			Type: api.WorkspaceType_REGULAR,
 		}
@@ -194,6 +195,7 @@ type BenchmarkScenario struct {
 	RunningTimeout  string                     `json:"waitForRunning"`
 	StoppingTimeout string                     `json:"waitForStopping"`
 	SuccessRate     float32                    `json:"successRate"`
+	WorkspaceClass  string                     `json:"workspaceClass"`
 }
 
 func handleWorkspaceDeletion(timeout string, executor loadgen.Executor) error {
@@ -235,5 +237,8 @@ func stopWorkspaces(timeout string, executor loadgen.Executor) error {
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), stopping)
 	defer cancel()
+	if err := executor.Dump("benchmark-result.json"); err != nil {
+		log.Warn("could not dump workspace state, trying to stop them anyway")
+	}
 	return executor.StopAll(ctx)
 }

dev/loadgen/pkg/loadgen/executor.go

@@ -6,9 +6,11 @@ package loadgen
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"math/rand"
+	"os"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -28,6 +30,9 @@ type Executor interface {
 
 	// StopAll stops all workspaces started by the executor
 	StopAll(ctx context.Context) error
+
+	// Dump dumps the executor state to a file
+	Dump(path string) error
 }
 
 // StartWorkspaceSpec specifies a workspace
@@ -112,7 +117,7 @@ type WsmanExecutor struct {
 
 // StartWorkspace starts a new workspace
 func (w *WsmanExecutor) StartWorkspace(spec *StartWorkspaceSpec) (callDuration time.Duration, err error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second)
 	defer cancel()
 
 	s := *spec
@@ -173,6 +178,14 @@ func (w *WsmanExecutor) StopAll(ctx context.Context) error {
 		s()
 	}
 
+	listReq := api.GetWorkspacesRequest{
+		MustMatch: &api.MetadataFilter{
+			Annotations: map[string]string{
+				loadgenAnnotation: "true",
+			},
+		},
+	}
+
 	log.Info("stopping workspaces")
 	for _, id := range w.workspaces {
 		stopReq := api.StopWorkspaceRequest{
@@ -188,14 +201,6 @@ func (w *WsmanExecutor) StopAll(ctx context.Context) error {
 
 	w.workspaces = make([]string, 0)
 
-	listReq := api.GetWorkspacesRequest{
-		MustMatch: &api.MetadataFilter{
-			Annotations: map[string]string{
-				loadgenAnnotation: "true",
-			},
-		},
-	}
-
 	for {
 		resp, err := w.C.GetWorkspaces(ctx, &listReq)
 		if len(resp.Status) == 0 {
@@ -216,3 +221,52 @@ func (w *WsmanExecutor) StopAll(ctx context.Context) error {
 
 	return nil
 }
+
+func (w *WsmanExecutor) Dump(path string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	listReq := api.GetWorkspacesRequest{
+		MustMatch: &api.MetadataFilter{
+			Annotations: map[string]string{
+				loadgenAnnotation: "true",
+			},
+		},
+	}
+
+	resp, err := w.C.GetWorkspaces(ctx, &listReq)
+	if err != nil {
+		return err
+	}
+
+	var wss []WorkspaceState
+	for _, status := range resp.GetStatus() {
+		ws := WorkspaceState{
+			WorkspaceName: status.Metadata.MetaId,
+			InstanceId:    status.Id,
+			Phase:         status.Phase,
+			Class:         status.Spec.Class,
+			NodeName:      status.Runtime.NodeName,
+			Pod:           status.Runtime.PodName,
+			Context:       status.Metadata.Annotations["context-url"],
+		}
+
+		wss = append(wss, ws)
+	}
+
+	fc, err := json.MarshalIndent(wss, "", "  ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, fc, 0644)
+}
+
+type WorkspaceState struct {
+	WorkspaceName string
+	InstanceId    string
+	Phase         api.WorkspacePhase
+	Class         string
+	NodeName      string
+	Pod           string
+	Context       string
+}

dev/loadgen/pkg/loadgen/generator.go

@@ -160,6 +160,7 @@ type WorkspaceCfg struct {
 	CloneTarget    string                     `json:"cloneTarget"`
 	Score          int                        `json:"score"`
 	Environment    []*api.EnvironmentVariable `json:"environment"`
+	WorkspaceClass string                     `json:"workspaceClass"`
 }
 
 type MultiWorkspaceGenerator struct {
@@ -202,6 +203,9 @@ func (f *MultiWorkspaceGenerator) Generate() (*StartWorkspaceSpec, error) {
 		},
 	}
 	out.Spec.WorkspaceImage = repo.WorkspaceImage
+	if len(repo.WorkspaceClass) > 0 {
+		out.Spec.Class = repo.WorkspaceClass
+	}
 	out.Spec.Envvars = append(out.Spec.Envvars, repo.Environment...)
 	r := StartWorkspaceSpec(*out)
 	return &r, nil

dev/loadgen/prod-benchmark.yaml

@@ -1,14 +1,15 @@
 ## start with
 ##    loadgen benchmark prod-benchmark.yaml
 
-workspaces: 10
+workspaces: 100
 ideImage: eu.gcr.io/gitpod-core-dev/build/ide/code:commit-ff263e14024f00d0ed78386b4417dfa6bcd4ae2f
 waitForRunning: "600s"
 waitForStopping: "600s"
-successRate: 0.80
+successRate: 1
 environment:
   - name: "THEIA_SUPERVISOR_TOKENS"
     value: '[{"token":"foobar","host":"gitpod-staging.com","scope":["function:getWorkspace","function:getLoggedInUser","function:getPortAuthenticationToken","function:getWorkspaceOwner","function:getWorkspaceUsers","function:isWorkspaceOwner","function:controlAdmission","function:setWorkspaceTimeout","function:getWorkspaceTimeout","function:sendHeartBeat","function:getOpenPorts","function:openPort","function:closePort","function:getLayout","function:generateNewGitpodToken","function:takeSnapshot","function:storeLayout","function:stopWorkspace","resource:workspace::fa498dcc-0a84-448f-9666-79f297ad821a::get/update","resource:workspaceInstance::e0a17083-6a78-441a-9b97-ef90d6aff463::get/update/delete","resource:snapshot::*::create/get","resource:gitpodToken::*::create","resource:userStorage::*::create/get/update"],"expiryDate":"2020-12-01T07:55:12.501Z","reuse":2}]'
+workspaceClass: ""
 repos:
   - cloneURL: https://github.com/Furisto/workspace-stress
     cloneTarget: main
@@ -58,7 +59,7 @@ repos:
       value: "600s"
     # backup
     - name: "BACKUP_SIZE" # size of the backup in gigabyte
-      value: 3
+      value: 12
     # tasks
     - name: "GITPOD_TASKS"
       value: "[{\"name\":\"start cpu stress\",\"command\":\"stress-ng --cpu ${CPU_COUNT:-3} --backoff ${CPU_BACKOFF:-10000000} --timeout ${CPU_TIMEOUT:-600s}\"},{\"name\":\"start io stress\",\"command\":\"fio --name io-stress --eta-newline=5s --filename=/workspace/gitpod.temp --rw=${DISK_IO_MODE} --size=${DISK_IO_FILE_SIZE:-2g} --io_size=${DISK_IO_TOTAL:-50g} --blocksize=${DISK_IO_BLOCKSIZE} --ioengine=libaio --fsync=${DISK_IO_FSYNC} --iodepth=${DISK_IO_DEPTH} --direct=1 --numjobs=${DISK_IO_JOBS} --runtime=${DISK_IO_TIMEOUT:-600}\"},{\"name\":\"start memory stress\",\"command\":\"stress-ng --vm 1 --vm-keep --vm-bytes ${MEMORY_BYTES:-6G} --timeout ${MEMORY_TIMEOUT:-600s}\"},{\"name\":\"create backup file\",\"command\":\"dd if=/dev/zero of=/workspace/benchmark-backup bs=1000M count=${BACKUP_SIZE:-2}\"}]"