From de1b8cd2410b4e938c9d3fba12e388c0b93ed67e Mon Sep 17 00:00:00 2001 From: Thomas Schubart Date: Wed, 1 Jun 2022 15:51:50 +0000 Subject: [PATCH 1/3] Support workspace classes and write results --- dev/loadgen/cmd/benchmark.go | 5 ++ dev/loadgen/pkg/loadgen/executor.go | 72 ++++++++++++++++++++++++---- dev/loadgen/pkg/loadgen/generator.go | 4 ++ 3 files changed, 72 insertions(+), 9 deletions(-) diff --git a/dev/loadgen/cmd/benchmark.go b/dev/loadgen/cmd/benchmark.go index 8462148b4a3876..d6161bd2ec40e1 100644 --- a/dev/loadgen/cmd/benchmark.go +++ b/dev/loadgen/cmd/benchmark.go @@ -78,6 +78,7 @@ var benchmarkCommand = &cobra.Command{ WorkspaceImage: "will-be-overriden", WorkspaceLocation: "workspace-stress", Envvars: scenario.Environment, + Class: scenario.WorkspaceClass, }, Type: api.WorkspaceType_REGULAR, } @@ -194,6 +195,7 @@ type BenchmarkScenario struct { RunningTimeout string `json:"waitForRunning"` StoppingTimeout string `json:"waitForStopping"` SuccessRate float32 `json:"successRate"` + WorkspaceClass string `json:"workspaceClass"` } func handleWorkspaceDeletion(timeout string, executor loadgen.Executor) error { @@ -235,5 +237,8 @@ func stopWorkspaces(timeout string, executor loadgen.Executor) error { } ctx, cancel := context.WithTimeout(context.Background(), stopping) defer cancel() + if err := executor.Dump("benchmark-result.json"); err != nil { + log.Warn("could not dump workspace state, trying to stop them anyway") + } return executor.StopAll(ctx) } diff --git a/dev/loadgen/pkg/loadgen/executor.go b/dev/loadgen/pkg/loadgen/executor.go index dec6d458ee3214..9f16f3f94372ae 100644 --- a/dev/loadgen/pkg/loadgen/executor.go +++ b/dev/loadgen/pkg/loadgen/executor.go @@ -6,9 +6,11 @@ package loadgen import ( "context" + "encoding/json" "fmt" "io" "math/rand" + "os" "time" log "github.com/sirupsen/logrus" @@ -28,6 +30,9 @@ type Executor interface { // StopAll stops all workspaces started by the executor StopAll(ctx context.Context) error + + // Dump dumps the executor state to a file + Dump(path string) error } // StartWorkspaceSpec specifies a workspace @@ -112,7 +117,7 @@ type WsmanExecutor struct { // StartWorkspace starts a new workspace func (w *WsmanExecutor) StartWorkspace(spec *StartWorkspaceSpec) (callDuration time.Duration, err error) { - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second) defer cancel() s := *spec @@ -173,6 +178,14 @@ func (w *WsmanExecutor) StopAll(ctx context.Context) error { s() } + listReq := api.GetWorkspacesRequest{ + MustMatch: &api.MetadataFilter{ + Annotations: map[string]string{ + loadgenAnnotation: "true", + }, + }, + } + log.Info("stopping workspaces") for _, id := range w.workspaces { stopReq := api.StopWorkspaceRequest{ @@ -188,14 +201,6 @@ func (w *WsmanExecutor) StopAll(ctx context.Context) error { w.workspaces = make([]string, 0) - listReq := api.GetWorkspacesRequest{ - MustMatch: &api.MetadataFilter{ - Annotations: map[string]string{ - loadgenAnnotation: "true", - }, - }, - } - for { resp, err := w.C.GetWorkspaces(ctx, &listReq) if len(resp.Status) == 0 { @@ -216,3 +221,52 @@ func (w *WsmanExecutor) StopAll(ctx context.Context) error { return nil } + +func (w *WsmanExecutor) Dump(path string) error { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + listReq := api.GetWorkspacesRequest{ + MustMatch: &api.MetadataFilter{ + Annotations: map[string]string{ + loadgenAnnotation: "true", + }, + }, + } + + resp, err := w.C.GetWorkspaces(ctx, &listReq) + if err != nil { + return err + } + + var wss []WorkspaceState + for _, status := range resp.GetStatus() { + ws := WorkspaceState{ + WorkspaceName: status.Metadata.MetaId, + InstanceId: status.Id, + Phase: status.Phase, + Class: status.Spec.Class, + NodeName: status.Runtime.NodeName, + Pod: status.Runtime.PodName, + Context: status.Metadata.Annotations["context-url"], + } + + wss = append(wss, ws) + } + + fc, err := json.MarshalIndent(wss, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, fc, 0644) +} + +type WorkspaceState struct { + WorkspaceName string + InstanceId string + Phase api.WorkspacePhase + Class string + NodeName string + Pod string + Context string +} diff --git a/dev/loadgen/pkg/loadgen/generator.go b/dev/loadgen/pkg/loadgen/generator.go index b9d03fc9fb5cda..dc2ba7f40f38b8 100644 --- a/dev/loadgen/pkg/loadgen/generator.go +++ b/dev/loadgen/pkg/loadgen/generator.go @@ -160,6 +160,7 @@ type WorkspaceCfg struct { CloneTarget string `json:"cloneTarget"` Score int `json:"score"` Environment []*api.EnvironmentVariable `json:"environment"` + WorkspaceClass string `json:"workspaceClass"` } type MultiWorkspaceGenerator struct { @@ -202,6 +203,9 @@ func (f *MultiWorkspaceGenerator) Generate() (*StartWorkspaceSpec, error) { }, } out.Spec.WorkspaceImage = repo.WorkspaceImage + if len(repo.WorkspaceClass) > 0 { + out.Spec.Class = repo.WorkspaceClass + } out.Spec.Envvars = append(out.Spec.Envvars, repo.Environment...) r := StartWorkspaceSpec(*out) return &r, nil From b1defc1b1ab4c070ae5ad06d6f0398a2f27326f5 Mon Sep 17 00:00:00 2001 From: Thomas Schubart Date: Wed, 1 Jun 2022 15:52:28 +0000 Subject: [PATCH 2/3] Document loadgen --- dev/loadgen/README.md | 58 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/dev/loadgen/README.md b/dev/loadgen/README.md index 65f205603d9179..8136cbcf76bb88 100644 --- a/dev/loadgen/README.md +++ b/dev/loadgen/README.md @@ -2,4 +2,60 @@ A load generator (framework) for Gitpod. -**Note:** this is a development tool only - there's no support for this. \ No newline at end of file +**Note:** this is a development tool only - there's no support for this. + +You can find a short explanation of this tool in this [loom video](https://www.loom.com/share/6487e3403c0746cc97bb3f766e15fab6). + +## How to run a benchmark +- Ensure your kubeconfig has the configuration for the cluster you want to benchmark. You can use kubecdl to update your kubeconfig with the cluster information +`kubecdl -p workspace-clusters [cluster-name]` +- Fetch the TLS config from ws-manager +`gpctl clusters get-tls-config` +- Port-forward ws-manager +`kubectl port-forward [ws-manager-pod] 12001:8080` +- Now you can start the benchmark with loadgen. If you want to keep the workspaces around after testing, add --interactive. Loadgen will then ask you before taking any destructive action. +`loadgen benchmark [config-file] --host localhost:12001 --tls ./wsman-tls --interactive` + +In order to configure the benchmark, you can use the configuration file + +| Parameter | Description | +| ------------- | ------------- | +| workspaces | The number of workspaces that will be started during the benchmark | +| ideImage | The image that will be used for the IDE | +| waitForRunning | How long to wait for workspaces to enter running state | +| waitForStopping | How long to wait until all workspaces are stopped | +| successRate | Percentage of started workspaces that should enter running state to count as a successful run +| environment | Global environment variables that will be set for all repositories | +| workspaceClass | The workspace class to use for workspaces. This setting can be overriden for individual repositories. +| repos | The repositories that will be used to create workspaces | +| repo.cloneURL | The url of the repository | +| repo.cloneTarget | The branch to clone from | +| repo.score | The score decides how often a repository is used for the creation of a workspace. | +| repo.workspaceImage | The docker image that will be used for the workspace | +| repo.environment | Environment variables that will only be set for this repository | +| repo.workspaceClass | The workspace class to use for the workspace that will be created for this repository | + +After the benchmark has completed, you will find a benchmark-result.json file in your working directory, that contains information about every started workspace. + +``` +[ + { + "WorkspaceName": "moccasin-lynx-aqjtmmi4", + "InstanceId": "d25c1a63-0319-4ecc-881d-68804a0d1e4a", + "Phase": 4, + "Class": "default", + "NodeName": "workspace-ws-ephemeral-fo3-pool-wtcj", + "Pod": "ws-d25c1a63-0319-4ecc-881d-68804a0d1e4a", + "Context": "https://github.com/gitpod-io/template-python-flask" + }, + { + "WorkspaceName": "black-wren-lqa4698w", + "InstanceId": "95f24d47-8c0c-4249-be24-fcc5d4d7b6fb", + "Phase": 4, + "Class": "default", + "NodeName": "workspace-ws-ephemeral-fo3-pool-wtcj", + "Pod": "ws-95f24d47-8c0c-4249-be24-fcc5d4d7b6fb", + "Context": "https://github.com/gitpod-io/template-python-django" + }, + ... +``` From 0e98af8ed425e0b2123e4f717f08e6571db09d9d Mon Sep 17 00:00:00 2001 From: Thomas Schubart Date: Wed, 1 Jun 2022 15:57:50 +0000 Subject: [PATCH 3/3] Adapt benchmark parameters --- dev/loadgen/prod-benchmark.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dev/loadgen/prod-benchmark.yaml b/dev/loadgen/prod-benchmark.yaml index 6b5c6fb62728b9..770456ad236dcb 100644 --- a/dev/loadgen/prod-benchmark.yaml +++ b/dev/loadgen/prod-benchmark.yaml @@ -1,14 +1,15 @@ ## start with ## loadgen benchmark prod-benchmark.yaml -workspaces: 10 +workspaces: 100 ideImage: eu.gcr.io/gitpod-core-dev/build/ide/code:commit-ff263e14024f00d0ed78386b4417dfa6bcd4ae2f waitForRunning: "600s" waitForStopping: "600s" -successRate: 0.80 +successRate: 1 environment: - name: "THEIA_SUPERVISOR_TOKENS" value: '[{"token":"foobar","host":"gitpod-staging.com","scope":["function:getWorkspace","function:getLoggedInUser","function:getPortAuthenticationToken","function:getWorkspaceOwner","function:getWorkspaceUsers","function:isWorkspaceOwner","function:controlAdmission","function:setWorkspaceTimeout","function:getWorkspaceTimeout","function:sendHeartBeat","function:getOpenPorts","function:openPort","function:closePort","function:getLayout","function:generateNewGitpodToken","function:takeSnapshot","function:storeLayout","function:stopWorkspace","resource:workspace::fa498dcc-0a84-448f-9666-79f297ad821a::get/update","resource:workspaceInstance::e0a17083-6a78-441a-9b97-ef90d6aff463::get/update/delete","resource:snapshot::*::create/get","resource:gitpodToken::*::create","resource:userStorage::*::create/get/update"],"expiryDate":"2020-12-01T07:55:12.501Z","reuse":2}]' +workspaceClass: "" repos: - cloneURL: https://github.com/Furisto/workspace-stress cloneTarget: main @@ -58,7 +59,7 @@ repos: value: "600s" # backup - name: "BACKUP_SIZE" # size of the backup in gigabyte - value: 3 + value: 12 # tasks - name: "GITPOD_TASKS" value: "[{\"name\":\"start cpu stress\",\"command\":\"stress-ng --cpu ${CPU_COUNT:-3} --backoff ${CPU_BACKOFF:-10000000} --timeout ${CPU_TIMEOUT:-600s}\"},{\"name\":\"start io stress\",\"command\":\"fio --name io-stress --eta-newline=5s --filename=/workspace/gitpod.temp --rw=${DISK_IO_MODE} --size=${DISK_IO_FILE_SIZE:-2g} --io_size=${DISK_IO_TOTAL:-50g} --blocksize=${DISK_IO_BLOCKSIZE} --ioengine=libaio --fsync=${DISK_IO_FSYNC} --iodepth=${DISK_IO_DEPTH} --direct=1 --numjobs=${DISK_IO_JOBS} --runtime=${DISK_IO_TIMEOUT:-600}\"},{\"name\":\"start memory stress\",\"command\":\"stress-ng --vm 1 --vm-keep --vm-bytes ${MEMORY_BYTES:-6G} --timeout ${MEMORY_TIMEOUT:-600s}\"},{\"name\":\"create backup file\",\"command\":\"dd if=/dev/zero of=/workspace/benchmark-backup bs=1000M count=${BACKUP_SIZE:-2}\"}]"