From 27a462d5a4d412ee2cd2302484681a63c8c3a985 Mon Sep 17 00:00:00 2001 From: Wouter Verlaek Date: Mon, 17 Oct 2022 15:59:43 +0000 Subject: [PATCH] chore: Create image-builder-mk3 readme --- components/image-builder-bob/README.md | 28 ++++- components/image-builder-mk3/.gitignore | 1 + components/image-builder-mk3/README.md | 72 +++++++++++ .../image-builder-mk3/config-schema.json | 114 ++++++++---------- .../image-builder-mk3/example-config.json | 39 +++--- .../image-builder.code-workspace | 4 +- components/image-builder-mk3/telepresence.sh | 23 ---- 7 files changed, 177 insertions(+), 104 deletions(-) create mode 100644 components/image-builder-mk3/.gitignore create mode 100644 components/image-builder-mk3/README.md delete mode 100755 components/image-builder-mk3/telepresence.sh diff --git a/components/image-builder-bob/README.md b/components/image-builder-bob/README.md index 831cf6f5484752..dab6116bed4704 100644 --- a/components/image-builder-bob/README.md +++ b/components/image-builder-bob/README.md @@ -1,7 +1,22 @@ +# Introduction to image-builder-bob + +## Overview + +Bob is a CLI responsible for building and pushing workspace images during workspace startup. + +For each image build, a headless workspace gets created in the meta cluster by `image-builder-mk3` ([#7845](https://github.com/gitpod-io/gitpod/issues/7845) will move the headless workspace and image-builder to the workspace cluster), in this headless workspace runs: +- `bob proxy`, which gets started by workspacekit in ring1, and receives credentials for pushing images to a docker registry. It proxies and authenticates the image pushes from `bob build`. +- `bob build` as a workspace task, which builds the + - **base layer**, if a custom Dockerfile is specified in `.gitpod.yaml`. If this base image has already been built for the workspace, this step is skipped, and the reference of the previously built image is used instead to build the workspace image next. + - **workspace image**, which builds an image from the base layer, where the base layer is either a previously built custom Dockerfile or a public image. + These images get pushed over `localhost` to `bob proxy`, as `bob build` does not receive the credentials to push to private registries. + + The built images do not include e.g. `supervisor` or the IDE, these layers will get added by [`registry-facade`](../registry-facade/README.md) during image pull. + ## How to try locally Prerequisite: make sure you have buildkit in the path -```bash +```console cd /tmp curl -OL https://github.com/moby/buildkit/releases/download/v0.10.0/buildkit-v0.10.0.linux-amd64.tar.gz tar xzfv buildkit-v0.10.0.linux-amd64.tar.gz @@ -9,7 +24,7 @@ sudo mv bin/* /usr/bin ``` Set things up -```bash +```console # install oci-tool for inspecting the built image go install github.com/csweichel/oci-tool@latest @@ -28,7 +43,7 @@ docker push localhost:5000/source:latest ``` Build and run -``` +```console # build and install bob (do this after every change) cd /workspace/gitpod/components/image-builder-bob go install @@ -39,3 +54,10 @@ BOB_BASE_REF=localhost:5000/source:latest BOB_TARGET_REF=localhost:5000/target:8 # debug using delve BOB_BASE_REF=localhost:5000/source:latest BOB_TARGET_REF=localhost:5000/target:83 sudo -E $(which dlv) --listen=:2345 --headless=true --api-version=2 exec $(which bob) build ``` + +## Run tests + +```console +cd /workspace/gitpod/components/image-builder-bob +go test -v ./... +``` diff --git a/components/image-builder-mk3/.gitignore b/components/image-builder-mk3/.gitignore new file mode 100644 index 00000000000000..38d3c615f564f0 --- /dev/null +++ b/components/image-builder-mk3/.gitignore @@ -0,0 +1 @@ +/wsman-tls/ diff --git a/components/image-builder-mk3/README.md b/components/image-builder-mk3/README.md new file mode 100644 index 00000000000000..0d6dcf6e4b2dd2 --- /dev/null +++ b/components/image-builder-mk3/README.md @@ -0,0 +1,72 @@ +# image-builder-mk3 + +`image-builder-mk3` is a service that runs in meta clusters, which provides APIs to create and list workspace image builds, resolve workspace docker image references, and listen to build updates and logs. + +> Once completed, [#7845](https://github.com/gitpod-io/gitpod/issues/7845) will move `image-builder-mk3` (and image builds) from meta to workspace clusters. + +See the [Architecture page](https://www.notion.so/gitpod/Architecture-0e39e570b10f4e8ba7b259629ee3cb74) for more details, including a diagram. + +Relevant: +* API definitions can be found [here](../image-builder-api/imgbuilder.proto) +* Installer k8s config can be found [here](../../install/installer/pkg/components/image-builder-mk3/) + +## Development + +To test local changes, the following script will compile and publish a docker image, +and restart the image-builder-mk3 k8s deployment at your current kube context with this image: + +```console +# Careful: check you're pointing at the right cluster! +components/image-builder-mk3/debug.sh +``` + +### Making changes to the protocol + +Protocol changes are to be made in the [`image-builder-api`](../image-builder-api/) component. + +### Updating the installer config + +Changes to the installed k8s config can be done [here](../../install/installer/pkg/components/image-builder-mk3/). + +### Running locally + +To run `image-builder-mk3` locally, the `example-config.json` can be used as follows: + +```console +cd /workspace/gitpod/components/image-builder-mk3 + +# Set up kube context, the image builder will connect to this cluster's ws-manager. +kubectx [cluster-name] + +# Fetch ws-manager TLS config. +gpctl clusters get-tls-config + +# Port forward to ws-manager. +kubectl port-forward deployment/ws-manager 12001:8080 + +# Run image-builder-mk3. +go run . run --config example-config.json +``` + +See metrics at `http://localhost:9500/metrics`. + +#### Invoking RPCs + +When `image-builder-mk3` is running locally, you can use `grpcurl` to make gRPC requests against the API: +```console +go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest + +# Call RPC, e.g. ListBuilds: +grpcurl -plaintext -proto /workspace/gitpod/components/image-builder-api/imgbuilder.proto -import-path=/workspace/gitpod/components/ localhost:8080 builder.ImageBuilder.ListBuilds +``` + +See https://github.com/fullstorydev/grpcurl#invoking-rpcs for how to make different types of requests with `grpcurl`. + + +### Running tests + +To run all `image-builder-mk3` tests: + +```console +go test -v ./... +``` diff --git a/components/image-builder-mk3/config-schema.json b/components/image-builder-mk3/config-schema.json index a13725d7af21b4..f707c87524a8c4 100644 --- a/components/image-builder-mk3/config-schema.json +++ b/components/image-builder-mk3/config-schema.json @@ -1,89 +1,40 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/config", + "$ref": "#/definitions/ServiceConfig", "title": "image-builder config schema - generated using img generate config", "definitions": { - "": { - "required": [ - "address", - "tls" - ], - "properties": { - "address": { - "type": "string" - }, - "tls": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/tlsConfig" - } - }, - "additionalProperties": false, - "type": "object" - }, "Configuration": { "required": [ + "wsman", "baseImageRepository", "workspaceImageRepository", - "gitpodLayerLoc", - "workdir" + "builderImage" ], "properties": { - "baseImageRepository": { - "type": "string" - }, - "dockerCfgFile": { - "type": "string" - }, - "gitpodLayerLoc": { - "type": "string" + "wsman": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/WorkspaceManagerConfig" }, - "imageBuildSalt": { + "pullSecret": { "type": "string" }, - "imagebuilderRef": { + "pullSecretFile": { "type": "string" }, - "workdir": { + "baseImageRepository": { "type": "string" }, "workspaceImageRepository": { "type": "string" - } - }, - "additionalProperties": false, - "type": "object" - }, - "config": { - "required": [ - "builder", - "service", - "prometheus", - "pprof" - ], - "properties": { - "builder": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/Configuration" - }, - "pprof": { - "$ref": "#/definitions/" - }, - "prometheus": { - "$ref": "#/definitions/" }, - "refCache": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/refcacheConfig" - }, - "service": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/" + "builderImage": { + "type": "string" } }, "additionalProperties": false, "type": "object" }, - "refcacheConfig": { + "RefCacheConfig": { "required": [ "interval", "refs" @@ -102,7 +53,28 @@ "additionalProperties": false, "type": "object" }, - "tlsConfig": { + "ServiceConfig": { + "required": [ + "orchestrator", + "server" + ], + "properties": { + "orchestrator": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/Configuration" + }, + "refCache": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/RefCacheConfig" + }, + "server": { + "$ref": "#/definitions/Configuration" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TLS": { "required": [ "ca", "crt", @@ -121,6 +93,22 @@ }, "additionalProperties": false, "type": "object" + }, + "WorkspaceManagerConfig": { + "required": [ + "address" + ], + "properties": { + "address": { + "type": "string" + }, + "tls": { + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/TLS" + } + }, + "additionalProperties": false, + "type": "object" } } -} \ No newline at end of file +} diff --git a/components/image-builder-mk3/example-config.json b/components/image-builder-mk3/example-config.json index 78c08bc18a7433..724e272f3680c0 100644 --- a/components/image-builder-mk3/example-config.json +++ b/components/image-builder-mk3/example-config.json @@ -1,19 +1,30 @@ { "$schema": "./config-schema.json", - "builder": { - "dockerCfgFile": "/home/gitpod/.docker/config.json", - "gitpodLayerLoc": "/tmp/build/components-image-builder-workspace-image-layer--pack.588fcb85389f5c10ae444b52674654698f446a10/pack.tar", - "baseImageRepository": "eu.gcr.io/gitpod-dev/base-images", - "workspaceImageRepository": "eu.gcr.io/gitpod-dev/workspace-images", - "imageBuildSalt": "001" + "orchestrator": { + "wsman": { + "address": "localhost:12001", + "tls": { + "ca": "./wsman-tls/ca.crt", + "crt": "./wsman-tls/tls.crt", + "key": "./wsman-tls/tls.key" + } + }, + "pullSecret": "gcp-sa-registry-auth", + "baseImageRepository": "eu.gcr.io/gitpod-core-dev/build/base-images", + "workspaceImageRepository": "eu.gcr.io/gitpod-core-dev/build/workspace-images", + "builderImage": "eu.gcr.io/gitpod-core-dev/build/image-builder-mk3/bob:4d9d04b1bf94218d47294f83b11eba652455a790" }, - "pprof": { - "address": ":9999" + "refCache": { + "interval": "6h0m0s", + "refs": [ + "docker.io/gitpod/workspace-full:latest" + ] }, - "prometheus": { - "address": ":9500" - }, - "service": { - "address": ":8080" + "server": { + "services": { + "grpc": { + "address": "0.0.0.0:8080" + } + } } -} \ No newline at end of file +} diff --git a/components/image-builder-mk3/image-builder.code-workspace b/components/image-builder-mk3/image-builder.code-workspace index a706e6032e50ee..fa7c86399910b7 100644 --- a/components/image-builder-mk3/image-builder.code-workspace +++ b/components/image-builder-mk3/image-builder.code-workspace @@ -1,11 +1,13 @@ { "folders": [ + { "path": "../image-builder-api/go" }, { "path": "../image-builder-bob" }, { "path": "../image-builder-mk3" }, { "path": "../ws-manager" }, { "path": "../server" }, { "path": "../../test" }, - { "path": "../../dev/gpctl" } + { "path": "../../dev/gpctl" }, + { "path": "../../install/installer" } ], "settings": { "typescript.tsdk": "gitpod/node_modules/typescript/lib", diff --git a/components/image-builder-mk3/telepresence.sh b/components/image-builder-mk3/telepresence.sh deleted file mode 100755 index 6e9abe629909e7..00000000000000 --- a/components/image-builder-mk3/telepresence.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -leeway build --save /tmp/gplayer.tgz components/image-builder/workspace-image-layer:pack -export GITPOD_LAYER_LOC=/tmp/gplayer.tgz - -if [ "$1" == "intp" ]; then - echo "starting delve" - dlv debug --listen=127.0.0.1:32991 --headless --api-version=2 github.com/gitpod-io/gitpod/image-builder -- run -v --config /tmp/imgblddebug/config/image-builder.json - exit $? -fi - -if [ "$1" == "run" ]; then - telepresence --mount /tmp/imgblddebug --swap-deployment image-builder --method vpn-tcp --run go run main.go run -v --config /tmp/imgblddebug/config/image-builder.json - exit $? -fi - -if [ "$1" == "debug" ]; then - telepresence --mount /tmp/imgblddebug --swap-deployment image-builder --method vpn-tcp --run "$0" intp - exit $? -fi - -echo "usage: $0 run|debug" -exit 1 \ No newline at end of file