Skip to content

.werft: Refactoring to use constant kubeconfig files instead of overrides #9016

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Apr 1, 2022
Merged

.werft: Refactoring to use constant kubeconfig files instead of overrides #9016

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a2c59bc
.werft/build: Decouple multiple kubeconfig paths
ArthurSens Mar 29, 2022
41f9366
.werft/deploy: Specify kubeconfig to every command
ArthurSens Mar 29, 2022
ea745b6
Fix kubeconfig on cleanup jobs
ArthurSens Mar 30, 2022
eb173bd
Fix certification issuing and installation
ArthurSens Mar 30, 2022
c649382
Fix observability deployment
ArthurSens Mar 31, 2022
bd1d2f2
Fix certificates on Harvester previews
ArthurSens Mar 31, 2022
21d9638
Fix helm-installer deployment iteration
ArthurSens Mar 31, 2022
71ec9ab
Fix clean-slate-deployment
ArthurSens Mar 31, 2022
540ada2
.werft/build/prepare: Split error throwing when configuring cluster c…
ArthurSens Mar 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .werft/jobs/build/const.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
export const GCLOUD_SERVICE_ACCOUNT_PATH = "/mnt/secrets/gcp-sa/service-account.json";
export const CORE_DEV_KUBECONFIG_PATH = "/workspace/gitpod/kubeconfigs/core-dev";
export const HARVESTER_KUBECONFIG_PATH = "/workspace/gitpod/kubeconfigs/harvester";
export const PREVIEW_K3S_KUBECONFIG_PATH = "/workspace/gitpod/kubeconfigs/k3s";
124 changes: 63 additions & 61 deletions .werft/jobs/build/deploy-to-preview-environment.ts
View file
Open in desktop

Large diffs are not rendered by default.

56 changes: 29 additions & 27 deletions .werft/jobs/build/installer/installer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ export type GitpodDaemonsetPorts = {

export type InstallerOptions = {
werft: Werft
configPath: string
installerConfigPath: string
kubeconfigPath: string
version: string
proxySecretName: string
domain: string
Expand All @@ -45,7 +46,7 @@ export class Installer {
this.options.werft.log(slice, "Downloading installer and initializing config file");
exec(`docker run --entrypoint sh --rm eu.gcr.io/gitpod-core-dev/build/installer:${this.options.version} -c "cat /app/installer" > /tmp/installer`, { slice: slice });
exec(`chmod +x /tmp/installer`, { slice: slice });
exec(`/tmp/installer init > ${this.options.configPath}`, { slice: slice });
exec(`/tmp/installer init > ${this.options.installerConfigPath}`, { slice: slice });
this.options.werft.done(slice);
}

Expand Down Expand Up @@ -75,30 +76,30 @@ export class Installer {
exec(`yq r ./.werft/jobs/build/helm/values.dev.yaml components.server.blockNewUsers | yq prefix - 'blockNewUsers' > ${BLOCK_NEW_USER_CONFIG_PATH}`, { slice: slice });
exec(`yq r ./.werft/jobs/build/helm/values.variant.cpuLimits.yaml workspaceSizing.dynamic.cpu.buckets | yq prefix - 'workspace.resources.dynamicLimits.cpu' > ${WORKSPACE_SIZE_CONFIG_PATH}`, { slice: slice });

exec(`yq m -i --overwrite ${this.options.configPath} ${BLOCK_NEW_USER_CONFIG_PATH}`, { slice: slice });
exec(`yq m -i ${this.options.configPath} ${WORKSPACE_SIZE_CONFIG_PATH}`, { slice: slice });
exec(`yq m -i --overwrite ${this.options.installerConfigPath} ${BLOCK_NEW_USER_CONFIG_PATH}`, { slice: slice });
exec(`yq m -i ${this.options.installerConfigPath} ${WORKSPACE_SIZE_CONFIG_PATH}`, { slice: slice });
}

private configureContainerRegistry(slice: string): void {
exec(`yq w -i ${this.options.configPath} certificate.name ${this.options.proxySecretName}`, { slice: slice });
exec(`yq w -i ${this.options.configPath} containerRegistry.inCluster false`, { slice: slice });
exec(`yq w -i ${this.options.configPath} containerRegistry.external.url ${CONTAINER_REGISTRY_URL}`, { slice: slice });
exec(`yq w -i ${this.options.configPath} containerRegistry.external.certificate.kind secret`, { slice: slice });
exec(`yq w -i ${this.options.configPath} containerRegistry.external.certificate.name ${this.options.imagePullSecretName}`, { slice: slice });
exec(`yq w -i ${this.options.installerConfigPath} certificate.name ${this.options.proxySecretName}`, { slice: slice });
exec(`yq w -i ${this.options.installerConfigPath} containerRegistry.inCluster false`, { slice: slice });
exec(`yq w -i ${this.options.installerConfigPath} containerRegistry.external.url ${CONTAINER_REGISTRY_URL}`, { slice: slice });
exec(`yq w -i ${this.options.installerConfigPath} containerRegistry.external.certificate.kind secret`, { slice: slice });
exec(`yq w -i ${this.options.installerConfigPath} containerRegistry.external.certificate.name ${this.options.imagePullSecretName}`, { slice: slice });
}

private configureDomain(slice: string) {
exec(`yq w -i ${this.options.configPath} domain ${this.options.domain}`, { slice: slice });
exec(`yq w -i ${this.options.installerConfigPath} domain ${this.options.domain}`, { slice: slice });
}

private configureWorkspaces(slice: string) {
exec(`yq w -i ${this.options.configPath} workspace.runtime.containerdRuntimeDir ${CONTAINERD_RUNTIME_DIR}`, { slice: slice });
exec(`yq w -i ${this.options.configPath} workspace.resources.requests.cpu "100m"`, { slice: slice });
exec(`yq w -i ${this.options.installerConfigPath} workspace.runtime.containerdRuntimeDir ${CONTAINERD_RUNTIME_DIR}`, { slice: slice });
exec(`yq w -i ${this.options.installerConfigPath} workspace.resources.requests.cpu "100m"`, { slice: slice });
}

private configureObservability(slice: string) {
const tracingEndpoint = exec(`yq r ./.werft/jobs/build/helm/values.tracing.yaml tracing.endpoint`, { slice: slice }).stdout.trim();
exec(`yq w -i ${this.options.configPath} observability.tracing.endpoint ${tracingEndpoint}`, { slice: slice });
exec(`yq w -i ${this.options.installerConfigPath} observability.tracing.endpoint ${tracingEndpoint}`, { slice: slice });
}

// auth-provider-secret.yml is a file generated by this job by reading a secret from core-dev cluster
Expand All @@ -114,14 +115,15 @@ export class Installer {
providerId=$(echo $row | base64 -d | jq -r '.value.id | ascii_downcase')
data=$(echo $row | base64 -d | yq r - value --prettyPrint)

yq w -i ${this.options.configPath} authProviders[$key].kind "secret"
yq w -i ${this.options.configPath} authProviders[$key].name "$providerId"
yq w -i ${this.options.installerConfigPath} authProviders[$key].kind "secret"
yq w -i ${this.options.installerConfigPath} authProviders[$key].name "$providerId"

kubectl create secret generic "$providerId" \
--namespace "${this.options.deploymentNamespace}" \
--kubeconfig "${this.options.kubeconfigPath}" \
--from-literal=provider="$data" \
--dry-run=client -o yaml | \
kubectl replace --force -f -
kubectl --kubeconfig "${this.options.kubeconfigPath}" replace --force -f -
done`, { slice: slice })
}

Expand All @@ -131,30 +133,30 @@ export class Installer {
| yq d - metadata.uid \
| yq d - metadata.resourceVersion \
| yq d - metadata.creationTimestamp \
| kubectl apply -f -`, { slice: slice })
exec(`yq w -i ${this.options.configPath} sshGatewayHostKey.kind "secret"`)
exec(`yq w -i ${this.options.configPath} sshGatewayHostKey.name "host-key"`)
| kubectl --kubeconfig ${this.options.kubeconfigPath} apply -f -`, { slice: slice })
exec(`yq w -i ${this.options.installerConfigPath} sshGatewayHostKey.kind "secret"`)
exec(`yq w -i ${this.options.installerConfigPath} sshGatewayHostKey.name "host-key"`)
}

private includeAnalytics(slice: string): void {
exec(`yq w -i ${this.options.configPath} analytics.writer segment`, { slice: slice });
exec(`yq w -i ${this.options.configPath} analytics.segmentKey ${this.options.analytics.token}`, { slice: slice });
exec(`yq w -i ${this.options.installerConfigPath} analytics.writer segment`, { slice: slice });
exec(`yq w -i ${this.options.installerConfigPath} analytics.segmentKey ${this.options.analytics.token}`, { slice: slice });
}

private dontIncludeAnalytics(slice: string): void {
exec(`yq w -i ${this.options.configPath} analytics.writer ""`, { slice: slice });
exec(`yq w -i ${this.options.installerConfigPath} analytics.writer ""`, { slice: slice });
}

validateConfiguration(slice: string): void {
this.options.werft.log(slice, "Validating configuration");
exec(`/tmp/installer validate config -c ${this.options.configPath}`, { slice: slice });
exec(`/tmp/installer validate cluster -c ${this.options.configPath} || true`, { slice: slice });
exec(`/tmp/installer validate config -c ${this.options.installerConfigPath}`, { slice: slice });
exec(`/tmp/installer validate cluster --kubeconfig ${this.options.kubeconfigPath} -c ${this.options.installerConfigPath} || true`, { slice: slice });
this.options.werft.done(slice)
}

render(slice: string): void {
this.options.werft.log(slice, "Rendering YAML manifests");
exec(`/tmp/installer render --namespace ${this.options.deploymentNamespace} --config ${this.options.configPath} > k8s.yaml`, { slice: slice });
exec(`/tmp/installer render --namespace ${this.options.deploymentNamespace} --config ${this.options.installerConfigPath} > k8s.yaml`, { slice: slice });
this.options.werft.done(slice)
}

Expand Down Expand Up @@ -199,9 +201,9 @@ export class Installer {

install(slice: string): void {
this.options.werft.log(slice, "Installing Gitpod");
exec(`kubectl delete -n ${this.options.deploymentNamespace} job migrations || true`, { silent: true });
exec(`kubectl --kubeconfig ${this.options.kubeconfigPath} delete -n ${this.options.deploymentNamespace} job migrations || true`, { silent: true });
// errors could result in outputing a secret to the werft log when kubernetes patches existing objects...
exec(`kubectl apply -f k8s.yaml`, { silent: true });
exec(`kubectl --kubeconfig ${this.options.kubeconfigPath} apply -f k8s.yaml`, { silent: true });

exec(`werft log result -d "dev installation" -c github-check-preview-env url https://${this.options.domain}/workspaces`);
this.options.werft.done(slice)
Expand Down
17 changes: 11 additions & 6 deletions .werft/jobs/build/prepare.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { exec } from '../../util/shell';
import { Werft } from "../../util/werft";
import * as VM from '../../vm/vm'
import { GCLOUD_SERVICE_ACCOUNT_PATH } from "./const";
import { CORE_DEV_KUBECONFIG_PATH, GCLOUD_SERVICE_ACCOUNT_PATH, HARVESTER_KUBECONFIG_PATH } from "./const";
import { JobConfig } from './job-config';

const phaseName = "prepare";
Expand All @@ -17,7 +17,7 @@ export async function prepare(werft: Werft, config: JobConfig) {
compareWerftAndGitpodImage()
activateCoreDevServiceAccount()
configureDocker()
configureCoreDevAccess()
configureStaticClustersAccess()
werft.done(prepareSlices.CONFIGURE_CORE_DEV)

decideHarvesterVMCreation(werft, config)
Expand Down Expand Up @@ -54,12 +54,17 @@ function configureDocker() {
}
}

function configureCoreDevAccess() {
const rc = exec('gcloud container clusters get-credentials core-dev --zone europe-west1-b --project gitpod-core-dev', { slice: prepareSlices.CONFIGURE_CORE_DEV }).code;

if (rc != 0) {
function configureStaticClustersAccess() {
const rcCoreDev = exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} gcloud container clusters get-credentials core-dev --zone europe-west1-b --project gitpod-core-dev`, { slice: prepareSlices.CONFIGURE_CORE_DEV }).code;
if (rcCoreDev != 0) {
throw new Error("Failed to get core-dev kubeconfig credentials.")
}

const rcHarvester = exec(`cp /mnt/secrets/harvester-kubeconfig/harvester-kubeconfig.yml ${HARVESTER_KUBECONFIG_PATH}`, { slice: prepareSlices.CONFIGURE_CORE_DEV }).code;

if (rcHarvester != 0) {
throw new Error("Failed to get Harvester kubeconfig credentials.")
}
}

function decideHarvesterVMCreation(werft: Werft, config: JobConfig) {
Expand Down
28 changes: 14 additions & 14 deletions .werft/observability/monitoring-satellite.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import * as fs from 'fs';
* Monitoring satellite deployment bits
*/
export class InstallMonitoringSatelliteParams {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would have been nicer to just have as a type instead of class.

pathToKubeConfig: string
kubeconfigPath: string
satelliteNamespace: string
clusterName: string
nodeExporterPort: number
Expand Down Expand Up @@ -76,39 +76,39 @@ export async function installMonitoringSatellite(params: InstallMonitoringSatell

// The correct kubectl context should already be configured prior to this step
// Only checks node-exporter readiness for harvester
ensureCorrectInstallationOrder(params.satelliteNamespace, params.withVM)
ensureCorrectInstallationOrder(params.kubeconfigPath, params.satelliteNamespace, params.withVM)
}

async function ensureCorrectInstallationOrder(namespace: string, checkNodeExporterStatus: boolean){
async function ensureCorrectInstallationOrder(kubeconfig: string, namespace: string, checkNodeExporterStatus: boolean){
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For this file you could have exposed a single class and then have all the functions as public/private methods. That way you wouldn't need to pass the kubeconfig around.

I don't think it's worth doing now, just as inspiration for future refactoring if you're feeling like it 😉

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, that definitely crossed my mind while doing this, maybe its worth raising an issue to refactor this later

const werft = getGlobalWerftInstance()

werft.log(sliceName, 'installing monitoring-satellite')
exec('cd observability && hack/deploy-satellite.sh', {slice: sliceName})
exec(`cd observability && hack/deploy-satellite.sh --kubeconfig ${kubeconfig}`, {slice: sliceName})

deployGitpodServiceMonitors()
checkReadiness(namespace, checkNodeExporterStatus)
deployGitpodServiceMonitors(kubeconfig)
checkReadiness(kubeconfig, namespace, checkNodeExporterStatus)
}

async function checkReadiness(namespace: string, checkNodeExporterStatus: boolean) {
async function checkReadiness(kubeconfig: string, namespace: string, checkNodeExporterStatus: boolean) {
// For some reason prometheus' statefulset always take quite some time to get created
// Therefore we wait a couple of seconds
exec(`sleep 30 && kubectl rollout status -n ${namespace} statefulset prometheus-k8s`, {slice: sliceName, async: true})
exec(`kubectl rollout status -n ${namespace} deployment grafana`, {slice: sliceName, async: true})
exec(`kubectl rollout status -n ${namespace} deployment kube-state-metrics`, {slice: sliceName, async: true})
exec(`kubectl rollout status -n ${namespace} deployment otel-collector`, {slice: sliceName, async: true})
exec(`sleep 30 && kubectl --kubeconfig ${kubeconfig} rollout status -n ${namespace} statefulset prometheus-k8s`, {slice: sliceName, async: true})
exec(`kubectl --kubeconfig ${kubeconfig} rollout status -n ${namespace} deployment grafana`, {slice: sliceName, async: true})
exec(`kubectl --kubeconfig ${kubeconfig} rollout status -n ${namespace} deployment kube-state-metrics`, {slice: sliceName, async: true})
exec(`kubectl --kubeconfig ${kubeconfig} rollout status -n ${namespace} deployment otel-collector`, {slice: sliceName, async: true})

// core-dev is just too unstable for node-exporter
// we don't guarantee that it will run at all
if(checkNodeExporterStatus) {
exec(`kubectl rollout status -n ${namespace} daemonset node-exporter`, {slice: sliceName, async: true})
exec(`kubectl --kubeconfig ${kubeconfig} rollout status -n ${namespace} daemonset node-exporter`, {slice: sliceName, async: true})
}
}

async function deployGitpodServiceMonitors() {
async function deployGitpodServiceMonitors(kubeconfig: string) {
const werft = getGlobalWerftInstance()

werft.log(sliceName, 'installing gitpod ServiceMonitor resources')
exec('kubectl apply -f observability/monitoring-satellite/manifests/gitpod/', {silent: true})
exec(`kubectl --kubeconfig ${kubeconfig} apply -f observability/monitoring-satellite/manifests/gitpod/`, {silent: true})
}

function postProcessManifests() {
Expand Down
7 changes: 4 additions & 3 deletions .werft/platform-delete-preview-environments-cron.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { SpanStatusCode } from '@opentelemetry/api';
import { wipePreviewEnvironmentAndNamespace, helmInstallName, listAllPreviewNamespaces } from './util/kubectl';
import { exec } from './util/shell';
import { previewNameFromBranchName } from './util/preview';
import { CORE_DEV_KUBECONFIG_PATH } from './jobs/build/const';

// Will be set once tracing has been initialized
let werft: Werft
Expand All @@ -30,7 +31,7 @@ async function deletePreviewEnvironments() {
try {
const GCLOUD_SERVICE_ACCOUNT_PATH = "/mnt/secrets/gcp-sa/service-account.json";
exec(`gcloud auth activate-service-account --key-file "${GCLOUD_SERVICE_ACCOUNT_PATH}"`);
exec('gcloud container clusters get-credentials core-dev --zone europe-west1-b --project gitpod-core-dev');
exec(`KUBECONFIG=${CORE_DEV_KUBECONFIG_PATH} gcloud container clusters get-credentials core-dev --zone europe-west1-b --project gitpod-core-dev`);
} catch (err) {
werft.fail("prep", err)
}
Expand All @@ -46,7 +47,7 @@ async function deletePreviewEnvironments() {
werft.phase("Fetching previews");
let previews: string[]
try {
previews = listAllPreviewNamespaces({});
previews = listAllPreviewNamespaces(CORE_DEV_KUBECONFIG_PATH, {});
previews.forEach(previewNs => werft.log("Fetching previews", previewNs))
} catch (err) {
werft.fail("Fetching previews", err)
Expand All @@ -57,7 +58,7 @@ async function deletePreviewEnvironments() {
try {
const previewsToDelete = previews.filter(ns => !expectedPreviewEnvironmentNamespaces.has(ns))
// Trigger namespace deletion in parallel
const promises = previewsToDelete.map(preview => wipePreviewEnvironmentAndNamespace(helmInstallName, preview, { slice: `Deleting preview ${preview}` }));
const promises = previewsToDelete.map(preview => wipePreviewEnvironmentAndNamespace(helmInstallName, preview, CORE_DEV_KUBECONFIG_PATH, { slice: `Deleting preview ${preview}` }));
// But wait for all of them to finish before (or one of them to fail) before we continue
await Promise.all(promises)
} catch (err) {
Expand Down
Loading