Skip to content

Improve handling of preview deployment failures on main #9837

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 6, 2022
Merged

Improve handling of preview deployment failures on main #9837

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c2b3876
Always log the error that caused the job to fail
mads-hartmann May 6, 2022
ea2d114
Handle preview deployment failures on main more robustly
mads-hartmann May 6, 2022
9b83bff
Don't end all spans when using .fail
mads-hartmann May 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions .werft/build.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ Tracing.initialize()
message: err
})

console.log('Error', err)

if (context.Repository.ref === "refs/heads/main") {
reportBuildFailureInSlack(context, err).catch((error: Error) => {
console.error("Failed to send message to Slack", error)
});
} else {
console.log('Error', err)
}

// Explicitly not using process.exit as we need to flush tracing, see tracing.js
Expand Down Expand Up @@ -60,6 +60,16 @@ async function run(context: any) {
return
}

await deployToPreviewEnvironment(werft, config)
try {
await deployToPreviewEnvironment(werft, config)
} catch (e) {
// We currently don't support concurrent deployments to the same preview environment.
// Until we do we don't want errors to mark the main build as failed.
if (config.mainBuild) {
return
}
throw e
}

await triggerIntegrationTests(werft, config, context.Owner)
}
60 changes: 12 additions & 48 deletions .werft/jobs/build/deploy-to-preview-environment.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,10 +142,7 @@ export async function deployToPreviewEnvironment(werft: Werft, jobConfig: JobCon
await installMetaCertificates(werft, jobConfig.repository.branch, withVM, 'default', PREVIEW_K3S_KUBECONFIG_PATH, vmSlices.COPY_CERT_MANAGER_RESOURCES)
werft.done(vmSlices.COPY_CERT_MANAGER_RESOURCES)
} catch (err) {
if (!jobConfig.mainBuild) {
werft.fail(vmSlices.COPY_CERT_MANAGER_RESOURCES, err);
}
exec('exit 0')
werft.fail(vmSlices.COPY_CERT_MANAGER_RESOURCES, err);
}

// Deploying monitoring satellite to VM-based preview environments is currently best-effort.
Expand Down Expand Up @@ -262,10 +259,7 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
}
werft.done(installerSlices.CLEAN_ENV_STATE);
} catch (err) {
if (!jobConfig.mainBuild) {
werft.fail(installerSlices.CLEAN_ENV_STATE, err);
}
exec('exit 0')
werft.fail(installerSlices.CLEAN_ENV_STATE, err);
}

if (!withVM) {
Expand All @@ -278,10 +272,7 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
await installMetaCertificates(werft, jobConfig.repository.branch, jobConfig.withVM, namespace, CORE_DEV_KUBECONFIG_PATH, installerSlices.COPY_CERTIFICATES);
werft.done(installerSlices.COPY_CERTIFICATES);
} catch (err) {
if (!jobConfig.mainBuild) {
werft.fail(installerSlices.COPY_CERTIFICATES, err);
}
exec('exit 0')
werft.fail(installerSlices.COPY_CERTIFICATES, err);
}
}

Expand All @@ -295,10 +286,7 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
exec(`kubectl --kubeconfig ${deploymentKubeconfig} create secret docker-registry ${IMAGE_PULL_SECRET_NAME} -n ${namespace} --from-file=.dockerconfigjson=./${IMAGE_PULL_SECRET_NAME}`, { slice: installerSlices.IMAGE_PULL_SECRET });
}
catch (err) {
if (!jobConfig.mainBuild) {
werft.fail(installerSlices.IMAGE_PULL_SECRET, err);
}
exec('exit 0')
werft.fail(installerSlices.IMAGE_PULL_SECRET, err);
}
}
werft.done(installerSlices.IMAGE_PULL_SECRET);
Expand Down Expand Up @@ -338,10 +326,7 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
installer.postProcessing(installerSlices.INSTALLER_POST_PROCESSING)
installer.install(installerSlices.APPLY_INSTALL_MANIFESTS)
} catch (err) {
if (!jobConfig.mainBuild) {
werft.fail(phases.DEPLOY, err)
}
exec('exit 0')
werft.fail(phases.DEPLOY, err)
}

werft.log(installerSlices.DEPLOYMENT_WAITING, "Waiting until all pods are ready.");
Expand Down Expand Up @@ -410,10 +395,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
await addDNSRecord(werft, deploymentConfig.namespace, deploymentConfig.domain, false, CORE_DEV_KUBECONFIG_PATH)
werft.done('prep');
} catch (err) {
if (!jobConfig.mainBuild) {
werft.fail('prep', err);
}
exec('exit 0')
werft.fail('prep', err);
}

// core-dev specific section start
Expand All @@ -435,10 +417,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
}` );
werft.done('secret');
} catch (err) {
if (!jobConfig.mainBuild) {
werft.fail('secret', err);
}
exec('exit 0')
werft.fail('secret', err);
}

werft.log("authProviders", "copy authProviders")
Expand All @@ -450,10 +429,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
exec(`yq merge --inplace .werft/jobs/build/helm/values.dev.yaml ./authProviders`, { slice: "authProviders" })
werft.done('authProviders');
} catch (err) {
if (!jobConfig.mainBuild) {
werft.fail('authProviders', err);
}
exec('exit 0')
werft.fail('authProviders', err);
}
// core-dev specific section end

Expand All @@ -478,10 +454,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
});
await installMonitoringSatellite.install()
} catch (err) {
if (!jobConfig.mainBuild) {
werft.fail('observability', err);
}
exec('exit 0')
werft.fail('observability', err);
}
} else {
exec(`echo '"with-observability" annotation not set, skipping...'`, { slice: `observability` })
Expand All @@ -500,10 +473,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
werft.log('helm', 'done');
werft.done('helm');
} catch (err) {
if (!jobConfig.mainBuild) {
werft.fail('deploy', err);
}
exec('exit 0')
werft.fail('deploy', err);
} finally {
// produce the result independently of Helm succeding, so that in case Helm fails we still have the URL.
exec(`werft log result -d "dev installation" -c github-check-preview-env url ${url}/workspaces`);
Expand Down Expand Up @@ -559,10 +529,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
try {
exec(`docker run --rm eu.gcr.io/gitpod-core-dev/build/versions:${version} cat /versions.yaml | tee versions.yaml`);
} catch (err) {
if (!jobConfig.mainBuild) {
werft.fail('helm', err);
}
exec('exit 0')
werft.fail('helm', err);
}
const pathToVersions = `${shell.pwd().toString()}/versions.yaml`;
flags += ` -f ${pathToVersions}`;
Expand Down Expand Up @@ -590,10 +557,7 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
await deleteNonNamespaceObjects(namespace, destname, CORE_DEV_KUBECONFIG_PATH, { ...shellOpts, slice: 'predeploy cleanup' });
werft.done('predeploy cleanup');
} catch (err) {
if (!jobConfig.mainBuild) {
werft.fail('predeploy cleanup', err);
}
exec('exit 0')
werft.fail('predeploy cleanup', err);
}
}
}
Expand Down
5 changes: 3 additions & 2 deletions .werft/util/werft.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,10 @@ export class Werft {
* Use this when you intend to fail the werft job
*/
public fail(slice, err) {
const span = this.sliceSpans[slice];
// Set the status on the span for the slice and also propagate the status to the phase and root span
// as well so we can query on all phases that had an error regardless of which slice produced the error.
[this.sliceSpans[slice], this.rootSpan, this.currentPhaseSpan].forEach((span: Span) => {
[span, this.rootSpan, this.currentPhaseSpan].forEach((span: Span) => {
if (!span) {
return
}
Expand All @@ -82,7 +83,7 @@ export class Werft {
})
})

this.endAllSpans()
span.end()

console.log(`[${slice}|FAIL] ${err}`);
throw err;
Expand Down