From 7c52bc03d372c4485fe2dea95bf01468076c26e9 Mon Sep 17 00:00:00 2001 From: utam0k Date: Wed, 22 Jun 2022 06:55:19 +0000 Subject: [PATCH 1/3] observability: Add GitpodWorkspaceTooLongTerminating alert. --- .../rules/components/workspaces/alerts.libsonnet | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/operations/observability/mixins/workspace/rules/components/workspaces/alerts.libsonnet b/operations/observability/mixins/workspace/rules/components/workspaces/alerts.libsonnet index 562d827fa9fdf7..e4dec9f12c1a05 100644 --- a/operations/observability/mixins/workspace/rules/components/workspaces/alerts.libsonnet +++ b/operations/observability/mixins/workspace/rules/components/workspaces/alerts.libsonnet @@ -134,6 +134,20 @@ gitpod_ws_manager_workspace_phase_total{phase="PENDING", type="PREBUILD"} > 20 |||, }, + { + alert: 'GitpodWorkspaceTooLongTerminating', + labels: { + severity: 'warning', + }, + annotations: { + runbook_url: 'https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceTooLongTerminating.md', + summary: 'workspace pods are too long in termintaing', + description: 'workspace pods are too long in termintaing', + }, + expr: ||| + sum(time() - kube_pod_deletion_timestamp{namespace="default", pod=~"^ws-.*"}) by (pod) > 24 * 60 * 60 + |||, + }, ], }, ], From eb6e195558ad36ab455ecb2a65048df1cf742bae Mon Sep 17 00:00:00 2001 From: Pavel Tumik <18602811+sagor999@users.noreply.github.com> Date: Wed, 22 Jun 2022 14:09:37 -0700 Subject: [PATCH 2/3] Apply suggestions from code review --- .../workspace/rules/components/workspaces/alerts.libsonnet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operations/observability/mixins/workspace/rules/components/workspaces/alerts.libsonnet b/operations/observability/mixins/workspace/rules/components/workspaces/alerts.libsonnet index e4dec9f12c1a05..bcea0939402639 100644 --- a/operations/observability/mixins/workspace/rules/components/workspaces/alerts.libsonnet +++ b/operations/observability/mixins/workspace/rules/components/workspaces/alerts.libsonnet @@ -141,8 +141,8 @@ }, annotations: { runbook_url: 'https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceTooLongTerminating.md', - summary: 'workspace pods are too long in termintaing', - description: 'workspace pods are too long in termintaing', + summary: 'workspace pods are too long in terminating', + description: 'workspace pods are too long in terminating', }, expr: ||| sum(time() - kube_pod_deletion_timestamp{namespace="default", pod=~"^ws-.*"}) by (pod) > 24 * 60 * 60 From 03002cfff3b07b43290e01ede0fbf2c72bd0563f Mon Sep 17 00:00:00 2001 From: Pavel Tumik <18602811+sagor999@users.noreply.github.com> Date: Wed, 22 Jun 2022 14:12:26 -0700 Subject: [PATCH 3/3] Apply suggestions from code review --- .../workspace/rules/components/workspaces/alerts.libsonnet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operations/observability/mixins/workspace/rules/components/workspaces/alerts.libsonnet b/operations/observability/mixins/workspace/rules/components/workspaces/alerts.libsonnet index bcea0939402639..b213bc6f44726a 100644 --- a/operations/observability/mixins/workspace/rules/components/workspaces/alerts.libsonnet +++ b/operations/observability/mixins/workspace/rules/components/workspaces/alerts.libsonnet @@ -141,8 +141,8 @@ }, annotations: { runbook_url: 'https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceTooLongTerminating.md', - summary: 'workspace pods are too long in terminating', - description: 'workspace pods are too long in terminating', + summary: 'workspace pods are terminating for too long', + description: 'workspace pods are terminating for too long', }, expr: ||| sum(time() - kube_pod_deletion_timestamp{namespace="default", pod=~"^ws-.*"}) by (pod) > 24 * 60 * 60