[ops] Add alert 'InstanceStartFailures'

Author: geropl

operations/observability/mixins/meta/rules/components/server/alerts.libsonnet

@@ -35,6 +35,20 @@
               description: 'Server has accumulated {{ printf "%.2f" $value }}ms event loop lag.',
             },
           },
+          {
+            alert: 'InstanceStartFailures',
+            // Reasoning: 1 failure every 120s should not trigger an incident: 1/120 = 0.00833.. => 0.01
+            expr: 'sum (irate(gitpod_server_instance_starts_failed_total[2m])) by (reason) > 0.01',
+            'for': '30s',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              runbook_url: 'https://github.com/gitpod-io/runbooks/blob/main/runbooks/InstanceStartFailures.md',
+              summary: 'Server tries to start an instance, but cannot for whatever reason. Investigation required.',
+              description: 'Server cannot start workspace instances on workspace clusters.',
+            },
+          },
         ],
       },
     ],