[observability] add alert for upstream down

Author: mustard-mh

operations/observability/mixins/IDE/rules/opensvx.yaml

@@ -26,6 +26,18 @@ spec:
       expr: |
           sum(rate(gitpod_vscode_extension_gallery_query_total{status="failure",errorCode!="canceled"}[5m])) by(cluster) / sum(rate(gitpod_vscode_extension_gallery_query_total[5m])) by(cluster) > 0.01
 
+    - alert: GitpodOpenVSXUpstreamDown
+      labels:
+        severity: critical
+      for: 20m
+      annotations:
+        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodOpenVSXUpstreamDown.md
+        summary: Open-VSX upstream is possibly down
+        description: Open-VSX upstream is possibly down.
+        dashboard_url: https://grafana.gitpod.io/d/WI_x6hN4k/openvsx-mirror?var-cluster=openvsx-mirror-us01&viewPanel=13
+      expr: |
+          sum(rate(http_client_requests_seconds_count{cluster="openvsx-mirror-us01",outcome!~"SUCCESS|REDIRECTION|CLIENT_ERROR"})[2m])/sum(rate(http_client_requests_seconds_count{cluster="openvsx-mirror-us01"})[2m]) > 0.01
+
     - alert: GitpodOpenVSXUnavailable
       labels:
         severity: warning