Add alerts to agent mixin used by Grafana cloud integration (#993)

* Add alerts to agent mixin used by Grafana integration

* Fix jsonnet lint

* Update alerts to be used in agent health integration

Author: Muhammad Shahzeb

grafana-agent-mixin/alerts/alerts.libsonnet

@@ -0,0 +1,78 @@
+{
+  prometheusAlerts+:: {
+    groups+: [
+      {
+        name: 'GrafanaAgentHealthChecks',
+        rules: [
+          {
+            alert: 'GrafanaAgentDown',
+            expr: |||
+              up{
+                job="integrations/agent",
+              } == 0
+            |||,
+            'for': '5m',
+            annotations: {
+              summary: 'Grafana agent is down.',
+              description: 'Grafana agent is down on {{ $labels.instance }} for the last 5 minutes',
+            },
+            labels: {
+              severity: 'critical',
+            },
+          },
+          {
+            alert: 'GrafanaAgentUnstable',
+            expr: |||
+              avg_over_time(up{
+                job="integrations/agent",
+              }[5m]) < 1
+            |||,
+            'for': '15m',
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              summary: 'Grafana agent is unstable.',
+              description: 'Grafana agent is unstable or restarting on {{ $labels.instance }} over the last 15 minutes',
+            },
+          },
+          {
+            alert: 'GrafanaAgentCPUHigh',
+            expr: |||
+              (
+                rate(process_cpu_seconds_total{
+                  job=~"integrations/agent"
+                }[5m]) > %(alertsCriticalCpuUsage5m)s / 100
+              )
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              summary: 'Grafana agent high CPU usage.',
+              description: 'Grafana agent is using more than %(alertsCriticalCpuUsage5m)s percent of CPU on {{ $labels.instance }} for the last 5 minutes' % $._config,
+            },
+          },
+          {
+            alert: 'GrafanaAgentMemHigh',
+            expr: |||
+              (
+                sum without (instance) (go_memstats_heap_inuse_bytes{job=~"integrations/agent"}) /
+                sum without (instance, instance_group_name) (agent_wal_storage_active_series{job=~"integrations/agent"}) / 1e3 > %(alertsCriticalMemUsage5m)s
+              )
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              summary: 'Grafana agent high memory usage.',
+              description: 'Grafana agent is using more than %(alertsCriticalMemUsage5m)s of memory on {{ $labels.instance }} for the last 5 minutes' % $._config,
+            },
+          },
+        ],
+      },
+    ],
+  },
+}

grafana-agent-mixin/config.libsonnet

@@ -1,10 +1,12 @@
 {
-  local makeGroupBy(groups) = std.join(', ', groups),
-
   _config+:: {
     dashboardTags: ['grafana-agent'],
     dashboardPeriod: 'now-1h',
     dashboardRefresh: '1m',
     dashboardTimezone: 'default',
+
+    //alert thresholds
+    alertsCriticalCpuUsage5m: 80,  //percent
+    alertsCriticalMemUsage5m: 100,  //kilo bytes per active series
   },
 }

grafana-agent-mixin/dashboards/grafana-agent-debugging.libsonnet

@@ -100,7 +100,7 @@ local instance_template = grafana.template.new(
           'rate(process_cpu_seconds_total{' + host_matcher + '}[$__rate_interval])',
           legendFormat='{{instance}}',
         )) +
-        utils.timeSeriesOverride(unit='percent');
+        utils.timeSeriesOverride(unit='percentunit');
 
       local TCPConnections =
         graphPanel.new(

grafana-agent-mixin/mixin.libsonnet

@@ -1,3 +1,3 @@
-{ grafanaDashboardFolder: 'Grafana Agent' }
-+ (import 'dashboards/dashboards.libsonnet')
+(import 'dashboards/dashboards.libsonnet')
++ (import 'alerts/alerts.libsonnet')
 + (import 'config.libsonnet')