Introduced firewall in the Alertmanager to block specific addresses in receiver integrations

Signed-off-by: Marco Pracucci <[email protected]>

Author: pracucci

CHANGELOG.md

@@ -53,6 +53,7 @@
 * [ENHANCEMENT] Ruler: Added `-ruler.enabled-tenants` and `-ruler.disabled-tenants` to explicitly enable or disable rules processing for specific tenants. #4074
 * [ENHANCEMENT] Block Storage Ingester: `/flush` now accepts two new parameters: `tenant` to specify tenant to flush and `wait=true` to make call synchronous. Multiple tenants can be specified by repeating `tenant` parameter. If no `tenant` is specified, all tenants are flushed, as before. #4073
 * [ENHANCEMENT] Alertmanager: validate configured `-alertmanager.web.external-url` and fail if ends with `/`. #4081
+* [ENHANCEMENT] Alertmanager: added `-alertmanager.receivers-firewall.block.cidrs` and `-alertmanager.receivers-firewall.block.private-addresses` to block specific network addresses in HTTP-based Alertmanager receiver integrations. #4085
 * [ENHANCEMENT] Allow configuration of Cassandra's host selection policy. #4069
 * [ENHANCEMENT] Store-gateway: retry synching blocks if a per-tenant sync fails. #3975 #4088
 * [ENHANCEMENT] Add metric `cortex_tcp_connections` exposing the current number of accepted TCP connections. #4099

docs/blocks-storage/production-tips.md

@@ -105,3 +105,14 @@ You can see that the initial migration is done by looking for the following mess
 The rule of thumb to ensure memcached is properly scaled is to make sure evictions happen infrequently. When that's not the case and they affect query performances, the suggestion is to scale out the memcached cluster adding more nodes or increasing the memory limit of existing ones.
 
 We also recommend to run a different memcached cluster for each cache type (metadata, index, chunks). It's not required, but suggested to not worry about the effect of memory pressure on a cache type against others.
+
+## Alertmanager
+
+### Ensure Alertmanager networking is hardened
+
+If the Alertmanager API is enabled and exposed to Cortex tenants, they can autonomously configure the Alertmanager, including receiver integrations (eg. webhook) that allow to issue network requests to the configured endpoint. If the Alertmanager network is not hardened, Cortex tenants may have the ability to issue network requests to any network endpoint including services running in the local network.
+
+Given hardening the Alertmanager is out of the scope of Cortex, we provide a basic built-in firewall to block connections creates by Alertmanager receiver integrations:
+
+- `-alertmanager.receivers-firewall.block.cidrs`
+- `-alertmanager.receivers-firewall.block.private-addresses`

docs/configuration/config-file-reference.md

@@ -1849,6 +1849,18 @@ The `alertmanager_config` configures the Cortex alertmanager.
 # CLI flag: -alertmanager.max-recv-msg-size
 [max_recv_msg_size: <int> | default = 16777216]
 
+receivers_firewall:
+  block:
+    # Comma-separated list of network CIDRs to block in Alertmanager receiver
+    # integrations.
+    # CLI flag: -alertmanager.receivers-firewall.block.cidrs
+    [cidrs: <string> | default = ""]
+
+    # True to block private and local addresses in Alertmanager receiver
+    # integrations.
+    # CLI flag: -alertmanager.receivers-firewall.block.private-addresses
+    [private_addresses: <boolean> | default = false]
+
 # Shard tenants across multiple alertmanager instances.
 # CLI flag: -alertmanager.sharding-enabled
 [sharding_enabled: <boolean> | default = false]

docs/configuration/v1-guarantees.md

@@ -41,7 +41,10 @@ Currently experimental features are:
 - Azure blob storage.
 - Zone awareness based replication.
 - Ruler API (to PUT rules).
-- Alertmanager API
+- Alertmanager:
+  - API (enabled via `-experimental.alertmanager.enable-api`)
+  - Sharding of tenants across multiple instances (enabled via `-alertmanager.sharding-enabled`)
+  - Receiver integrations firewall (configured via `-alertmanager.receivers-firewal.*`)
 - Memcached client DNS-based service discovery.
 - Delete series APIs.
 - In-memory (FIFO) and Redis cache.
@@ -61,7 +64,6 @@ Currently experimental features are:
   - The bucket index support in the querier and store-gateway (enabled via `-blocks-storage.bucket-store.bucket-index.enabled=true`) is experimental
   - The block deletion marks migration support in the compactor (`-compactor.block-deletion-marks-migration-enabled`) is temporarily and will be removed in future versions
 - Querier: tenant federation
-- Alertmanager: Sharding of tenants across multiple instances
 - The thanosconvert tool for converting Thanos block metadata to Cortex
 - HA Tracker: cleanup of old replicas from KV Store.
 - Flags for configuring whether blocks-ingester streams samples or chunks are temporary, and will be removed when feature is tested:

pkg/alertmanager/alertmanager.go

@@ -59,12 +59,13 @@ const (
 
 // Config configures an Alertmanager.
 type Config struct {
-	UserID      string
-	Logger      log.Logger
-	Peer        *cluster.Peer
-	PeerTimeout time.Duration
-	Retention   time.Duration
-	ExternalURL *url.URL
+	UserID            string
+	Logger            log.Logger
+	Peer              *cluster.Peer
+	PeerTimeout       time.Duration
+	Retention         time.Duration
+	ExternalURL       *url.URL
+	ReceiversFirewall FirewallConfig
 
 	// Tenant-specific local directory where AM can store its state (notifications, silences, templates). When AM is stopped, entire dir is removed.
 	TenantDataDir string
@@ -279,6 +280,8 @@ func clusterWait(position func() int, timeout time.Duration) func() time.Duratio
 
 // ApplyConfig applies a new configuration to an Alertmanager.
 func (am *Alertmanager) ApplyConfig(userID string, conf *config.Config, rawCfg string) error {
+	conf = injectFirewallToAlertmanagerConfig(conf, am.cfg.ReceiversFirewall)
+
 	templateFiles := make([]string, len(conf.Templates))
 	if len(conf.Templates) > 0 {
 		for i, t := range conf.Templates {

pkg/alertmanager/firewall.go

@@ -0,0 +1,76 @@
+package alertmanager
+
+import (
+	"flag"
+	"fmt"
+
+	"github.com/prometheus/alertmanager/config"
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/cortexproject/cortex/pkg/util/flagext"
+	netutil "github.com/cortexproject/cortex/pkg/util/net"
+)
+
+type FirewallConfig struct {
+	Block FirewallHostsSpec `yaml:"block"`
+}
+
+func (cfg *FirewallConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	cfg.Block.RegisterFlagsWithPrefix(prefix+".block", "block", f)
+}
+
+type FirewallHostsSpec struct {
+	CIDRs   flagext.CIDRSliceCSV `yaml:"cidrs"`
+	Private bool                 `yaml:"private_addresses"`
+}
+
+func (cfg *FirewallHostsSpec) RegisterFlagsWithPrefix(prefix, action string, f *flag.FlagSet) {
+	f.Var(&cfg.CIDRs, prefix+".cidrs", fmt.Sprintf("Comma-separated list of network CIDRs to %s in Alertmanager receiver integrations.", action))
+	f.BoolVar(&cfg.Private, prefix+".private-addresses", false, fmt.Sprintf("True to %s private and local addresses in Alertmanager receiver integrations.", action))
+}
+
+func injectFirewallToAlertmanagerConfig(conf *config.Config, firewallCfg FirewallConfig) *config.Config {
+	firewall := netutil.NewFirewallDialer(netutil.FirewallDialerConfig{
+		BlockCIDRs:   firewallCfg.Block.CIDRs,
+		BlockPrivate: firewallCfg.Block.Private,
+	})
+
+	conf.Global.HTTPConfig = injectFirewallToHTTPConfig(conf.Global.HTTPConfig, firewall)
+
+	for _, receiver := range conf.Receivers {
+		for _, rc := range receiver.WebhookConfigs {
+			rc.HTTPConfig = injectFirewallToHTTPConfig(rc.HTTPConfig, firewall)
+		}
+		for _, rc := range receiver.SlackConfigs {
+			rc.HTTPConfig = injectFirewallToHTTPConfig(rc.HTTPConfig, firewall)
+		}
+		for _, rc := range receiver.PushoverConfigs {
+			rc.HTTPConfig = injectFirewallToHTTPConfig(rc.HTTPConfig, firewall)
+		}
+		for _, rc := range receiver.PagerdutyConfigs {
+			rc.HTTPConfig = injectFirewallToHTTPConfig(rc.HTTPConfig, firewall)
+		}
+		for _, rc := range receiver.OpsGenieConfigs {
+			rc.HTTPConfig = injectFirewallToHTTPConfig(rc.HTTPConfig, firewall)
+		}
+		for _, rc := range receiver.WechatConfigs {
+			rc.HTTPConfig = injectFirewallToHTTPConfig(rc.HTTPConfig, firewall)
+		}
+		for _, rc := range receiver.VictorOpsConfigs {
+			rc.HTTPConfig = injectFirewallToHTTPConfig(rc.HTTPConfig, firewall)
+		}
+	}
+
+	return conf
+}
+
+func injectFirewallToHTTPConfig(conf *commoncfg.HTTPClientConfig, firewall *netutil.FirewallDialer) *commoncfg.HTTPClientConfig {
+	if conf == nil {
+		return &commoncfg.HTTPClientConfig{
+			DialContext: firewall.DialContext,
+		}
+	}
+
+	conf.DialContext = firewall.DialContext
+	return conf
+}

pkg/alertmanager/multitenant.go

@@ -102,11 +102,12 @@ func init() {
 
 // MultitenantAlertmanagerConfig is the configuration for a multitenant Alertmanager.
 type MultitenantAlertmanagerConfig struct {
-	DataDir        string           `yaml:"data_dir"`
-	Retention      time.Duration    `yaml:"retention"`
-	ExternalURL    flagext.URLValue `yaml:"external_url"`
-	PollInterval   time.Duration    `yaml:"poll_interval"`
-	MaxRecvMsgSize int64            `yaml:"max_recv_msg_size"`
+	DataDir           string           `yaml:"data_dir"`
+	Retention         time.Duration    `yaml:"retention"`
+	ExternalURL       flagext.URLValue `yaml:"external_url"`
+	PollInterval      time.Duration    `yaml:"poll_interval"`
+	MaxRecvMsgSize    int64            `yaml:"max_recv_msg_size"`
+	ReceiversFirewall FirewallConfig   `yaml:"receivers_firewall"`
 
 	// Enable sharding for the Alertmanager
 	ShardingEnabled bool       `yaml:"sharding_enabled"`
@@ -158,9 +159,8 @@ func (cfg *MultitenantAlertmanagerConfig) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&cfg.ShardingEnabled, "alertmanager.sharding-enabled", false, "Shard tenants across multiple alertmanager instances.")
 
 	cfg.AlertmanagerClient.RegisterFlagsWithPrefix("alertmanager.alertmanager-client", f)
-
 	cfg.Persister.RegisterFlagsWithPrefix("alertmanager", f)
-
+	cfg.ReceiversFirewall.RegisterFlagsWithPrefix("alertmanager.receivers-firewall", f)
 	cfg.ShardingRing.RegisterFlags(f)
 	cfg.Store.RegisterFlags(f)
 	cfg.Cluster.RegisterFlags(f)
@@ -873,6 +873,7 @@ func (am *MultitenantAlertmanager) newAlertmanager(userID string, amConfig *amco
 		ReplicationFactor: am.cfg.ShardingRing.ReplicationFactor,
 		Store:             am.store,
 		PersisterConfig:   am.cfg.Persister,
+		ReceiversFirewall: am.cfg.ReceiversFirewall,
 	}, reg)
 	if err != nil {
 		return nil, fmt.Errorf("unable to start Alertmanager for user %v: %v", userID, err)

pkg/alertmanager/multitenant_test.go

@@ -24,11 +24,13 @@ import (
 	"github.com/prometheus/alertmanager/types"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/weaveworks/common/httpgrpc"
 	"github.com/weaveworks/common/user"
+	"go.uber.org/atomic"
 	"google.golang.org/grpc"
 
 	"github.com/cortexproject/cortex/pkg/alertmanager/alertmanagerpb"
@@ -277,6 +279,175 @@ templates:
 	`), "cortex_alertmanager_config_last_reload_successful"))
 }
 
+func TestMultitenantAlertmanager_FirewallShouldBlockHTTPBasedReceiversWhenEnabled(t *testing.T) {
+	tests := map[string]struct {
+		getAlertmanagerConfig func(backendURL string) string
+	}{
+		"webhook": {
+			getAlertmanagerConfig: func(backendURL string) string {
+				return fmt.Sprintf(`
+route:
+  receiver: webhook
+  group_wait: 0s
+
+receivers:
+  - name: webhook
+    webhook_configs:
+      - url: %s
+`, backendURL)
+			},
+		},
+		"pagerduty": {
+			getAlertmanagerConfig: func(backendURL string) string {
+				return fmt.Sprintf(`
+route:
+  receiver: pagerduty
+  group_wait: 0s
+
+receivers:
+  - name: pagerduty
+    pagerduty_configs:
+      - url: %s
+        routing_key: secret
+`, backendURL)
+			},
+		},
+		"slack": {
+			getAlertmanagerConfig: func(backendURL string) string {
+				return fmt.Sprintf(`
+route:
+  receiver: slack
+  group_wait: 0s
+
+receivers:
+  - name: slack
+    slack_configs:
+      - api_url: %s
+        channel: test
+`, backendURL)
+			},
+		},
+		"opsgenie": {
+			getAlertmanagerConfig: func(backendURL string) string {
+				return fmt.Sprintf(`
+route:
+  receiver: opsgenie
+  group_wait: 0s
+
+receivers:
+  - name: opsgenie
+    opsgenie_configs:
+      - api_url: %s
+        api_key: secret
+`, backendURL)
+			},
+		},
+		"wechat": {
+			getAlertmanagerConfig: func(backendURL string) string {
+				return fmt.Sprintf(`
+route:
+  receiver: wechat
+  group_wait: 0s
+
+receivers:
+  - name: wechat
+    wechat_configs:
+      - api_url: %s
+        api_secret: secret
+        corp_id: babycorp
+`, backendURL)
+			},
+		},
+	}
+
+	for receiverName, testData := range tests {
+		for _, firewallEnabled := range []bool{true, false} {
+			t.Run(fmt.Sprintf("%s firewall: %v", receiverName, firewallEnabled), func(t *testing.T) {
+				ctx := context.Background()
+				userID := "user-1"
+				serverInvoked := atomic.NewBool(false)
+
+				// Create a local HTTP server to test whether the request is received.
+				server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+					serverInvoked.Store(true)
+					writer.WriteHeader(http.StatusOK)
+				}))
+				defer server.Close()
+
+				// Create the alertmanager config.
+				alertmanagerCfg := testData.getAlertmanagerConfig(fmt.Sprintf("http://%s", server.Listener.Addr().String()))
+
+				// Store the alertmanager config in the bucket.
+				store := prepareInMemoryAlertStore()
+				require.NoError(t, store.SetAlertConfig(ctx, alertspb.AlertConfigDesc{
+					User:      userID,
+					RawConfig: alertmanagerCfg,
+				}))
+
+				// Prepare the alertmanager config.
+				cfg := mockAlertmanagerConfig(t)
+				cfg.ReceiversFirewall.Block.Private = firewallEnabled
+
+				// Start the alertmanager.
+				reg := prometheus.NewPedanticRegistry()
+				am, err := createMultitenantAlertmanager(cfg, nil, nil, store, nil, log.NewNopLogger(), reg)
+				require.NoError(t, err)
+				require.NoError(t, services.StartAndAwaitRunning(ctx, am))
+				t.Cleanup(func() {
+					require.NoError(t, services.StopAndAwaitTerminated(ctx, am))
+				})
+
+				// Ensure the configs are synced correctly.
+				assert.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(`
+		# HELP cortex_alertmanager_config_last_reload_successful Boolean set to 1 whenever the last configuration reload attempt was successful.
+		# TYPE cortex_alertmanager_config_last_reload_successful gauge
+		cortex_alertmanager_config_last_reload_successful{user="user-1"} 1
+	`), "cortex_alertmanager_config_last_reload_successful"))
+
+				// Create an alert to push.
+				alerts := types.Alerts(&types.Alert{
+					Alert: model.Alert{
+						Labels:   map[model.LabelName]model.LabelValue{model.AlertNameLabel: "test"},
+						StartsAt: time.Now().Add(-time.Minute),
+						EndsAt:   time.Now().Add(time.Minute),
+					},
+					UpdatedAt: time.Now(),
+					Timeout:   false,
+				})
+
+				alertsPayload, err := json.Marshal(alerts)
+				require.NoError(t, err)
+
+				// Push an alert.
+				req := httptest.NewRequest(http.MethodPost, cfg.ExternalURL.String()+"/api/v1/alerts", bytes.NewReader(alertsPayload))
+				req.Header.Set("content-type", "application/json")
+				reqCtx := user.InjectOrgID(req.Context(), userID)
+				{
+					w := httptest.NewRecorder()
+					am.ServeHTTP(w, req.WithContext(reqCtx))
+
+					resp := w.Result()
+					_, err := ioutil.ReadAll(resp.Body)
+					require.NoError(t, err)
+					assert.Equal(t, http.StatusOK, w.Code)
+				}
+
+				// Ensure the server endpoint has not been called if firewall is enabled. Since the alert is delivered
+				// asynchronously, we should pool it for a short period.
+				deadline := time.Now().Add(time.Second)
+				for {
+					if time.Now().After(deadline) || serverInvoked.Load() {
+						break
+					}
+					time.Sleep(100 * time.Millisecond)
+				}
+
+				assert.Equal(t, !firewallEnabled, serverInvoked.Load())
+			})
+		}
+	}
+}
+
 func TestMultitenantAlertmanager_migrateStateFilesToPerTenantDirectories(t *testing.T) {
 	ctx := context.Background()