Panics when using Purger with Cassandra #2772
Description
level=info ts=2020-06-24T06:30:45.832053361Z caller=main.go:133 msg="Starting Cortex" version="(version=1.1.0, branch=r93, revision=ecad1573f)"
level=warn ts=2020-06-24T06:30:45.832128263Z caller=experimental.go:19 msg="experimental feature in use" feature="Cassandra Backend"
level=info ts=2020-06-24T06:30:45.869170931Z caller=events.go:247 module=gocql client=index-read msg=Session.handleNodeUp ip=52.176.155.127 port=10350
level=info ts=2020-06-24T06:30:45.920053787Z caller=events.go:247 module=gocql client=index-write msg=Session.handleNodeUp ip=52.176.155.127 port=10350
level=info ts=2020-06-24T06:30:45.933321962Z caller=server.go:192 http=[::]:80 grpc=[::]:9095 msg="server listening on addresses"
level=warn ts=2020-06-24T06:30:45.933795672Z caller=experimental.go:19 msg="experimental feature in use" feature="Cassandra Backend"
panic: duplicate metrics collector registration attempted
goroutine 1 [running]:
github.com/prometheus/client_golang/prometheus.(*wrappingRegisterer).MustRegister(0xc000b6a3f0, 0xc00068f7f0, 0x1, 0x1)
/go/src/github.com/cortexproject/cortex/vendor/github.com/prometheus/client_golang/prometheus/wrap.go:102 +0x195
github.com/prometheus/client_golang/prometheus/promauto.Factory.NewGaugeFunc(0x3dc1a80, 0xc000b6a3f0, 0x0, 0x0, 0x0, 0x0, 0x367085e, 0x1b, 0x36c024b, 0x2f, ...)
/go/src/github.com/cortexproject/cortex/vendor/github.com/prometheus/client_golang/prometheus/promauto/auto.go:319 +0x13c
github.com/gocql/gocql.newPolicyConnPool(0x3d661c0, 0xc000b6a390, 0x3dc1a80, 0xc000b6a3f0, 0xc000b2b000, 0xc000b57560)
/go/src/github.com/cortexproject/cortex/vendor/github.com/gocql/gocql/connectionpool.go:132 +0x22f
github.com/gocql/gocql.PoolConfig.buildPool(...)
/go/src/github.com/cortexproject/cortex/vendor/github.com/gocql/gocql/cluster.go:28
github.com/gocql/gocql.NewSession(0xc00068f760, 0x1, 0x1, 0x362f7c1, 0x5, 0x0, 0x77359400, 0x12a05f200, 0x286e, 0x7ffe8900bcb0, ...)
/go/src/github.com/cortexproject/cortex/vendor/github.com/gocql/gocql/session.go:136 +0x55f
github.com/gocql/gocql.(*ClusterConfig).CreateSession(...)
/go/src/github.com/cortexproject/cortex/vendor/github.com/gocql/gocql/cluster.go:207
github.com/cortexproject/cortex/pkg/chunk/cassandra.(*Config).session(0xc000c18fa0, 0x363ae17, 0xa, 0x0, 0x0, 0xc000b58ac0)
/go/src/github.com/cortexproject/cortex/pkg/chunk/cassandra/storage_client.go:125 +0x6da
github.com/cortexproject/cortex/pkg/chunk/cassandra.NewStorageClient(0x7ffe8900bc08, 0x2f, 0x286e, 0x7ffe8900bcb0, 0x6, 0x3631695, 0x6, 0x3, 0x101, 0x0, ...)
/go/src/github.com/cortexproject/cortex/pkg/chunk/cassandra/storage_client.go:228 +0x6e
github.com/cortexproject/cortex/pkg/chunk/storage.NewIndexClient(0xc000315990, 0x9, 0x3631c71, 0x6, 0x0, 0x4000000000000000, 0x4024000000000000, 0x0, 0x0, 0x186a0, ...)
/go/src/github.com/cortexproject/cortex/pkg/chunk/storage/factory.go:211 +0x909
github.com/cortexproject/cortex/pkg/chunk/storage.NewStore(0x3631c71, 0x6, 0x0, 0x4000000000000000, 0x4024000000000000, 0x0, 0x0, 0x186a0, 0x3ff4cccccccccccd, 0x3ff0000000000000, ...)
/go/src/github.com/cortexproject/cortex/pkg/chunk/storage/factory.go:156 +0x53e
github.com/cortexproject/cortex/pkg/cortex.(*Cortex).initStore(0xc0008f2800, 0x0, 0x0, 0x0, 0xc000100a80)
/go/src/github.com/cortexproject/cortex/pkg/cortex/modules.go:272 +0x201
github.com/cortexproject/cortex/pkg/util/modules.(*Manager).InitModuleServices(0xc00011e800, 0x7ffe8900be35, 0xb, 0x3d98c00, 0xc000a8b5c0, 0x3e109e0)
/go/src/github.com/cortexproject/cortex/pkg/util/modules/modules.go:72 +0x26f
github.com/cortexproject/cortex/pkg/cortex.(*Cortex).Run(0xc0008f2800, 0xc000599c40, 0x4)
/go/src/github.com/cortexproject/cortex/pkg/cortex/cortex.go:294 +0x5b
main.main()
/go/src/github.com/cortexproject/cortex/cmd/cortex/main.go:135 +0xa18
Config options:
-azure.account-key=<hidden>
-azure.account-name=<hidden>
-azure.container-name=<hidden>
-cassandra.addresses=<hidden>
-cassandra.auth=true
-cassandra.disable-initial-host-lookup=true
-cassandra.host-verification=false
-cassandra.keyspace=cortex
-cassandra.password=<hidden>
-cassandra.port=10350
-cassandra.replication-factor=3
-cassandra.ssl=true
-cassandra.username=<hidden>
-deletes.store=cassandra
-purger.enable=true
-purger.num-workers=2
-purger.object-store-type=azure
-schema-config-file=/etc/cortex/schema/config.yaml
-target=data-purger