From 2b925079fc67ac8887f58f0ea0ae18a19621682c Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Tue, 7 Jul 2020 16:27:51 -0400
Subject: [PATCH 01/21] Moved queryable and engine creation into
 initStoreQueryables

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 .../.data-minio/.gitignore                     |  1 +
 .../.data-minio/cortex-rules/.gitignore        |  2 ++
 .../config/cortex.yaml                         | 11 +++++++++++
 pkg/cortex/cortex.go                           |  7 ++++---
 pkg/cortex/modules.go                          | 18 ++++++++++--------
 5 files changed, 28 insertions(+), 11 deletions(-)
 create mode 100644 development/tsdb-blocks-storage-s3-single-binary/.data-minio/cortex-rules/.gitignore

diff --git a/development/tsdb-blocks-storage-s3-single-binary/.data-minio/.gitignore b/development/tsdb-blocks-storage-s3-single-binary/.data-minio/.gitignore
index 7efb0530d7d..dfd690b42f6 100644
--- a/development/tsdb-blocks-storage-s3-single-binary/.data-minio/.gitignore
+++ b/development/tsdb-blocks-storage-s3-single-binary/.data-minio/.gitignore
@@ -1,3 +1,4 @@
 *
+!cortex-rules
 !cortex-tsdb
 !.gitignore
diff --git a/development/tsdb-blocks-storage-s3-single-binary/.data-minio/cortex-rules/.gitignore b/development/tsdb-blocks-storage-s3-single-binary/.data-minio/cortex-rules/.gitignore
new file mode 100644
index 00000000000..d6b7ef32c84
--- /dev/null
+++ b/development/tsdb-blocks-storage-s3-single-binary/.data-minio/cortex-rules/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/development/tsdb-blocks-storage-s3-single-binary/config/cortex.yaml b/development/tsdb-blocks-storage-s3-single-binary/config/cortex.yaml
index f7c64f642b4..f1ed5be2e84 100644
--- a/development/tsdb-blocks-storage-s3-single-binary/config/cortex.yaml
+++ b/development/tsdb-blocks-storage-s3-single-binary/config/cortex.yaml
@@ -56,3 +56,14 @@ tsdb:
 
 storage:
   engine: tsdb
+
+ruler:
+  enable_api: true
+  enable_sharding: false
+  poll_interval: 2s
+  storage:
+    type: s3
+    s3:
+      bucketnames: cortex-rules
+      s3forcepathstyle: true
+      s3: http://cortex:supersecret@minio.:9000
\ No newline at end of file
diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go
index 23b9689496f..5dc9d933b5f 100644
--- a/pkg/cortex/cortex.go
+++ b/pkg/cortex/cortex.go
@@ -50,6 +50,8 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/runtimeconfig"
 	"github.com/cortexproject/cortex/pkg/util/services"
 	"github.com/cortexproject/cortex/pkg/util/validation"
+	"github.com/prometheus/prometheus/promql"
+	prom_storage "github.com/prometheus/prometheus/storage"
 )
 
 // The design pattern for Cortex is a series of config objects, which are
@@ -219,9 +221,8 @@ type Cortex struct {
 	StoreGateway *storegateway.StoreGateway
 	MemberlistKV *memberlist.KVInitService
 
-	// Queryables that the querier should use to query the long
-	// term storage. It depends on the storage engine used.
-	StoreQueryables []querier.QueryableWithFilter
+	Queryable prom_storage.SampleAndChunkQueryable
+	Engine    *promql.Engine
 }
 
 // New makes a new Cortex.
diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go
index 7100608d853..27c61ed9a65 100644
--- a/pkg/cortex/modules.go
+++ b/pkg/cortex/modules.go
@@ -169,8 +169,6 @@ func (t *Cortex) initDistributor() (serv services.Service, err error) {
 }
 
 func (t *Cortex) initQuerier() (serv services.Service, err error) {
-	queryable, engine := querier.New(t.Cfg.Querier, t.Overrides, t.Distributor, t.StoreQueryables, t.TombstonesLoader, prometheus.DefaultRegisterer)
-
 	// Prometheus histograms for requests to the querier.
 	querierRequestDuration := promauto.With(prometheus.DefaultRegisterer).NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: "cortex",
@@ -181,7 +179,7 @@ func (t *Cortex) initQuerier() (serv services.Service, err error) {
 
 	// if we are not configured for single binary mode then the querier needs to register its paths externally
 	registerExternally := t.Cfg.Target != All
-	handler := t.API.RegisterQuerier(queryable, engine, t.Distributor, registerExternally, t.TombstonesLoader, querierRequestDuration)
+	handler := t.API.RegisterQuerier(t.Queryable, t.Engine, t.Distributor, registerExternally, t.TombstonesLoader, querierRequestDuration)
 
 	// single binary mode requires a properly configured worker.  if the operator did not attempt to configure the
 	//  worker we will attempt an automatic configuration here
@@ -203,12 +201,13 @@ func (t *Cortex) initQuerier() (serv services.Service, err error) {
 
 func (t *Cortex) initStoreQueryables() (services.Service, error) {
 	var servs []services.Service
+	var storeQueryables []querier.QueryableWithFilter
 
 	//nolint:golint // I prefer this form over removing 'else', because it allows q to have smaller scope.
 	if q, err := initQueryableForEngine(t.Cfg.Storage.Engine, t.Cfg, t.Store, prometheus.DefaultRegisterer); err != nil {
 		return nil, fmt.Errorf("failed to initialize querier for engine '%s': %v", t.Cfg.Storage.Engine, err)
 	} else {
-		t.StoreQueryables = append(t.StoreQueryables, querier.UseAlwaysQueryable(q))
+		storeQueryables = append(storeQueryables, querier.UseAlwaysQueryable(q))
 		if s, ok := q.(services.Service); ok {
 			servs = append(servs, s)
 		}
@@ -224,13 +223,17 @@ func (t *Cortex) initStoreQueryables() (services.Service, error) {
 			return nil, fmt.Errorf("failed to initialize querier for engine '%s': %v", t.Cfg.Querier.SecondStoreEngine, err)
 		}
 
-		t.StoreQueryables = append(t.StoreQueryables, querier.UseBeforeTimestampQueryable(sq, time.Time(t.Cfg.Querier.UseSecondStoreBeforeTime)))
+		storeQueryables = append(storeQueryables, querier.UseBeforeTimestampQueryable(sq, time.Time(t.Cfg.Querier.UseSecondStoreBeforeTime)))
 
 		if s, ok := sq.(services.Service); ok {
 			servs = append(servs, s)
 		}
 	}
 
+	queryable, engine := querier.New(t.Cfg.Querier, t.Overrides, t.Distributor, storeQueryables, t.TombstonesLoader, prometheus.DefaultRegisterer)
+	t.Queryable = queryable
+	t.Engine = engine
+
 	// Return service, if any.
 	switch len(servs) {
 	case 0:
@@ -451,9 +454,8 @@ func (t *Cortex) initTableManager() (services.Service, error) {
 func (t *Cortex) initRuler() (serv services.Service, err error) {
 	t.Cfg.Ruler.Ring.ListenPort = t.Cfg.Server.GRPCListenPort
 	t.Cfg.Ruler.Ring.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
-	queryable, engine := querier.New(t.Cfg.Querier, t.Overrides, t.Distributor, t.StoreQueryables, t.TombstonesLoader, prometheus.DefaultRegisterer)
 
-	t.Ruler, err = ruler.NewRuler(t.Cfg.Ruler, engine, queryable, t.Distributor, prometheus.DefaultRegisterer, util.Logger)
+	t.Ruler, err = ruler.NewRuler(t.Cfg.Ruler, t.Engine, t.Queryable, t.Distributor, prometheus.DefaultRegisterer, util.Logger)
 	if err != nil {
 		return
 	}
@@ -596,7 +598,7 @@ func (t *Cortex) setupModuleManager() error {
 		Compactor:      {API},
 		StoreGateway:   {API},
 		Purger:         {Store, DeleteRequestsStore, API},
-		All:            {QueryFrontend, Querier, Ingester, Distributor, TableManager, Purger, StoreGateway},
+		All:            {QueryFrontend, Querier, Ingester, Distributor, TableManager, Purger, StoreGateway, Ruler},
 	}
 	for mod, targets := range deps {
 		if err := mm.AddDependency(mod, targets...); err != nil {

From ad832c7b6eee8dd5abd0ced0559aa37921ff35d1 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Wed, 8 Jul 2020 08:32:59 -0400
Subject: [PATCH 02/21] Enabled ruler sharding in local dev mode

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 .../config/cortex.yaml                                   | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/development/tsdb-blocks-storage-s3-single-binary/config/cortex.yaml b/development/tsdb-blocks-storage-s3-single-binary/config/cortex.yaml
index f1ed5be2e84..93e8aa8f9d9 100644
--- a/development/tsdb-blocks-storage-s3-single-binary/config/cortex.yaml
+++ b/development/tsdb-blocks-storage-s3-single-binary/config/cortex.yaml
@@ -59,11 +59,16 @@ storage:
 
 ruler:
   enable_api: true
-  enable_sharding: false
+  enable_sharding: true
   poll_interval: 2s
   storage:
     type: s3
     s3:
       bucketnames: cortex-rules
       s3forcepathstyle: true
-      s3: http://cortex:supersecret@minio.:9000
\ No newline at end of file
+      s3: http://cortex:supersecret@minio.:9000
+  ring:
+    kvstore:
+      store: consul
+      consul:
+        host: consul:8500

From 822c2441647b4ec3ff7fd02004e55ab60406ae8a Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Wed, 8 Jul 2020 09:53:26 -0400
Subject: [PATCH 03/21] Added local filesystem option.  Added ruler config to
 single binary.

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 .../single-process-config-blocks-gossip-1.yaml     | 14 +++++++++++++-
 .../single-process-config-blocks-gossip-2.yaml     | 14 +++++++++++++-
 .../single-process-config-blocks-tls.yaml          |  9 +++++++++
 .../single-process-config-blocks.yaml              | 11 ++++++++++-
 docs/configuration/single-process-config.yaml      | 11 ++++++++++-
 integration/e2ecortex/services.go                  |  3 +++
 pkg/ruler/storage.go                               | 10 ++++++++--
 7 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/docs/configuration/single-process-config-blocks-gossip-1.yaml b/docs/configuration/single-process-config-blocks-gossip-1.yaml
index 3b13746d569..92e27c85dc3 100644
--- a/docs/configuration/single-process-config-blocks-gossip-1.yaml
+++ b/docs/configuration/single-process-config-blocks-gossip-1.yaml
@@ -80,4 +80,16 @@ tsdb:
     dir: /tmp/cortex/storage
 
 frontend_worker:
-  match_max_concurrent: true
\ No newline at end of file
+  match_max_concurrent: true
+
+ruler:
+  enable_api: true
+  enable_sharding: true
+  poll_interval: 2s
+  storage:
+    type: local
+    local:
+      directory: /tmp/cortex/rules
+  ring:
+    kvstore:
+      store: memberlist
\ No newline at end of file
diff --git a/docs/configuration/single-process-config-blocks-gossip-2.yaml b/docs/configuration/single-process-config-blocks-gossip-2.yaml
index b3867c49169..fc1f4c8c86c 100644
--- a/docs/configuration/single-process-config-blocks-gossip-2.yaml
+++ b/docs/configuration/single-process-config-blocks-gossip-2.yaml
@@ -79,4 +79,16 @@ tsdb:
     dir: /tmp/cortex/storage
 
 frontend_worker:
-  match_max_concurrent: true
\ No newline at end of file
+  match_max_concurrent: true
+
+ruler:
+  enable_api: true
+  enable_sharding: true
+  poll_interval: 2s
+  storage:
+    type: local
+    local:
+      directory: /tmp/cortex/rules
+  ring:
+    kvstore:
+      store: memberlist
\ No newline at end of file
diff --git a/docs/configuration/single-process-config-blocks-tls.yaml b/docs/configuration/single-process-config-blocks-tls.yaml
index 3a5b9214c31..5bc03888f89 100644
--- a/docs/configuration/single-process-config-blocks-tls.yaml
+++ b/docs/configuration/single-process-config-blocks-tls.yaml
@@ -98,3 +98,12 @@ frontend_worker:
     tls_cert_path: "client.crt"
     tls_key_path: "client.key"
     tls_ca_path: "root.crt"
+
+ruler:
+  enable_api: true
+  enable_sharding: false
+  poll_interval: 2s
+  storage:
+    type: local
+    local:
+      directory: /tmp/cortex/rules
\ No newline at end of file
diff --git a/docs/configuration/single-process-config-blocks.yaml b/docs/configuration/single-process-config-blocks.yaml
index 033ec1618fa..1abb0c1fd6d 100644
--- a/docs/configuration/single-process-config-blocks.yaml
+++ b/docs/configuration/single-process-config-blocks.yaml
@@ -84,4 +84,13 @@ compactor:
       store: inmemory
 
 frontend_worker:
-  match_max_concurrent: true
\ No newline at end of file
+  match_max_concurrent: true
+
+ruler:
+  enable_api: true
+  enable_sharding: false
+  poll_interval: 2s
+  storage:
+    type: local
+    local:
+      directory: /tmp/cortex/rules
\ No newline at end of file
diff --git a/docs/configuration/single-process-config.yaml b/docs/configuration/single-process-config.yaml
index 665a9645cb3..1363a651dcf 100644
--- a/docs/configuration/single-process-config.yaml
+++ b/docs/configuration/single-process-config.yaml
@@ -82,4 +82,13 @@ purger:
   object_store_type: filesystem
 
 frontend_worker:
-  match_max_concurrent: true
\ No newline at end of file
+  match_max_concurrent: true
+
+ruler:
+  enable_api: true
+  enable_sharding: false
+  poll_interval: 2s
+  storage:
+    type: local
+    local:
+      directory: /tmp/cortex/rules
\ No newline at end of file
diff --git a/integration/e2ecortex/services.go b/integration/e2ecortex/services.go
index b0eed836bb1..fbc44fada9e 100644
--- a/integration/e2ecortex/services.go
+++ b/integration/e2ecortex/services.go
@@ -237,6 +237,9 @@ func NewSingleBinary(name string, flags map[string]string, image string, otherPo
 			"-ingester.concurrent-flushes":   "10",
 			"-ingester.max-transfer-retries": "10",
 			"-ingester.num-tokens":           "512",
+			// Ruler
+			"-ruler.storage.type":                  "local",
+			"-ruler.storage.local.chunk-directory": "/tmp/cortex/rules",
 		}, flags))...),
 		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299),
 		httpPort,
diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go
index 8ab6657b7c3..e3a85f9f9a4 100644
--- a/pkg/ruler/storage.go
+++ b/pkg/ruler/storage.go
@@ -11,6 +11,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/chunk/aws"
 	"github.com/cortexproject/cortex/pkg/chunk/azure"
 	"github.com/cortexproject/cortex/pkg/chunk/gcp"
+	"github.com/cortexproject/cortex/pkg/chunk/local"
 	"github.com/cortexproject/cortex/pkg/chunk/openstack"
 	"github.com/cortexproject/cortex/pkg/configs/client"
 	"github.com/cortexproject/cortex/pkg/ruler/rules"
@@ -27,6 +28,7 @@ type RuleStoreConfig struct {
 	GCS   gcp.GCSConfig           `yaml:"gcs"`
 	S3    aws.S3Config            `yaml:"s3"`
 	Swift openstack.SwiftConfig   `yaml:"swift"`
+	Local local.FSConfig          `yaml:"local"`
 
 	mock rules.RuleStore `yaml:"-"`
 }
@@ -38,7 +40,9 @@ func (cfg *RuleStoreConfig) RegisterFlags(f *flag.FlagSet) {
 	cfg.GCS.RegisterFlagsWithPrefix("ruler.storage.", f)
 	cfg.S3.RegisterFlagsWithPrefix("ruler.storage.", f)
 	cfg.Swift.RegisterFlagsWithPrefix("ruler.storage.", f)
-	f.StringVar(&cfg.Type, "ruler.storage.type", "configdb", "Method to use for backend rule storage (configdb, azure, gcs, s3)")
+	cfg.Local.RegisterFlagsWithPrefix("ruler.storage.", f)
+
+	f.StringVar(&cfg.Type, "ruler.storage.type", "configdb", "Method to use for backend rule storage (configdb, azure, gcs, s3, swift, local)")
 }
 
 // Validate config and returns error on failure
@@ -72,8 +76,10 @@ func NewRuleStorage(cfg RuleStoreConfig) (rules.RuleStore, error) {
 		return newObjRuleStore(aws.NewS3ObjectClient(cfg.S3, ""))
 	case "swift":
 		return newObjRuleStore(openstack.NewSwiftObjectClient(cfg.Swift, ""))
+	case "local":
+		return newObjRuleStore(local.NewFSObjectClient(cfg.Local))
 	default:
-		return nil, fmt.Errorf("Unrecognized rule storage mode %v, choose one of: configdb, gcs, s3, swift, azure", cfg.Type)
+		return nil, fmt.Errorf("Unrecognized rule storage mode %v, choose one of: configdb, gcs, s3, swift, azure, local", cfg.Type)
 	}
 }
 

From 8b01b6b4db63fb3f36bde82955757bd536a996e8 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Wed, 8 Jul 2020 09:58:54 -0400
Subject: [PATCH 04/21] StoreQueryable => Queryable

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 pkg/cortex/modules.go | 42 +++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go
index 27c61ed9a65..d96a7e3a138 100644
--- a/pkg/cortex/modules.go
+++ b/pkg/cortex/modules.go
@@ -51,7 +51,7 @@ const (
 	Ingester            string = "ingester"
 	Flusher             string = "flusher"
 	Querier             string = "querier"
-	StoreQueryable      string = "store-queryable"
+	Queryable           string = "queryable"
 	QueryFrontend       string = "query-frontend"
 	Store               string = "store"
 	DeleteRequestsStore string = "delete-requests-store"
@@ -199,7 +199,7 @@ func (t *Cortex) initQuerier() (serv services.Service, err error) {
 	return worker, nil
 }
 
-func (t *Cortex) initStoreQueryables() (services.Service, error) {
+func (t *Cortex) initQueryable() (services.Service, error) {
 	var servs []services.Service
 	var storeQueryables []querier.QueryableWithFilter
 
@@ -568,7 +568,7 @@ func (t *Cortex) setupModuleManager() error {
 	mm.RegisterModule(Ingester, t.initIngester)
 	mm.RegisterModule(Flusher, t.initFlusher)
 	mm.RegisterModule(Querier, t.initQuerier)
-	mm.RegisterModule(StoreQueryable, t.initStoreQueryables, modules.UserInvisibleModule)
+	mm.RegisterModule(Queryable, t.initQueryable, modules.UserInvisibleModule)
 	mm.RegisterModule(QueryFrontend, t.initQueryFrontend)
 	mm.RegisterModule(TableManager, t.initTableManager)
 	mm.RegisterModule(Ruler, t.initRuler)
@@ -581,24 +581,24 @@ func (t *Cortex) setupModuleManager() error {
 
 	// Add dependencies
 	deps := map[string][]string{
-		API:            {Server},
-		Ring:           {API, RuntimeConfig, MemberlistKV},
-		Overrides:      {RuntimeConfig},
-		Distributor:    {Ring, API, Overrides},
-		Store:          {Overrides, DeleteRequestsStore},
-		Ingester:       {Overrides, Store, API, RuntimeConfig, MemberlistKV},
-		Flusher:        {Store, API},
-		Querier:        {Overrides, Distributor, Store, Ring, API, StoreQueryable},
-		StoreQueryable: {Store},
-		QueryFrontend:  {API, Overrides, DeleteRequestsStore},
-		TableManager:   {API},
-		Ruler:          {Overrides, Distributor, Store, StoreQueryable},
-		Configs:        {API},
-		AlertManager:   {API},
-		Compactor:      {API},
-		StoreGateway:   {API},
-		Purger:         {Store, DeleteRequestsStore, API},
-		All:            {QueryFrontend, Querier, Ingester, Distributor, TableManager, Purger, StoreGateway, Ruler},
+		API:           {Server},
+		Ring:          {API, RuntimeConfig, MemberlistKV},
+		Overrides:     {RuntimeConfig},
+		Distributor:   {Ring, API, Overrides},
+		Store:         {Overrides, DeleteRequestsStore},
+		Ingester:      {Overrides, Store, API, RuntimeConfig, MemberlistKV},
+		Flusher:       {Store, API},
+		Querier:       {Overrides, Distributor, Store, Ring, API, Queryable},
+		Queryable:     {Store},
+		QueryFrontend: {API, Overrides, DeleteRequestsStore},
+		TableManager:  {API},
+		Ruler:         {Overrides, Distributor, Store, Queryable},
+		Configs:       {API},
+		AlertManager:  {API},
+		Compactor:     {API},
+		StoreGateway:  {API},
+		Purger:        {Store, DeleteRequestsStore, API},
+		All:           {QueryFrontend, Querier, Ingester, Distributor, TableManager, Purger, StoreGateway, Ruler},
 	}
 	for mod, targets := range deps {
 		if err := mm.AddDependency(mod, targets...); err != nil {

From 74391b79a2a38b3ca6d128cd6477cc33186048a3 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Wed, 8 Jul 2020 10:27:50 -0400
Subject: [PATCH 05/21] Made queryable dependent on the distributor

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 pkg/cortex/modules.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go
index d96a7e3a138..9203aa76d34 100644
--- a/pkg/cortex/modules.go
+++ b/pkg/cortex/modules.go
@@ -588,11 +588,11 @@ func (t *Cortex) setupModuleManager() error {
 		Store:         {Overrides, DeleteRequestsStore},
 		Ingester:      {Overrides, Store, API, RuntimeConfig, MemberlistKV},
 		Flusher:       {Store, API},
-		Querier:       {Overrides, Distributor, Store, Ring, API, Queryable},
-		Queryable:     {Store},
+		Querier:       {Overrides, Ring, API, Queryable},
+		Queryable:     {Store, Distributor},
 		QueryFrontend: {API, Overrides, DeleteRequestsStore},
 		TableManager:  {API},
-		Ruler:         {Overrides, Distributor, Store, Queryable},
+		Ruler:         {Overrides, Queryable},
 		Configs:       {API},
 		AlertManager:  {API},
 		Compactor:     {API},

From c3b066c01e1d38ae1e082f65306c52fdea764d0a Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Wed, 8 Jul 2020 10:38:30 -0400
Subject: [PATCH 06/21] lint + doc

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 docs/configuration/config-file-reference.md | 8 +++++++-
 pkg/cortex/cortex.go                        | 5 +++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index 12e3c9a7722..fc73ec55692 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -816,7 +816,8 @@ ruler_client:
 [poll_interval: <duration> | default = 1m]
 
 storage:
-  # Method to use for backend rule storage (configdb, azure, gcs, s3)
+  # Method to use for backend rule storage (configdb, azure, gcs, s3, swift,
+  # local)
   # CLI flag: -ruler.storage.type
   [type: <string> | default = "configdb"]
 
@@ -958,6 +959,11 @@ storage:
     # CLI flag: -ruler.storage.swift.container-name
     [container_name: <string> | default = "cortex"]
 
+  local:
+    # Directory to store chunks in.
+    # CLI flag: -ruler.storage.local.chunk-directory
+    [directory: <string> | default = ""]
+
 # file path to store temporary rule files for the prometheus rule managers
 # CLI flag: -ruler.rule-path
 [rule_path: <string> | default = "/rules"]
diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go
index 5dc9d933b5f..4a5c09880a2 100644
--- a/pkg/cortex/cortex.go
+++ b/pkg/cortex/cortex.go
@@ -20,6 +20,9 @@ import (
 	"google.golang.org/grpc/health/grpc_health_v1"
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/prometheus/promql"
+	prom_storage "github.com/prometheus/prometheus/storage"
+
 	"github.com/cortexproject/cortex/pkg/alertmanager"
 	"github.com/cortexproject/cortex/pkg/api"
 	"github.com/cortexproject/cortex/pkg/chunk"
@@ -50,8 +53,6 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/runtimeconfig"
 	"github.com/cortexproject/cortex/pkg/util/services"
 	"github.com/cortexproject/cortex/pkg/util/validation"
-	"github.com/prometheus/prometheus/promql"
-	prom_storage "github.com/prometheus/prometheus/storage"
 )
 
 // The design pattern for Cortex is a series of config objects, which are

From 6fab147cc3f7425248b52d468c8a8cc61f58c9d5 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Wed, 8 Jul 2020 10:41:11 -0400
Subject: [PATCH 07/21] changelog

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0b40cf74e04..115480d8ad5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -37,6 +37,7 @@
 * [ENHANCEMENT] Experimental TSDB: Added support to enforce max query time range length via `-store.max-query-length`. #2826
 * [ENHANCEMENT] Ingester: Added new metric `cortex_ingester_flush_series_in_progress` that reports number of ongoing flush-series operations. Useful when calling `/flush` handler: if `cortex_ingester_flush_queue_length + cortex_ingester_flush_series_in_progress` is 0, all flushes are finished. #2778
 * [ENHANCEMENT] Memberlist members can join cluster via SRV records. #2788
+* [ENHANCEMENT] Added ruler to the single binary.  #2854
 * [BUGFIX] Fixed a bug in the index intersect code causing storage to return more chunks/series than required. #2796
 * [BUGFIX] Fixed the number of reported keys in the background cache queue. #2764
 * [BUGFIX] Fix race in processing of headers in sharded queries. #2762

From a3222bf2451a661d430524bbd6c650d17fab5096 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Wed, 8 Jul 2020 11:54:56 -0400
Subject: [PATCH 08/21] Fixed test

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 pkg/cortex/cortex_test.go | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/pkg/cortex/cortex_test.go b/pkg/cortex/cortex_test.go
index 14c44e21a27..27af6dfcb22 100644
--- a/pkg/cortex/cortex_test.go
+++ b/pkg/cortex/cortex_test.go
@@ -1,20 +1,27 @@
 package cortex
 
 import (
+	"net/url"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/cortexproject/cortex/pkg/chunk/aws"
 	"github.com/cortexproject/cortex/pkg/chunk/storage"
 	"github.com/cortexproject/cortex/pkg/ingester"
 	"github.com/cortexproject/cortex/pkg/ring"
 	"github.com/cortexproject/cortex/pkg/ring/kv"
+	"github.com/cortexproject/cortex/pkg/ruler"
 	"github.com/cortexproject/cortex/pkg/storage/backend/s3"
 	"github.com/cortexproject/cortex/pkg/storage/tsdb"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/services"
 )
 
 func TestCortex(t *testing.T) {
+	rulerUrl, err := url.Parse("inmemory:///rules")
+	require.NoError(t, err)
+
 	cfg := Config{
 		Storage: storage.Config{
 			Engine: storage.StorageEngineTSDB, // makes config easier
@@ -47,6 +54,16 @@ func TestCortex(t *testing.T) {
 				},
 			},
 		},
+		Ruler: ruler.Config{
+			StoreConfig: ruler.RuleStoreConfig{
+				Type: "s3",
+				S3: aws.S3Config{
+					S3: flagext.URLValue{
+						URL: rulerUrl,
+					},
+				},
+			},
+		},
 		Target: All,
 	}
 

From 0782259daf38e2bb4e4b2794b1a3e4b7a2b78482 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Wed, 8 Jul 2020 13:20:51 -0400
Subject: [PATCH 09/21] Url => URL

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 pkg/cortex/cortex_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/cortex/cortex_test.go b/pkg/cortex/cortex_test.go
index 27af6dfcb22..61be1742bb2 100644
--- a/pkg/cortex/cortex_test.go
+++ b/pkg/cortex/cortex_test.go
@@ -19,7 +19,7 @@ import (
 )
 
 func TestCortex(t *testing.T) {
-	rulerUrl, err := url.Parse("inmemory:///rules")
+	rulerURL, err := url.Parse("inmemory:///rules")
 	require.NoError(t, err)
 
 	cfg := Config{

From 9347b3a69e40c1815cdc60b4debdb988036628c2 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Wed, 8 Jul 2020 13:26:23 -0400
Subject: [PATCH 10/21] Url => URL

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 pkg/cortex/cortex_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/cortex/cortex_test.go b/pkg/cortex/cortex_test.go
index 61be1742bb2..bd41d0d6c62 100644
--- a/pkg/cortex/cortex_test.go
+++ b/pkg/cortex/cortex_test.go
@@ -59,7 +59,7 @@ func TestCortex(t *testing.T) {
 				Type: "s3",
 				S3: aws.S3Config{
 					S3: flagext.URLValue{
-						URL: rulerUrl,
+						URL: rulerURL,
 					},
 				},
 			},

From b698acb5317bb5a5fb3d857b1c00cdf7a5ec8a39 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Thu, 9 Jul 2020 10:28:11 -0400
Subject: [PATCH 11/21] Added local storage option for ruler and added tests

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 integration/api_ruler_test.go       |  37 ++++++++
 integration/configs.go              |  12 +++
 integration/e2ecortex/services.go   |   4 +-
 pkg/ruler/rules/local/local.go      | 136 ++++++++++++++++++++++++++++
 pkg/ruler/rules/local/local_test.go |  68 ++++++++++++++
 pkg/ruler/storage.go                |   6 +-
 6 files changed, 258 insertions(+), 5 deletions(-)
 create mode 100644 pkg/ruler/rules/local/local.go
 create mode 100644 pkg/ruler/rules/local/local_test.go

diff --git a/integration/api_ruler_test.go b/integration/api_ruler_test.go
index 8da8eb2c123..f464daff945 100644
--- a/integration/api_ruler_test.go
+++ b/integration/api_ruler_test.go
@@ -3,6 +3,8 @@
 package main
 
 import (
+	"path"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -94,3 +96,38 @@ func TestRulerAPI(t *testing.T) {
 	// Ensure no service-specific metrics prefix is used by the wrong service.
 	assertServiceMetricsPrefixes(t, Ruler, ruler)
 }
+
+func TestRulerAPISingleBinary(t *testing.T) {
+	s, err := e2e.NewScenario(networkName)
+	require.NoError(t, err)
+	defer s.Close()
+
+	namespace := "ns"
+	user := "fake"
+
+	configOverrides := map[string]string{
+		"-ruler.storage.local.directory": filepath.Join(e2e.ContainerSharedDir, "ruler_configs"),
+	}
+
+	// Start Cortex components.
+	require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config.yaml", cortexConfigFile))
+	require.NoError(t, writeFileToSharedDir(s, path.Join("ruler_configs", user, namespace), []byte(cortexRulerUserConfigYaml)))
+	cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex", cortexConfigFile, configOverrides, "", 9009, 9095)
+	require.NoError(t, s.StartAndWaitReady(cortex))
+
+	// Create a client with the ruler address configured
+	c, err := e2ecortex.NewClient("", "", "", cortex.HTTPEndpoint(), "")
+	require.NoError(t, err)
+
+	// Wait until the user manager is created
+	require.NoError(t, cortex.WaitSumMetrics(e2e.Equals(1), "cortex_ruler_managers_total"))
+
+	// Check to ensure the rules running in the cortex match what was set
+	rgs, err := c.GetRuleGroups()
+	require.NoError(t, err)
+
+	retrievedNamespace, exists := rgs[namespace]
+	require.True(t, exists)
+	require.Len(t, retrievedNamespace, 1)
+	require.Equal(t, retrievedNamespace[0].Name, "rule")
+}
diff --git a/integration/configs.go b/integration/configs.go
index 162512aa82c..130836d077a 100644
--- a/integration/configs.go
+++ b/integration/configs.go
@@ -45,6 +45,18 @@ const (
 receivers:
   - name: "example_receiver"
 `
+
+	cortexRulerUserConfigYaml = `groups:
+- name: rule
+  interval: 100s
+  rules:
+  - record: test_rule
+    alert: ""
+    expr: up
+    for: 0s
+    labels: {}
+    annotations: {}	
+`
 )
 
 var (
diff --git a/integration/e2ecortex/services.go b/integration/e2ecortex/services.go
index fbc44fada9e..1a2d021cff7 100644
--- a/integration/e2ecortex/services.go
+++ b/integration/e2ecortex/services.go
@@ -238,8 +238,8 @@ func NewSingleBinary(name string, flags map[string]string, image string, otherPo
 			"-ingester.max-transfer-retries": "10",
 			"-ingester.num-tokens":           "512",
 			// Ruler
-			"-ruler.storage.type":                  "local",
-			"-ruler.storage.local.chunk-directory": "/tmp/cortex/rules",
+			"-ruler.storage.type":            "local",
+			"-ruler.storage.local.directory": "/tmp/cortex/rules",
 		}, flags))...),
 		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299),
 		httpPort,
diff --git a/pkg/ruler/rules/local/local.go b/pkg/ruler/rules/local/local.go
new file mode 100644
index 00000000000..d15dd6182ec
--- /dev/null
+++ b/pkg/ruler/rules/local/local.go
@@ -0,0 +1,136 @@
+package local
+
+import (
+	"context"
+	"flag"
+	"io/ioutil"
+	"path"
+
+	"github.com/pkg/errors"
+
+	rulefmt "github.com/cortexproject/cortex/pkg/ruler/legacy_rulefmt"
+	"github.com/cortexproject/cortex/pkg/ruler/rules"
+)
+
+type Config struct {
+	Directory string `yaml:"directory"`
+}
+
+// RegisterFlags registers flags.
+func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.StringVar(&cfg.Directory, prefix+"local.directory", "", "Directory to scan for rules")
+}
+
+// LocalClient expects to load already existing rules located at:
+//  cfg.Directory / userID / namespace
+type LocalClient struct {
+	cfg Config
+}
+
+func NewLocalRulesClient(cfg Config) (*LocalClient, error) {
+	if cfg.Directory == "" {
+		return nil, errors.New("directory required for local rules config")
+	}
+
+	return &LocalClient{
+		cfg: cfg,
+	}, nil
+}
+
+// ListAllRuleGroups implements RuleStore
+func (l *LocalClient) ListAllRuleGroups(ctx context.Context) (map[string]rules.RuleGroupList, error) {
+	lists := make(map[string]rules.RuleGroupList)
+
+	root := l.cfg.Directory
+	infos, err := ioutil.ReadDir(root)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to read dir "+root)
+	}
+
+	for _, info := range infos {
+		if !info.IsDir() {
+			continue
+		}
+
+		list, err := l.listAllRulesGroupsForUser(ctx, info.Name())
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to list rule group")
+		}
+
+		lists[info.Name()] = list
+	}
+
+	return lists, nil
+}
+
+// ListRuleGroups implements RuleStore
+func (l *LocalClient) ListRuleGroups(ctx context.Context, userID string, namespace string) (rules.RuleGroupList, error) {
+	if namespace != "" {
+		return l.listAllRulesGroupsForUserAndNamespace(ctx, userID, namespace)
+	}
+
+	return l.listAllRulesGroupsForUser(ctx, userID)
+}
+
+// GetRuleGroup implements RuleStore
+func (l *LocalClient) GetRuleGroup(ctx context.Context, userID, namespace, group string) (*rules.RuleGroupDesc, error) {
+	return nil, errors.New("GetRuleGroup unsupported in rule local store")
+}
+
+// SetRuleGroup implements RuleStore
+func (l *LocalClient) SetRuleGroup(ctx context.Context, userID, namespace string, group *rules.RuleGroupDesc) error {
+	return errors.New("SetRuleGroup unsupported in rule local store")
+}
+
+// DeleteRuleGroup implements RuleStore
+func (l *LocalClient) DeleteRuleGroup(ctx context.Context, userID, namespace string, group string) error {
+	return errors.New("DeleteRuleGroup unsupported in rule local store")
+}
+
+func (l *LocalClient) listAllRulesGroupsForUser(ctx context.Context, userID string) (rules.RuleGroupList, error) {
+	var allLists rules.RuleGroupList
+
+	root := path.Join(l.cfg.Directory, userID)
+	infos, err := ioutil.ReadDir(root)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to read dir "+root)
+	}
+
+	for _, info := range infos {
+		if info.IsDir() {
+			continue
+		}
+
+		list, err := l.ListRuleGroups(ctx, userID, info.Name())
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to list rule group")
+		}
+
+		allLists = append(allLists, list...)
+	}
+
+	return allLists, nil
+}
+
+func (l *LocalClient) listAllRulesGroupsForUserAndNamespace(ctx context.Context, userID string, namespace string) (rules.RuleGroupList, error) {
+	filename := path.Join(l.cfg.Directory, userID, namespace)
+
+	rulegroups, allErrors := rulefmt.ParseFile(filename)
+	if len(allErrors) > 0 {
+		return nil, errors.Wrap(allErrors[0], "error parsing "+filename)
+	}
+
+	allErrors = rulegroups.Validate()
+	if len(allErrors) > 0 {
+		return nil, errors.Wrap(allErrors[0], "error validating "+filename)
+	}
+
+	var list rules.RuleGroupList
+
+	for _, group := range rulegroups.Groups {
+		desc := rules.ToProto(userID, namespace, group)
+		list = append(list, desc)
+	}
+
+	return list, nil
+}
diff --git a/pkg/ruler/rules/local/local_test.go b/pkg/ruler/rules/local/local_test.go
new file mode 100644
index 00000000000..d7d08fb0df7
--- /dev/null
+++ b/pkg/ruler/rules/local/local_test.go
@@ -0,0 +1,68 @@
+package local
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	rulefmt "github.com/cortexproject/cortex/pkg/ruler/legacy_rulefmt"
+	"github.com/cortexproject/cortex/pkg/ruler/rules"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
+)
+
+func TestLoadingRules(t *testing.T) {
+	user := "user"
+	namespace := "ns"
+
+	dir, err := ioutil.TempDir("", "")
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	ruleGroups := rulefmt.RuleGroups{
+		Groups: []rulefmt.RuleGroup{
+			{
+				Name:     "rule",
+				Interval: model.Duration(100 * time.Second),
+				Rules: []rulefmt.Rule{
+					{
+						Record: "test_rule",
+						Expr:   "up",
+					},
+				},
+			},
+		},
+	}
+
+	b, err := yaml.Marshal(ruleGroups)
+	require.NoError(t, err)
+
+	err = os.MkdirAll(path.Join(dir, user), 0777)
+	require.NoError(t, err)
+
+	err = ioutil.WriteFile(path.Join(dir, user, namespace), b, 0777)
+	require.NoError(t, err)
+
+	client, err := NewLocalRulesClient(Config{
+		Directory: dir,
+	})
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	userMap, err := client.ListAllRuleGroups(ctx)
+	require.NoError(t, err)
+
+	actual, found := userMap[user]
+	require.True(t, found)
+
+	require.Equal(t, len(ruleGroups.Groups), len(actual))
+	for i, actualGroup := range actual {
+		expected := rules.ToProto(user, namespace, ruleGroups.Groups[i])
+
+		require.Equal(t, expected, actualGroup)
+	}
+}
diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go
index e3a85f9f9a4..dd82a0ba25f 100644
--- a/pkg/ruler/storage.go
+++ b/pkg/ruler/storage.go
@@ -11,10 +11,10 @@ import (
 	"github.com/cortexproject/cortex/pkg/chunk/aws"
 	"github.com/cortexproject/cortex/pkg/chunk/azure"
 	"github.com/cortexproject/cortex/pkg/chunk/gcp"
-	"github.com/cortexproject/cortex/pkg/chunk/local"
 	"github.com/cortexproject/cortex/pkg/chunk/openstack"
 	"github.com/cortexproject/cortex/pkg/configs/client"
 	"github.com/cortexproject/cortex/pkg/ruler/rules"
+	"github.com/cortexproject/cortex/pkg/ruler/rules/local"
 	"github.com/cortexproject/cortex/pkg/ruler/rules/objectclient"
 )
 
@@ -28,7 +28,7 @@ type RuleStoreConfig struct {
 	GCS   gcp.GCSConfig           `yaml:"gcs"`
 	S3    aws.S3Config            `yaml:"s3"`
 	Swift openstack.SwiftConfig   `yaml:"swift"`
-	Local local.FSConfig          `yaml:"local"`
+	Local local.Config            `yaml:"local"`
 
 	mock rules.RuleStore `yaml:"-"`
 }
@@ -77,7 +77,7 @@ func NewRuleStorage(cfg RuleStoreConfig) (rules.RuleStore, error) {
 	case "swift":
 		return newObjRuleStore(openstack.NewSwiftObjectClient(cfg.Swift, ""))
 	case "local":
-		return newObjRuleStore(local.NewFSObjectClient(cfg.Local))
+		return local.NewLocalRulesClient(cfg.Local)
 	default:
 		return nil, fmt.Errorf("Unrecognized rule storage mode %v, choose one of: configdb, gcs, s3, swift, azure, local", cfg.Type)
 	}

From befddc87ba97f1ead0774843142f88379d0d88eb Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Thu, 9 Jul 2020 10:40:28 -0400
Subject: [PATCH 12/21] lint

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 pkg/ruler/rules/local/local.go      | 22 +++++++++++-----------
 pkg/ruler/rules/local/local_test.go |  5 +++--
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/pkg/ruler/rules/local/local.go b/pkg/ruler/rules/local/local.go
index d15dd6182ec..f2e48a5ff3b 100644
--- a/pkg/ruler/rules/local/local.go
+++ b/pkg/ruler/rules/local/local.go
@@ -21,24 +21,24 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.StringVar(&cfg.Directory, prefix+"local.directory", "", "Directory to scan for rules")
 }
 
-// LocalClient expects to load already existing rules located at:
+// Client expects to load already existing rules located at:
 //  cfg.Directory / userID / namespace
-type LocalClient struct {
+type Client struct {
 	cfg Config
 }
 
-func NewLocalRulesClient(cfg Config) (*LocalClient, error) {
+func NewLocalRulesClient(cfg Config) (*Client, error) {
 	if cfg.Directory == "" {
 		return nil, errors.New("directory required for local rules config")
 	}
 
-	return &LocalClient{
+	return &Client{
 		cfg: cfg,
 	}, nil
 }
 
 // ListAllRuleGroups implements RuleStore
-func (l *LocalClient) ListAllRuleGroups(ctx context.Context) (map[string]rules.RuleGroupList, error) {
+func (l *Client) ListAllRuleGroups(ctx context.Context) (map[string]rules.RuleGroupList, error) {
 	lists := make(map[string]rules.RuleGroupList)
 
 	root := l.cfg.Directory
@@ -64,7 +64,7 @@ func (l *LocalClient) ListAllRuleGroups(ctx context.Context) (map[string]rules.R
 }
 
 // ListRuleGroups implements RuleStore
-func (l *LocalClient) ListRuleGroups(ctx context.Context, userID string, namespace string) (rules.RuleGroupList, error) {
+func (l *Client) ListRuleGroups(ctx context.Context, userID string, namespace string) (rules.RuleGroupList, error) {
 	if namespace != "" {
 		return l.listAllRulesGroupsForUserAndNamespace(ctx, userID, namespace)
 	}
@@ -73,21 +73,21 @@ func (l *LocalClient) ListRuleGroups(ctx context.Context, userID string, namespa
 }
 
 // GetRuleGroup implements RuleStore
-func (l *LocalClient) GetRuleGroup(ctx context.Context, userID, namespace, group string) (*rules.RuleGroupDesc, error) {
+func (l *Client) GetRuleGroup(ctx context.Context, userID, namespace, group string) (*rules.RuleGroupDesc, error) {
 	return nil, errors.New("GetRuleGroup unsupported in rule local store")
 }
 
 // SetRuleGroup implements RuleStore
-func (l *LocalClient) SetRuleGroup(ctx context.Context, userID, namespace string, group *rules.RuleGroupDesc) error {
+func (l *Client) SetRuleGroup(ctx context.Context, userID, namespace string, group *rules.RuleGroupDesc) error {
 	return errors.New("SetRuleGroup unsupported in rule local store")
 }
 
 // DeleteRuleGroup implements RuleStore
-func (l *LocalClient) DeleteRuleGroup(ctx context.Context, userID, namespace string, group string) error {
+func (l *Client) DeleteRuleGroup(ctx context.Context, userID, namespace string, group string) error {
 	return errors.New("DeleteRuleGroup unsupported in rule local store")
 }
 
-func (l *LocalClient) listAllRulesGroupsForUser(ctx context.Context, userID string) (rules.RuleGroupList, error) {
+func (l *Client) listAllRulesGroupsForUser(ctx context.Context, userID string) (rules.RuleGroupList, error) {
 	var allLists rules.RuleGroupList
 
 	root := path.Join(l.cfg.Directory, userID)
@@ -112,7 +112,7 @@ func (l *LocalClient) listAllRulesGroupsForUser(ctx context.Context, userID stri
 	return allLists, nil
 }
 
-func (l *LocalClient) listAllRulesGroupsForUserAndNamespace(ctx context.Context, userID string, namespace string) (rules.RuleGroupList, error) {
+func (l *Client) listAllRulesGroupsForUserAndNamespace(ctx context.Context, userID string, namespace string) (rules.RuleGroupList, error) {
 	filename := path.Join(l.cfg.Directory, userID, namespace)
 
 	rulegroups, allErrors := rulefmt.ParseFile(filename)
diff --git a/pkg/ruler/rules/local/local_test.go b/pkg/ruler/rules/local/local_test.go
index d7d08fb0df7..630237397e0 100644
--- a/pkg/ruler/rules/local/local_test.go
+++ b/pkg/ruler/rules/local/local_test.go
@@ -8,11 +8,12 @@ import (
 	"testing"
 	"time"
 
-	rulefmt "github.com/cortexproject/cortex/pkg/ruler/legacy_rulefmt"
-	"github.com/cortexproject/cortex/pkg/ruler/rules"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+
+	rulefmt "github.com/cortexproject/cortex/pkg/ruler/legacy_rulefmt"
+	"github.com/cortexproject/cortex/pkg/ruler/rules"
 )
 
 func TestLoadingRules(t *testing.T) {

From c3f651e7197d9b0a9513988dbb48bc69d0b5e0a5 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Thu, 9 Jul 2020 10:51:30 -0400
Subject: [PATCH 13/21] make doc

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 docs/configuration/config-file-reference.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index fc73ec55692..27348a6974f 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -960,8 +960,8 @@ storage:
     [container_name: <string> | default = "cortex"]
 
   local:
-    # Directory to store chunks in.
-    # CLI flag: -ruler.storage.local.chunk-directory
+    # Directory to scan for rules
+    # CLI flag: -ruler.storage.local.directory
     [directory: <string> | default = ""]
 
 # file path to store temporary rule files for the prometheus rule managers

From 207d7f692691fdd96cb3efa72fdbcc636456cf60 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Thu, 9 Jul 2020 13:27:58 -0400
Subject: [PATCH 14/21] Split promql query engine in two

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 CHANGELOG.md                  |  1 +
 integration/api_ruler_test.go |  4 +++
 pkg/cortex/cortex.go          |  8 ++---
 pkg/cortex/modules.go         | 60 +++++++++++++++++------------------
 4 files changed, 38 insertions(+), 35 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 515187a2e0c..9f7b4f19dee 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -40,6 +40,7 @@
 * [ENHANCEMENT] Ingester: Added new metric `cortex_ingester_flush_series_in_progress` that reports number of ongoing flush-series operations. Useful when calling `/flush` handler: if `cortex_ingester_flush_queue_length + cortex_ingester_flush_series_in_progress` is 0, all flushes are finished. #2778
 * [ENHANCEMENT] Memberlist members can join cluster via SRV records. #2788
 * [ENHANCEMENT] Added ruler to the single binary.  #2854
+  * Two sets of Prometheus query engine metrics now appear in the single binary.  The `engine` label has been added to distinguish between `ruler` and `querier` metrics.
 * [ENHANCEMENT] Added configuration options for chunks s3 client. #2831
   * `s3.endpoint`
   * `s3.region`
diff --git a/integration/api_ruler_test.go b/integration/api_ruler_test.go
index f464daff945..9c339c12fa0 100644
--- a/integration/api_ruler_test.go
+++ b/integration/api_ruler_test.go
@@ -130,4 +130,8 @@ func TestRulerAPISingleBinary(t *testing.T) {
 	require.True(t, exists)
 	require.Len(t, retrievedNamespace, 1)
 	require.Equal(t, retrievedNamespace[0].Name, "rule")
+
+	// Check to make sure prometheus engine metrics are available for both engine types
+	require.NoError(t, cortex.WaitForMetricWithLabels(e2e.EqualsSingle(0), "prometheus_engine_queries", map[string]string{"engine": "querier"}))
+	require.NoError(t, cortex.WaitForMetricWithLabels(e2e.EqualsSingle(0), "prometheus_engine_queries", map[string]string{"engine": "ruler"}))
 }
diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go
index 4a5c09880a2..23b9689496f 100644
--- a/pkg/cortex/cortex.go
+++ b/pkg/cortex/cortex.go
@@ -20,9 +20,6 @@ import (
 	"google.golang.org/grpc/health/grpc_health_v1"
 	"gopkg.in/yaml.v2"
 
-	"github.com/prometheus/prometheus/promql"
-	prom_storage "github.com/prometheus/prometheus/storage"
-
 	"github.com/cortexproject/cortex/pkg/alertmanager"
 	"github.com/cortexproject/cortex/pkg/api"
 	"github.com/cortexproject/cortex/pkg/chunk"
@@ -222,8 +219,9 @@ type Cortex struct {
 	StoreGateway *storegateway.StoreGateway
 	MemberlistKV *memberlist.KVInitService
 
-	Queryable prom_storage.SampleAndChunkQueryable
-	Engine    *promql.Engine
+	// Queryables that the querier should use to query the long
+	// term storage. It depends on the storage engine used.
+	StoreQueryables []querier.QueryableWithFilter
 }
 
 // New makes a new Cortex.
diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go
index c13c3f00e2e..1ea91ed85ef 100644
--- a/pkg/cortex/modules.go
+++ b/pkg/cortex/modules.go
@@ -51,7 +51,7 @@ const (
 	Ingester            string = "ingester"
 	Flusher             string = "flusher"
 	Querier             string = "querier"
-	Queryable           string = "queryable"
+	StoreQueryable      string = "store-queryable"
 	QueryFrontend       string = "query-frontend"
 	Store               string = "store"
 	DeleteRequestsStore string = "delete-requests-store"
@@ -169,6 +169,9 @@ func (t *Cortex) initDistributor() (serv services.Service, err error) {
 }
 
 func (t *Cortex) initQuerier() (serv services.Service, err error) {
+	querierRegisterer := prometheus.WrapRegistererWith(prometheus.Labels{"engine": "querier"}, prometheus.DefaultRegisterer)
+	queryable, engine := querier.New(t.Cfg.Querier, t.Overrides, t.Distributor, t.StoreQueryables, t.TombstonesLoader, querierRegisterer)
+
 	// Prometheus histograms for requests to the querier.
 	querierRequestDuration := promauto.With(prometheus.DefaultRegisterer).NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: "cortex",
@@ -179,7 +182,7 @@ func (t *Cortex) initQuerier() (serv services.Service, err error) {
 
 	// if we are not configured for single binary mode then the querier needs to register its paths externally
 	registerExternally := t.Cfg.Target != All
-	handler := t.API.RegisterQuerier(t.Queryable, t.Engine, t.Distributor, registerExternally, t.TombstonesLoader, querierRequestDuration)
+	handler := t.API.RegisterQuerier(queryable, engine, t.Distributor, registerExternally, t.TombstonesLoader, querierRequestDuration)
 
 	// single binary mode requires a properly configured worker.  if the operator did not attempt to configure the
 	//  worker we will attempt an automatic configuration here
@@ -199,15 +202,14 @@ func (t *Cortex) initQuerier() (serv services.Service, err error) {
 	return worker, nil
 }
 
-func (t *Cortex) initQueryable() (services.Service, error) {
+func (t *Cortex) initStoreQueryables() (services.Service, error) {
 	var servs []services.Service
-	var storeQueryables []querier.QueryableWithFilter
 
 	//nolint:golint // I prefer this form over removing 'else', because it allows q to have smaller scope.
 	if q, err := initQueryableForEngine(t.Cfg.Storage.Engine, t.Cfg, t.Store, prometheus.DefaultRegisterer); err != nil {
 		return nil, fmt.Errorf("failed to initialize querier for engine '%s': %v", t.Cfg.Storage.Engine, err)
 	} else {
-		storeQueryables = append(storeQueryables, querier.UseAlwaysQueryable(q))
+		t.StoreQueryables = append(t.StoreQueryables, querier.UseAlwaysQueryable(q))
 		if s, ok := q.(services.Service); ok {
 			servs = append(servs, s)
 		}
@@ -223,17 +225,13 @@ func (t *Cortex) initQueryable() (services.Service, error) {
 			return nil, fmt.Errorf("failed to initialize querier for engine '%s': %v", t.Cfg.Querier.SecondStoreEngine, err)
 		}
 
-		storeQueryables = append(storeQueryables, querier.UseBeforeTimestampQueryable(sq, time.Time(t.Cfg.Querier.UseSecondStoreBeforeTime)))
+		t.StoreQueryables = append(t.StoreQueryables, querier.UseBeforeTimestampQueryable(sq, time.Time(t.Cfg.Querier.UseSecondStoreBeforeTime)))
 
 		if s, ok := sq.(services.Service); ok {
 			servs = append(servs, s)
 		}
 	}
 
-	queryable, engine := querier.New(t.Cfg.Querier, t.Overrides, t.Distributor, storeQueryables, t.TombstonesLoader, prometheus.DefaultRegisterer)
-	t.Queryable = queryable
-	t.Engine = engine
-
 	// Return service, if any.
 	switch len(servs) {
 	case 0:
@@ -454,8 +452,10 @@ func (t *Cortex) initTableManager() (services.Service, error) {
 func (t *Cortex) initRuler() (serv services.Service, err error) {
 	t.Cfg.Ruler.Ring.ListenPort = t.Cfg.Server.GRPCListenPort
 	t.Cfg.Ruler.Ring.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
+	rulerRegisterer := prometheus.WrapRegistererWith(prometheus.Labels{"engine": "ruler"}, prometheus.DefaultRegisterer)
+	queryable, engine := querier.New(t.Cfg.Querier, t.Overrides, t.Distributor, t.StoreQueryables, t.TombstonesLoader, rulerRegisterer)
 
-	t.Ruler, err = ruler.NewRuler(t.Cfg.Ruler, t.Engine, t.Queryable, t.Distributor, prometheus.DefaultRegisterer, util.Logger)
+	t.Ruler, err = ruler.NewRuler(t.Cfg.Ruler, engine, queryable, t.Distributor, prometheus.DefaultRegisterer, util.Logger)
 	if err != nil {
 		return
 	}
@@ -569,7 +569,7 @@ func (t *Cortex) setupModuleManager() error {
 	mm.RegisterModule(Ingester, t.initIngester)
 	mm.RegisterModule(Flusher, t.initFlusher)
 	mm.RegisterModule(Querier, t.initQuerier)
-	mm.RegisterModule(Queryable, t.initQueryable, modules.UserInvisibleModule)
+	mm.RegisterModule(StoreQueryable, t.initStoreQueryables, modules.UserInvisibleModule)
 	mm.RegisterModule(QueryFrontend, t.initQueryFrontend)
 	mm.RegisterModule(TableManager, t.initTableManager)
 	mm.RegisterModule(Ruler, t.initRuler)
@@ -582,24 +582,24 @@ func (t *Cortex) setupModuleManager() error {
 
 	// Add dependencies
 	deps := map[string][]string{
-		API:           {Server},
-		Ring:          {API, RuntimeConfig, MemberlistKV},
-		Overrides:     {RuntimeConfig},
-		Distributor:   {Ring, API, Overrides},
-		Store:         {Overrides, DeleteRequestsStore},
-		Ingester:      {Overrides, Store, API, RuntimeConfig, MemberlistKV},
-		Flusher:       {Store, API},
-		Querier:       {Overrides, Ring, API, Queryable},
-		Queryable:     {Store, Distributor},
-		QueryFrontend: {API, Overrides, DeleteRequestsStore},
-		TableManager:  {API},
-		Ruler:         {Overrides, Queryable},
-		Configs:       {API},
-		AlertManager:  {API},
-		Compactor:     {API},
-		StoreGateway:  {API},
-		Purger:        {Store, DeleteRequestsStore, API},
-		All:           {QueryFrontend, Querier, Ingester, Distributor, TableManager, Purger, StoreGateway, Ruler},
+		API:            {Server},
+		Ring:           {API, RuntimeConfig, MemberlistKV},
+		Overrides:      {RuntimeConfig},
+		Distributor:    {Ring, API, Overrides},
+		Store:          {Overrides, DeleteRequestsStore},
+		Ingester:       {Overrides, Store, API, RuntimeConfig, MemberlistKV},
+		Flusher:        {Store, API},
+		Querier:        {Overrides, Distributor, Store, Ring, API, StoreQueryable},
+		StoreQueryable: {Store},
+		QueryFrontend:  {API, Overrides, DeleteRequestsStore},
+		TableManager:   {API},
+		Ruler:          {Overrides, Distributor, Store, StoreQueryable},
+		Configs:        {API},
+		AlertManager:   {API},
+		Compactor:      {API},
+		StoreGateway:   {API},
+		Purger:         {Store, DeleteRequestsStore, API},
+		All:            {QueryFrontend, Querier, Ingester, Distributor, TableManager, Purger, StoreGateway, Ruler},
 	}
 	for mod, targets := range deps {
 		if err := mm.AddDependency(mod, targets...); err != nil {

From ad38df0f2a0c5363c3f99c5b96d31278c28e21a5 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Thu, 9 Jul 2020 13:43:27 -0400
Subject: [PATCH 15/21] Addressed review concerns

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 integration/api_ruler_test.go       |  3 +--
 pkg/ruler/rules/local/local.go      | 20 ++++++++++----------
 pkg/ruler/rules/local/local_test.go |  2 +-
 3 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/integration/api_ruler_test.go b/integration/api_ruler_test.go
index 9c339c12fa0..8ec3dfe0aee 100644
--- a/integration/api_ruler_test.go
+++ b/integration/api_ruler_test.go
@@ -3,7 +3,6 @@
 package main
 
 import (
-	"path"
 	"path/filepath"
 	"testing"
 
@@ -111,7 +110,7 @@ func TestRulerAPISingleBinary(t *testing.T) {
 
 	// Start Cortex components.
 	require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config.yaml", cortexConfigFile))
-	require.NoError(t, writeFileToSharedDir(s, path.Join("ruler_configs", user, namespace), []byte(cortexRulerUserConfigYaml)))
+	require.NoError(t, writeFileToSharedDir(s, filepath.Join("ruler_configs", user, namespace), []byte(cortexRulerUserConfigYaml)))
 	cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex", cortexConfigFile, configOverrides, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex))
 
diff --git a/pkg/ruler/rules/local/local.go b/pkg/ruler/rules/local/local.go
index f2e48a5ff3b..c4c39963547 100644
--- a/pkg/ruler/rules/local/local.go
+++ b/pkg/ruler/rules/local/local.go
@@ -4,7 +4,7 @@ import (
 	"context"
 	"flag"
 	"io/ioutil"
-	"path"
+	"path/filepath"
 
 	"github.com/pkg/errors"
 
@@ -44,7 +44,7 @@ func (l *Client) ListAllRuleGroups(ctx context.Context) (map[string]rules.RuleGr
 	root := l.cfg.Directory
 	infos, err := ioutil.ReadDir(root)
 	if err != nil {
-		return nil, errors.Wrap(err, "unable to read dir "+root)
+		return nil, errors.Wrapf(err, "unable to read dir %s", root)
 	}
 
 	for _, info := range infos {
@@ -54,7 +54,7 @@ func (l *Client) ListAllRuleGroups(ctx context.Context) (map[string]rules.RuleGr
 
 		list, err := l.listAllRulesGroupsForUser(ctx, info.Name())
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to list rule group")
+			return nil, errors.Wrapf(err, "failed to list rule groups for user %s", info.Name())
 		}
 
 		lists[info.Name()] = list
@@ -90,10 +90,10 @@ func (l *Client) DeleteRuleGroup(ctx context.Context, userID, namespace string,
 func (l *Client) listAllRulesGroupsForUser(ctx context.Context, userID string) (rules.RuleGroupList, error) {
 	var allLists rules.RuleGroupList
 
-	root := path.Join(l.cfg.Directory, userID)
+	root := filepath.Join(l.cfg.Directory, userID)
 	infos, err := ioutil.ReadDir(root)
 	if err != nil {
-		return nil, errors.Wrap(err, "unable to read dir "+root)
+		return nil, errors.Wrapf(err, "unable to read dir %s", root)
 	}
 
 	for _, info := range infos {
@@ -101,9 +101,9 @@ func (l *Client) listAllRulesGroupsForUser(ctx context.Context, userID string) (
 			continue
 		}
 
-		list, err := l.ListRuleGroups(ctx, userID, info.Name())
+		list, err := l.listAllRulesGroupsForUserAndNamespace(ctx, userID, info.Name())
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to list rule group")
+			return nil, errors.Wrapf(err, "failed to list rule group for user %s and namespace %s", userID, info.Name())
 		}
 
 		allLists = append(allLists, list...)
@@ -113,16 +113,16 @@ func (l *Client) listAllRulesGroupsForUser(ctx context.Context, userID string) (
 }
 
 func (l *Client) listAllRulesGroupsForUserAndNamespace(ctx context.Context, userID string, namespace string) (rules.RuleGroupList, error) {
-	filename := path.Join(l.cfg.Directory, userID, namespace)
+	filename := filepath.Join(l.cfg.Directory, userID, namespace)
 
 	rulegroups, allErrors := rulefmt.ParseFile(filename)
 	if len(allErrors) > 0 {
-		return nil, errors.Wrap(allErrors[0], "error parsing "+filename)
+		return nil, errors.Wrapf(allErrors[0], "error parsing %s", filename)
 	}
 
 	allErrors = rulegroups.Validate()
 	if len(allErrors) > 0 {
-		return nil, errors.Wrap(allErrors[0], "error validating "+filename)
+		return nil, errors.Wrapf(allErrors[0], "error validating %s", filename)
 	}
 
 	var list rules.RuleGroupList
diff --git a/pkg/ruler/rules/local/local_test.go b/pkg/ruler/rules/local/local_test.go
index 630237397e0..202942bf0d5 100644
--- a/pkg/ruler/rules/local/local_test.go
+++ b/pkg/ruler/rules/local/local_test.go
@@ -16,7 +16,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/ruler/rules"
 )
 
-func TestLoadingRules(t *testing.T) {
+func TestClient_ListAllRuleGroups(t *testing.T) {
 	user := "user"
 	namespace := "ns"
 

From e3ded38489eb5a08e42faa3de24961722e45fa03 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Thu, 9 Jul 2020 14:17:36 -0400
Subject: [PATCH 16/21] removed 2s poll interval from standard configs and
 moved to test

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 .../single-process-config-blocks-gossip-1.yaml         |  1 -
 .../single-process-config-blocks-gossip-2.yaml         |  1 -
 .../single-process-config-blocks-tls.yaml              |  1 -
 docs/configuration/single-process-config-blocks.yaml   |  1 -
 docs/configuration/single-process-config.md            | 10 ++++++++++
 docs/configuration/single-process-config.yaml          |  1 -
 integration/api_ruler_test.go                          |  1 +
 7 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/docs/configuration/single-process-config-blocks-gossip-1.yaml b/docs/configuration/single-process-config-blocks-gossip-1.yaml
index 92e27c85dc3..892cdf3133b 100644
--- a/docs/configuration/single-process-config-blocks-gossip-1.yaml
+++ b/docs/configuration/single-process-config-blocks-gossip-1.yaml
@@ -85,7 +85,6 @@ frontend_worker:
 ruler:
   enable_api: true
   enable_sharding: true
-  poll_interval: 2s
   storage:
     type: local
     local:
diff --git a/docs/configuration/single-process-config-blocks-gossip-2.yaml b/docs/configuration/single-process-config-blocks-gossip-2.yaml
index fc1f4c8c86c..6f70e23870a 100644
--- a/docs/configuration/single-process-config-blocks-gossip-2.yaml
+++ b/docs/configuration/single-process-config-blocks-gossip-2.yaml
@@ -84,7 +84,6 @@ frontend_worker:
 ruler:
   enable_api: true
   enable_sharding: true
-  poll_interval: 2s
   storage:
     type: local
     local:
diff --git a/docs/configuration/single-process-config-blocks-tls.yaml b/docs/configuration/single-process-config-blocks-tls.yaml
index 5bc03888f89..72f4630b891 100644
--- a/docs/configuration/single-process-config-blocks-tls.yaml
+++ b/docs/configuration/single-process-config-blocks-tls.yaml
@@ -102,7 +102,6 @@ frontend_worker:
 ruler:
   enable_api: true
   enable_sharding: false
-  poll_interval: 2s
   storage:
     type: local
     local:
diff --git a/docs/configuration/single-process-config-blocks.yaml b/docs/configuration/single-process-config-blocks.yaml
index 1abb0c1fd6d..ccc3f64d8e8 100644
--- a/docs/configuration/single-process-config-blocks.yaml
+++ b/docs/configuration/single-process-config-blocks.yaml
@@ -89,7 +89,6 @@ frontend_worker:
 ruler:
   enable_api: true
   enable_sharding: false
-  poll_interval: 2s
   storage:
     type: local
     local:
diff --git a/docs/configuration/single-process-config.md b/docs/configuration/single-process-config.md
index 291d10847b6..08463bfc918 100644
--- a/docs/configuration/single-process-config.md
+++ b/docs/configuration/single-process-config.md
@@ -76,4 +76,14 @@ storage:
 #  to max_concurrent on the queriers.
 frontend_worker:
   match_max_concurrent: true
+
+# Configure the ruler to scan the /tmp/cortex/rules directory for prometheus
+# rules: https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules
+ruler:
+  enable_api: true
+  enable_sharding: false
+  storage:
+    type: local
+    local:
+      directory: /tmp/cortex/rules
 ```
diff --git a/docs/configuration/single-process-config.yaml b/docs/configuration/single-process-config.yaml
index 1363a651dcf..d56294e3273 100644
--- a/docs/configuration/single-process-config.yaml
+++ b/docs/configuration/single-process-config.yaml
@@ -87,7 +87,6 @@ frontend_worker:
 ruler:
   enable_api: true
   enable_sharding: false
-  poll_interval: 2s
   storage:
     type: local
     local:
diff --git a/integration/api_ruler_test.go b/integration/api_ruler_test.go
index 8ec3dfe0aee..0f1e02aa3f3 100644
--- a/integration/api_ruler_test.go
+++ b/integration/api_ruler_test.go
@@ -106,6 +106,7 @@ func TestRulerAPISingleBinary(t *testing.T) {
 
 	configOverrides := map[string]string{
 		"-ruler.storage.local.directory": filepath.Join(e2e.ContainerSharedDir, "ruler_configs"),
+		"-ruler.poll-interval":           "2s",
 	}
 
 	// Start Cortex components.

From fd63adc4eeb408368e742b6bc71836e6e36f804b Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Thu, 9 Jul 2020 14:56:52 -0400
Subject: [PATCH 17/21] Added notes about local storage

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 docs/guides/sharded_ruler.md | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/docs/guides/sharded_ruler.md b/docs/guides/sharded_ruler.md
index e2705c898d4..e246c7901ce 100644
--- a/docs/guides/sharded_ruler.md
+++ b/docs/guides/sharded_ruler.md
@@ -26,3 +26,23 @@ In addition the ruler requires it's own ring to be configured, for instance:
 The only configuration that is required is to enable sharding and configure a key value store. From there the rulers will shard and handle the division of rules automatically.
 
 Unlike ingesters, rulers do not hand over responsibility: all rules are re-sharded randomly every time a ruler is added to or removed from the ring.
+
+## Ruler Storage
+
+The ruler supports six kinds of storage (configdb, azure, gcs, s3, swift, local).  Most kinds of storage work with the sharded ruler configuration in an obvious way.  i.e. Configure all rulers to use the same backend.
+
+The local implementation reads [Prometheus recording rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) off of the local filesystem.  This is a read only backend that does not support the creation and deletion of rules.  Despite the fact that it reads the local filesystem this method can still be used in the a sharded ruler configuration if the operator takes care to load the same rules to every ruler.  For instance this could be accomplished by mounting a [Kubernetes ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) onto every ruler pod.
+
+A typical local config may look something like:
+```
+  -ruler.storage.type=local
+	-ruler.storage.local.directory=/tmp/cortex/rules
+```
+
+With the above configuration the ruler would expect the following layout:
+```
+/tmp/cortex/rules/<tenant id>/rules1.yaml
+                             /rules2.yaml
+```
+Yaml files are expected to be in the [Prometheus format](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules).
+

From 569599d5fe53a16a1e368300c7907a01c89c4c58 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Thu, 9 Jul 2020 15:02:42 -0400
Subject: [PATCH 18/21] Clarified creation/deletion

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 docs/guides/sharded_ruler.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/guides/sharded_ruler.md b/docs/guides/sharded_ruler.md
index e246c7901ce..252be171bbb 100644
--- a/docs/guides/sharded_ruler.md
+++ b/docs/guides/sharded_ruler.md
@@ -31,7 +31,7 @@ Unlike ingesters, rulers do not hand over responsibility: all rules are re-shard
 
 The ruler supports six kinds of storage (configdb, azure, gcs, s3, swift, local).  Most kinds of storage work with the sharded ruler configuration in an obvious way.  i.e. Configure all rulers to use the same backend.
 
-The local implementation reads [Prometheus recording rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) off of the local filesystem.  This is a read only backend that does not support the creation and deletion of rules.  Despite the fact that it reads the local filesystem this method can still be used in the a sharded ruler configuration if the operator takes care to load the same rules to every ruler.  For instance this could be accomplished by mounting a [Kubernetes ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) onto every ruler pod.
+The local implementation reads [Prometheus recording rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) off of the local filesystem.  This is a read only backend that does not support the creation and deletion of rules through [the API](https://cortexmetrics.io/docs/apis/#ruler).  Despite the fact that it reads the local filesystem this method can still be used in a sharded ruler configuration if the operator takes care to load the same rules to every ruler.  For instance this could be accomplished by mounting a [Kubernetes ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) onto every ruler pod.
 
 A typical local config may look something like:
 ```

From d0e125414e9db69910a9c8f35a4e0f21959250f7 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Tue, 14 Jul 2020 09:58:13 -0400
Subject: [PATCH 19/21] readme cleanup

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 docs/guides/sharded_ruler.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/guides/sharded_ruler.md b/docs/guides/sharded_ruler.md
index 252be171bbb..5158222c6ee 100644
--- a/docs/guides/sharded_ruler.md
+++ b/docs/guides/sharded_ruler.md
@@ -29,14 +29,14 @@ Unlike ingesters, rulers do not hand over responsibility: all rules are re-shard
 
 ## Ruler Storage
 
-The ruler supports six kinds of storage (configdb, azure, gcs, s3, swift, local).  Most kinds of storage work with the sharded ruler configuration in an obvious way.  i.e. Configure all rulers to use the same backend.
+The ruler supports six kinds of storage (configdb, azure, gcs, s3, swift, local).  Most kinds of storage work with the sharded ruler configuration in an obvious way.  i.e. configure all rulers to use the same backend.
 
 The local implementation reads [Prometheus recording rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) off of the local filesystem.  This is a read only backend that does not support the creation and deletion of rules through [the API](https://cortexmetrics.io/docs/apis/#ruler).  Despite the fact that it reads the local filesystem this method can still be used in a sharded ruler configuration if the operator takes care to load the same rules to every ruler.  For instance this could be accomplished by mounting a [Kubernetes ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) onto every ruler pod.
 
 A typical local config may look something like:
 ```
   -ruler.storage.type=local
-	-ruler.storage.local.directory=/tmp/cortex/rules
+  -ruler.storage.local.directory=/tmp/cortex/rules
 ```
 
 With the above configuration the ruler would expect the following layout:

From 2eab3d7713e2ffc6c16b487987b755b351869c67 Mon Sep 17 00:00:00 2001
From: Joe Elliott <number101010@gmail.com>
Date: Tue, 14 Jul 2020 11:27:49 -0400
Subject: [PATCH 20/21] Removed ruler config from integration tests.  Allow
 single binary to start without a ruler config

Signed-off-by: Joe Elliott <number101010@gmail.com>
---
 integration/e2ecortex/services.go | 3 ---
 pkg/cortex/modules.go             | 9 +++++++++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/integration/e2ecortex/services.go b/integration/e2ecortex/services.go
index 00c03d818bc..c22afcab962 100644
--- a/integration/e2ecortex/services.go
+++ b/integration/e2ecortex/services.go
@@ -237,9 +237,6 @@ func NewSingleBinary(name string, flags map[string]string, image string, otherPo
 			"-ingester.concurrent-flushes":   "10",
 			"-ingester.max-transfer-retries": "10",
 			"-ingester.num-tokens":           "512",
-			// Ruler
-			"-ruler.storage.type":            "local",
-			"-ruler.storage.local.directory": "/tmp/cortex/rules",
 		}, flags))...),
 		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299),
 		httpPort,
diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go
index 1ea91ed85ef..efbf2b15836 100644
--- a/pkg/cortex/modules.go
+++ b/pkg/cortex/modules.go
@@ -450,6 +450,15 @@ func (t *Cortex) initTableManager() (services.Service, error) {
 }
 
 func (t *Cortex) initRuler() (serv services.Service, err error) {
+	// if the ruler is not configured and we're in single binary then let's just log an error and continue
+	// unfortunately there is no way to generate a "default" config and compare default against actual
+	// to determine if it's unconfigured.  the following check, however, correctly tests this.
+	// Single binary integration tests will break if this ever drifts
+	if t.Cfg.Target == All && t.Cfg.Ruler.StoreConfig.Type == "configdb" && t.Cfg.Ruler.StoreConfig.ConfigDB.ConfigsAPIURL.URL == nil {
+		level.Info(util.Logger).Log("msg", "Ruler is not configured in single binary mode and will not be started.")
+		return nil, nil
+	}
+
 	t.Cfg.Ruler.Ring.ListenPort = t.Cfg.Server.GRPCListenPort
 	t.Cfg.Ruler.Ring.KVStore.MemberlistKV = t.MemberlistKV.GetMemberlistKV
 	rulerRegisterer := prometheus.WrapRegistererWith(prometheus.Labels{"engine": "ruler"}, prometheus.DefaultRegisterer)

From 41a65f51ae094d3f5c1a060015c3389945a0988e Mon Sep 17 00:00:00 2001
From: Marco Pracucci <marco@pracucci.com>
Date: Thu, 16 Jul 2020 16:54:16 +0200
Subject: [PATCH 21/21] Reworked the CHANGELOG

Signed-off-by: Marco Pracucci <marco@pracucci.com>
---
 CHANGELOG.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9f7b4f19dee..4c5e194a03a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,9 +13,12 @@
 * [CHANGE] Ingester: Chunks flushed via /flush stay in memory until retention period is reached. This affects `cortex_ingester_memory_chunks` metric. #2778
 * [CHANGE] Querier: the error message returned when the query time range exceeds `-store.max-query-length` has changed from `invalid query, length > limit (X > Y)` to `the query time range exceeds the limit (query length: X, limit: Y)`. #2826
 * [CHANGE] KV: The `role` label which was a label of `multi` KV store client only has been added to metrics of every KV store client. If KV store client is not `multi`, then the value of `role` label is `primary`. #2837
+* [CHANGE] Added the `engine` label to the metrics exposed by the Prometheus query engine, to distinguish between `ruler` and `querier` metrics. #2854
+* [CHANGE] Added ruler to the single binary when started with `-target=all` (default). #2854
 * [FEATURE] Introduced `ruler.for-outage-tolerance`, Max time to tolerate outage for restoring "for" state of alert. #2783
 * [FEATURE] Introduced `ruler.for-grace-period`, Minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period. #2783
 * [FEATURE] Introduced `ruler.resend-delay`, Minimum amount of time to wait before resending an alert to Alertmanager. #2783
+* [FEATURE] Ruler: added `local` filesystem support to store rules (read-only). #2854
 * [ENHANCEMENT] Upgraded Docker base images to `alpine:3.12`. #2862
 * [ENHANCEMENT] Experimental: Querier can now optionally query secondary store. This is specified by using `-querier.second-store-engine` option, with values `chunks` or `tsdb`. Standard configuration options for this store are used. Additionally, this querying can be configured to happen only for queries that need data older than `-querier.use-second-store-before-time`. Default value of zero will always query secondary store. #2747
 * [ENHANCEMENT] Query-tee: increased the `cortex_querytee_request_duration_seconds` metric buckets granularity. #2799
@@ -39,8 +42,6 @@
 * [ENHANCEMENT] Experimental TSDB: Added support to enforce max query time range length via `-store.max-query-length`. #2826
 * [ENHANCEMENT] Ingester: Added new metric `cortex_ingester_flush_series_in_progress` that reports number of ongoing flush-series operations. Useful when calling `/flush` handler: if `cortex_ingester_flush_queue_length + cortex_ingester_flush_series_in_progress` is 0, all flushes are finished. #2778
 * [ENHANCEMENT] Memberlist members can join cluster via SRV records. #2788
-* [ENHANCEMENT] Added ruler to the single binary.  #2854
-  * Two sets of Prometheus query engine metrics now appear in the single binary.  The `engine` label has been added to distinguish between `ruler` and `querier` metrics.
 * [ENHANCEMENT] Added configuration options for chunks s3 client. #2831
   * `s3.endpoint`
   * `s3.region`