Report metrics from custom-plugin-monitor

Xuewei Zhang · Xuewei Zhang · commit b1d33da49bfd · 2019-07-24T16:20:18.000-07:00
diff --git a/config/custom-plugin-monitor.json b/config/custom-plugin-monitor.json
@@ -8,6 +8,7 @@
     "enable_message_change_based_condition_update": false
   },
   "source": "ntp-custom-plugin-monitor",
+  "metricsReporting": true,
   "conditions": [
     {
       "type": "NTPProblem",
diff --git a/config/kernel-monitor-counter.json b/config/kernel-monitor-counter.json
@@ -7,6 +7,7 @@
     "concurrency": 1
   },
   "source": "kernel-monitor",
+  "metricsReporting": true,
   "conditions": [
     {
       "type": "FrequentUnregisterNetDevice",
diff --git a/config/network-problem-monitor.json b/config/network-problem-monitor.json
@@ -7,6 +7,7 @@
     "concurrency": 3
   },
   "source": "network-custom-plugin-monitor",
+  "metricsReporting": true,
   "conditions": [],
   "rules": [
     {
diff --git a/config/systemd-monitor-counter.json b/config/systemd-monitor-counter.json
@@ -7,6 +7,7 @@
     "concurrency": 1
   },
   "source": "systemd-monitor",
+  "metricsReporting": true,
   "conditions": [
     {
       "type": "FrequentKubeletRestart",
diff --git a/pkg/custompluginmonitor/custom_plugin_monitor.go b/pkg/custompluginmonitor/custom_plugin_monitor.go
@@ -26,6 +26,7 @@ import (
 	"k8s.io/node-problem-detector/pkg/custompluginmonitor/plugin"
 	cpmtypes "k8s.io/node-problem-detector/pkg/custompluginmonitor/types"
 	"k8s.io/node-problem-detector/pkg/problemdaemon"
+	"k8s.io/node-problem-detector/pkg/problemmetrics"
 	"k8s.io/node-problem-detector/pkg/types"
 	"k8s.io/node-problem-detector/pkg/util"
 	"k8s.io/node-problem-detector/pkg/util/tomb"
@@ -80,9 +81,31 @@ func NewCustomPluginMonitorOrDie(configPath string) types.Monitor {
 	c.plugin = plugin.NewPlugin(c.config)
 	// A 1000 size channel should be big enough.
 	c.statusChan = make(chan *types.Status, 1000)
+
+	if *c.config.EnableMetricsReporting {
+		initializeProblemMetricsOrDie(c.config.Rules)
+	}
 	return c
 }
 
+// initializeProblemMetricsOrDie creates problem metrics for all problems and set the value to 0,
+// panic if error occurs.
+func initializeProblemMetricsOrDie(rules []*cpmtypes.CustomRule) {
+	for _, rule := range rules {
+		if rule.Type == types.Perm {
+			err := problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(rule.Condition, rule.Reason, false)
+			if err != nil {
+				glog.Fatalf("Failed to initialize problem gauge metrics for problem %q, reason %q: %v",
+					rule.Condition, rule.Reason, err)
+			}
+		}
+		err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(rule.Reason, 0)
+		if err != nil {
+			glog.Fatalf("Failed to initialize problem counter metrics for %q: %v", rule.Reason, err)
+		}
+	}
+}
+
 func (c *customPluginMonitor) Start() (<-chan *types.Status, error) {
 	glog.Info("Start custom plugin monitor")
 	go c.plugin.Run()
@@ -130,6 +153,13 @@ func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Stat
 				Reason:    result.Rule.Reason,
 				Message:   result.Message,
 			})
+
+			if *c.config.EnableMetricsReporting {
+				err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(result.Rule.Reason, 1)
+				if err != nil {
+					glog.Errorf("Failed to update problem counter metrics for %q: %v", result.Rule.Reason, err)
+				}
+			}
 		}
 	} else {
 		// For permanent error changes the condition
@@ -161,6 +191,14 @@ func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Stat
 					condition.Status = status
 					condition.Message = defaultConditionMessage
 					condition.Reason = defaultConditionReason
+
+					if *c.config.EnableMetricsReporting {
+						err := problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(result.Rule.Condition, result.Rule.Reason, false)
+						if err != nil {
+							glog.Errorf("Failed to update problem gauge metrics for problem %q, reason %q: %v",
+								result.Rule.Condition, result.Rule.Reason, err)
+						}
+					}
 				} else if condition.Status != types.True && status == types.True {
 					// change 2: Condition status change from False/Unknown to True
 					condition.Transition = timestamp
@@ -174,6 +212,18 @@ func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Stat
 
 					condition.Status = status
 					condition.Reason = result.Rule.Reason
+
+					if *c.config.EnableMetricsReporting {
+						err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(result.Rule.Reason, 1)
+						if err != nil {
+							glog.Errorf("Failed to update problem counter metrics for %q: %v", result.Rule.Reason, err)
+						}
+						err = problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(result.Rule.Condition, result.Rule.Reason, true)
+						if err != nil {
+							glog.Errorf("Failed to update problem gauge metrics for problem %q, reason %q: %v",
+								result.Rule.Condition, result.Rule.Reason, err)
+						}
+					}
 				} else if condition.Status != status {
 					// change 3: Condition status change from False to Unknown or vice versa
 					condition.Transition = timestamp
@@ -202,6 +252,19 @@ func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Stat
 						condition.Reason,
 						timestamp,
 					))
+
+					// update metrics only when condition reason changes.
+					if condition.Reason != result.Rule.Reason && *c.config.EnableMetricsReporting {
+						err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(result.Rule.Reason, 1)
+						if err != nil {
+							glog.Errorf("Failed to update problem counter metrics for %q: %v", result.Rule.Reason, err)
+						}
+						err = problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(result.Rule.Condition, result.Rule.Reason, true)
+						if err != nil {
+							glog.Errorf("Failed to update problem gauge metrics for problem %q, reason %q: %v",
+								result.Rule.Condition, result.Rule.Reason, err)
+						}
+					}
 				}
 
 				break
diff --git a/pkg/custompluginmonitor/types/config.go b/pkg/custompluginmonitor/types/config.go
@@ -32,6 +32,7 @@ var (
 	defaultMaxOutputLength                   = 80
 	defaultConcurrency                       = 3
 	defaultMessageChangeBasedConditionUpdate = false
+	defaultEnableMetricsReporting            = true
 
 	customPluginName = "custom"
 )
@@ -66,6 +67,8 @@ type CustomPluginConfig struct {
 	DefaultConditions []types.Condition `json:"conditions"`
 	// Rules are the rules custom plugin monitor will follow to parse and invoke plugins.
 	Rules []*CustomRule `json:"rules"`
+	// EnableMetricsReporting describes whether to report problems as metrics or not.
+	EnableMetricsReporting *bool `json:"metricsReporting,omitempty"`
 }
 
 // ApplyConfiguration applies default configurations.
@@ -112,6 +115,10 @@ func (cpc *CustomPluginConfig) ApplyConfiguration() error {
 		}
 	}
 
+	if cpc.EnableMetricsReporting == nil {
+		cpc.EnableMetricsReporting = &defaultEnableMetricsReporting
+	}
+
 	return nil
 }
 
diff --git a/pkg/custompluginmonitor/types/config_test.go b/pkg/custompluginmonitor/types/config_test.go
@@ -30,6 +30,7 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 	maxOutputLength := 79
 	concurrency := 2
 	messageChangeBasedConditionUpdate := true
+	disableMetricsReporting := false
 
 	ruleTimeout := 1 * time.Second
 	ruleTimeoutString := ruleTimeout.String()
@@ -60,6 +61,7 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 					Concurrency:                             &defaultConcurrency,
 					EnableMessageChangeBasedConditionUpdate: &defaultMessageChangeBasedConditionUpdate,
 				},
+				EnableMetricsReporting: &defaultEnableMetricsReporting,
 				Rules: []*CustomRule{
 					{
 						Path: "../plugin/test-data/ok.sh",
@@ -88,6 +90,7 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 					Concurrency:                             &defaultConcurrency,
 					EnableMessageChangeBasedConditionUpdate: &defaultMessageChangeBasedConditionUpdate,
 				},
+				EnableMetricsReporting: &defaultEnableMetricsReporting,
 			},
 		},
 		"custom default timeout": {
@@ -106,6 +109,7 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 					Concurrency:                             &defaultConcurrency,
 					EnableMessageChangeBasedConditionUpdate: &defaultMessageChangeBasedConditionUpdate,
 				},
+				EnableMetricsReporting: &defaultEnableMetricsReporting,
 			},
 		},
 		"custom max output length": {
@@ -124,6 +128,7 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 					Concurrency:                             &defaultConcurrency,
 					EnableMessageChangeBasedConditionUpdate: &defaultMessageChangeBasedConditionUpdate,
 				},
+				EnableMetricsReporting: &defaultEnableMetricsReporting,
 			},
 		},
 		"custom concurrency": {
@@ -142,6 +147,7 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 					Concurrency:                             &concurrency,
 					EnableMessageChangeBasedConditionUpdate: &defaultMessageChangeBasedConditionUpdate,
 				},
+				EnableMetricsReporting: &defaultEnableMetricsReporting,
 			},
 		},
 		"custom message change based condition update": {
@@ -160,6 +166,24 @@ func TestCustomPluginConfigApplyConfiguration(t *testing.T) {
 					Concurrency:                             &defaultConcurrency,
 					EnableMessageChangeBasedConditionUpdate: &messageChangeBasedConditionUpdate,
 				},
+				EnableMetricsReporting: &defaultEnableMetricsReporting,
+			},
+		},
+		"disable metrics reporting": {
+			Orig: CustomPluginConfig{
+				EnableMetricsReporting: &disableMetricsReporting,
+			},
+			Wanted: CustomPluginConfig{
+				PluginGlobalConfig: pluginGlobalConfig{
+					InvokeIntervalString:                    &defaultInvokeIntervalString,
+					InvokeInterval:                          &defaultInvokeInterval,
+					TimeoutString:                           &defaultGlobalTimeoutString,
+					Timeout:                                 &defaultGlobalTimeout,
+					MaxOutputLength:                         &defaultMaxOutputLength,
+					Concurrency:                             &defaultConcurrency,
+					EnableMessageChangeBasedConditionUpdate: &defaultMessageChangeBasedConditionUpdate,
+				},
+				EnableMetricsReporting: &disableMetricsReporting,
 			},
 		},
 	}

Original file line number	Diff line number	Diff line change
`@@ -8,6 +8,7 @@`
`8`	`8`	`"enable_message_change_based_condition_update": false`
`9`	`9`	`},`
`10`	`10`	`"source": "ntp-custom-plugin-monitor",`
	`11`	`+ "metricsReporting": true,`
`11`	`12`	`"conditions": [`
`12`	`13`	`{`
`13`	`14`	`"type": "NTPProblem",`
Original file line number	Diff line number	Diff line change
`@@ -7,6 +7,7 @@`
`7`	`7`	`"concurrency": 1`
`8`	`8`	`},`
`9`	`9`	`"source": "kernel-monitor",`
	`10`	`+ "metricsReporting": true,`
`10`	`11`	`"conditions": [`
`11`	`12`	`{`
`12`	`13`	`"type": "FrequentUnregisterNetDevice",`
Original file line number	Diff line number	Diff line change
`@@ -7,6 +7,7 @@`
`7`	`7`	`"concurrency": 3`
`8`	`8`	`},`
`9`	`9`	`"source": "network-custom-plugin-monitor",`
	`10`	`+ "metricsReporting": true,`
`10`	`11`	`"conditions": [],`
`11`	`12`	`"rules": [`
`12`	`13`	`{`
Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,7 @@ var (`
`32`	`32`	`defaultMaxOutputLength = 80`
`33`	`33`	`defaultConcurrency = 3`
`34`	`34`	`defaultMessageChangeBasedConditionUpdate = false`
	`35`	`+ defaultEnableMetricsReporting = true`
`35`	`36`
`36`	`37`	`customPluginName = "custom"`
`37`	`38`	`)`
`@@ -66,6 +67,8 @@ type CustomPluginConfig struct {`
`66`	`67`	DefaultConditions []types.Condition `json:"conditions"`
`67`	`68`	`// Rules are the rules custom plugin monitor will follow to parse and invoke plugins.`
`68`	`69`	Rules []*CustomRule `json:"rules"`
	`70`	`+ // EnableMetricsReporting describes whether to report problems as metrics or not.`
	`71`	+ EnableMetricsReporting *bool `json:"metricsReporting,omitempty"`
`69`	`72`	`}`
`70`	`73`
`71`	`74`	`// ApplyConfiguration applies default configurations.`
`@@ -112,6 +115,10 @@ func (cpc *CustomPluginConfig) ApplyConfiguration() error {`
`112`	`115`	`}`
`113`	`116`	`}`
`114`	`117`
	`118`	`+ if cpc.EnableMetricsReporting == nil {`
	`119`	`+ cpc.EnableMetricsReporting = &defaultEnableMetricsReporting`
	`120`	`+ }`
	`121`	`+`
`115`	`122`	`return nil`
`116`	`123`	`}`
`117`	`124`