Report metrics from custom-plugin-monitor

Author: Xuewei Zhang

config/custom-plugin-monitor.json

@@ -8,6 +8,7 @@
     "enable_message_change_based_condition_update": false
   },
   "source": "ntp-custom-plugin-monitor",
+  "metricsReporting": true,
   "conditions": [
     {
       "type": "NTPProblem",

config/kernel-monitor-counter.json

@@ -7,6 +7,7 @@
     "concurrency": 1
   },
   "source": "kernel-monitor",
+  "metricsReporting": true,
   "conditions": [
     {
       "type": "FrequentUnregisterNetDevice",

config/network-problem-monitor.json

@@ -7,6 +7,7 @@
     "concurrency": 3
   },
   "source": "network-custom-plugin-monitor",
+  "metricsReporting": true,
   "conditions": [],
   "rules": [
     {

config/systemd-monitor-counter.json

@@ -7,6 +7,7 @@
     "concurrency": 1
   },
   "source": "systemd-monitor",
+  "metricsReporting": true,
   "conditions": [
     {
       "type": "FrequentKubeletRestart",

pkg/custompluginmonitor/custom_plugin_monitor.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/node-problem-detector/pkg/custompluginmonitor/plugin"
 	cpmtypes "k8s.io/node-problem-detector/pkg/custompluginmonitor/types"
 	"k8s.io/node-problem-detector/pkg/problemdaemon"
+	"k8s.io/node-problem-detector/pkg/problemmetrics"
 	"k8s.io/node-problem-detector/pkg/types"
 	"k8s.io/node-problem-detector/pkg/util"
 	"k8s.io/node-problem-detector/pkg/util/tomb"
@@ -80,9 +81,31 @@ func NewCustomPluginMonitorOrDie(configPath string) types.Monitor {
 	c.plugin = plugin.NewPlugin(c.config)
 	// A 1000 size channel should be big enough.
 	c.statusChan = make(chan *types.Status, 1000)
+
+	if *c.config.EnableMetricsReporting {
+		initializeProblemMetricsOrDie(c.config.Rules)
+	}
 	return c
 }
 
+// initializeProblemMetricsOrDie creates problem metrics for all problems and set the value to 0,
+// panic if error occurs.
+func initializeProblemMetricsOrDie(rules []*cpmtypes.CustomRule) {
+	for _, rule := range rules {
+		if rule.Type == types.Perm {
+			err := problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(rule.Condition, rule.Reason, false)
+			if err != nil {
+				glog.Fatalf("Failed to initialize problem gauge metrics for problem %q, reason %q: %v",
+					rule.Condition, rule.Reason, err)
+			}
+		}
+		err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(rule.Reason, 0)
+		if err != nil {
+			glog.Fatalf("Failed to initialize problem counter metrics for %q: %v", rule.Reason, err)
+		}
+	}
+}
+
 func (c *customPluginMonitor) Start() (<-chan *types.Status, error) {
 	glog.Info("Start custom plugin monitor")
 	go c.plugin.Run()
@@ -130,6 +153,13 @@ func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Stat
 				Reason:    result.Rule.Reason,
 				Message:   result.Message,
 			})
+
+			if *c.config.EnableMetricsReporting {
+				err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(result.Rule.Reason, 1)
+				if err != nil {
+					glog.Errorf("Failed to update problem counter metrics for %q: %v", result.Rule.Reason, err)
+				}
+			}
 		}
 	} else {
 		// For permanent error changes the condition
@@ -161,6 +191,14 @@ func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Stat
 					condition.Status = status
 					condition.Message = defaultConditionMessage
 					condition.Reason = defaultConditionReason
+
+					if *c.config.EnableMetricsReporting {
+						err := problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(result.Rule.Condition, result.Rule.Reason, false)
+						if err != nil {
+							glog.Errorf("Failed to update problem gauge metrics for problem %q, reason %q: %v",
+								result.Rule.Condition, result.Rule.Reason, err)
+						}
+					}
 				} else if condition.Status != types.True && status == types.True {
 					// change 2: Condition status change from False/Unknown to True
 					condition.Transition = timestamp
@@ -174,6 +212,18 @@ func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Stat
 
 					condition.Status = status
 					condition.Reason = result.Rule.Reason
+
+					if *c.config.EnableMetricsReporting {
+						err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(result.Rule.Reason, 1)
+						if err != nil {
+							glog.Errorf("Failed to update problem counter metrics for %q: %v", result.Rule.Reason, err)
+						}
+						err = problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(result.Rule.Condition, result.Rule.Reason, true)
+						if err != nil {
+							glog.Errorf("Failed to update problem gauge metrics for problem %q, reason %q: %v",
+								result.Rule.Condition, result.Rule.Reason, err)
+						}
+					}
 				} else if condition.Status != status {
 					// change 3: Condition status change from False to Unknown or vice versa
 					condition.Transition = timestamp
@@ -202,6 +252,19 @@ func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Stat
 						condition.Reason,
 						timestamp,
 					))
+
+					// update metrics only when condition reason changes.
+					if condition.Reason != result.Rule.Reason && *c.config.EnableMetricsReporting {
+						err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(result.Rule.Reason, 1)
+						if err != nil {
+							glog.Errorf("Failed to update problem counter metrics for %q: %v", result.Rule.Reason, err)
+						}
+						err = problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(result.Rule.Condition, result.Rule.Reason, true)
+						if err != nil {
+							glog.Errorf("Failed to update problem gauge metrics for problem %q, reason %q: %v",
+								result.Rule.Condition, result.Rule.Reason, err)
+						}
+					}
 				}
 
 				break

pkg/custompluginmonitor/types/config.go

@@ -32,6 +32,7 @@ var (
 	defaultMaxOutputLength                   = 80
 	defaultConcurrency                       = 3
 	defaultMessageChangeBasedConditionUpdate = false
+	defaultEnableMetricsReporting            = true
 
 	customPluginName = "custom"
 )
@@ -66,6 +67,8 @@ type CustomPluginConfig struct {
 	DefaultConditions []types.Condition `json:"conditions"`
 	// Rules are the rules custom plugin monitor will follow to parse and invoke plugins.
 	Rules []*CustomRule `json:"rules"`
+	// EnableMetricsReporting describes whether to report problems as metrics or not.
+	EnableMetricsReporting *bool `json:"metricsReporting,omitempty"`
 }
 
 // ApplyConfiguration applies default configurations.
@@ -112,6 +115,10 @@ func (cpc *CustomPluginConfig) ApplyConfiguration() error {
 		}
 	}
 
+	if cpc.EnableMetricsReporting == nil {
+		cpc.EnableMetricsReporting = &defaultEnableMetricsReporting
+	}
+
 	return nil
 }