test commit

Author: jason1028kr

.gitignore

@@ -7,3 +7,4 @@ junit*.xml
 debug.test
 /output/
 coverage.out
+.idea

Dockerfile

@@ -27,5 +27,8 @@ COPY ./bin/node-problem-detector /node-problem-detector
 ARG LOGCOUNTER
 COPY ./bin/health-checker ${LOGCOUNTER} /home/kubernetes/bin/
 
+COPY ./config/test_condition_fail.sh /home/kubernetes/scripts/
+COPY ./config/test_condition_succeed.sh /home/kubernetes/scripts/
+
 COPY config /config
 ENTRYPOINT ["/node-problem-detector", "--config.system-log-monitor=/config/kernel-monitor.json"]

Makefile

@@ -32,7 +32,8 @@ VERSION?=$(shell if [ -d .git ]; then echo `git describe --tags --dirty`; else e
 TAG?=$(VERSION)
 
 # REGISTRY is the container registry to push into.
-REGISTRY?=gcr.io/k8s-staging-npd
+#REGISTRY?=gcr.io/k8s-staging-npd
+REGISTRY?=jasonjungacr.azurecr.io/k8s-staging-npd
 
 # UPLOAD_PATH is the cloud storage path to upload release tar.
 UPLOAD_PATH?=gs://kubernetes-release
@@ -62,7 +63,7 @@ IMAGE:=$(REGISTRY)/node-problem-detector:$(TAG)
 
 # ENABLE_JOURNALD enables build journald support or not. Building journald
 # support needs libsystemd-dev or libsystemd-journal-dev.
-ENABLE_JOURNALD?=1
+ENABLE_JOURNALD?=0
 
 ifeq ($(go env GOHOSTOS), darwin)
 ENABLE_JOURNALD=0
@@ -260,7 +261,6 @@ build-in-docker: clean docker-builder
 		-c 'cd /gopath/src/k8s.io/node-problem-detector/ && make build-binaries'
 
 push-container: build-container
-	gcloud auth configure-docker
 	docker push $(IMAGE)
 
 push-tar: build-tar

config/custom-plugin-monitor-test.json

@@ -0,0 +1,32 @@
+{
+    "plugin": "custom",
+    "pluginConfig": {
+      "invoke_interval": "60s",
+      "timeout": "60s"
+    },
+    "source": "test-custom-plugin-monitor",
+    "metricsReporting": true,
+    "conditions": [
+      {
+        "type": "TestConditionHasProblem",
+        "reason": "TestIsOK",
+        "message": "test condition is OK"
+      }
+    ],
+    "rules": [
+      {
+        "type": "permanent",
+        "condition": "TestConditionHasProblem",
+        "reason": "TestIsIntentionallyFailing",
+        "path": "./config/plugin/test/test_condition_fail.sh",
+        "timeout": "60s"
+      },
+      {
+        "type": "permanent",
+        "condition": "TestConditionHasProblem",
+        "reason": "TestIsIntentionallySucceeding",
+        "path": "./config/plugin/test/test_condition_succeed.sh",
+        "timeout": "60s"
+      }
+    ]
+  }

config/test_condition_fail.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exit 1

config/test_condition_succeed.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exit 0

deployment/node-problem-detector-config.yaml

@@ -64,25 +64,58 @@ data:
             }
         ]
     }
-  docker-monitor.json: |
+  cpm-test.json: |
     {
-        "plugin": "journald",
-        "pluginConfig": {
-            "source": "dockerd"
+      "plugin": "custom",
+      "pluginConfig": {
+        "invoke_interval": "60s",
+        "timeout": "60s"
+      },
+      "source": "test-custom-plugin-monitor",
+      "metricsReporting": true,
+      "conditions": [
+        {
+          "type": "TestConditionHasProblem",
+          "reason": "TestIsOK",
+          "message": "test condition is OK"
+        }
+      ],
+      "rules": [
+        {
+          "type": "permanent",
+          "condition": "TestConditionHasProblem",
+          "reason": "TestIsIntentionallyFailing",
+          "path": "/home/kubernetes/scripts//test_condition_fail.sh",
+          "timeout": "60s"
         },
-        "logPath": "/var/log/journal",
-        "lookback": "5m",
-        "bufferSize": 10,
-        "source": "docker-monitor",
-        "conditions": [],
-        "rules": [
-            {
-                "type": "temporary",
-                "reason": "CorruptDockerImage",
-                "pattern": "Error trying v2 registry: failed to register layer: rename /var/lib/docker/image/(.+) /var/lib/docker/image/(.+): directory not empty.*"
-            }
-        ]
+        {
+          "type": "permanent",
+          "condition": "TestConditionHasProblem",
+          "reason": "TestIsIntentionallySucceeding",
+          "path": "/home/kubernetes/scripts//test_condition_succeed.sh",
+          "timeout": "60s"
+        }
+      ]
     }
+#  docker-monitor.json: |
+#    {
+#        "plugin": "journald",
+#        "pluginConfig": {
+#            "source": "dockerd"
+#        },
+#        "logPath": "/var/log/journal",
+#        "lookback": "5m",
+#        "bufferSize": 10,
+#        "source": "docker-monitor",
+#        "conditions": [],
+#        "rules": [
+#            {
+#                "type": "temporary",
+#                "reason": "CorruptDockerImage",
+#                "pattern": "Error trying v2 registry: failed to register layer: rename /var/lib/docker/image/(.+) /var/lib/docker/image/(.+): directory not empty.*"
+#            }
+#        ]
+#    }
 kind: ConfigMap
 metadata:
   name: node-problem-detector-config

deployment/node-problem-detector.yaml

@@ -28,8 +28,9 @@ spec:
         command:
         - /node-problem-detector
         - --logtostderr
-        - --config.system-log-monitor=/config/kernel-monitor.json,/config/docker-monitor.json
-        image: k8s.gcr.io/node-problem-detector/node-problem-detector:v0.8.7
+        #- --config.system-log-monitor=/config/kernel-monitor.json,/config/docker-monitor.json
+        - --config.custom-plugin-monitor=/config/cpm-test.json
+        image: jasonjungacr.azurecr.io/k8s-staging-npd/node-problem-detector:v0.8.10-10-g4412a2b-dirty
         resources:
           limits:
             cpu: 10m
@@ -77,8 +78,10 @@ spec:
           items:
           - key: kernel-monitor.json
             path: kernel-monitor.json
-          - key: docker-monitor.json
-            path: docker-monitor.json
+#          - key: docker-monitor.json
+#            path: docker-monitor.json
+          - key: cpm-test.json
+            path: cpm-test.json
       tolerations:
         - effect: NoSchedule
           operator: Exists

pkg/custompluginmonitor/custom_plugin_monitor.go

@@ -126,17 +126,35 @@ func (c *customPluginMonitor) monitorLoop() {
 
 	resultChan := c.plugin.GetResultChan()
 
+	// runRules done for the interval
+	//intervalEndChan := c.plugin.GetIntervalEndChan()
+	var intervalResults []cpmtypes.Result
+
 	for {
 		select {
 		case result, ok := <-resultChan:
 			if !ok {
 				glog.Errorf("Result channel closed: %s", c.configPath)
 				return
 			}
+
 			glog.V(3).Infof("Receive new plugin result for %s: %+v", c.configPath, result)
-			status := c.generateStatus(result)
-			glog.V(3).Infof("New status generated: %+v", status)
-			c.statusChan <- status
+			// gather results for single rule interval loop
+			intervalResults = append(intervalResults)
+
+			//glog.V(3).Infof("Receive new plugin result for %s: %+v", c.configPath, result)
+			//status := c.generateStatus(result)
+			//glog.V(3).Infof("New status generated: %+v", status)
+			//c.statusChan <- status
+
+		case _, ok := <-intervalEndChan:
+			if !ok {
+				glog.Errorf("Interval End Channel closed: %s", c.configPath)
+				return
+			}
+			//status := c.generateStatus(intervalResults)
+			//glog.V(3).Infof("New status generated: %+v", status)
+			//c.statusChan <- status
 		case <-c.tomb.Stopping():
 			c.plugin.Stop()
 			glog.Infof("Custom plugin monitor stopped: %s", c.configPath)
@@ -146,8 +164,158 @@ func (c *customPluginMonitor) monitorLoop() {
 	}
 }
 
+func (c *customPluginMonitor) generateStatus(results []cpmtypes.Result) *types.Status {
+	timestamp := time.Now()
+	var activeProblemEvents []types.Event
+	var inactiveProblemEvents []types.Event
+
+	var unProcessedResults []cpmtypes.Result
+
+	for _, result := range results {
+		status := toConditionStatus(result.ExitStatus)
+		if result.Rule.Type == types.Temp {
+			// For temporary error only generate event when exit status is above warning
+			if result.ExitStatus >= cpmtypes.NonOK {
+				activeProblemEvents = append(activeProblemEvents, types.Event{
+					Severity:  types.Warn,
+					Timestamp: timestamp,
+					Reason:    result.Rule.Reason,
+					Message:   result.Message,
+				})
+			}
+		} else {
+			// we skip result that sets condition true, and result that sets condition false/unknown but with a different reason
+			if status == types.True {
+				unProcessedResults = append(unProcessedResults, result)
+				continue
+			}
+
+			for i := range c.conditions {
+				condition := &c.conditions[i]
+
+				// if appropriate (current condition's reason changes to false/unknown), unset(set to false/unknown) condition first.
+				// In case there are multiple reasons per condition, this will prevent ignoring new reason that sets
+				// condition true (since original condition reason takes precedence) or flapping (condition set to false
+				// by current reason, then to true by another reason)
+				if condition.Type == result.Rule.Condition && condition.Reason == result.Rule.Reason {
+					// The condition reason specified in the rule and the result message
+					// represent the problem happened. We need to know the default condition
+					// from the config, so that we can set the new condition reason/message
+					// back when such problem goes away.
+					var defaultConditionReason string
+					var defaultConditionMessage string
+					for j := range c.config.DefaultConditions {
+						defaultCondition := &c.config.DefaultConditions[j]
+						if defaultCondition.Type == result.Rule.Condition {
+							defaultConditionReason = defaultCondition.Reason
+							defaultConditionMessage = defaultCondition.Message
+							break
+						}
+					}
+
+					var newReason string
+					var newMessage string
+					newReason = defaultConditionReason
+					if status == types.False {
+						newMessage = defaultConditionMessage
+					} else {
+						// When status unknown, the result's message is important for debug
+						newMessage = result.Message
+					}
+
+					condition.Transition = timestamp
+					condition.Status = status
+					condition.Reason = newReason
+					condition.Message = newMessage
+
+					break
+				}
+			}
+		}
+	}
+
+	for _, result := range unProcessedResults {
+		status := toConditionStatus(result.ExitStatus)
+		// we iterate through results that sets condition true for different reasons
+		// whatever result that went through result channel first takes precedence
+		for i := range c.conditions {
+			condition := &c.conditions[i]
+			if condition.Type == result.Rule.Condition {
+				if condition.Status != types.True ||
+					(condition.Reason == result.Rule.Reason && *c.config.PluginGlobalConfig.EnableMessageChangeBasedConditionUpdate) {
+					// update condition only when condition is currently false, or message based condition update is enabled.
+					// for each condition, this if-block will be reached once
+					condition.Transition = timestamp
+					condition.Status = status
+					condition.Reason = result.Rule.Reason
+					condition.Message = result.Message
+
+					updateEvent := util.GenerateConditionChangeEvent(
+						condition.Type,
+						status,
+						result.Rule.Reason,
+						timestamp,
+					)
+
+					activeProblemEvents = append(activeProblemEvents, updateEvent)
+				}
+
+				break
+			}
+		}
+	}
+
+	for i := range c.conditions {
+		// check for conditions that are still false/unknown
+		condition := &c.conditions[i]
+		if condition.Status != types.True {
+			updateEvent := util.GenerateConditionChangeEvent(
+				condition.Type,
+				condition.Status,
+				condition.Reason,
+				timestamp,
+			)
+
+			inactiveProblemEvents = append(inactiveProblemEvents, updateEvent)
+		}
+	}
+
+	if *c.config.EnableMetricsReporting {
+		// Increment problem counter only for active problems which just got detected.
+		for _, event := range activeProblemEvents {
+			err := problemmetrics.GlobalProblemMetricsManager.IncrementProblemCounter(
+				event.Reason, 1)
+			if err != nil {
+				glog.Errorf("Failed to update problem counter metrics for %q: %v",
+					event.Reason, err)
+			}
+		}
+		for _, condition := range c.conditions {
+			err := problemmetrics.GlobalProblemMetricsManager.SetProblemGauge(
+				condition.Type, condition.Reason, condition.Status == types.True)
+			if err != nil {
+				glog.Errorf("Failed to update problem gauge metrics for problem %q, reason %q: %v",
+					condition.Type, condition.Reason, err)
+			}
+		}
+	}
+	status := &types.Status{
+		Source: c.config.Source,
+		// TODO(random-liu): Aggregate events and conditions and then do periodically report.
+		Events:     append(activeProblemEvents, inactiveProblemEvents...),
+		Conditions: c.conditions,
+	}
+	// Log only if condition has changed
+	if len(activeProblemEvents) != 0 || len(inactiveProblemEvents) != 0 {
+		glog.V(0).Infof("New status generated: %+v", status)
+	}
+	return status
+}
+
+/*
 // generateStatus generates status from the plugin check result.
 func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Status {
+
 	timestamp := time.Now()
 	var activeProblemEvents []types.Event
 	var inactiveProblemEvents []types.Event
@@ -277,6 +445,7 @@ func (c *customPluginMonitor) generateStatus(result cpmtypes.Result) *types.Stat
 	}
 	return status
 }
+*/
 
 func toConditionStatus(s cpmtypes.Status) types.ConditionStatus {
 	switch s {

pkg/custompluginmonitor/custom_plugin_monitor_test.go

@@ -29,3 +29,7 @@ func TestRegistration(t *testing.T) {
 		func() { problemdaemon.GetProblemDaemonHandlerOrDie("custom-plugin-monitor") },
 		"Custom plugin monitor failed to register itself as a problem daemon.")
 }
+
+func TestGenerateStatus(t *testing.T) {
+
+}