From 7ab5e27909c5b72363641ed6e683be42488e0365 Mon Sep 17 00:00:00 2001 From: Katarzyna Kujawa Date: Thu, 16 Apr 2020 09:22:14 +0200 Subject: [PATCH] Add referenced bytes metric for containers see: https://github.com/brendangregg/wss#wsspl-referenced-page-flag Signed-off-by: Katarzyna Kujawa --- cmd/cadvisor.go | 4 +- cmd/cadvisor_test.go | 7 ++ container/factory.go | 2 + container/libcontainer/handler.go | 115 +++++++++++++++++++- container/libcontainer/handler_test.go | 85 +++++++++++++++ container/libcontainer/helpers_test.go | 15 +++ container/libcontainer/testdata/clear_refs4 | 1 + container/libcontainer/testdata/clear_refs6 | 1 + container/libcontainer/testdata/clear_refs8 | 1 + container/libcontainer/testdata/smaps4 | 84 ++++++++++++++ container/libcontainer/testdata/smaps6 | 21 ++++ container/libcontainer/testdata/smaps8 | 21 ++++ docs/storage/prometheus.md | 1 + info/v1/container.go | 3 + info/v1/test/datagen.go | 1 + info/v2/container.go | 4 + info/v2/conversion.go | 4 +- info/v2/conversion_test.go | 6 +- metrics/prometheus.go | 12 ++ metrics/prometheus_fake.go | 1 + metrics/testdata/prometheus_metrics | 3 + 21 files changed, 386 insertions(+), 6 deletions(-) create mode 100644 container/libcontainer/testdata/clear_refs4 create mode 100644 container/libcontainer/testdata/clear_refs6 create mode 100644 container/libcontainer/testdata/clear_refs8 create mode 100644 container/libcontainer/testdata/smaps4 create mode 100644 container/libcontainer/testdata/smaps6 create mode 100644 container/libcontainer/testdata/smaps8 diff --git a/cmd/cadvisor.go b/cmd/cadvisor.go index bda4fa3685..d0e26f719c 100644 --- a/cmd/cadvisor.go +++ b/cmd/cadvisor.go @@ -87,6 +87,7 @@ var ( container.ProcessSchedulerMetrics: struct{}{}, container.ProcessMetrics: struct{}{}, container.HugetlbUsageMetrics: struct{}{}, + container.ReferencedMemoryMetrics: struct{}{}, }} // List of metrics that can be ignored. @@ -101,6 +102,7 @@ var ( container.ProcessSchedulerMetrics: struct{}{}, container.ProcessMetrics: struct{}{}, container.HugetlbUsageMetrics: struct{}{}, + container.ReferencedMemoryMetrics: struct{}{}, } ) @@ -132,7 +134,7 @@ func (ml *metricSetValue) Set(value string) error { } func init() { - flag.Var(&ignoreMetrics, "disable_metrics", "comma-separated list of `metrics` to be disabled. Options are 'disk', 'diskIO', 'network', 'tcp', 'udp', 'percpu', 'sched', 'process', 'hugetlb'.") + flag.Var(&ignoreMetrics, "disable_metrics", "comma-separated list of `metrics` to be disabled. Options are 'disk', 'diskIO', 'network', 'tcp', 'udp', 'percpu', 'sched', 'process', 'hugetlb', 'referenced_memory'.") // Default logging verbosity to V(2) flag.Set("v", "2") diff --git a/cmd/cadvisor_test.go b/cmd/cadvisor_test.go index 60315d7c5b..4b417fce89 100644 --- a/cmd/cadvisor_test.go +++ b/cmd/cadvisor_test.go @@ -40,6 +40,12 @@ func TestUdpMetricsAreDisabledByDefault(t *testing.T) { assert.True(t, ignoreMetrics.Has(container.NetworkUdpUsageMetrics)) } +func TestReferencedMemoryMetricsIsDisabledByDefault(t *testing.T) { + assert.True(t, ignoreMetrics.Has(container.ReferencedMemoryMetrics)) + flag.Parse() + assert.True(t, ignoreMetrics.Has(container.ReferencedMemoryMetrics)) +} + func TestIgnoreMetrics(t *testing.T) { tests := []struct { value string @@ -86,6 +92,7 @@ func TestToIncludedMetrics(t *testing.T) { container.AppMetrics: struct{}{}, container.HugetlbUsageMetrics: struct{}{}, container.PerfMetrics: struct{}{}, + container.ReferencedMemoryMetrics: struct{}{}, }, container.AllMetrics, {}, diff --git a/container/factory.go b/container/factory.go index 759a39af0c..8507cd3b71 100644 --- a/container/factory.go +++ b/container/factory.go @@ -59,6 +59,7 @@ const ( ProcessMetrics MetricKind = "process" HugetlbUsageMetrics MetricKind = "hugetlb" PerfMetrics MetricKind = "perf_event" + ReferencedMemoryMetrics MetricKind = "referenced_memory" ) // AllMetrics represents all kinds of metrics that cAdvisor supported. @@ -79,6 +80,7 @@ var AllMetrics = MetricSet{ AppMetrics: struct{}{}, HugetlbUsageMetrics: struct{}{}, PerfMetrics: struct{}{}, + ReferencedMemoryMetrics: struct{}{}, } func (mk MetricKind) String() string { diff --git a/container/libcontainer/handler.go b/container/libcontainer/handler.go index f19cd82e6b..b568b72b24 100644 --- a/container/libcontainer/handler.go +++ b/container/libcontainer/handler.go @@ -17,6 +17,7 @@ package libcontainer import ( "bufio" "encoding/json" + "flag" "fmt" "io" "io/ioutil" @@ -38,16 +39,27 @@ import ( "k8s.io/klog/v2" ) +var ( + whitelistedUlimits = [...]string{"max_open_files"} + referencedResetInterval = flag.Uint64("referenced_reset_interval", 0, + "Reset interval for referenced bytes (container_referenced_bytes metric), number of measurement cycles after which referenced bytes are cleared, if set to 0 referenced bytes are never cleared (default: 0)") + + smapsFilePathPattern = "/proc/%d/smaps" + clearRefsFilePathPattern = "/proc/%d/clear_refs" + + referencedRegexp = regexp.MustCompile(`Referenced:\s*([0-9]+)\s*kB`) + isDigitRegExp = regexp.MustCompile("\\d+") +) + type Handler struct { cgroupManager cgroups.Manager rootFs string pid int includedMetrics container.MetricSet pidMetricsCache map[int]*info.CpuSchedstat + cycles uint64 } -var whitelistedUlimits = [...]string{"max_open_files"} - func NewHandler(cgroupManager cgroups.Manager, rootFs string, pid int, includedMetrics container.MetricSet) *Handler { return &Handler{ cgroupManager: cgroupManager, @@ -81,6 +93,19 @@ func (h *Handler) GetStats() (*info.ContainerStats, error) { } } + if h.includedMetrics.Has(container.ReferencedMemoryMetrics) { + h.cycles++ + pids, err := h.cgroupManager.GetPids() + if err != nil { + klog.V(4).Infof("Could not get PIDs for container %d: %v", h.pid, err) + } else { + stats.ReferencedMemory, err = referencedBytesStat(pids, h.cycles, *referencedResetInterval) + if err != nil { + klog.V(4).Infof("Unable to get referenced bytes: %v", err) + } + } + } + // If we know the pid then get network stats from /proc//net/dev if h.pid == 0 { return stats, nil @@ -318,6 +343,92 @@ func schedulerStatsFromProcs(rootFs string, pids []int, pidMetricsCache map[int] return schedstats, nil } +// referencedBytesStat gets and clears referenced bytes +// see: https://github.com/brendangregg/wss#wsspl-referenced-page-flag +func referencedBytesStat(pids []int, cycles uint64, resetInterval uint64) (uint64, error) { + referencedKBytes, err := getReferencedKBytes(pids) + if err != nil { + return uint64(0), err + } + + err = clearReferencedBytes(pids, cycles, resetInterval) + if err != nil { + return uint64(0), err + } + return referencedKBytes * 1024, nil +} + +func getReferencedKBytes(pids []int) (uint64, error) { + referencedKBytes := uint64(0) + readSmapsContent := false + foundMatch := false + for _, pid := range pids { + smapsFilePath := fmt.Sprintf(smapsFilePathPattern, pid) + smapsContent, err := ioutil.ReadFile(smapsFilePath) + if err != nil { + klog.V(5).Infof("Cannot read %s file, err: %s", smapsFilePath, err) + if os.IsNotExist(err) { + continue //smaps file does not exists for all PIDs + } + return 0, err + } + readSmapsContent = true + + allMatches := referencedRegexp.FindAllSubmatch(smapsContent, -1) + if len(allMatches) == 0 { + klog.V(5).Infof("Not found any information about referenced bytes in %s file", smapsFilePath) + continue // referenced bytes may not exist in smaps file + } + + for _, matches := range allMatches { + if len(matches) != 2 { + return 0, fmt.Errorf("failed to match regexp in output: %s", string(smapsContent)) + } + foundMatch = true + referenced, err := strconv.ParseUint(string(matches[1]), 10, 64) + if err != nil { + return 0, err + } + referencedKBytes += referenced + } + } + + if len(pids) != 0 { + if !readSmapsContent { + klog.Warningf("Cannot read smaps files for any PID from %s", "CONTAINER") + } else if !foundMatch { + klog.Warningf("Not found any information about referenced bytes in smaps files for any PID from %s", "CONTAINER") + } + } + return referencedKBytes, nil +} + +func clearReferencedBytes(pids []int, cycles uint64, resetInterval uint64) error { + if resetInterval == 0 { + return nil + } + + if cycles%resetInterval == 0 { + for _, pid := range pids { + clearRefsFilePath := fmt.Sprintf(clearRefsFilePathPattern, pid) + clerRefsFile, err := os.OpenFile(clearRefsFilePath, os.O_WRONLY, 0644) + if err != nil { + // clear_refs file may not exist for all PIDs + continue + } + _, err = clerRefsFile.WriteString("1\n") + if err != nil { + return err + } + err = clerRefsFile.Close() + if err != nil { + return err + } + } + } + return nil +} + func networkStatsFromProc(rootFs string, pid int) ([]info.InterfaceStats, error) { netStatsFile := path.Join(rootFs, "proc", strconv.Itoa(pid), "/net/dev") diff --git a/container/libcontainer/handler_test.go b/container/libcontainer/handler_test.go index 3f942ed551..10db6ee07b 100644 --- a/container/libcontainer/handler_test.go +++ b/container/libcontainer/handler_test.go @@ -22,6 +22,7 @@ import ( info "github.com/google/cadvisor/info/v1" "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/system" + "github.com/stretchr/testify/assert" ) func TestScanInterfaceStats(t *testing.T) { @@ -215,3 +216,87 @@ func TestParseLimitsFile(t *testing.T) { } } } + +func TestReferencedBytesStat(t *testing.T) { + //overwrite package variables + smapsFilePathPattern = "testdata/smaps%d" + clearRefsFilePathPattern = "testdata/clear_refs%d" + + pids := []int{4, 6, 8} + stat, err := referencedBytesStat(pids, 1, 3) + assert.Nil(t, err) + assert.Equal(t, uint64(416*1024), stat) + + clearRefsFiles := []string{ + "testdata/clear_refs4", + "testdata/clear_refs6", + "testdata/clear_refs8"} + + //check if clear_refs files have proper values + assert.Equal(t, "0\n", getFileContent(t, clearRefsFiles[0])) + assert.Equal(t, "0\n", getFileContent(t, clearRefsFiles[1])) + assert.Equal(t, "0\n", getFileContent(t, clearRefsFiles[2])) +} + +func TestReferencedBytesStatWhenNeverCleared(t *testing.T) { + //overwrite package variables + smapsFilePathPattern = "testdata/smaps%d" + clearRefsFilePathPattern = "testdata/clear_refs%d" + + pids := []int{4, 6, 8} + stat, err := referencedBytesStat(pids, 1, 0) + assert.Nil(t, err) + assert.Equal(t, uint64(416*1024), stat) + + clearRefsFiles := []string{ + "testdata/clear_refs4", + "testdata/clear_refs6", + "testdata/clear_refs8"} + + //check if clear_refs files have proper values + assert.Equal(t, "0\n", getFileContent(t, clearRefsFiles[0])) + assert.Equal(t, "0\n", getFileContent(t, clearRefsFiles[1])) + assert.Equal(t, "0\n", getFileContent(t, clearRefsFiles[2])) +} + +func TestReferencedBytesStatWhenResetIsNeeded(t *testing.T) { + //overwrite package variables + smapsFilePathPattern = "testdata/smaps%d" + clearRefsFilePathPattern = "testdata/clear_refs%d" + + pids := []int{4, 6, 8} + stat, err := referencedBytesStat(pids, 1, 1) + assert.Nil(t, err) + assert.Equal(t, uint64(416*1024), stat) + + clearRefsFiles := []string{ + "testdata/clear_refs4", + "testdata/clear_refs6", + "testdata/clear_refs8"} + + //check if clear_refs files have proper values + assert.Equal(t, "1\n", getFileContent(t, clearRefsFiles[0])) + assert.Equal(t, "1\n", getFileContent(t, clearRefsFiles[1])) + assert.Equal(t, "1\n", getFileContent(t, clearRefsFiles[2])) + + clearTestData(t, clearRefsFiles) +} + +func TestGetReferencedKBytesWhenSmapsMissing(t *testing.T) { + //overwrite package variable + smapsFilePathPattern = "testdata/smaps%d" + + pids := []int{10} + referenced, err := getReferencedKBytes(pids) + assert.Nil(t, err) + assert.Equal(t, uint64(0), referenced) +} + +func TestClearReferencedBytesWhenClearRefsMissing(t *testing.T) { + //overwrite package variable + clearRefsFilePathPattern = "testdata/clear_refs%d" + + pids := []int{10} + err := clearReferencedBytes(pids, 0, 1) + assert.Nil(t, err) +} diff --git a/container/libcontainer/helpers_test.go b/container/libcontainer/helpers_test.go index 5c6a005ddb..a516be41df 100644 --- a/container/libcontainer/helpers_test.go +++ b/container/libcontainer/helpers_test.go @@ -16,6 +16,7 @@ package libcontainer import ( "fmt" + "io/ioutil" "path/filepath" "reflect" "sort" @@ -23,6 +24,7 @@ import ( "testing" "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/stretchr/testify/assert" ) var defaultCgroupSubsystems = []string{ @@ -130,3 +132,16 @@ func assertCgroupSubsystemsEqual(t *testing.T, expected, actual CgroupSubsystems t.Fatalf("%s Expected %v == %v", message, expected.Mounts, actual.Mounts) } } + +func getFileContent(t *testing.T, filePath string) string { + fileContent, err := ioutil.ReadFile(filePath) + assert.Nil(t, err) + return string(fileContent) +} + +func clearTestData(t *testing.T, clearRefsPaths []string) { + for _, clearRefsPath := range clearRefsPaths { + err := ioutil.WriteFile(clearRefsPath, []byte("0\n"), 0644) + assert.Nil(t, err) + } +} diff --git a/container/libcontainer/testdata/clear_refs4 b/container/libcontainer/testdata/clear_refs4 new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/container/libcontainer/testdata/clear_refs4 @@ -0,0 +1 @@ +0 diff --git a/container/libcontainer/testdata/clear_refs6 b/container/libcontainer/testdata/clear_refs6 new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/container/libcontainer/testdata/clear_refs6 @@ -0,0 +1 @@ +0 diff --git a/container/libcontainer/testdata/clear_refs8 b/container/libcontainer/testdata/clear_refs8 new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/container/libcontainer/testdata/clear_refs8 @@ -0,0 +1 @@ +0 diff --git a/container/libcontainer/testdata/smaps4 b/container/libcontainer/testdata/smaps4 new file mode 100644 index 0000000000..67e257b81f --- /dev/null +++ b/container/libcontainer/testdata/smaps4 @@ -0,0 +1,84 @@ +55f523c9f000-55f523cc1000 r-xp 00000000 08:02 5505067 /sbin/cgmanager +Size: 136 kB +KernelPageSize: 4 kB +MMUPageSize: 4 kB +Rss: 132 kB +Pss: 132 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 132 kB +Private_Dirty: 0 kB +Referenced: 132 kB +Anonymous: 0 kB +LazyFree: 0 kB +AnonHugePages: 0 kB +ShmemPmdMapped: 0 kB +Shared_Hugetlb: 0 kB +Private_Hugetlb: 0 kB +Swap: 0 kB +SwapPss: 0 kB +Locked: 0 kB +VmFlags: rd ex mr mw me dw sd +55f523ec0000-55f523ec2000 r--p 00021000 08:02 5505067 /sbin/cgmanager +Size: 8 kB +KernelPageSize: 4 kB +MMUPageSize: 4 kB +Rss: 8 kB +Pss: 8 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 0 kB +Private_Dirty: 8 kB +Referenced: 4 kB +Anonymous: 8 kB +LazyFree: 0 kB +AnonHugePages: 0 kB +ShmemPmdMapped: 0 kB +Shared_Hugetlb: 0 kB +Private_Hugetlb: 0 kB +Swap: 0 kB +SwapPss: 0 kB +Locked: 0 kB +VmFlags: rd mr mw me dw ac sd +55f523ec2000-55f523ec3000 rw-p 00023000 08:02 5505067 /sbin/cgmanager +Size: 4 kB +KernelPageSize: 4 kB +MMUPageSize: 4 kB +Rss: 4 kB +Pss: 4 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 0 kB +Private_Dirty: 4 kB +Referenced: 0 kB +Anonymous: 4 kB +LazyFree: 0 kB +AnonHugePages: 0 kB +ShmemPmdMapped: 0 kB +Shared_Hugetlb: 0 kB +Private_Hugetlb: 0 kB +Swap: 0 kB +SwapPss: 0 kB +Locked: 0 kB +VmFlags: rd wr mr mw me dw ac sd +55f52478d000-55f5247ae000 rw-p 00000000 00:00 0 [heap] +Size: 132 kB +KernelPageSize: 4 kB +MMUPageSize: 4 kB +Rss: 16 kB +Pss: 16 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 0 kB +Private_Dirty: 16 kB +Referenced: 16 kB +Anonymous: 16 kB +LazyFree: 0 kB +AnonHugePages: 0 kB +ShmemPmdMapped: 0 kB +Shared_Hugetlb: 0 kB +Private_Hugetlb: 0 kB +Swap: 0 kB +SwapPss: 0 kB +Locked: 0 kB +VmFlags: rd wr mr mw me ac sd diff --git a/container/libcontainer/testdata/smaps6 b/container/libcontainer/testdata/smaps6 new file mode 100644 index 0000000000..cec4092716 --- /dev/null +++ b/container/libcontainer/testdata/smaps6 @@ -0,0 +1,21 @@ +55f523c9f000-55f523cc1000 r-xp 00000000 08:02 5505067 /sbin/cgmanager +Size: 136 kB +KernelPageSize: 4 kB +MMUPageSize: 4 kB +Rss: 132 kB +Pss: 132 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 132 kB +Private_Dirty: 0 kB +Referenced: 132 kB +Anonymous: 0 kB +LazyFree: 0 kB +AnonHugePages: 0 kB +ShmemPmdMapped: 0 kB +Shared_Hugetlb: 0 kB +Private_Hugetlb: 0 kB +Swap: 0 kB +SwapPss: 0 kB +Locked: 0 kB +VmFlags: rd ex mr mw me dw sd diff --git a/container/libcontainer/testdata/smaps8 b/container/libcontainer/testdata/smaps8 new file mode 100644 index 0000000000..cec4092716 --- /dev/null +++ b/container/libcontainer/testdata/smaps8 @@ -0,0 +1,21 @@ +55f523c9f000-55f523cc1000 r-xp 00000000 08:02 5505067 /sbin/cgmanager +Size: 136 kB +KernelPageSize: 4 kB +MMUPageSize: 4 kB +Rss: 132 kB +Pss: 132 kB +Shared_Clean: 0 kB +Shared_Dirty: 0 kB +Private_Clean: 132 kB +Private_Dirty: 0 kB +Referenced: 132 kB +Anonymous: 0 kB +LazyFree: 0 kB +AnonHugePages: 0 kB +ShmemPmdMapped: 0 kB +Shared_Hugetlb: 0 kB +Private_Hugetlb: 0 kB +Swap: 0 kB +SwapPss: 0 kB +Locked: 0 kB +VmFlags: rd ex mr mw me dw sd diff --git a/docs/storage/prometheus.md b/docs/storage/prometheus.md index 0f5c91258f..0cde28b7fd 100644 --- a/docs/storage/prometheus.md +++ b/docs/storage/prometheus.md @@ -70,6 +70,7 @@ Metric name | Type | Description | Unit (where applicable) `container_network_udp_usage_total` | Gauge | udp connection usage statistic for container | `container_network_udp6_usage_total` | Gauge | udp6 connection usage statistic for container | `container_processes` | Gauge | Number of processes running inside the container | +`container_referenced_bytes` | Gauge | Container referenced bytes during last measurements cycle based on Referenced field in /proc/smaps file, with /proc/PIDs/clear_refs set to 1 after defined number of cycles configured through `referenced_reset_interval` cAdvisor parameter.
Warning: this is intrusive collection because can influence kernel page reclaim policy and add latency. Refer to https://github.com/brendangregg/wss#wsspl-referenced-page-flag for more details. | bytes `container_spec_cpu_period` | Gauge | CPU period of the container | `container_spec_cpu_quota` | Gauge | CPU quota of the container | `container_spec_cpu_shares` | Gauge | CPU share of the container | diff --git a/info/v1/container.go b/info/v1/container.go index bac9030f90..5b95ddbd6d 100644 --- a/info/v1/container.go +++ b/info/v1/container.go @@ -899,6 +899,9 @@ type ContainerStats struct { // Statistics originating from perf events PerfStats []PerfStat `json:"perf_stats,omitempty"` + + // Referenced memory + ReferencedMemory uint64 `json:"referenced_memory,omitempty"` } func timeEq(t1, t2 time.Time, tolerance time.Duration) bool { diff --git a/info/v1/test/datagen.go b/info/v1/test/datagen.go index 8b02cd8c73..af6ec180ed 100644 --- a/info/v1/test/datagen.go +++ b/info/v1/test/datagen.go @@ -47,6 +47,7 @@ func GenerateRandomStats(numStats, numCores int, duration time.Duration) []*info stats.Memory.Cache = uint64(rand.Int63n(4096)) stats.Memory.RSS = uint64(rand.Int63n(4096)) stats.Memory.MappedFile = uint64(rand.Int63n(4096)) + stats.ReferencedMemory = uint64(rand.Int63n(1000)) ret[i] = stats } return ret diff --git a/info/v2/container.go b/info/v2/container.go index 247fe8f058..ac87fa08af 100644 --- a/info/v2/container.go +++ b/info/v2/container.go @@ -136,6 +136,8 @@ type DeprecatedContainerStats struct { // Custom Metrics HasCustomMetrics bool `json:"has_custom_metrics"` CustomMetrics map[string][]v1.MetricVal `json:"custom_metrics,omitempty"` + // Referenced memory + ReferencedMemory uint64 `json:"referenced_memory,omitempty"` } type ContainerStats struct { @@ -166,6 +168,8 @@ type ContainerStats struct { CustomMetrics map[string][]v1.MetricVal `json:"custom_metrics,omitempty"` // Perf events counters PerfStats []v1.PerfStat `json:"perf_stats,omitempty"` + // Referenced memory + ReferencedMemory uint64 `json:"referenced_memory,omitempty"` } type Percentiles struct { diff --git a/info/v2/conversion.go b/info/v2/conversion.go index 031d5e65fd..400e01adf9 100644 --- a/info/v2/conversion.go +++ b/info/v2/conversion.go @@ -101,7 +101,8 @@ func ContainerStatsFromV1(containerName string, spec *v1.ContainerSpec, stats [] var last *v1.ContainerStats for _, val := range stats { stat := &ContainerStats{ - Timestamp: val.Timestamp, + Timestamp: val.Timestamp, + ReferencedMemory: val.ReferencedMemory, } if spec.HasCpu { stat.Cpu = &val.Cpu @@ -172,6 +173,7 @@ func DeprecatedStatsFromV1(cont *v1.ContainerInfo) []DeprecatedContainerStats { HasFilesystem: cont.Spec.HasFilesystem, HasDiskIo: cont.Spec.HasDiskIo, HasCustomMetrics: cont.Spec.HasCustomMetrics, + ReferencedMemory: val.ReferencedMemory, } if stat.HasCpu { stat.Cpu = val.Cpu diff --git a/info/v2/conversion_test.go b/info/v2/conversion_test.go index 332f7626e9..050249f239 100644 --- a/info/v2/conversion_test.go +++ b/info/v2/conversion_test.go @@ -208,6 +208,7 @@ func TestContainerStatsFromV1(t *testing.T) { Name: "cycles", }, }, + ReferencedMemory: uint64(1234), } expectedV2Stats := ContainerStats{ Timestamp: timestamp, @@ -224,8 +225,9 @@ func TestContainerStatsFromV1(t *testing.T) { BaseUsageBytes: &v1Stats.Filesystem[0].BaseUsage, InodeUsage: &v1Stats.Filesystem[0].Inodes, }, - Accelerators: v1Stats.Accelerators, - PerfStats: v1Stats.PerfStats, + Accelerators: v1Stats.Accelerators, + PerfStats: v1Stats.PerfStats, + ReferencedMemory: v1Stats.ReferencedMemory, } v2Stats := ContainerStatsFromV1("test", &v1Spec, []*v1.ContainerStats{&v1Stats}) diff --git a/metrics/prometheus.go b/metrics/prometheus.go index b088597eab..7e38579990 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -1581,6 +1581,18 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri }, }...) } + if includedMetrics.Has(container.ReferencedMemoryMetrics) { + c.containerMetrics = append(c.containerMetrics, []containerMetric{ + { + name: "container_referenced_bytes", + help: "Container referenced bytes during last measurements cycle", + valueType: prometheus.GaugeValue, + getValues: func(s *info.ContainerStats) metricValues { + return metricValues{{value: float64(s.ReferencedMemory), timestamp: s.Timestamp}} + }, + }, + }...) + } return c } diff --git a/metrics/prometheus_fake.go b/metrics/prometheus_fake.go index 1eabcfb204..433bd60041 100644 --- a/metrics/prometheus_fake.go +++ b/metrics/prometheus_fake.go @@ -437,6 +437,7 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container Cpu: 1, }, }, + ReferencedMemory: 1234, }, }, }, diff --git a/metrics/testdata/prometheus_metrics b/metrics/testdata/prometheus_metrics index ce3e3d145f..572ea0aacb 100644 --- a/metrics/testdata/prometheus_metrics +++ b/metrics/testdata/prometheus_metrics @@ -342,6 +342,9 @@ container_perf_metric_scaling_ratio{container_env_foo_env="prod",container_label # HELP container_processes Number of processes running inside the container. # TYPE container_processes gauge container_processes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1 1395066363000 +# HELP container_referenced_bytes Container referenced bytes during last measurements cycle +# TYPE container_referenced_bytes gauge +container_referenced_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1234 1395066363000 # HELP container_scrape_error 1 if there was an error while getting container metrics, 0 otherwise # TYPE container_scrape_error gauge container_scrape_error 0