Skip to content

Export entire smaps memory metrics instead of only referenced_memory #2767

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Export entire smaps memory metrics instead of only referenced_memory #2767

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
79bde2d
Export entire smaps memory metrics
JensErat Dec 17, 2020
a6e5fd7
make tidy
JensErat Dec 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions cmd/cadvisor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ var (
container.ProcessSchedulerMetrics: struct{}{},
container.ProcessMetrics: struct{}{},
container.HugetlbUsageMetrics: struct{}{},
container.ReferencedMemoryMetrics: struct{}{},
container.SmapsMemoryMetrics: struct{}{},
container.CPUTopologyMetrics: struct{}{},
container.ResctrlMetrics: struct{}{},
}}
Expand All @@ -107,7 +107,7 @@ var (
container.ProcessSchedulerMetrics: struct{}{},
container.ProcessMetrics: struct{}{},
container.HugetlbUsageMetrics: struct{}{},
container.ReferencedMemoryMetrics: struct{}{},
container.SmapsMemoryMetrics: struct{}{},
container.CPUTopologyMetrics: struct{}{},
container.ResctrlMetrics: struct{}{},
}
Expand Down Expand Up @@ -141,7 +141,7 @@ func (ml *metricSetValue) Set(value string) error {
}

func init() {
flag.Var(&ignoreMetrics, "disable_metrics", "comma-separated list of `metrics` to be disabled. Options are 'accelerator', 'cpu_topology','disk', 'diskIO', 'memory_numa', 'network', 'tcp', 'udp', 'percpu', 'sched', 'process', 'hugetlb', 'referenced_memory', 'resctrl'.")
flag.Var(&ignoreMetrics, "disable_metrics", "comma-separated list of `metrics` to be disabled. Options are 'accelerator', 'cpu_topology','disk', 'diskIO', 'memory_numa', 'network', 'tcp', 'udp', 'percpu', 'sched', 'process', 'hugetlb', 'resctrl', 'smaps_memory'.")

// Default logging verbosity to V(2)
flag.Set("v", "2")
Expand Down
8 changes: 4 additions & 4 deletions cmd/cadvisor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,10 @@ func TestUdpMetricsAreDisabledByDefault(t *testing.T) {
assert.True(t, ignoreMetrics.Has(container.NetworkUdpUsageMetrics))
}

func TestReferencedMemoryMetricsIsDisabledByDefault(t *testing.T) {
assert.True(t, ignoreMetrics.Has(container.ReferencedMemoryMetrics))
func TestSmapsMemoryMetricsIsDisabledByDefault(t *testing.T) {
assert.True(t, ignoreMetrics.Has(container.SmapsMemoryMetrics))
flag.Parse()
assert.True(t, ignoreMetrics.Has(container.ReferencedMemoryMetrics))
assert.True(t, ignoreMetrics.Has(container.SmapsMemoryMetrics))
}

func TestCPUTopologyMetricsAreDisabledByDefault(t *testing.T) {
Expand Down Expand Up @@ -105,7 +105,7 @@ func TestToIncludedMetrics(t *testing.T) {
container.AppMetrics: struct{}{},
container.HugetlbUsageMetrics: struct{}{},
container.PerfMetrics: struct{}{},
container.ReferencedMemoryMetrics: struct{}{},
container.SmapsMemoryMetrics: struct{}{},
container.CPUTopologyMetrics: struct{}{},
container.ResctrlMetrics: struct{}{},
},
Expand Down
20 changes: 14 additions & 6 deletions cmd/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Microsoft/go-winio v0.4.15 h1:qkLXKzb1QoVatRyd/YlXZ/Kg0m5K3SPuoD82jjSOaBc=
github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk=
github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
github.com/Rican7/retry v0.1.1-0.20160712041035-272ad122d6e5 h1:6olZmdYuK84eO0PeCQX1iy2EFWlOl8G+JNBi4vFmcU8=
github.com/Rican7/retry v0.1.1-0.20160712041035-272ad122d6e5/go.mod h1:FgOROf8P5bebcC1DS0PdOQiqGUridaZvikzUmkFW6gg=
github.com/SeanDolphin/bqschema v0.0.0-20150424181127-f92a08f515e1 h1:4EBKNUkI0tKxZb75f41jFGQQBPG4A/qbbgmgJ1MTTvw=
Expand Down Expand Up @@ -110,6 +110,8 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand All @@ -118,8 +120,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible h1:dvc1KSkIYTVjZgHf/CTC2diTYC8PzhaA5sFISRfNVrE=
github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible h1:SiUATuP//KecDjpOK2tvZJgeScYAklvyjfK8JZlU6fo=
github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v20.10.0+incompatible h1:4g8Xjho+7quMwzsTrhtrWpdQU9UTc2rX57A3iALaBmE=
github.com/docker/docker v20.10.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
Expand Down Expand Up @@ -334,6 +336,8 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/moby/sys/mountinfo v0.1.3 h1:KIrhRO14+AkwKvG/g2yIpNMOUVZ02xNhOw8KY1WsLOI=
github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o=
github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk=
github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
Expand Down Expand Up @@ -460,6 +464,7 @@ github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
Expand Down Expand Up @@ -646,6 +651,7 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down Expand Up @@ -682,6 +688,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
Expand Down Expand Up @@ -844,8 +851,9 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
Expand Down
10 changes: 6 additions & 4 deletions cmd/internal/storage/influxdb/influxdb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,14 +94,14 @@ const (
setHugetlbFailcnt = "hugetlb_failcnt"
// Perf statistics
serPerfStat = "perf_stat"
// Referenced memory
serReferencedMemory = "referenced_memory"
// Resctrl - Total memory bandwidth
serResctrlMemoryBandwidthTotal = "resctrl_memory_bandwidth_total"
// Resctrl - Local memory bandwidth
serResctrlMemoryBandwidthLocal = "resctrl_memory_bandwidth_local"
// Resctrl - Last level cache usage
serResctrlLLCOccupancy = "resctrl_llc_occupancy"
// Smaps memory prefix
serSmapsPrefix = "smaps_"
)

func new() (storage.StorageDriver, error) {
Expand Down Expand Up @@ -230,8 +230,10 @@ func (s *influxdbStorage) containerStatsToPoints(
points = append(points, makePoint(serTxBytes, stats.Network.TxBytes))
points = append(points, makePoint(serTxErrors, stats.Network.TxErrors))

// Referenced Memory
points = append(points, makePoint(serReferencedMemory, stats.ReferencedMemory))
// Smaps Memory
for k, v := range stats.SmapsMemory {
points = append(points, makePoint(serSmapsPrefix+k, v))
}

s.tagPoints(cInfo, stats, points)

Expand Down
30 changes: 26 additions & 4 deletions cmd/internal/storage/influxdb/influxdb_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ func TestContainerStatsToPoints(t *testing.T) {

// Then
assert.NotEmpty(t, points)
assert.Len(t, points, 34+len(stats.Cpu.Usage.PerCpu))
assert.Len(t, points, 52+len(stats.Cpu.Usage.PerCpu))

// CPU stats
assertContainsPointWithValue(t, points, serCpuUsageTotal, stats.Cpu.Usage.Total)
Expand Down Expand Up @@ -276,7 +276,9 @@ func TestContainerStatsToPoints(t *testing.T) {
}

// Reference memory
assertContainsPointWithValue(t, points, serReferencedMemory, stats.ReferencedMemory)
for k, v := range stats.SmapsMemory {
assertContainsPointWithValue(t, points, serSmapsPrefix+k, v)
}

// Resource Control stats - memory bandwidth
for _, rdtMemoryBandwidth := range stats.Resctrl.MemoryBandwidth {
Expand Down Expand Up @@ -359,8 +361,28 @@ func createTestStats() (*info.ContainerInfo, *info.ContainerStats) {
"1GB": {Usage: 1234, MaxUsage: 5678, Failcnt: 9},
"2GB": {Usage: 9876, MaxUsage: 5432, Failcnt: 1},
},
ReferencedMemory: 12345,
PerfStats: []info.PerfStat{{Cpu: 1, Name: "cycles", ScalingRatio: 1.5, Value: 4589}},
SmapsMemory: map[string]uint64{
"AnonHugePages": 1,
"Anonymous": 2,
"KernelPageSize": 3,
"LazyFree": 4,
"Locked": 5,
"MMUPageSize": 6,
"Private_Clean": 7,
"Private_Dirty": 8,
"Private_Hugetlb": 9,
"Pss": 10,
"Referenced": 11,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removing ContainerStats.ReferencedMemory breaks backward compatibility. I'm afraid we will have to keep it, for a while at least.

Copy link
Contributor Author

@JensErat JensErat Dec 17, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might even be reasonable to exclude it from the smaps metrics, or keep it duplicate forever. From what I understand, all other smaps metrics do not get reset using this referenced-cycle-count-reset-logic, but I'll have to investigate further. So referenced is somewhat different from all the other metrics. Everything else has "proper" bookkeeping, the referenced counter is somewhat an estimate based on page access.

Adding the referenced metric again is a matter of minutes and adjusting some tests. I think some early feedback on the general approach (and matters like compatiblity) is very important though.

"Rss": 12,
"Shared_Clean": 13,
"Shared_Dirty": 14,
"Shared_Hugetlb": 15,
"ShmemPmdMapped": 16,
"Size": 17,
"Swap": 18,
"SwapPss": 19,
},
PerfStats: []info.PerfStat{{Cpu: 1, PerfValue: info.PerfValue{Name: "cycles", ScalingRatio: 1.5, Value: 4589}}},
Resctrl: info.ResctrlStats{
MemoryBandwidth: []info.MemoryBandwidthStats{
{TotalBytes: 11234, LocalBytes: 4567},
Expand Down
10 changes: 6 additions & 4 deletions cmd/internal/storage/statsd/statsd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,14 +83,14 @@ const (
setHugetlbFailcnt string = "hugetlb_failcnt"
// Perf statistics
serPerfStat string = "perf_stat"
// Referenced memory
serReferencedMemory string = "referenced_memory"
// Resctrl - Total memory bandwidth
serResctrlMemoryBandwidthTotal string = "resctrl_memory_bandwidth_total"
// Resctrl - Local memory bandwidth
serResctrlMemoryBandwidthLocal string = "resctrl_memory_bandwidth_local"
// Resctrl - Last level cache usage
serResctrlLLCOccupancy string = "resctrl_llc_occupancy"
// Smaps prefix
serSmaps string = "smaps_"
)

func new() (storage.StorageDriver, error) {
Expand Down Expand Up @@ -126,8 +126,10 @@ func (s *statsdStorage) containerStatsToValues(stats *info.ContainerStats) (seri
series[serTxBytes] = stats.Network.TxBytes
series[serTxErrors] = stats.Network.TxErrors

// Referenced Memory
series[serReferencedMemory] = stats.ReferencedMemory
// Smaps
for k, v := range stats.SmapsMemory {
series[serSmaps+k] = v
}

return series
}
Expand Down
10 changes: 6 additions & 4 deletions cmd/internal/storage/stdout/stdout.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,14 +85,14 @@ const (
setHugetlbFailcnt string = "hugetlb_failcnt"
// Perf statistics
serPerfStat string = "perf_stat"
// Referenced memory
serReferencedMemory string = "referenced_memory"
// Resctrl - Total memory bandwidth
serResctrlMemoryBandwidthTotal string = "resctrl_memory_bandwidth_total"
// Resctrl - Local memory bandwidth
serResctrlMemoryBandwidthLocal string = "resctrl_memory_bandwidth_local"
// Resctrl - Last level cache usage
serResctrlLLCOccupancy string = "resctrl_llc_occupancy"
// Smaps series
serSmapsPrefix string = "smaps_"
)

func new() (storage.StorageDriver, error) {
Expand Down Expand Up @@ -131,8 +131,10 @@ func (driver *stdoutStorage) containerStatsToValues(stats *info.ContainerStats)
series[serTxBytes] = stats.Network.TxBytes
series[serTxErrors] = stats.Network.TxErrors

// Referenced Memory
series[serReferencedMemory] = stats.ReferencedMemory
// Smaps Memory
for k, v := range stats.SmapsMemory {
series[serSmapsPrefix+k] = v
}

return series
}
Expand Down
4 changes: 2 additions & 2 deletions container/factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ const (
ProcessMetrics MetricKind = "process"
HugetlbUsageMetrics MetricKind = "hugetlb"
PerfMetrics MetricKind = "perf_event"
ReferencedMemoryMetrics MetricKind = "referenced_memory"
SmapsMemoryMetrics MetricKind = "smaps_memory"
CPUTopologyMetrics MetricKind = "cpu_topology"
ResctrlMetrics MetricKind = "resctrl"
)
Expand All @@ -84,7 +84,7 @@ var AllMetrics = MetricSet{
AppMetrics: struct{}{},
HugetlbUsageMetrics: struct{}{},
PerfMetrics: struct{}{},
ReferencedMemoryMetrics: struct{}{},
SmapsMemoryMetrics: struct{}{},
CPUTopologyMetrics: struct{}{},
ResctrlMetrics: struct{}{},
}
Expand Down
43 changes: 22 additions & 21 deletions container/libcontainer/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ var (
smapsFilePathPattern = "/proc/%d/smaps"
clearRefsFilePathPattern = "/proc/%d/clear_refs"

referencedRegexp = regexp.MustCompile(`Referenced:\s*([0-9]+)\s*kB`)
smapsRegexp = regexp.MustCompile(`(?m)^(.+):\s*([0-9]+)\s*kB$`)
)

type Handler struct {
Expand Down Expand Up @@ -104,13 +104,13 @@ func (h *Handler) GetStats() (*info.ContainerStats, error) {
}
}

if h.includedMetrics.Has(container.ReferencedMemoryMetrics) {
if h.includedMetrics.Has(container.SmapsMemoryMetrics) {
h.cycles++
pids, err := h.cgroupManager.GetPids()
if err != nil {
klog.V(4).Infof("Could not get PIDs for container %d: %v", h.pid, err)
} else {
stats.ReferencedMemory, err = referencedBytesStat(pids, h.cycles, *referencedResetInterval)
stats.SmapsMemory, err = smapsStat(pids, h.cycles, *referencedResetInterval)
if err != nil {
klog.V(4).Infof("Unable to get referenced bytes: %v", err)
}
Expand Down Expand Up @@ -358,23 +358,23 @@ func schedulerStatsFromProcs(rootFs string, pids []int, pidMetricsCache map[int]
return schedstats, nil
}

// referencedBytesStat gets and clears referenced bytes
// smapsStat gets smaps metrics and clears referenced bytes
// see: https://github.com/brendangregg/wss#wsspl-referenced-page-flag
func referencedBytesStat(pids []int, cycles uint64, resetInterval uint64) (uint64, error) {
referencedKBytes, err := getReferencedKBytes(pids)
func smapsStat(pids []int, cycles uint64, resetInterval uint64) (map[string]uint64, error) {
smapsBytes, err := getSmapsBytes(pids)
if err != nil {
return uint64(0), err
return nil, err
}

err = clearReferencedBytes(pids, cycles, resetInterval)
if err != nil {
return uint64(0), err
return nil, err
}
return referencedKBytes * 1024, nil
return smapsBytes, nil
}

func getReferencedKBytes(pids []int) (uint64, error) {
referencedKBytes := uint64(0)
func getSmapsBytes(pids []int) (map[string]uint64, error) {
smapsBytes := map[string]uint64{}
readSmapsContent := false
foundMatch := false
for _, pid := range pids {
Expand All @@ -385,37 +385,38 @@ func getReferencedKBytes(pids []int) (uint64, error) {
if os.IsNotExist(err) {
continue //smaps file does not exists for all PIDs
}
return 0, err
return nil, err
}
readSmapsContent = true

allMatches := referencedRegexp.FindAllSubmatch(smapsContent, -1)
allMatches := smapsRegexp.FindAllSubmatch(smapsContent, -1)
if len(allMatches) == 0 {
klog.V(5).Infof("Not found any information about referenced bytes in %s file", smapsFilePath)
klog.V(5).Infof("Not found any smaps information in %s file", smapsFilePath)
continue // referenced bytes may not exist in smaps file
}

for _, matches := range allMatches {
if len(matches) != 2 {
return 0, fmt.Errorf("failed to match regexp in output: %s", string(smapsContent))
if len(matches) != 3 {
return nil, fmt.Errorf("failed to match regexp in output: %s", string(smapsContent))
}
foundMatch = true
referenced, err := strconv.ParseUint(string(matches[1]), 10, 64)
key := string(matches[1])
kbytes, err := strconv.ParseUint(string(matches[2]), 10, 64)
if err != nil {
return 0, err
return nil, err
}
referencedKBytes += referenced
smapsBytes[key] = smapsBytes[key] + kbytes*1024
}
}

if len(pids) != 0 {
if !readSmapsContent {
klog.Warningf("Cannot read smaps files for any PID from %s", "CONTAINER")
} else if !foundMatch {
klog.Warningf("Not found any information about referenced bytes in smaps files for any PID from %s", "CONTAINER")
klog.Warningf("Not found any information in smaps files for any PID from %s", "CONTAINER")
}
}
return referencedKBytes, nil
return smapsBytes, nil
}

func clearReferencedBytes(pids []int, cycles uint64, resetInterval uint64) error {
Expand Down
Loading