From f6c270e4680f4baba3bf9696e470129b9195d23c Mon Sep 17 00:00:00 2001 From: Beyang Liu Date: Sun, 2 Aug 2020 17:16:34 -0700 Subject: [PATCH 1/5] remove gitserver memory usage alert It is always triggered. See https://www.bwplotka.dev/2019/golang-memory-monitoring. --- doc/admin/observability/alert_solutions.md | 35 ---------------------- monitoring/git_server.go | 3 -- 2 files changed, 38 deletions(-) diff --git a/doc/admin/observability/alert_solutions.md b/doc/admin/observability/alert_solutions.md index cd47b4fcc0ef..b5f91bc92db2 100644 --- a/doc/admin/observability/alert_solutions.md +++ b/doc/admin/observability/alert_solutions.md @@ -366,17 +366,6 @@ This alert is triggered via the `triggerObservabilityTestAlert` GraphQL endpoint - **Kubernetes:** Consider increasing CPU limits in the the relevant `Deployment.yaml`. - **Docker Compose:** Consider increasing `cpus:` of the gitserver container in `docker-compose.yml`. -# gitserver: container_memory_usage - -**Descriptions:** - -- _gitserver: 99%+ container memory usage by instance_ (`warning_gitserver_container_memory_usage`) - -**Possible solutions:** - -- **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. -- **Docker Compose:** Consider increasing `memory:` of gitserver container in `docker-compose.yml`. - # gitserver: container_restarts **Descriptions:** @@ -416,19 +405,6 @@ This alert is triggered via the `triggerObservabilityTestAlert` GraphQL endpoint - **Docker Compose:** Consider descreasing `cpus:` of the gitserver container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. -# gitserver: provisioning_container_memory_usage_7d - -**Descriptions:** - -- _gitserver: 80%+ or less than 30% container memory usage (7d maximum) by instance_ (`warning_gitserver_provisioning_container_memory_usage_7d`) - -**Possible solutions:** - -- If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of gitserver container in `docker-compose.yml`. -- If usage is low, consider decreasing the above values. - # gitserver: provisioning_container_cpu_usage_5m **Descriptions:** @@ -440,17 +416,6 @@ This alert is triggered via the `triggerObservabilityTestAlert` GraphQL endpoint - **Kubernetes:** Consider increasing CPU limits in the the relevant `Deployment.yaml`. - **Docker Compose:** Consider increasing `cpus:` of the gitserver container in `docker-compose.yml`. -# gitserver: provisioning_container_memory_usage_5m - -**Descriptions:** - -- _gitserver: 90%+ container memory usage (5m maximum) by instance_ (`warning_gitserver_provisioning_container_memory_usage_5m`) - -**Possible solutions:** - -- **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. -- **Docker Compose:** Consider increasing `memory:` of gitserver container in `docker-compose.yml`. - # github-proxy: container_cpu_usage **Descriptions:** diff --git a/monitoring/git_server.go b/monitoring/git_server.go index 019825169138..81784e7db1ae 100644 --- a/monitoring/git_server.go +++ b/monitoring/git_server.go @@ -92,7 +92,6 @@ func GitServer() *Container { Rows: []Row{ { sharedContainerCPUUsage("gitserver"), - sharedContainerMemoryUsage("gitserver"), }, { sharedContainerRestarts("gitserver"), @@ -106,11 +105,9 @@ func GitServer() *Container { Rows: []Row{ { sharedProvisioningCPUUsage7d("gitserver"), - sharedProvisioningMemoryUsage7d("gitserver"), }, { sharedProvisioningCPUUsage5m("gitserver"), - sharedProvisioningMemoryUsage5m("gitserver"), }, }, }, From 5f8eb4497588ea106566b097c9d9d540d1258fe3 Mon Sep 17 00:00:00 2001 From: Robert Lin Date: Thu, 6 Aug 2020 10:18:45 +0800 Subject: [PATCH 2/5] add back sharedContainerMemoryUsage --- doc/admin/observability/alert_solutions.md | 11 +++++++++++ monitoring/git_server.go | 1 + 2 files changed, 12 insertions(+) diff --git a/doc/admin/observability/alert_solutions.md b/doc/admin/observability/alert_solutions.md index acdd3c7f6923..9dc34cde5151 100644 --- a/doc/admin/observability/alert_solutions.md +++ b/doc/admin/observability/alert_solutions.md @@ -377,6 +377,17 @@ This alert is triggered via the `triggerObservabilityTestAlert` GraphQL endpoint - **Kubernetes:** Consider increasing CPU limits in the the relevant `Deployment.yaml`. - **Docker Compose:** Consider increasing `cpus:` of the gitserver container in `docker-compose.yml`. +# gitserver: container_memory_usage + +**Descriptions:** + +- _gitserver: 99%+ container memory usage by instance_ (`warning_gitserver_container_memory_usage`) + +**Possible solutions:** + +- **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. +- **Docker Compose:** Consider increasing `memory:` of gitserver container in `docker-compose.yml`. + # gitserver: container_restarts **Descriptions:** diff --git a/monitoring/git_server.go b/monitoring/git_server.go index 81784e7db1ae..9f1614cf225d 100644 --- a/monitoring/git_server.go +++ b/monitoring/git_server.go @@ -92,6 +92,7 @@ func GitServer() *Container { Rows: []Row{ { sharedContainerCPUUsage("gitserver"), + sharedContainerMemoryUsage("gitserver"), }, { sharedContainerRestarts("gitserver"), From 4c0125929da0124617fa8aec90b15bee30956fde Mon Sep 17 00:00:00 2001 From: Robert Lin Date: Thu, 6 Aug 2020 10:22:43 +0800 Subject: [PATCH 3/5] add lower-bound 7d memory for gitserver --- monitoring/git_server.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/monitoring/git_server.go b/monitoring/git_server.go index 9f1614cf225d..7f7f244aba8f 100644 --- a/monitoring/git_server.go +++ b/monitoring/git_server.go @@ -1,5 +1,10 @@ package main +import ( + "fmt" + "time" +) + func GitServer() *Container { return &Container{ Name: "gitserver", @@ -106,6 +111,21 @@ func GitServer() *Container { Rows: []Row{ { sharedProvisioningCPUUsage7d("gitserver"), + { + Name: "provisioning_container_memory_usage_7d", + Description: "container memory usage (7d maximum) by instance", + Query: fmt.Sprintf(`max_over_time(cadvisor_container_memory_usage_percentage_total{%s}[7d])`, promCadvisorContainerMatchers("gitserver")), + DataMayNotExist: true, + // gitserver generally uses up all the memory it gets, so + // alerting on long-term high memory usage is not very useful + Warning: Alert{LessOrEqual: 30, For: 14 * 24 * time.Hour}, + PanelOptions: PanelOptions().LegendFormat("{{name}}").Unit(Percentage).Max(100).Min(0), + Owner: ObservableOwnerDistribution, + PossibleSolutions: ` + - **Kubernetes:** Consider decreasing memory limit in relevant 'Deployment.yaml'. + - **Docker Compose:** Consider decreasing 'memory:' of gitserver container in 'docker-compose.yml'. + `, + }, }, { sharedProvisioningCPUUsage5m("gitserver"), From 06f20814ab46ee989ad063b33b58950024b10899 Mon Sep 17 00:00:00 2001 From: Robert Lin Date: Thu, 6 Aug 2020 10:22:50 +0800 Subject: [PATCH 4/5] fix wording of 7d provisioning --- monitoring/shared.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/shared.go b/monitoring/shared.go index 713392a759bb..d793325a8aad 100644 --- a/monitoring/shared.go +++ b/monitoring/shared.go @@ -197,8 +197,8 @@ var sharedProvisioningMemoryUsage7d sharedObservable = func(containerName string Owner: ObservableOwnerDistribution, PossibleSolutions: strings.Replace(` - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant 'Deployment.yaml'. - - **Docker Compose:** Consider decreasing 'memory:' of {{CONTAINER_NAME}} container in 'docker-compose.yml'. + - **Kubernetes:** Consider increasing memory limit in relevant 'Deployment.yaml'. + - **Docker Compose:** Consider increasing 'memory:' of {{CONTAINER_NAME}} container in 'docker-compose.yml'. - If usage is low, consider decreasing the above values. `, "{{CONTAINER_NAME}}", containerName, -1), } From 7e6b50961a11671393cb0e3f8f781353608595e4 Mon Sep 17 00:00:00 2001 From: Robert Lin Date: Thu, 6 Aug 2020 10:24:50 +0800 Subject: [PATCH 5/5] generate --- doc/admin/observability/alert_solutions.md | 67 +++++++++++++--------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/doc/admin/observability/alert_solutions.md b/doc/admin/observability/alert_solutions.md index 9dc34cde5151..50f5dd6595e5 100644 --- a/doc/admin/observability/alert_solutions.md +++ b/doc/admin/observability/alert_solutions.md @@ -258,8 +258,8 @@ This alert is triggered via the `triggerObservabilityTestAlert` GraphQL endpoint **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of frontend container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of frontend container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # frontend: provisioning_container_cpu_usage_5m @@ -427,6 +427,17 @@ This alert is triggered via the `triggerObservabilityTestAlert` GraphQL endpoint - **Docker Compose:** Consider descreasing `cpus:` of the gitserver container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. +# gitserver: provisioning_container_memory_usage_7d + +**Descriptions:** + +- _gitserver: less than 30% container memory usage (7d maximum) by instance_ (`warning_gitserver_provisioning_container_memory_usage_7d`) + +**Possible solutions:** + +- **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. +- **Docker Compose:** Consider decreasing `memory:` of gitserver container in `docker-compose.yml`. + # gitserver: provisioning_container_cpu_usage_5m **Descriptions:** @@ -528,8 +539,8 @@ Try restarting the pod to get a different public IP. **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of github-proxy container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of github-proxy container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # github-proxy: provisioning_container_cpu_usage_5m @@ -653,8 +664,8 @@ Try restarting the pod to get a different public IP. **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of precise-code-intel-bundle-manager container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of precise-code-intel-bundle-manager container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # precise-code-intel-bundle-manager: provisioning_container_cpu_usage_5m @@ -765,8 +776,8 @@ Try restarting the pod to get a different public IP. **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of precise-code-intel-worker container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of precise-code-intel-worker container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # precise-code-intel-worker: provisioning_container_cpu_usage_5m @@ -877,8 +888,8 @@ Try restarting the pod to get a different public IP. **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of precise-code-intel-indexer container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of precise-code-intel-indexer container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # precise-code-intel-indexer: provisioning_container_cpu_usage_5m @@ -989,8 +1000,8 @@ Try restarting the pod to get a different public IP. **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of query-runner container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of query-runner container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # query-runner: provisioning_container_cpu_usage_5m @@ -1101,8 +1112,8 @@ Try restarting the pod to get a different public IP. **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of replacer container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of replacer container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # replacer: provisioning_container_cpu_usage_5m @@ -1213,8 +1224,8 @@ Try restarting the pod to get a different public IP. **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of repo-updater container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of repo-updater container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # repo-updater: provisioning_container_cpu_usage_5m @@ -1325,8 +1336,8 @@ Try restarting the pod to get a different public IP. **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of searcher container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of searcher container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # searcher: provisioning_container_cpu_usage_5m @@ -1437,8 +1448,8 @@ Try restarting the pod to get a different public IP. **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of symbols container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of symbols container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # symbols: provisioning_container_cpu_usage_5m @@ -1533,8 +1544,8 @@ Try restarting the pod to get a different public IP. **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of syntect-server container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of syntect-server container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # syntect-server: provisioning_container_cpu_usage_5m @@ -1629,8 +1640,8 @@ Try restarting the pod to get a different public IP. **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of zoekt-indexserver container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of zoekt-indexserver container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # zoekt-indexserver: provisioning_container_cpu_usage_5m @@ -1725,8 +1736,8 @@ Try restarting the pod to get a different public IP. **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of zoekt-webserver container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of zoekt-webserver container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # zoekt-webserver: provisioning_container_cpu_usage_5m @@ -1831,8 +1842,8 @@ Ensure that your [`observability.alerts` configuration](https://docs.sourcegraph **Possible solutions:** - If usage is high: - - **Kubernetes:** Consider decreasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider decreasing `memory:` of prometheus container in `docker-compose.yml`. + - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. + - **Docker Compose:** Consider increasing `memory:` of prometheus container in `docker-compose.yml`. - If usage is low, consider decreasing the above values. # prometheus: provisioning_container_cpu_usage_5m