Skip to content

Commit 4ff0c7e

Browse files
akosyakovroboquat
akosyakov
authored and
roboquat
committed
[jb]: monitor low memory notifications and gc overhead
1 parent eec9a93 commit 4ff0c7e

File tree

5 files changed

+285
-10
lines changed

5 files changed

+285
-10
lines changed

‎components/ide/jetbrains/backend-plugin/launch-dev-server.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,4 +49,6 @@ export GIT_EDITOR="$EDITOR --wait"
4949
export GP_PREVIEW_BROWSER="$IDEA_CLI_DEV_PATH preview"
5050
export GP_EXTERNAL_BROWSER="$IDEA_CLI_DEV_PATH preview"
5151

52+
export JETBRAINS_GITPOD_BACKEND_KIND=intellij
53+
5254
$TEST_BACKEND_DIR/bin/remote-dev-server.sh run $TEST_DIR

‎components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodCLIService.kt

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,27 +10,48 @@ import com.intellij.ide.CommandLineProcessor
1010
import com.intellij.openapi.application.ApplicationManager
1111
import com.intellij.openapi.client.ClientSession
1212
import com.intellij.openapi.client.ClientSessionsManager
13+
import com.intellij.openapi.components.service
1314
import com.intellij.openapi.diagnostic.thisLogger
1415
import com.intellij.openapi.project.Project
16+
import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream
1517
import com.intellij.openapi.util.io.FileUtilRt
1618
import com.intellij.util.application
19+
import io.netty.buffer.Unpooled
1720
import io.netty.channel.ChannelHandlerContext
1821
import io.netty.handler.codec.http.FullHttpRequest
1922
import io.netty.handler.codec.http.QueryStringDecoder
23+
import io.prometheus.client.exporter.common.TextFormat
2024
import org.jetbrains.ide.RestService
25+
import org.jetbrains.io.response
26+
import java.io.OutputStreamWriter
2127
import java.nio.file.InvalidPathException
2228
import java.nio.file.Path
2329

2430
@Suppress("UnstableApiUsage")
2531
class GitpodCLIService : RestService() {
2632

33+
private val manager = service<GitpodManager>()
34+
2735
override fun getServiceName() = SERVICE_NAME
2836

2937
override fun execute(urlDecoder: QueryStringDecoder, request: FullHttpRequest, context: ChannelHandlerContext): String? {
38+
val operation = getStringParameter("op", urlDecoder)
3039
if (application.isHeadlessEnvironment) {
3140
return "not supported in headless mode"
3241
}
33-
val operation = getStringParameter("op", urlDecoder)
42+
/**
43+
* prod: curl http://localhost:63342/api/gitpod/cli?op=metrics
44+
* dev: curl http://localhost:63343/api/gitpod/cli?op=metrics
45+
*/
46+
if (operation == "metrics") {
47+
val out = BufferExposingByteArrayOutputStream()
48+
val writer = OutputStreamWriter(out)
49+
TextFormat.write004(writer, manager.registry.metricFamilySamples())
50+
writer.close()
51+
val response = response(TextFormat.CONTENT_TYPE_004, Unpooled.wrappedBuffer(out.internalBuffer, 0, out.size()))
52+
sendResponse(request, context, response)
53+
return null
54+
}
3455
if (operation == "open") {
3556
val fileStr = getStringParameter("file", urlDecoder)
3657
if (fileStr.isNullOrBlank()) {

‎components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodManager.kt

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import com.intellij.openapi.Disposable
1212
import com.intellij.openapi.components.Service
1313
import com.intellij.openapi.diagnostic.thisLogger
1414
import com.intellij.openapi.extensions.PluginId
15+
import com.intellij.openapi.util.LowMemoryWatcher
1516
import com.intellij.remoteDev.util.onTerminationOrNow
1617
import com.intellij.util.application
1718
import com.jetbrains.rd.util.lifetime.Lifetime
@@ -28,6 +29,7 @@ import io.grpc.ManagedChannelBuilder
2829
import io.grpc.stub.ClientCallStreamObserver
2930
import io.grpc.stub.ClientResponseObserver
3031
import io.prometheus.client.CollectorRegistry
32+
import io.prometheus.client.Counter
3133
import io.prometheus.client.Gauge
3234
import io.prometheus.client.exporter.PushGateway
3335
import kotlinx.coroutines.GlobalScope
@@ -64,13 +66,29 @@ class GitpodManager : Disposable {
6466
lifetime.terminate()
6567
}
6668

69+
val registry = CollectorRegistry()
70+
6771
init {
68-
val monitoringJob = GlobalScope.launch {
72+
// Rate of low memory after GC notifications in the last 5 minutes:
73+
// rate(gitpod_jb_backend_low_memory_after_gc_total[5m])
74+
val lowMemoryCounter = Counter.build()
75+
.name("gitpod_jb_backend_low_memory_after_gc")
76+
.help("Low memory notifications after GC")
77+
.labelNames("product", "qualifier")
78+
.register(registry)
79+
LowMemoryWatcher.register({
80+
lowMemoryCounter.labels(backendKind, backendQualifier).inc()
81+
}, LowMemoryWatcher.LowMemoryWatcherType.ONLY_AFTER_GC, this)
82+
}
83+
84+
init {
85+
val monitoringJob = GlobalScope.launch {
6986
if (application.isHeadlessEnvironment) {
7087
return@launch
7188
}
72-
val pg = PushGateway("localhost:22999")
73-
val registry = CollectorRegistry()
89+
val pg = if(devMode) null else PushGateway("localhost:22999")
90+
// Heap usage at any time in the last 5 minutes:
91+
// max_over_time(gitpod_jb_backend_memory_used_bytes[5m:])/max_over_time(gitpod_jb_backend_memory_max_bytes[5m:])
7492
val allocatedGauge = Gauge.build()
7593
.name("gitpod_jb_backend_memory_max_bytes")
7694
.help("Total allocated memory of JB backend in bytes.")
@@ -81,13 +99,14 @@ class GitpodManager : Disposable {
8199
.help("Used memory of JB backend in bytes.")
82100
.labelNames("product", "qualifier")
83101
.register(registry)
84-
while(isActive) {
102+
103+
while (isActive) {
85104
val totalMemory = Runtime.getRuntime().totalMemory()
86105
allocatedGauge.labels(backendKind, backendQualifier).set(totalMemory.toDouble())
87106
val usedMemory = (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())
88107
usedGauge.labels(backendKind, backendQualifier).set(usedMemory.toDouble())
89108
try {
90-
pg.push(registry, "jb_backend")
109+
pg?.push(registry, "jb_backend")
91110
} catch (t: Throwable) {
92111
thisLogger().error("gitpod: failed to push monitoring metrics:", t)
93112
}
@@ -108,7 +127,7 @@ class GitpodManager : Disposable {
108127
.connectTimeout(Duration.ofSeconds(5))
109128
.build()
110129
val httpRequest = HttpRequest.newBuilder()
111-
.uri(URI.create("http://localhost:24000/gatewayLink?backendPort=${backendPort}"))
130+
.uri(URI.create("http://localhost:24000/gatewayLink?backendPort=$backendPort"))
112131
.GET()
113132
.build()
114133
val response =
@@ -260,14 +279,14 @@ class GitpodManager : Disposable {
260279
tokenResponse.token
261280
)
262281
} finally {
263-
Thread.currentThread().contextClassLoader = originalClassLoader;
282+
Thread.currentThread().contextClassLoader = originalClassLoader
264283
}
265284
}
266285

267286
val minReconnectionDelay = 2 * 1000L
268287
val maxReconnectionDelay = 30 * 1000L
269-
val reconnectionDelayGrowFactor = 1.5;
270-
var reconnectionDelay = minReconnectionDelay;
288+
val reconnectionDelayGrowFactor = 1.5
289+
var reconnectionDelay = minReconnectionDelay
271290
val gitpodHost = info.gitpodApi.host
272291
var closeReason: Any = "cancelled"
273292
try {

‎operations/observability/mixins/IDE/dashboards.libsonnet

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
// 'my-new-dashboard.json': (import 'dashboards/components/new-component.json'),
1111
'gitpod-component-openvsx-proxy.json': (import 'dashboards/components/openvsx-proxy.json'),
1212
'gitpod-component-ssh-gateway.json': (import 'dashboards/components/ssh-gateway.json'),
13+
'gitpod-component-jb.json': (import 'dashboards/components/jb.json'),
1314

1415
},
1516
}

‎operations/observability/mixins/IDE/dashboards/components/jb.json

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
{
2+
"annotations": {
3+
"list": [
4+
{
5+
"builtIn": 1,
6+
"datasource": {
7+
"type": "grafana",
8+
"uid": "-- Grafana --"
9+
},
10+
"enable": true,
11+
"hide": true,
12+
"iconColor": "rgba(0, 211, 255, 1)",
13+
"name": "Annotations & Alerts",
14+
"target": {
15+
"limit": 100,
16+
"matchAny": false,
17+
"tags": [],
18+
"type": "dashboard"
19+
},
20+
"type": "dashboard"
21+
}
22+
]
23+
},
24+
"editable": true,
25+
"fiscalYearStartMonth": 0,
26+
"graphTooltip": 0,
27+
"id": 63,
28+
"links": [],
29+
"liveNow": false,
30+
"panels": [
31+
{
32+
"datasource": {
33+
"type": "prometheus",
34+
"uid": "P1809F7CD0C75ACF3"
35+
},
36+
"fieldConfig": {
37+
"defaults": {
38+
"color": {
39+
"mode": "palette-classic"
40+
},
41+
"custom": {
42+
"axisLabel": "notification/second",
43+
"axisPlacement": "auto",
44+
"barAlignment": 0,
45+
"drawStyle": "line",
46+
"fillOpacity": 0,
47+
"gradientMode": "none",
48+
"hideFrom": {
49+
"legend": false,
50+
"tooltip": false,
51+
"viz": false
52+
},
53+
"lineInterpolation": "linear",
54+
"lineWidth": 1,
55+
"pointSize": 5,
56+
"scaleDistribution": {
57+
"type": "linear"
58+
},
59+
"showPoints": "auto",
60+
"spanNulls": false,
61+
"stacking": {
62+
"group": "A",
63+
"mode": "none"
64+
},
65+
"thresholdsStyle": {
66+
"mode": "off"
67+
}
68+
},
69+
"mappings": [],
70+
"thresholds": {
71+
"mode": "absolute",
72+
"steps": [
73+
{
74+
"color": "green",
75+
"value": null
76+
},
77+
{
78+
"color": "red",
79+
"value": 80
80+
}
81+
]
82+
}
83+
},
84+
"overrides": []
85+
},
86+
"gridPos": {
87+
"h": 11,
88+
"w": 12,
89+
"x": 0,
90+
"y": 0
91+
},
92+
"id": 2,
93+
"options": {
94+
"legend": {
95+
"calcs": [
96+
"max",
97+
"mean"
98+
],
99+
"displayMode": "table",
100+
"placement": "bottom"
101+
},
102+
"tooltip": {
103+
"mode": "single",
104+
"sort": "none"
105+
}
106+
},
107+
"targets": [
108+
{
109+
"datasource": {
110+
"type": "prometheus",
111+
"uid": "P1809F7CD0C75ACF3"
112+
},
113+
"editorMode": "code",
114+
"expr": "sum(rate(gitpod_jb_backend_low_memory_after_gc_total[5m]))",
115+
"legendFormat": "rate",
116+
"range": true,
117+
"refId": "A"
118+
}
119+
],
120+
"title": "Total low memory after GC",
121+
"type": "timeseries"
122+
},
123+
{
124+
"datasource": {
125+
"type": "prometheus",
126+
"uid": "P1809F7CD0C75ACF3"
127+
},
128+
"fieldConfig": {
129+
"defaults": {
130+
"color": {
131+
"mode": "palette-classic"
132+
},
133+
"custom": {
134+
"axisLabel": "notification/second",
135+
"axisPlacement": "auto",
136+
"barAlignment": 0,
137+
"drawStyle": "line",
138+
"fillOpacity": 0,
139+
"gradientMode": "none",
140+
"hideFrom": {
141+
"legend": false,
142+
"tooltip": false,
143+
"viz": false
144+
},
145+
"lineInterpolation": "linear",
146+
"lineWidth": 1,
147+
"pointSize": 5,
148+
"scaleDistribution": {
149+
"type": "linear"
150+
},
151+
"showPoints": "auto",
152+
"spanNulls": false,
153+
"stacking": {
154+
"group": "A",
155+
"mode": "none"
156+
},
157+
"thresholdsStyle": {
158+
"mode": "off"
159+
}
160+
},
161+
"mappings": [],
162+
"thresholds": {
163+
"mode": "absolute",
164+
"steps": [
165+
{
166+
"color": "green",
167+
"value": null
168+
},
169+
{
170+
"color": "red",
171+
"value": 80
172+
}
173+
]
174+
}
175+
},
176+
"overrides": []
177+
},
178+
"gridPos": {
179+
"h": 11,
180+
"w": 12,
181+
"x": 12,
182+
"y": 0
183+
},
184+
"id": 4,
185+
"options": {
186+
"legend": {
187+
"calcs": [
188+
"max",
189+
"mean"
190+
],
191+
"displayMode": "table",
192+
"placement": "right"
193+
},
194+
"tooltip": {
195+
"mode": "single",
196+
"sort": "none"
197+
}
198+
},
199+
"targets": [
200+
{
201+
"datasource": {
202+
"type": "prometheus",
203+
"uid": "P1809F7CD0C75ACF3"
204+
},
205+
"editorMode": "code",
206+
"expr": "topk(10, sum(rate(gitpod_jb_backend_low_memory_after_gc_total[5m])) by (pod))",
207+
"legendFormat": "{{pod}}",
208+
"range": true,
209+
"refId": "A"
210+
}
211+
],
212+
"title": "Top low memory after GC",
213+
"type": "timeseries"
214+
}
215+
],
216+
"schemaVersion": 36,
217+
"style": "dark",
218+
"tags": [],
219+
"templating": {
220+
"list": []
221+
},
222+
"time": {
223+
"from": "now-6h",
224+
"to": "now"
225+
},
226+
"timepicker": {},
227+
"timezone": "",
228+
"title": "JetBrains Overview",
229+
"uid": "oamBLUC7k",
230+
"version": 8,
231+
"weekStart": ""
232+
}

0 commit comments

Comments
 (0)