IDE metrics server

.github/CODEOWNERS

@@ -19,6 +19,8 @@
 /components/gitpod-protocol @gitpod-io/engineering-webapp
 /components/gitpod-protocol/java @gitpod-io/engineering-ide
 /components/ide @gitpod-io/engineering-ide
+/components/ide-metrics @gitpod-io/engineering-ide
+/components/ide-metrics-api @gitpod-io/engineering-ide
 /components/ide-proxy @gitpod-io/engineering-ide
 /components/image-builder-api @csweichel @geropl
 /components/image-builder-bob @gitpod-io/engineering-workspace
@@ -33,6 +35,7 @@
 /install/installer/pkg/components/components-workspace @gitpod-io/engineering-workspace
 /install/installer/pkg/components/content-service @gitpod-io/engineering-workspace
 /install/installer/pkg/components/dashboard @gitpod-io/engineering-webapp
+/install/installer/pkg/components/ide-metrics @gitpod-io/engineering-ide
 /install/installer/pkg/components/ide-proxy @gitpod-io/engineering-ide
 /install/installer/pkg/components/image-builder-mk3 @gitpod-io/engineering-workspace
 /install/installer/pkg/components/openvsx-proxy @gitpod-io/engineering-ide

components/BUILD.yaml

@@ -71,6 +71,7 @@ packages:
       - components/ws-manager:docker
       - components/ws-proxy:docker
       - components/ide-proxy:docker
+      - components/ide-metrics:docker
       - components/kots-config-check/database:docker
       - components/kots-config-check/registry:docker
       - components/kots-config-check/storage:docker
@@ -85,6 +86,8 @@ packages:
   - name: publish-api
     type: generic
     deps:
+      - components/ide-metrics-api/typescript-grpc:publish
+      - components/ide-metrics-api/typescript-grpcweb:publish
       - components/gitpod-protocol:publish
       - components/local-app-api/typescript-grpcweb:publish
       - components/supervisor-api/typescript-grpc:publish
@@ -114,6 +117,7 @@ packages:
       - components/ws-daemon:app
       - components/ws-manager-bridge:app
       - components/ws-manager:app
+      - components/ide-metrics:app
 scripts:
   - name: update-license-header
     description: Updates the license header in all source files

components/gitpod-protocol/src/util/gitpod-host-url.ts

@@ -175,4 +175,21 @@ export class GitpodHostUrl {
     asApiLogout(): GitpodHostUrl {
         return this.withApi((url) => ({ pathname: "/logout/" }));
     }
+
+    asIDEProxy(): GitpodHostUrl {
+        const hostSegments = this.url.host.split(".");
+        if (hostSegments[0] === "ide") {
+            return this;
+        }
+        return this.with((url) => ({ host: "ide." + url.host }));
+    }
+
+    asIDEMetrics(): GitpodHostUrl {
+        let newUrl: GitpodHostUrl = this;
+        const hostSegments = this.url.host.split(".");
+        if (hostSegments[0] !== "ide") {
+            newUrl = newUrl.asIDEProxy();
+        }
+        return newUrl.with((url) => ({ pathname: "/metrics-api" }));
+    }
 }

components/ide-metrics-api/.gitignore

@@ -0,0 +1 @@
+typescript-rest/lib/

components/ide-metrics-api/BUILD.yaml

@@ -0,0 +1,8 @@
+packages:
+  - name: proto
+    type: generic
+    srcs:
+      - "*.proto"
+    config:
+      commands:
+        - ["echo"]

components/ide-metrics-api/generate.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+if [ -n "$DEBUG" ]; then
+  set -x
+fi
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)/../../
+COMPONENTS_DIR=$ROOT_DIR/components
+
+# include protoc bash functions
+# shellcheck disable=SC1090,SC1091
+source "$ROOT_DIR"/scripts/protoc-generator.sh
+
+# TODO (aledbf): refactor to avoid duplication
+local_go_protoc() {
+    local ROOT_DIR=$1
+    # shellcheck disable=2035
+    protoc \
+        -I/usr/lib/protoc/include -I"$COMPONENTS_DIR" -I. \
+        --go_out=go \
+        --go_opt=paths=source_relative \
+        --go-grpc_out=go \
+        --go-grpc_opt=paths=source_relative \
+        *.proto
+}
+
+go_protoc_gateway() {
+    # shellcheck disable=2035
+    protoc \
+        -I/usr/lib/protoc/include -I"$COMPONENTS_DIR" -I. \
+        --grpc-gateway_out=logtostderr=true,paths=source_relative:go \
+        *.proto
+}
+
+local_java_protoc() {
+    protoc \
+        -I /usr/lib/protoc/include -I"$COMPONENTS_DIR" -I. \
+        --plugin=protoc-gen-grpc-java=/tmp/protoc-gen-grpc-java \
+        --grpc-java_out=java/src/main/java \
+        --java_out=java/src/main/java \
+        ./*.proto
+    # remove trailing spaces
+    find "$COMPONENTS_DIR"/ide-metrics-api/java/src/main/java/io/gitpod/idemetrics/api -maxdepth 1 -name "*.java" -exec sed -i -e "s/[[:space:]]*$//" {} \;
+}
+
+install_dependencies
+local_go_protoc "$COMPONENTS_DIR"
+go_protoc_gateway "$COMPONENTS_DIR"
+local_java_protoc "$COMPONENTS_DIR"
+update_license

components/ide-metrics-api/go.mod

@@ -0,0 +1,3 @@
+module github.com/gitpod-io/generated_code_dependencies
+
+go 1.18

components/ide-metrics-api/go/BUILD.yaml

@@ -0,0 +1,12 @@
+packages:
+  - name: lib
+    type: go
+    deps:
+      - components/common-go:lib
+    srcs:
+      - "**/*.go"
+      - "go.mod"
+      - "go.sum"
+    config:
+      packaging: library
+      dontTest: false

components/ide-metrics-api/go/config/config.go

@@ -0,0 +1,70 @@
+// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License-AGPL.txt in the project root for license information.
+
+package config
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+
+	"github.com/gitpod-io/gitpod/common-go/grpc"
+	"golang.org/x/xerrors"
+)
+
+type LabelAllowList struct {
+	Name        string   `json:"name"`
+	AllowValues []string `json:"allowValues"`
+}
+
+type MetricsServerConfiguration struct {
+	Port             int                             `json:"port"`
+	RateLimits       map[string]grpc.RateLimit       `json:"ratelimits"`
+	CounterMetrics   []CounterMetricsConfiguration   `json:"counterMetrics"`
+	HistogramMetrics []HistogramMetricsConfiguration `json:"histogramMetrics"`
+}
+
+type CounterMetricsConfiguration struct {
+	Name   string           `json:"name"`
+	Help   string           `json:"help"`
+	Labels []LabelAllowList `json:"labels"`
+}
+
+type HistogramMetricsConfiguration struct {
+	Name    string           `json:"name"`
+	Help    string           `json:"help"`
+	Labels  []LabelAllowList `json:"labels"`
+	Buckets []float64        `json:"buckets"`
+}
+
+type ServiceConfiguration struct {
+	Server MetricsServerConfiguration `json:"server"`
+
+	Debug bool `json:"debug"`
+
+	PProf struct {
+		Addr string `json:"addr"`
+	} `json:"pprof"`
+
+	Prometheus struct {
+		Addr string `json:"addr"`
+	} `json:"prometheus"`
+}
+
+func Read(fn string) (*ServiceConfiguration, error) {
+	ctnt, err := os.ReadFile(fn)
+	if err != nil {
+		return nil, xerrors.Errorf("cannot read config file: %w", err)
+	}
+
+	var cfg ServiceConfiguration
+	dec := json.NewDecoder(bytes.NewReader(ctnt))
+	dec.DisallowUnknownFields()
+	err = dec.Decode(&cfg)
+	if err != nil {
+		return nil, xerrors.Errorf("cannot parse config file: %w", err)
+	}
+
+	return &cfg, nil
+}

components/ide-metrics-api/go/go.mod

@@ -0,0 +1,83 @@
+module github.com/gitpod-io/gitpod/ide-metrics-api
+
+go 1.18
+
+require (
+	github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f
+	google.golang.org/grpc v1.45.0
+	google.golang.org/protobuf v1.28.0
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/prometheus/client_golang v1.12.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
+	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
+	google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced // indirect
+)
+
+replace github.com/gitpod-io/gitpod/common-go => ../../common-go // leeway
+
+replace k8s.io/api => k8s.io/api v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/apimachinery => k8s.io/apimachinery v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/apiserver => k8s.io/apiserver v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/client-go => k8s.io/client-go v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/code-generator => k8s.io/code-generator v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/component-base => k8s.io/component-base v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/cri-api => k8s.io/cri-api v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/kubelet => k8s.io/kubelet v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/metrics => k8s.io/metrics v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/component-helpers => k8s.io/component-helpers v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/controller-manager => k8s.io/controller-manager v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/kubectl => k8s.io/kubectl v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/mount-utils => k8s.io/mount-utils v0.23.5 // leeway indirect from components/common-go:lib
+
+replace k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.5 // leeway indirect from components/common-go:lib