diff --git a/docker-compose-local.yaml b/docker-compose-local.yaml index 643384b8..adf88d8a 100644 --- a/docker-compose-local.yaml +++ b/docker-compose-local.yaml @@ -23,6 +23,7 @@ services: - --web.enable-remote-write-receiver - --enable-feature=native-histograms - --config.file=/etc/prometheus/prometheus.yml + - --enable-feature=exemplar-storage ports: - "9090:9090" @@ -33,12 +34,9 @@ services: tempo: image: grafana/tempo:2.1.0 - command: - - "-storage.trace.backend=local" # tell tempo where to permanently put traces - - "-storage.trace.local.path=/tmp/tempo/traces" - - "-storage.trace.wal.path=/tmp/tempo/wal" # tell tempo where to store the wal - - "-auth.enabled=false" # disables the requirement for the X-Scope-OrgID header - - "-server.http-listen-port=3200" + command: [ "-config.file=/etc/tempo.yaml" ] + volumes: + - ./tempo/tempo.yaml:/etc/tempo.yaml ports: - "3200:3200" - "4317:4317" diff --git a/grafana/datasources/datasource.yaml b/grafana/datasources/datasource.yaml index 67b63c61..b07be269 100644 --- a/grafana/datasources/datasource.yaml +++ b/grafana/datasources/datasource.yaml @@ -6,6 +6,7 @@ apiVersion: 1 datasources: - name: prometheus type: prometheus + uid: prometheus access: proxy orgId: 1 url: http://prometheus:9090 @@ -14,7 +15,11 @@ datasources: jsonData: tlsAuth: false tlsAuthWithCACert: false + exemplarTraceIdDestinations: + - name: TraceID + datasourceUid: tempo editable: false + - name: Tempo type: tempo access: proxy @@ -26,6 +31,34 @@ datasources: editable: false apiVersion: 1 uid: tempo + jsonData: + httpMethod: GET + tracesToLogsV2: + datasourceUid: 'loki' + spanStartTimeShift: '1h' + spanEndTimeShift: '-1h' + tags: [{ key: 'job', value: 'job' }] + filterByTraceID: false + filterBySpanID: false + customQuery: true + query: '{compose_service="quickpizza"} |= `$${__span.traceId}`' + tracesToMetrics: + datasourceUid: 'prometheus' + spanStartTimeShift: '1h' + spanEndTimeShift: '-1h' + tags: [{ key: 'job', value: 'service.job' }, { key: 'instance'}, { key: 'method'}, { key: 'status'}, { key: 'path'}] + queries: + - name: 'Request Rate' + query: 'sum by(client, server) (rate(traces_service_graph_request_total[$$__rate_interval]))' + - name: 'Error Rate' + query: 'sum by(client, server) (rate(traces_service_graph_request_failed_total[$$__rate_interval]))' + nodeGraph: + enabled: true + lokiSearch: + datasourceUid: 'loki' + serviceMap: + datasourceUid: 'prometheus' + - name: Pyroscope type: 'phlare' access: 'proxy' @@ -34,8 +67,10 @@ datasources: url: http://pyroscope:4040 isDefault: false editable: false + - name: Loki type: loki + uid: loki access: proxy orgId: 1 url: http://loki:3100 diff --git a/tempo/tempo.yaml b/tempo/tempo.yaml new file mode 100644 index 00000000..670f056f --- /dev/null +++ b/tempo/tempo.yaml @@ -0,0 +1,53 @@ +server: + http_listen_port: 3200 + +query_frontend: + search: + duration_slo: 5s + throughput_bytes_slo: 1.073741824e+09 + trace_by_id: + duration_slo: 5s + +distributor: + receivers: # this configuration will listen on all ports and protocols that tempo is capable of. + jaeger: # the receives all come from the OpenTelemetry collector. more configuration information can + protocols: # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver + thrift_http: # + grpc: # for a production deployment you should only enable the receivers you need! + thrift_binary: + thrift_compact: + zipkin: + otlp: + protocols: + http: + grpc: + opencensus: + +ingester: + max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally + +compactor: + compaction: + block_retention: 1h # overall Tempo trace retention. set for demo purposes + +metrics_generator: + registry: + external_labels: + source: tempo + cluster: docker-compose + storage: + path: /tmp/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + +storage: + trace: + backend: local # backend configuration to use + wal: + path: /tmp/tempo/wal # where to store the the wal locally + local: + path: /tmp/tempo/traces + +overrides: + metrics_generator_processors: [service-graphs, span-metrics] \ No newline at end of file