Exemplar storage added. Configuration moved from docker-compose to lo…

docker-compose-local.yaml

@@ -23,6 +23,7 @@ services:
       - --web.enable-remote-write-receiver
       - --enable-feature=native-histograms
       - --config.file=/etc/prometheus/prometheus.yml
+      - --enable-feature=exemplar-storage
     ports:
       - "9090:9090"
 
@@ -33,12 +34,9 @@ services:
 
   tempo:
     image: grafana/tempo:2.1.0
-    command:
-      - "-storage.trace.backend=local"                  # tell tempo where to permanently put traces
-      - "-storage.trace.local.path=/tmp/tempo/traces"
-      - "-storage.trace.wal.path=/tmp/tempo/wal"        # tell tempo where to store the wal
-      - "-auth.enabled=false"                           # disables the requirement for the X-Scope-OrgID header
-      - "-server.http-listen-port=3200"
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./tempo/tempo.yaml:/etc/tempo.yaml
     ports:
       - "3200:3200"
       - "4317:4317"

grafana/datasources/datasource.yaml

@@ -6,6 +6,7 @@ apiVersion: 1
 datasources:
   - name: prometheus
     type: prometheus
+    uid: prometheus
     access: proxy
     orgId: 1
     url: http://prometheus:9090
@@ -14,7 +15,11 @@ datasources:
     jsonData:
       tlsAuth: false
       tlsAuthWithCACert: false
+      exemplarTraceIdDestinations:
+      - name: TraceID
+        datasourceUid: tempo
     editable: false
+
   - name: Tempo
     type: tempo
     access: proxy
@@ -26,6 +31,34 @@ datasources:
     editable: false
     apiVersion: 1
     uid: tempo
+    jsonData:
+      httpMethod: GET
+      tracesToLogsV2:
+        datasourceUid: 'loki'
+        spanStartTimeShift: '1h'
+        spanEndTimeShift: '-1h'
+        tags: [{ key: 'job', value: 'job' }] 
+        filterByTraceID: false
+        filterBySpanID: false
+        customQuery: true
+        query: '{compose_service="quickpizza"} |= `$${__span.traceId}`'
+      tracesToMetrics:
+        datasourceUid: 'prometheus'
+        spanStartTimeShift: '1h'
+        spanEndTimeShift: '-1h'
+        tags: [{ key: 'job', value: 'service.job' }, { key: 'instance'}, { key: 'method'}, { key: 'status'}, { key: 'path'}] 
+        queries:
+        - name: 'Request Rate'
+          query: 'sum by(client, server) (rate(traces_service_graph_request_total[$$__rate_interval]))'
+        - name: 'Error Rate'
+          query: 'sum by(client, server) (rate(traces_service_graph_request_failed_total[$$__rate_interval]))'
+      nodeGraph:
+        enabled: true
+      lokiSearch:
+        datasourceUid: 'loki'
+      serviceMap:
+        datasourceUid: 'prometheus'
+
   - name: Pyroscope
     type: 'phlare'
     access: 'proxy'
@@ -34,8 +67,10 @@ datasources:
     url: http://pyroscope:4040
     isDefault: false
     editable: false
+
   - name: Loki
     type: loki
+    uid: loki
     access: proxy 
     orgId: 1
     url: http://loki:3100

tempo/tempo.yaml

@@ -0,0 +1,53 @@
+server:
+  http_listen_port: 3200
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:                           # this configuration will listen on all ports and protocols that tempo is capable of.
+    jaeger:                            # the receives all come from the OpenTelemetry collector.  more configuration information can
+      protocols:                       # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver
+        thrift_http:                   #
+        grpc:                          # for a production deployment you should only enable the receivers you need!
+        thrift_binary:
+        thrift_compact:
+    zipkin:
+    otlp:
+      protocols:
+        http:
+        grpc:
+    opencensus:
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h                # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /tmp/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /tmp/tempo/wal             # where to store the the wal locally
+    local:
+      path: /tmp/tempo/traces
+
+overrides:
+  metrics_generator_processors: [service-graphs, span-metrics]