feat: Add proxied tools functionality, specifically make sure it proxies tempo correctly

.github/workflows/e2e.yml

@@ -59,6 +59,8 @@ jobs:
         run: go build -o dist/mcp-grafana ./cmd/mcp-grafana
 
       - name: Start MCP server in background
+        env:
+          GRAFANA_URL: http://localhost:3000
         if: matrix.transport != 'stdio'
         run: nohup ./dist/mcp-grafana -t ${{ matrix.transport }} > mcp.log 2>&1 &
 

.gitignore

@@ -13,3 +13,6 @@ wheels/
 # Virtual environments
 .venv
 .envrc
+
+# Temporary test files
+.tmp

Makefile

@@ -47,6 +47,11 @@ test-python-e2e: ## Run Python E2E tests (requires docker-compose services and S
 	cd tests && uv sync --all-groups
 	cd tests && uv run pytest
 
+.PHONY: test-python-e2e-local
+test-python-e2e-local: ## Run Python E2E tests excluding those requiring external API keys (claude model tests).
+	cd tests && uv sync --all-groups
+	cd tests && uv run pytest -k "not claude" --tb=short
+
 .PHONY: run
 run: ## Run the MCP server in stdio mode.
 	go run ./cmd/mcp-grafana
@@ -61,4 +66,32 @@ run-streamable-http: ## Run the MCP server in StreamableHTTP mode.
 
 .PHONY: run-test-services
 run-test-services: ## Run the docker-compose services required for the unit and integration tests.
-	docker-compose up -d --build
+	docker compose up -d --build
+
+.PHONY: test-e2e-full
+test-e2e-full: ## Run full E2E test workflow: start services, rebuild server, run tests.
+	@echo "Starting full E2E test workflow..."
+	@mkdir -p .tmp
+	@echo "Ensuring Docker services are running..."
+	$(MAKE) run-test-services
+	@echo "Stopping any existing MCP server processes..."
+	-pkill -f "mcp-grafana.*sse" || true
+	@echo "Building fresh MCP server binary..."
+	$(MAKE) build
+	@echo "Starting MCP server in background..."
+	@GRAFANA_URL=http://localhost:3000 ./dist/mcp-grafana --transport sse --log-level debug --debug > .tmp/server.log 2>&1 & echo $$! > .tmp/mcp-server.pid
+	@sleep 5
+	@echo "Running Python E2E tests..."
+	@$(MAKE) test-python-e2e-local; \
+	test_result=$$?; \
+	echo "Cleaning up MCP server..."; \
+	kill `cat .tmp/mcp-server.pid 2>/dev/null` 2>/dev/null || true; \
+	rm -rf .tmp; \
+	exit $$test_result
+
+.PHONY: test-e2e-cleanup
+test-e2e-cleanup: ## Clean up any leftover E2E test processes and files.
+	@echo "Cleaning up any leftover E2E test processes and files..."
+	-pkill -f "mcp-grafana.*sse" || true
+	-rm -rf .tmp
+	@echo "Cleanup complete."

README.md

@@ -78,6 +78,12 @@ _The following features are currently available in MCP server. This list is for
   - **Time range support:** Add time range parameters to links (`from=now-1h&to=now`)
   - **Custom parameters:** Include additional query parameters like dashboard variables or refresh intervals
 
+### Proxied Tools
+
+- **Dynamic tool discovery:** Automatically discover and expose tools from external MCP servers connected as Grafana datasources
+- **Tempo support:** Query traces and perform trace analysis through Tempo datasources that have MCP capabilities
+- **Extensible architecture:** Support for additional datasource types can be added by implementing the ProxyHandler interface
+
 The list of tools is configurable, so you can choose which tools you want to make available to the MCP client.
 This is useful if you don't use certain functionality or if you don't want to take up too much of the context window.
 To disable a category of tools, use the `--disable-<category>` flag when starting the server. For example, to disable

cmd/mcp-grafana/main.go

@@ -6,8 +6,11 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"os/signal"
 	"slices"
 	"strings"
+	"syscall"
+	"time"
 
 	"github.com/mark3labs/mcp-go/server"
 
@@ -35,7 +38,7 @@ type disabledTools struct {
 	search, datasource, incident,
 	prometheus, loki, alerting,
 	dashboard, oncall, asserts, sift, admin,
-	pyroscope, navigation bool
+	pyroscope, navigation, proxied bool
 }
 
 // Configuration for the Grafana client.
@@ -51,8 +54,7 @@ type grafanaConfig struct {
 }
 
 func (dt *disabledTools) addFlags() {
-	flag.StringVar(&dt.enabledTools, "enabled-tools", "search,datasource,incident,prometheus,loki,alerting,dashboard,oncall,asserts,sift,admin,pyroscope,navigation", "A comma separated list of tools enabled for this server. Can be overwritten entirely or by disabling specific components, e.g. --disable-search.")
-
+	flag.StringVar(&dt.enabledTools, "enabled-tools", "search,datasource,incident,prometheus,loki,alerting,dashboard,oncall,asserts,sift,admin,pyroscope,navigation,proxied", "A comma separated list of tools enabled for this server. Can be overwritten entirely or by disabling specific components, e.g. --disable-search.")
 	flag.BoolVar(&dt.search, "disable-search", false, "Disable search tools")
 	flag.BoolVar(&dt.datasource, "disable-datasource", false, "Disable datasource tools")
 	flag.BoolVar(&dt.incident, "disable-incident", false, "Disable incident tools")
@@ -66,6 +68,7 @@ func (dt *disabledTools) addFlags() {
 	flag.BoolVar(&dt.admin, "disable-admin", false, "Disable admin tools")
 	flag.BoolVar(&dt.pyroscope, "disable-pyroscope", false, "Disable pyroscope tools")
 	flag.BoolVar(&dt.navigation, "disable-navigation", false, "Disable navigation tools")
+	flag.BoolVar(&dt.proxied, "disable-proxied", false, "Disable proxied tools (tools from external MCP servers)")
 }
 
 func (gc *grafanaConfig) addFlags() {
@@ -93,6 +96,7 @@ func (dt *disabledTools) addTools(s *server.MCPServer) {
 	maybeAddTools(s, tools.AddAdminTools, enabledTools, dt.admin, "admin")
 	maybeAddTools(s, tools.AddPyroscopeTools, enabledTools, dt.pyroscope, "pyroscope")
 	maybeAddTools(s, tools.AddNavigationTools, enabledTools, dt.navigation, "navigation")
+	maybeAddTools(s, tools.AddProxiedTools, enabledTools, dt.proxied, "proxied")
 }
 
 func newServer(dt disabledTools) *server.MCPServer {
@@ -110,6 +114,7 @@ func newServer(dt disabledTools) *server.MCPServer {
 	- Admin: List teams and perform administrative tasks.
 	- Pyroscope: Profile applications and fetch profiling data.
 	- Navigation: Generate deeplink URLs for Grafana resources like dashboards, panels, and Explore queries.
+	- Proxied Tools: Access tools from external MCP servers (like Tempo) through dynamic discovery.
 	`))
 	dt.addTools(s)
 	return s
@@ -119,6 +124,23 @@ func run(transport, addr, basePath, endpointPath string, logLevel slog.Level, dt
 	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel})))
 	s := newServer(dt)
 
+	// Create a context that will be cancelled on shutdown
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Set up signal handling for graceful shutdown
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
+
+	// Handle shutdown in a goroutine
+	go func() {
+		<-sigChan
+		slog.Info("Received shutdown signal, cleaning up...")
+		cancel() // This will cause servers to stop
+	}()
+
+	// Start the appropriate server
+	var serverErr error
 	switch transport {
 	case "stdio":
 		srv := server.NewStdioServer(s)
@@ -134,6 +156,7 @@ func run(transport, addr, basePath, endpointPath string, logLevel slog.Level, dt
 		if err := srv.Start(addr); err != nil {
 			return fmt.Errorf("server error: %v", err)
 		}
+		<-ctx.Done()
 	case "streamable-http":
 		srv := server.NewStreamableHTTPServer(s, server.WithHTTPContextFunc(mcpgrafana.ComposedHTTPContextFunc(gc)),
 			server.WithStateLess(true),
@@ -143,13 +166,18 @@ func run(transport, addr, basePath, endpointPath string, logLevel slog.Level, dt
 		if err := srv.Start(addr); err != nil {
 			return fmt.Errorf("server error: %v", err)
 		}
+		<-ctx.Done()
 	default:
-		return fmt.Errorf(
-			"invalid transport type: %s. Must be 'stdio', 'sse' or 'streamable-http'",
-			transport,
-		)
+		return fmt.Errorf("invalid transport type: %s. Must be 'stdio', 'sse' or 'streamable-http'", transport)
 	}
-	return nil
+
+	// Cleanup after server stops
+	tools.StopProxiedTools()
+
+	// Give a bit of time for cleanup and log flushing
+	time.Sleep(100 * time.Millisecond)
+
+	return serverErr
 }
 
 func main() {

docker-compose.yaml

@@ -44,3 +44,22 @@ services:
     image: grafana/pyroscope:1.13.4
     ports:
       - 4040:4040
+
+  # We're using a specific digest of Tempo that includes the MCP server functionality.
+  # This feature is not available in older releases like 2.3.1.
+  # Pinned to the digest from July 25, 2025 to prevent unexpected changes.
+  tempo:
+    image: grafana/tempo@sha256:3a6738580f7babc11104f8617c892fae8aa27b19ec3bfcbf44fbb1c343ae50fc
+    command: [ "-config.file=/etc/tempo/tempo-config.yaml" ]
+    volumes:
+      - ./testdata/tempo-config.yaml:/etc/tempo/tempo-config.yaml
+    ports:
+      - "3200:3200"    # tempo
+
+  tempo2:
+    image: grafana/tempo@sha256:3a6738580f7babc11104f8617c892fae8aa27b19ec3bfcbf44fbb1c343ae50fc
+    command: [ "-config.file=/etc/tempo/tempo-config.yaml" ]
+    volumes:
+      - ./testdata/tempo-config-2.yaml:/etc/tempo/tempo-config.yaml
+    ports:
+      - "3201:3201"    # tempo instance 2

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
-	github.com/cenkalti/backoff/v5 v5.0.2 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cheekybits/genny v1.0.0 // indirect
 	github.com/chromedp/cdproto v0.0.0-20250429231605-6ed5b53462d4 // indirect

go.sum

@@ -21,6 +21,8 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMU
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
 github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=

testdata/provisioning/datasources/datasources.yaml

@@ -27,3 +27,17 @@ datasources:
     access: proxy
     url: http://pyroscope:4040
     isDefault: false
+  - name: Tempo
+    id: 4
+    uid: tempo
+    type: tempo
+    access: proxy
+    url: http://tempo:3200
+    isDefault: false
+  - name: Tempo Secondary
+    id: 5
+    uid: tempo-secondary
+    type: tempo
+    access: proxy
+    url: http://tempo2:3201
+    isDefault: false

testdata/tempo-config-2.yaml

@@ -0,0 +1,29 @@
+server:
+  http_listen_port: 3201
+  log_level: debug
+
+query_frontend:
+  mcp_server:
+    enabled: true
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+        grpc:
+
+ingester:
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    block_retention: 1h
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /tmp/tempo2/blocks
+    wal:
+      path: /tmp/tempo2/wal

testdata/tempo-config.yaml

@@ -0,0 +1,29 @@
+server:
+  http_listen_port: 3200
+  log_level: debug
+
+query_frontend:
+  mcp_server:
+    enabled: true
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+        grpc:
+
+ingester:
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    block_retention: 1h
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /tmp/tempo/blocks
+    wal:
+      path: /tmp/tempo/wal