fix: remove broken tracing middleware

llama_stack/core/server/server.py

@@ -69,7 +69,6 @@
 
 from .auth import AuthenticationMiddleware
 from .quota import QuotaMiddleware
-from .tracing import TracingMiddleware
 
 REPO_ROOT = Path(__file__).parent.parent.parent.parent
 
@@ -411,6 +410,7 @@ def create_app() -> StackApp:
 
     if Api.telemetry in impls:
         setup_logger(impls[Api.telemetry])
+        TelemetryAdapter.fastapi_middleware(app)  # hold us over until we can move to programmatic instrumentation
     else:
         setup_logger(TelemetryAdapter(TelemetryConfig(), {}))
 
@@ -470,8 +470,6 @@ def create_app() -> StackApp:
     app.exception_handler(RequestValidationError)(global_exception_handler)
     app.exception_handler(Exception)(global_exception_handler)
 
-    app.add_middleware(TracingMiddleware, impls=impls, external_apis=external_apis)
-
     return app
 
 

llama_stack/providers/inline/telemetry/meta_reference/sqlite_span_processor.py

@@ -112,6 +112,12 @@ def on_end(self, span: Span):
             if parent_context:
                 parent_span_id = format_span_id(parent_context.span_id)
 
+            # Determine if this span should be marked as root:
+            # 1. If it has LOCAL_ROOT_SPAN_MARKER attribute (explicitly marked by start_trace())
+            # 2. If it has no parent (implicit root span from FastAPI instrumentation)
+            is_root_span = span.attributes.get(LOCAL_ROOT_SPAN_MARKER) or parent_span_id is None
+            root_span_id_value = span_id if is_root_span else None
+
             # Insert into traces
             cursor.execute(
                 """
@@ -126,7 +132,7 @@ def on_end(self, span: Span):
                 (
                     trace_id,
                     service_name,
-                    (span_id if span.attributes.get(LOCAL_ROOT_SPAN_MARKER) else None),
+                    root_span_id_value,
                     datetime.fromtimestamp(span.start_time / 1e9, UTC).isoformat(),
                     datetime.fromtimestamp(span.end_time / 1e9, UTC).isoformat(),
                 ),

llama_stack/providers/inline/telemetry/meta_reference/telemetry.py

@@ -8,9 +8,11 @@
 import threading
 from typing import Any
 
+from fastapi import FastAPI
 from opentelemetry import metrics, trace
 from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from opentelemetry.sdk.metrics import MeterProvider
 from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
 from opentelemetry.sdk.resources import Resource
@@ -362,3 +364,8 @@ async def get_span_tree(
                 max_depth=max_depth,
             )
         )
+
+    @staticmethod
+    def fastapi_middleware(app: FastAPI):
+        """Inject telemetry middleware into the FastAPI app"""
+        FastAPIInstrumentor.instrument_app(app)

llama_stack/providers/utils/telemetry/sqlite_trace_store.py

@@ -19,6 +19,7 @@
     QueryCondition,
     QueryMetricsResponse,
     Span,
+    SpanStatus,
     SpanWithStatus,
     Trace,
 )
@@ -343,6 +344,11 @@ async def get_span_tree(
                     raise ValueError(f"Span {span_id} not found")
 
                 for row in rows:
+                    # Map OpenTelemetry status codes to API enum values
+                    # OTEL has: UNSET, OK, ERROR -> Map to SpanStatus enum
+                    otel_status = row["status"].upper()
+                    status = SpanStatus.ERROR if otel_status == "ERROR" else SpanStatus.OK
+
                     span = SpanWithStatus(
                         span_id=row["span_id"],
                         trace_id=row["trace_id"],
@@ -351,7 +357,7 @@ async def get_span_tree(
                         start_time=datetime.fromisoformat(row["start_time"]),
                         end_time=datetime.fromisoformat(row["end_time"]),
                         attributes=json.loads(row["filtered_attributes"]),
-                        status=row["status"].lower(),
+                        status=status,
                     )
 
                     spans_by_id[span.span_id] = span

pyproject.toml

@@ -25,14 +25,14 @@ classifiers = [
 ]
 dependencies = [
     "aiohttp",
-    "fastapi>=0.115.0,<1.0",                  # server
-    "fire",                                   # for MCP in LLS client
+    "fastapi>=0.115.0,<1.0", # server
+    "fire", # for MCP in LLS client
     "httpx",
     "huggingface-hub>=0.34.0,<1.0",
     "jinja2>=3.1.6",
     "jsonschema",
     "llama-stack-client>=0.2.23",
-    "openai>=1.107",                                # for expires_after support
+    "openai>=1.107", # for expires_after support
     "prompt-toolkit",
     "python-dotenv",
     "python-jose[cryptography]",
@@ -43,13 +43,14 @@ dependencies = [
     "tiktoken",
     "pillow",
     "h11>=0.16.0",
-    "python-multipart>=0.0.20",               # For fastapi Form
-    "uvicorn>=0.34.0",                        # server
-    "opentelemetry-sdk>=1.30.0",              # server
+    "python-multipart>=0.0.20", # For fastapi Form
+    "uvicorn>=0.34.0", # server
+    "opentelemetry-sdk>=1.30.0", # server
     "opentelemetry-exporter-otlp-proto-http>=1.30.0", # server
-    "aiosqlite>=0.21.0",                      # server - for metadata store
-    "asyncpg",                                # for metadata store
-    "sqlalchemy[asyncio]>=2.0.41",           # server - for conversations
+    "aiosqlite>=0.21.0", # server - for metadata store
+    "asyncpg", # for metadata store
+    "sqlalchemy[asyncio]>=2.0.41", # server - for conversations
+    "opentelemetry-instrumentation-fastapi>=0.57b0",
 ]
 
 [project.optional-dependencies]