[wip] LLM Observability SDK merge branch

.github/CODEOWNERS

@@ -74,3 +74,9 @@ dd-trace-core/src/main/java/datadog/trace/core/datastreams                  @Dat
 dd-trace-core/src/test/groovy/datadog/trace/core/datastreams                @DataDog/data-streams-monitoring
 internal-api/src/main/java/datadog/trace/api/datastreams                    @DataDog/data-streams-monitoring
 internal-api/src/test/groovy/datadog/trace/api/datastreams                  @DataDog/data-streams-monitoring
+
+# @DataDog/ml-observability
+dd-trace-api/src/main/java/datadog/trace/api/llmobs/ @DataDog/ml-observability
+dd-java-agent/agent-llmobs/                          @DataDog/ml-observability
+dd-trace-core/src/main/java/datadog/trace/llmobs/    @DataDog/ml-observability
+dd-trace-core/src/test/groovy/datadog/trace/llmobs/  @DataDog/ml-observability

dd-trace-api/build.gradle

@@ -32,6 +32,12 @@ excludedClassesCoverage += [
   'datadog.trace.api.profiling.ProfilingScope',
   'datadog.trace.api.profiling.ProfilingContext',
   'datadog.trace.api.profiling.ProfilingContextAttribute.NoOp',
+  'datadog.trace.api.llmobs.LLMObs',
+  'datadog.trace.api.llmobs.LLMObs.LLMMessage',
+  'datadog.trace.api.llmobs.LLMObs.ToolCall',
+  'datadog.trace.api.llmobs.LLMObsSpan',
+  'datadog.trace.api.llmobs.noop.NoOpLLMObsSpan',
+  'datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory',
   'datadog.trace.api.experimental.DataStreamsCheckpointer',
   'datadog.trace.api.experimental.DataStreamsCheckpointer.NoOp',
   'datadog.trace.api.experimental.DataStreamsContextCarrier',

dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java

@@ -0,0 +1,136 @@
+package datadog.trace.api.llmobs;
+
+import datadog.trace.api.llmobs.noop.NoOpLLMObsSpanFactory;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+public class LLMObs {
+  protected LLMObs() {}
+
+  protected static LLMObsSpanFactory SPAN_FACTORY = NoOpLLMObsSpanFactory.INSTANCE;
+
+  public static LLMObsSpan startLLMSpan(
+      String spanName,
+      String modelName,
+      String modelProvider,
+      @Nullable String mlApp,
+      @Nullable String sessionId) {
+
+    return SPAN_FACTORY.startLLMSpan(spanName, modelName, modelProvider, mlApp, sessionId);
+  }
+
+  public static LLMObsSpan startAgentSpan(
+      String spanName, @Nullable String mlApp, @Nullable String sessionId) {
+
+    return SPAN_FACTORY.startAgentSpan(spanName, mlApp, sessionId);
+  }
+
+  public static LLMObsSpan startToolSpan(
+      String spanName, @Nullable String mlApp, @Nullable String sessionId) {
+
+    return SPAN_FACTORY.startToolSpan(spanName, mlApp, sessionId);
+  }
+
+  public static LLMObsSpan startTaskSpan(
+      String spanName, @Nullable String mlApp, @Nullable String sessionId) {
+
+    return SPAN_FACTORY.startTaskSpan(spanName, mlApp, sessionId);
+  }
+
+  public static LLMObsSpan startWorkflowSpan(
+      String spanName, @Nullable String mlApp, @Nullable String sessionId) {
+
+    return SPAN_FACTORY.startWorkflowSpan(spanName, mlApp, sessionId);
+  }
+
+  public interface LLMObsSpanFactory {
+    LLMObsSpan startLLMSpan(
+        String spanName,
+        String modelName,
+        String modelProvider,
+        @Nullable String mlApp,
+        @Nullable String sessionId);
+
+    LLMObsSpan startAgentSpan(String spanName, @Nullable String mlApp, @Nullable String sessionId);
+
+    LLMObsSpan startToolSpan(String spanName, @Nullable String mlApp, @Nullable String sessionId);
+
+    LLMObsSpan startTaskSpan(String spanName, @Nullable String mlApp, @Nullable String sessionId);
+
+    LLMObsSpan startWorkflowSpan(
+        String spanName, @Nullable String mlApp, @Nullable String sessionId);
+  }
+
+  public static class ToolCall {
+    private String name;
+    private String type;
+    private String toolId;
+    private Map<String, Object> arguments;
+
+    public static ToolCall from(
+        String name, String type, String toolId, Map<String, Object> arguments) {
+      return new ToolCall(name, type, toolId, arguments);
+    }
+
+    private ToolCall(String name, String type, String toolId, Map<String, Object> arguments) {
+      this.name = name;
+      this.type = type;
+      this.toolId = toolId;
+      this.arguments = arguments;
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    public String getType() {
+      return type;
+    }
+
+    public String getToolId() {
+      return toolId;
+    }
+
+    public Map<String, Object> getArguments() {
+      return arguments;
+    }
+  }
+
+  public static class LLMMessage {
+    private String role;
+    private String content;
+    private List<ToolCall> toolCalls;
+
+    public static LLMMessage from(String role, String content, List<ToolCall> toolCalls) {
+      return new LLMMessage(role, content, toolCalls);
+    }
+
+    public static LLMMessage from(String role, String content) {
+      return new LLMMessage(role, content);
+    }
+
+    private LLMMessage(String role, String content, List<ToolCall> toolCalls) {
+      this.role = role;
+      this.content = content;
+      this.toolCalls = toolCalls;
+    }
+
+    private LLMMessage(String role, String content) {
+      this.role = role;
+      this.content = content;
+    }
+
+    public String getRole() {
+      return role;
+    }
+
+    public String getContent() {
+      return content;
+    }
+
+    public List<ToolCall> getToolCalls() {
+      return toolCalls;
+    }
+  }
+}

dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java

@@ -0,0 +1,145 @@
+package datadog.trace.api.llmobs;
+
+import java.util.List;
+import java.util.Map;
+
+/** This interface represent an individual LLM Obs span. */
+public interface LLMObsSpan {
+
+  /**
+   * Annotate the span with inputs and outputs for LLM spans
+   *
+   * @param inputMessages The input messages of the span in the form of a list
+   * @param outputMessages The output messages of the span in the form of a list
+   */
+  void annotateIO(List<LLMObs.LLMMessage> inputMessages, List<LLMObs.LLMMessage> outputMessages);
+
+  /**
+   * Annotate the span with inputs and outputs
+   *
+   * @param inputData The input data of the span in the form of a string
+   * @param outputData The output data of the span in the form of a string
+   */
+  void annotateIO(String inputData, String outputData);
+
+  /**
+   * Annotate the span with metadata
+   *
+   * @param metadata A map of JSON serializable key-value pairs that contains metadata information
+   *     relevant to the input or output operation described by the span
+   */
+  void setMetadata(Map<String, Object> metadata);
+
+  /**
+   * Annotate the span with metrics
+   *
+   * @param metrics A map of JSON serializable keys and numeric values that users can add as metrics
+   *     relevant to the operation described by the span (input_tokens, output_tokens, total_tokens,
+   *     etc.).
+   */
+  void setMetrics(Map<String, Number> metrics);
+
+  /**
+   * Annotate the span with a single metric key value pair for the span’s context (number of tokens
+   * document length, etc).
+   *
+   * @param key the name of the metric
+   * @param value the value of the metric
+   */
+  void setMetric(CharSequence key, int value);
+
+  /**
+   * Annotate the span with a single metric key value pair for the span’s context (number of tokens
+   * document length, etc).
+   *
+   * @param key the name of the metric
+   * @param value the value of the metric
+   */
+  void setMetric(CharSequence key, long value);
+
+  /**
+   * Annotate the span with a single metric key value pair for the span’s context (number of tokens
+   * document length, etc).
+   *
+   * @param key the name of the metric
+   * @param value the value of the metric
+   */
+  void setMetric(CharSequence key, double value);
+
+  /**
+   * Annotate the span with tags
+   *
+   * @param tags An map of JSON serializable key-value pairs that users can add as tags regarding
+   *     the span’s context (session, environment, system, versioning, etc.).
+   */
+  void setTags(Map<String, Object> tags);
+
+  /**
+   * Annotate the span with a single tag key value pair as a tag regarding the span’s context
+   * (session, environment, system, versioning, etc.).
+   *
+   * @param key the key of the tag
+   * @param value the value of the tag
+   */
+  void setTag(String key, String value);
+
+  /**
+   * Annotate the span with a single tag key value pair as a tag regarding the span’s context
+   * (session, environment, system, versioning, etc.).
+   *
+   * @param key the key of the tag
+   * @param value the value of the tag
+   */
+  void setTag(String key, boolean value);
+
+  /**
+   * Annotate the span with a single tag key value pair as a tag regarding the span’s context
+   * (session, environment, system, versioning, etc.).
+   *
+   * @param key the key of the tag
+   * @param value the value of the tag
+   */
+  void setTag(String key, int value);
+
+  /**
+   * Annotate the span with a single tag key value pair as a tag regarding the span’s context
+   * (session, environment, system, versioning, etc.).
+   *
+   * @param key the key of the tag
+   * @param value the value of the tag
+   */
+  void setTag(String key, long value);
+
+  /**
+   * Annotate the span with a single tag key value pair as a tag regarding the span’s context
+   * (session, environment, system, versioning, etc.).
+   *
+   * @param key the key of the tag
+   * @param value the value of the tag
+   */
+  void setTag(String key, double value);
+
+  /**
+   * Annotate the span to indicate that an error occurred
+   *
+   * @param error whether an error occurred
+   */
+  void setError(boolean error);
+
+  /**
+   * Annotate the span with an error message
+   *
+   * @param errorMessage the message of the error
+   */
+  void setErrorMessage(String errorMessage);
+
+  /**
+   * Annotate the span with a throwable
+   *
+   * @param throwable the errored throwable
+   */
+  void addThrowable(Throwable throwable);
+
+  /** Finishes (closes) a span */
+  void finish();
+}

dd-trace-api/src/main/java/datadog/trace/api/llmobs/noop/NoOpLLMObsSpan.java

@@ -0,0 +1,61 @@
+package datadog.trace.api.llmobs.noop;
+
+import datadog.trace.api.llmobs.LLMObs;
+import datadog.trace.api.llmobs.LLMObsSpan;
+import java.util.List;
+import java.util.Map;
+
+public class NoOpLLMObsSpan implements LLMObsSpan {
+  public static final LLMObsSpan INSTANCE = new NoOpLLMObsSpan();
+
+  @Override
+  public void annotateIO(List<LLMObs.LLMMessage> inputData, List<LLMObs.LLMMessage> outputData) {}
+
+  @Override
+  public void annotateIO(String inputData, String outputData) {}
+
+  @Override
+  public void setMetadata(Map<String, Object> metadata) {}
+
+  @Override
+  public void setMetrics(Map<String, Number> metrics) {}
+
+  @Override
+  public void setMetric(CharSequence key, int value) {}
+
+  @Override
+  public void setMetric(CharSequence key, long value) {}
+
+  @Override
+  public void setMetric(CharSequence key, double value) {}
+
+  @Override
+  public void setTags(Map<String, Object> tags) {}
+
+  @Override
+  public void setTag(String key, String value) {}
+
+  @Override
+  public void setTag(String key, boolean value) {}
+
+  @Override
+  public void setTag(String key, int value) {}
+
+  @Override
+  public void setTag(String key, long value) {}
+
+  @Override
+  public void setTag(String key, double value) {}
+
+  @Override
+  public void setError(boolean error) {}
+
+  @Override
+  public void setErrorMessage(String errorMessage) {}
+
+  @Override
+  public void addThrowable(Throwable throwable) {}
+
+  @Override
+  public void finish() {}
+}