Add OpenTelemetry support for Java Azure Functions (#819)

* working version using agent

* working using shading

* make sure that the app setting is processed everywhere

* update span name

* remove exporters from worker

* add unit tests

* update app setting

* test paths where otel is enabled/disabled

* code cleanup

* change tracer name to match conventions

* remove middleware from worker and rely on serviceloader to load it in from the new otel library

* set class loader consistently

* remove otel dependencies

* move initialization logic back to where it used to be

* revert back to using appsettings to turn on capabilities

* update logic for setting capabilities

* remove unnecessary changes

* revert log line back to its original state

Author: ahmedmuhsin

pom.xml

@@ -111,6 +111,8 @@
       <artifactId>javax.annotation-api</artifactId>
       <version>1.3.2</version>
     </dependency>
+
+    <!-- test -->
     <dependency>
       <groupId>com.microsoft.azure.functions</groupId>
       <artifactId>azure-functions-java-library</artifactId>

src/main/java/com/microsoft/azure/functions/worker/Constants.java

@@ -15,4 +15,6 @@ private Constants(){}
     public final static String JAVA_LIBRARY_DIRECTORY = "/annotationLib";
     public final static String JAVA_LIBRARY_ARTIFACT_ID = "azure-functions-java-library";
     public final static String HAS_IMPLICIT_OUTPUT_QUALIFIED_NAME = "com.microsoft.azure.functions.annotation.HasImplicitOutput";
+    public static final String JAVA_ENABLE_OPENTELEMETRY = "JAVA_ENABLE_OPENTELEMETRY";
+    public static final String JAVA_APPLICATIONINSIGHTS_ENABLE_TELEMETRY = "JAVA_APPLICATIONINSIGHTS_ENABLE_TELEMETRY";
 }

src/main/java/com/microsoft/azure/functions/worker/broker/JavaFunctionBroker.java

@@ -42,12 +42,13 @@ public class JavaFunctionBroker {
 	private volatile InvocationChainFactory invocationChainFactory;
 	private volatile FunctionInstanceInjector functionInstanceInjector;
 	private final Object oneTimeLogicInitializationLock = new Object();
-	private List<Middleware> baseMiddlewares = new ArrayList<>();
+	private List<Middleware> serviceLoadedMiddlewares = new ArrayList<>();
 	private final Map<String, InvocationChainFactory> functionFactories = new ConcurrentHashMap<>();
 	private final SdkParameterAnalyzer sdkParameterAnalyzer = new SdkParameterAnalyzer();
 	private final WorkerObjectCache<CacheKey> workerObjectCache;
 	private static final boolean JAVA_ENABLE_SDK_TYPES_FLAG =
 			Boolean.parseBoolean(System.getenv("JAVA_ENABLE_SDK_TYPES"));
+	private ClassLoader userContextClassLoader;
 
 	private FunctionInstanceInjector newInstanceInjector() {
 		return new FunctionInstanceInjector() {
@@ -86,17 +87,16 @@ private void createInvocationChainFactory(FunctionDefinition functionDefinition,
 		SdkParameterAnalysisResult sdkParameterAnalysisResult =
 				this.sdkParameterAnalyzer.analyze(functionDefinition.getCandidate().getMethod());
 
-		ClassLoader classLoader = this.classLoaderProvider.createClassLoader();
-		List<Middleware> functionMws = new ArrayList<>(this.baseMiddlewares);
+		List<Middleware> functionMws = new ArrayList<>(this.serviceLoadedMiddlewares);
 		boolean hasAnySdkTypes = sdkParameterAnalysisResult.hasAnySdkTypes();
 
 		if (hasAnySdkTypes) {
-			functionMws.add(new SdkTypeMiddleware(classLoader,
+			functionMws.add(new SdkTypeMiddleware(userContextClassLoader,
 					sdkParameterAnalysisResult.getSdkTypesMetaData(),
 					this.sdkParameterAnalyzer.getRegistry()));
 		}
 
-		functionMws.add(getFunctionExecutionMiddleWare(classLoader));
+		functionMws.add(getFunctionExecutionMiddleWare(userContextClassLoader));
 
 		InvocationChainFactory factory = new InvocationChainFactory(functionMws);
 		String functionId = functionDefinition.getDescriptor().getId();
@@ -110,15 +110,16 @@ private void initializeOneTimeLogics() {
 		if (!oneTimeLogicInitialized) {
 			synchronized (oneTimeLogicInitializationLock) {
 				if (!oneTimeLogicInitialized) {
+					userContextClassLoader = classLoaderProvider.createClassLoader();
 
 					if (JAVA_ENABLE_SDK_TYPES_FLAG) {
 						loadGlobalMiddlewares();
 					} else {
 						initializeInvocationChainFactory();
 					}
 
-					initializeFunctionInstanceInjector();
 					oneTimeLogicInitialized = true;
+					initializeFunctionInstanceInjector();
 				}
 			}
 		}
@@ -128,9 +129,9 @@ private void loadGlobalMiddlewares() {
 		ClassLoader prevContextClassLoader = Thread.currentThread().getContextClassLoader();
 		try {
 			//ServiceLoader will use thread context classloader to verify loaded class
-			Thread.currentThread().setContextClassLoader(classLoaderProvider.createClassLoader());
+			Thread.currentThread().setContextClassLoader(userContextClassLoader);
 			for (Middleware middleware : ServiceLoader.load(Middleware.class)) {
-				this.baseMiddlewares.add(middleware);
+				this.serviceLoadedMiddlewares.add(middleware);
 				WorkerLogManager.getSystemLogger().info("Loading discovered middleware " + middleware.getClass().getSimpleName());
 			}
 		} finally {
@@ -139,20 +140,20 @@ private void loadGlobalMiddlewares() {
 	}
 
 	private void initializeInvocationChainFactory() {
-		ArrayList<Middleware> middlewares = new ArrayList<>();
-		ClassLoader prevContextClassLoader = Thread.currentThread().getContextClassLoader();
-		ClassLoader newContextClassLoader = classLoaderProvider.createClassLoader();
+        ClassLoader prevContextClassLoader = Thread.currentThread().getContextClassLoader();
 		try {
 			//ServiceLoader will use thread context classloader to verify loaded class
-			Thread.currentThread().setContextClassLoader(newContextClassLoader);
+			Thread.currentThread().setContextClassLoader(userContextClassLoader);
 			for (Middleware middleware : ServiceLoader.load(Middleware.class)) {
-				middlewares.add(middleware);
+				this.serviceLoadedMiddlewares.add(middleware);
 				WorkerLogManager.getSystemLogger().info("Load middleware " + middleware.getClass().getSimpleName());
 			}
 		} finally {
 			Thread.currentThread().setContextClassLoader(prevContextClassLoader);
 		}
-		middlewares.add(getFunctionExecutionMiddleWare(newContextClassLoader));
+
+		ArrayList<Middleware> middlewares = new ArrayList<>(this.serviceLoadedMiddlewares);
+		middlewares.add(getFunctionExecutionMiddleWare(userContextClassLoader));
 		this.invocationChainFactory = new InvocationChainFactory(middlewares);
 	}
 

src/main/java/com/microsoft/azure/functions/worker/handler/WorkerInitRequestHandler.java

@@ -6,6 +6,9 @@
 
 import java.util.logging.Level;
 
+import static com.microsoft.azure.functions.worker.Constants.JAVA_APPLICATIONINSIGHTS_ENABLE_TELEMETRY;
+import static com.microsoft.azure.functions.worker.Constants.JAVA_ENABLE_OPENTELEMETRY;
+
 public class WorkerInitRequestHandler extends MessageHandler<WorkerInitRequest, WorkerInitResponse.Builder> {
     public WorkerInitRequestHandler(JavaFunctionBroker broker) {
         super(StreamingMessage::getWorkerInitRequest,
@@ -26,7 +29,15 @@ String execute(WorkerInitRequest request, WorkerInitResponse.Builder response) {
         response.putCapabilities("RpcHttpTriggerMetadataRemoved", "RpcHttpTriggerMetadataRemoved");
         response.putCapabilities("HandlesWorkerTerminateMessage", "HandlesWorkerTerminateMessage");
         response.putCapabilities("HandlesWorkerWarmupMessage", "HandlesWorkerWarmupMessage");
+
+        if (Boolean.parseBoolean(System.getenv(JAVA_ENABLE_OPENTELEMETRY)) ||
+                Boolean.parseBoolean(System.getenv(JAVA_APPLICATIONINSIGHTS_ENABLE_TELEMETRY))) {
+            response.putCapabilities("WorkerOpenTelemetryEnabled", "true");
+            response.putCapabilities("WorkerApplicationInsightsLoggingEnabled", "true");
+        }
+
         response.setWorkerMetadata(composeWorkerMetadata());
+
         return "Worker initialized";
     }