Performance Test the invocation loop (fix #377) (#542)

sebsto · web-flow · commit 11bea7b2eefc · 2025-08-05T08:39:20.000+02:00
re-implement MAX_INVOCATIONS and fix shell script Fix #377 ### Motivation: In v1, there was a script measuring the performance of the invocation loop. Re-instate this script to allow users and developers to measure the performance impact of their changes. ### Modifications: I re-implemented MAX_INVOCATIONS, to avoid the client looping against the Mock Server. But this time, MAX_INVOCATIONS is handled on the server, not on the client. I slightly modified the script to work with v2 and the new MockServer. ### Result: The script works. This PR has a dependency on #465
diff --git a/Sources/MockServer/MockHTTPServer.swift b/Sources/MockServer/MockHTTPServer.swift
@@ -35,7 +35,7 @@ struct HttpServer {
     private let eventLoopGroup: MultiThreadedEventLoopGroup
     /// the mode. Are we mocking a server for a Lambda function that expects a String or a JSON document? (default: string)
     private let mode: Mode
-    /// the number of connections this server must accept before shutting down (default: 1)
+    /// the number of invocations this server must accept before shutting down (default: 1)
     private let maxInvocations: Int
     /// the logger (control verbosity with LOG_LEVEL environment variable)
     private let logger: Logger
@@ -91,10 +91,6 @@ struct HttpServer {
             ]
         )
 
-        // This counter is used to track the number of incoming connections.
-        // This mock servers accepts n TCP connection then shutdowns
-        let connectionCounter = SharedCounter(maxValue: self.maxInvocations)
-
         // We are handling each incoming connection in a separate child task. It is important
         // to use a discarding task group here which automatically discards finished child tasks.
         // A normal task group retains all child tasks and their outputs in memory until they are
@@ -105,21 +101,15 @@ struct HttpServer {
             try await channel.executeThenClose { inbound in
                 for try await connectionChannel in inbound {
 
-                    let counter = connectionCounter.current()
-                    logger.trace("Handling new connection", metadata: ["connectionNumber": "\(counter)"])
-
                     group.addTask {
-                        await self.handleConnection(channel: connectionChannel)
-                        logger.trace("Done handling connection", metadata: ["connectionNumber": "\(counter)"])
+                        await self.handleConnection(channel: connectionChannel, maxInvocations: self.maxInvocations)
+                        logger.trace("Done handling connection")
                     }
 
-                    if connectionCounter.increment() {
-                        logger.info(
-                            "Maximum number of connections reached, shutting down after current connection",
-                            metadata: ["maxConnections": "\(self.maxInvocations)"]
-                        )
-                        break  // this causes the server to shutdown after handling the connection
-                    }
+                    // This mock server only accepts one connection
+                    // the Lambda Function Runtime will send multiple requests on that single connection
+                    // This Mock Server closes the connection when MAX_INVOCATION is reached
+                    break
                 }
             }
         }
@@ -131,17 +121,19 @@ struct HttpServer {
     /// It handles two requests: one for the next invocation and one for the response.
     /// when the maximum number of requests is reached, it closes the connection.
     private func handleConnection(
-        channel: NIOAsyncChannel<HTTPServerRequestPart, HTTPServerResponsePart>
+        channel: NIOAsyncChannel<HTTPServerRequestPart, HTTPServerResponsePart>,
+        maxInvocations: Int
     ) async {
 
         var requestHead: HTTPRequestHead!
         var requestBody: ByteBuffer?
 
-        // each Lambda invocation results in TWO HTTP requests (next and response)
-        let requestCount = SharedCounter(maxValue: 2)
+        // each Lambda invocation results in TWO HTTP requests (GET /next and POST /response)
+        let maxRequests = maxInvocations * 2
+        let requestCount = SharedCounter(maxValue: maxRequests)
 
         // Note that this method is non-throwing and we are catching any error.
-        // We do this since we don't want to tear down the whole server when a single connection
+        // We do this since we don't want to tear down the whole server when a single request
         // encounters an error.
         do {
             try await channel.executeThenClose { inbound, outbound in
@@ -178,7 +170,7 @@ struct HttpServer {
                         if requestCount.increment() {
                             logger.info(
                                 "Maximum number of requests reached, closing this connection",
-                                metadata: ["maxRequest": "2"]
+                                metadata: ["maxRequest": "\(maxRequests)"]
                             )
                             break  // this finishes handiling request on this connection
                         }
diff --git a/scripts/performance_test.sh b/scripts/performance_test.sh
@@ -3,7 +3,7 @@
 ##
 ## This source file is part of the SwiftAWSLambdaRuntime open source project
 ##
-## Copyright (c) 2017-2022 Apple Inc. and the SwiftAWSLambdaRuntime project authors
+## Copyright (c) 2017-2025 Apple Inc. and the SwiftAWSLambdaRuntime project authors
 ## Licensed under Apache License v2.0
 ##
 ## See LICENSE.txt for license information
@@ -13,67 +13,89 @@
 ##
 ##===----------------------------------------------------------------------===##
 
-set -eu
+log() { printf -- "** %s\n" "$*" >&2; }
+error() { printf -- "** ERROR: %s\n" "$*" >&2; }
+fatal() { error "$@"; exit 1; }
 
 export HOST=127.0.0.1
-export PORT=3000
+export PORT=7000
 export AWS_LAMBDA_RUNTIME_API="$HOST:$PORT"
-export LOG_LEVEL=warning # important, otherwise log becomes a bottleneck
+export LOG_LEVEL=error # important, otherwise log becomes a bottleneck
+
+DATE_CMD="date"
+# using gdate on darwin for nanoseconds
+# gdate is installed by coreutils on macOS
+if [[ $(uname -s) == "Darwin" ]]; then
+  if ! command -v gdate &> /dev/null; then
+    # shellcheck disable=SC2006 # we explicitly want to use backticks here
+    fatal "gdate could not be found. Please \`brew install coreutils\` to proceed."
+  fi
+  DATE_CMD="gdate"
+fi
+echo "⏱️ using $DATE_CMD to count time"
 
-# using gdate on mdarwin for nanoseconds
-if [[ $(uname -s) == "Linux" ]]; then
-  shopt -s expand_aliases
-  alias gdate="date"
+if ! command -v "$DATE_CMD" &> /dev/null; then
+  fatal "$DATE_CMD could not be found. Please install $DATE_CMD to proceed."
 fi
 
+echo "🏗️ Building library and test functions"
 swift build -c release -Xswiftc -g
 LAMBDA_USE_LOCAL_DEPS=../.. swift build --package-path Examples/HelloWorld -c release -Xswiftc -g
 LAMBDA_USE_LOCAL_DEPS=../.. swift build --package-path Examples/HelloJSON -c release -Xswiftc -g
 
 cleanup() {
-  kill -9 $server_pid # ignore-unacceptable-language
+  pkill -9 MockServer && echo "killed previous mock server" # ignore-unacceptable-language
 }
 
-trap "cleanup" ERR
+# start a mock server
+start_mockserver() {
+    if [ $# -ne 2 ]; then
+        fatal "Usage: $0 <mode> <invocations>"
+    fi
+    MODE=$1
+    INVOCATIONS=$2
+    pkill -9 MockServer && echo "killed previous mock server" && sleep 1 # ignore-unacceptable-language
+    echo "👨‍🔧 starting server in $MODE mode for $INVOCATIONS invocations"
+    (MAX_INVOCATIONS="$INVOCATIONS" MODE="$MODE" ./.build/release/MockServer) &
+    server_pid=$!
+    sleep 1
+    kill -0 $server_pid # check server is alive # ignore-unacceptable-language
+}
 
-cold_iterations=1000
-warm_iterations=10000
+cold_iterations=100
+warm_iterations=1000
 results=()
 
 #------------------
 # string
 #------------------
 
-export MODE=string
+MODE=string
 
-# start (fork) mock server
-pkill -9 MockServer && echo "killed previous servers" && sleep 1 # ignore-unacceptable-language
-echo "starting server in $MODE mode"
-(./.build/release/MockServer) &
-server_pid=$!
-sleep 1
-kill -0 $server_pid # check server is alive # ignore-unacceptable-language
+# Start mock server
+start_mockserver "$MODE" "$cold_iterations"
 
 # cold start
-echo "running $MODE mode cold test"
+echo "🚀❄️ running $MODE mode $cold_iterations cold test"
 cold=()
-export MAX_REQUESTS=1
 for (( i=0; i<cold_iterations; i++ )); do
-  start=$(gdate +%s%N)
+  start=$("$DATE_CMD" +%s%N)
   ./Examples/HelloWorld/.build/release/MyLambda
-  end=$(gdate +%s%N)
+  end=$("$DATE_CMD" +%s%N)
   cold+=( $((end-start)) )
 done
 sum_cold=$(IFS=+; echo "$((${cold[*]}))")
 avg_cold=$((sum_cold/cold_iterations))
 results+=( "$MODE, cold: $avg_cold (ns)" )
 
+# reset mock server 
+start_mockserver "$MODE" "$warm_iterations"
+
 # normal calls
-echo "running $MODE mode warm test"
-export MAX_REQUESTS=$warm_iterations
-start=$(gdate +%s%N)
+echo "🚀🌤️ running $MODE mode warm test"
+start=$("$DATE_CMD" +%s%N)
 ./Examples/HelloWorld/.build/release/MyLambda
-end=$(gdate +%s%N)
+end=$("$DATE_CMD" +%s%N)
 sum_warm=$((end-start-avg_cold)) # substract by avg cold since the first call is cold
 avg_warm=$((sum_warm/(warm_iterations-1))) # substract since the first call is cold
 results+=( "$MODE, warm: $avg_warm (ns)" )
@@ -84,34 +106,30 @@ results+=( "$MODE, warm: $avg_warm (ns)" )
 
 export MODE=json
 
-# start (fork) mock server
-pkill -9 MockServer && echo "killed previous servers" && sleep 1 # ignore-unacceptable-language
-echo "starting server in $MODE mode"
-(./.build/release/MockServer) &
-server_pid=$!
-sleep 1
-kill -0 $server_pid # check server is alive # ignore-unacceptable-language
+# Start mock server
+start_mockserver "$MODE" "$cold_iterations"
 
 # cold start
-echo "running $MODE mode cold test"
+echo "🚀❄️ running $MODE mode cold test"
 cold=()
-export MAX_REQUESTS=1
 for (( i=0; i<cold_iterations; i++ )); do
-  start=$(gdate +%s%N)
-  ./Examples/HelloJSON/.build/release/MyLambda
-  end=$(gdate +%s%N)
+  start=$("$DATE_CMD" +%s%N)
+  ./Examples/HelloJSON/.build/release/HelloJSON
+  end=$("$DATE_CMD" +%s%N)
   cold+=( $((end-start)) )
 done
 sum_cold=$(IFS=+; echo "$((${cold[*]}))")
 avg_cold=$((sum_cold/cold_iterations))
 results+=( "$MODE, cold: $avg_cold (ns)" )
 
+# reset mock server 
+start_mockserver "$MODE" "$warm_iterations"
+
 # normal calls
-echo "running $MODE mode warm test"
-export MAX_REQUESTS=$warm_iterations
-start=$(gdate +%s%N)
-./Examples/HelloJSON/.build/release/MyLambda
-end=$(gdate +%s%N)
+echo "🚀🌤️ running $MODE mode warm test"
+start=$("$DATE_CMD" +%s%N)
+./Examples/HelloJSON/.build/release/HelloJSON
+end=$("$DATE_CMD" +%s%N)
 sum_warm=$((end-start-avg_cold)) # substract by avg cold since the first call is cold
 avg_warm=$((sum_warm/(warm_iterations-1))) # substract since the first call is cold
 results+=( "$MODE, warm: $avg_warm (ns)" )
