Skip to content

Performance Test the invocation loop (fix #377) #542

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Aug 5, 2025
Merged

Performance Test the invocation loop (fix #377) #542

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5df07d8
fix performance test script
sebsto Aug 1, 2025
e8cfd8b
swift format
sebsto Aug 1, 2025
55badb2
shell check
sebsto Aug 1, 2025
5cd8a1c
shell check 2
sebsto Aug 1, 2025
9506682
check if gdate is installed + cleanup
sebsto Aug 1, 2025
47f159b
shell check 3
sebsto Aug 1, 2025
8f1b23d
Merge branch 'main' into sebsto/performance_tests
sebsto Aug 3, 2025
af245c4
Merge branch 'main' into sebsto/performance_tests
sebsto Aug 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 14 additions & 22 deletions Sources/MockServer/MockHTTPServer.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ struct HttpServer {
private let eventLoopGroup: MultiThreadedEventLoopGroup
/// the mode. Are we mocking a server for a Lambda function that expects a String or a JSON document? (default: string)
private let mode: Mode
/// the number of connections this server must accept before shutting down (default: 1)
/// the number of invocations this server must accept before shutting down (default: 1)
private let maxInvocations: Int
/// the logger (control verbosity with LOG_LEVEL environment variable)
private let logger: Logger
Expand Down Expand Up @@ -91,10 +91,6 @@ struct HttpServer {
]
)

// This counter is used to track the number of incoming connections.
// This mock servers accepts n TCP connection then shutdowns
let connectionCounter = SharedCounter(maxValue: self.maxInvocations)

// We are handling each incoming connection in a separate child task. It is important
// to use a discarding task group here which automatically discards finished child tasks.
// A normal task group retains all child tasks and their outputs in memory until they are
Expand All @@ -105,21 +101,15 @@ struct HttpServer {
try await channel.executeThenClose { inbound in
for try await connectionChannel in inbound {

let counter = connectionCounter.current()
logger.trace("Handling new connection", metadata: ["connectionNumber": "\(counter)"])

group.addTask {
await self.handleConnection(channel: connectionChannel)
logger.trace("Done handling connection", metadata: ["connectionNumber": "\(counter)"])
await self.handleConnection(channel: connectionChannel, maxInvocations: self.maxInvocations)
logger.trace("Done handling connection")
}

if connectionCounter.increment() {
logger.info(
"Maximum number of connections reached, shutting down after current connection",
metadata: ["maxConnections": "\(self.maxInvocations)"]
)
break // this causes the server to shutdown after handling the connection
}
// This mock server only accepts one connection
// the Lambda Function Runtime will send multiple requests on that single connection
// This Mock Server closes the connection when MAX_INVOCATION is reached
break
}
}
}
Expand All @@ -131,17 +121,19 @@ struct HttpServer {
/// It handles two requests: one for the next invocation and one for the response.
/// when the maximum number of requests is reached, it closes the connection.
private func handleConnection(
channel: NIOAsyncChannel<HTTPServerRequestPart, HTTPServerResponsePart>
channel: NIOAsyncChannel<HTTPServerRequestPart, HTTPServerResponsePart>,
maxInvocations: Int
) async {

var requestHead: HTTPRequestHead!
var requestBody: ByteBuffer?

// each Lambda invocation results in TWO HTTP requests (next and response)
let requestCount = SharedCounter(maxValue: 2)
// each Lambda invocation results in TWO HTTP requests (GET /next and POST /response)
let maxRequests = maxInvocations * 2
let requestCount = SharedCounter(maxValue: maxRequests)

// Note that this method is non-throwing and we are catching any error.
// We do this since we don't want to tear down the whole server when a single connection
// We do this since we don't want to tear down the whole server when a single request
// encounters an error.
do {
try await channel.executeThenClose { inbound, outbound in
Expand Down Expand Up @@ -178,7 +170,7 @@ struct HttpServer {
if requestCount.increment() {
logger.info(
"Maximum number of requests reached, closing this connection",
metadata: ["maxRequest": "2"]
metadata: ["maxRequest": "\(maxRequests)"]
)
break // this finishes handiling request on this connection
}
Expand Down
108 changes: 63 additions & 45 deletions scripts/performance_test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
##
## This source file is part of the SwiftAWSLambdaRuntime open source project
##
## Copyright (c) 2017-2022 Apple Inc. and the SwiftAWSLambdaRuntime project authors
## Copyright (c) 2017-2025 Apple Inc. and the SwiftAWSLambdaRuntime project authors
## Licensed under Apache License v2.0
##
## See LICENSE.txt for license information
Expand All @@ -13,67 +13,89 @@
##
##===----------------------------------------------------------------------===##

set -eu
log() { printf -- "** %s\n" "$*" >&2; }
error() { printf -- "** ERROR: %s\n" "$*" >&2; }
fatal() { error "$@"; exit 1; }

export HOST=127.0.0.1
export PORT=3000
export PORT=7000
export AWS_LAMBDA_RUNTIME_API="$HOST:$PORT"
export LOG_LEVEL=warning # important, otherwise log becomes a bottleneck
export LOG_LEVEL=error # important, otherwise log becomes a bottleneck

DATE_CMD="date"
# using gdate on darwin for nanoseconds
# gdate is installed by coreutils on macOS
if [[ $(uname -s) == "Darwin" ]]; then
if ! command -v gdate &> /dev/null; then
# shellcheck disable=SC2006 # we explicitly want to use backticks here
fatal "gdate could not be found. Please \`brew install coreutils\` to proceed."
fi
DATE_CMD="gdate"
fi
echo "⏱️ using $DATE_CMD to count time"

# using gdate on mdarwin for nanoseconds
if [[ $(uname -s) == "Linux" ]]; then
shopt -s expand_aliases
alias gdate="date"
if ! command -v "$DATE_CMD" &> /dev/null; then
fatal "$DATE_CMD could not be found. Please install $DATE_CMD to proceed."
fi

echo "🏗️ Building library and test functions"
swift build -c release -Xswiftc -g
LAMBDA_USE_LOCAL_DEPS=../.. swift build --package-path Examples/HelloWorld -c release -Xswiftc -g
LAMBDA_USE_LOCAL_DEPS=../.. swift build --package-path Examples/HelloJSON -c release -Xswiftc -g

cleanup() {
kill -9 $server_pid # ignore-unacceptable-language
pkill -9 MockServer && echo "killed previous mock server" # ignore-unacceptable-language
}

trap "cleanup" ERR
# start a mock server
start_mockserver() {
if [ $# -ne 2 ]; then
fatal "Usage: $0 <mode> <invocations>"
fi
MODE=$1
INVOCATIONS=$2
pkill -9 MockServer && echo "killed previous mock server" && sleep 1 # ignore-unacceptable-language
echo "👨‍🔧 starting server in $MODE mode for $INVOCATIONS invocations"
(MAX_INVOCATIONS="$INVOCATIONS" MODE="$MODE" ./.build/release/MockServer) &
server_pid=$!
sleep 1
kill -0 $server_pid # check server is alive # ignore-unacceptable-language
}

cold_iterations=1000
warm_iterations=10000
cold_iterations=100
warm_iterations=1000
results=()

#------------------
# string
#------------------

export MODE=string
MODE=string

# start (fork) mock server
pkill -9 MockServer && echo "killed previous servers" && sleep 1 # ignore-unacceptable-language
echo "starting server in $MODE mode"
(./.build/release/MockServer) &
server_pid=$!
sleep 1
kill -0 $server_pid # check server is alive # ignore-unacceptable-language
# Start mock server
start_mockserver "$MODE" "$cold_iterations"

# cold start
echo "running $MODE mode cold test"
echo "🚀❄️ running $MODE mode $cold_iterations cold test"
cold=()
export MAX_REQUESTS=1
for (( i=0; i<cold_iterations; i++ )); do
start=$(gdate +%s%N)
start=$("$DATE_CMD" +%s%N)
./Examples/HelloWorld/.build/release/MyLambda
end=$(gdate +%s%N)
end=$("$DATE_CMD" +%s%N)
cold+=( $((end-start)) )
done
sum_cold=$(IFS=+; echo "$((${cold[*]}))")
avg_cold=$((sum_cold/cold_iterations))
results+=( "$MODE, cold: $avg_cold (ns)" )

# reset mock server
start_mockserver "$MODE" "$warm_iterations"

# normal calls
echo "running $MODE mode warm test"
export MAX_REQUESTS=$warm_iterations
start=$(gdate +%s%N)
echo "🚀🌤️ running $MODE mode warm test"
start=$("$DATE_CMD" +%s%N)
./Examples/HelloWorld/.build/release/MyLambda
end=$(gdate +%s%N)
end=$("$DATE_CMD" +%s%N)
sum_warm=$((end-start-avg_cold)) # substract by avg cold since the first call is cold
avg_warm=$((sum_warm/(warm_iterations-1))) # substract since the first call is cold
results+=( "$MODE, warm: $avg_warm (ns)" )
Expand All @@ -84,34 +106,30 @@ results+=( "$MODE, warm: $avg_warm (ns)" )

export MODE=json

# start (fork) mock server
pkill -9 MockServer && echo "killed previous servers" && sleep 1 # ignore-unacceptable-language
echo "starting server in $MODE mode"
(./.build/release/MockServer) &
server_pid=$!
sleep 1
kill -0 $server_pid # check server is alive # ignore-unacceptable-language
# Start mock server
start_mockserver "$MODE" "$cold_iterations"

# cold start
echo "running $MODE mode cold test"
echo "🚀❄️ running $MODE mode cold test"
cold=()
export MAX_REQUESTS=1
for (( i=0; i<cold_iterations; i++ )); do
start=$(gdate +%s%N)
./Examples/HelloJSON/.build/release/MyLambda
end=$(gdate +%s%N)
start=$("$DATE_CMD" +%s%N)
./Examples/HelloJSON/.build/release/HelloJSON
end=$("$DATE_CMD" +%s%N)
cold+=( $((end-start)) )
done
sum_cold=$(IFS=+; echo "$((${cold[*]}))")
avg_cold=$((sum_cold/cold_iterations))
results+=( "$MODE, cold: $avg_cold (ns)" )

# reset mock server
start_mockserver "$MODE" "$warm_iterations"

# normal calls
echo "running $MODE mode warm test"
export MAX_REQUESTS=$warm_iterations
start=$(gdate +%s%N)
./Examples/HelloJSON/.build/release/MyLambda
end=$(gdate +%s%N)
echo "🚀🌤️ running $MODE mode warm test"
start=$("$DATE_CMD" +%s%N)
./Examples/HelloJSON/.build/release/HelloJSON
end=$("$DATE_CMD" +%s%N)
sum_warm=$((end-start-avg_cold)) # substract by avg cold since the first call is cold
avg_warm=$((sum_warm/(warm_iterations-1))) # substract since the first call is cold
results+=( "$MODE, warm: $avg_warm (ns)" )
Expand Down