diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 05a5adfcbb4..8b621365070 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -15,7 +15,7 @@ script: - export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}" - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - - git clone --branch dd-trace-java/tracer-benchmarks https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform + - git clone --branch dd-trace-java/tracer-benchmarks-parallel https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform artifacts: name: "reports" paths: diff --git a/benchmark/benchmarks.sh b/benchmark/benchmarks.sh index fac7c3f3fbb..0b245038afa 100755 --- a/benchmark/benchmarks.sh +++ b/benchmark/benchmarks.sh @@ -33,10 +33,6 @@ if [[ ! -f "${TRACER}" ]]; then cd "${SCRIPT_DIR}" fi -# Cleanup previous reports -rm -rf "${REPORTS_DIR}" -mkdir -p "${REPORTS_DIR}" - if [[ "$#" == '0' ]]; then for type in 'startup' 'load' 'dacapo'; do run_benchmarks "$type" diff --git a/benchmark/load/insecure-bank/benchmark.json b/benchmark/load/insecure-bank/benchmark.json deleted file mode 100644 index 89785199d61..00000000000 --- a/benchmark/load/insecure-bank/benchmark.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "name": "load_insecure-bank", - "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"", - "service": "bash -c \"${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080/login ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", - "run": "bash -c \"java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log\"", - "timeout": 150, - "iterations": 1, - "variants": { - "${NO_AGENT_VARIANT}": { - "env": { - "VARIANT": "${NO_AGENT_VARIANT}", - "JAVA_OPTS": "" - } - }, - "tracing": { - "env": { - "VARIANT": "tracing", - "JAVA_OPTS": "-javaagent:${TRACER}" - } - }, - "iast": { - "env": { - "VARIANT": "iast", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true" - } - }, - "iast_GLOBAL": { - "env": { - "VARIANT": "iast_GLOBAL", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL" - } - }, - "iast_FULL": { - "env": { - "VARIANT": "iast_FULL", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL" - } - } - } -} diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 993ac6639f7..2dc5ce582c6 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -1,18 +1,62 @@ import http from 'k6/http'; import {checkResponse, isOk, isRedirect} from "../../utils/k6.js"; -const baseUrl = 'http://localhost:8080'; +const variants = { + "no_agent": { + "APP_URL": 'http://localhost:8080', + }, + "tracing": { + "APP_URL": 'http://localhost:8081', + }, + "profiling": { + "APP_URL": 'http://localhost:8082', + }, + "iast": { + "APP_URL": 'http://localhost:8083', + }, + "iast_GLOBAL": { + "APP_URL": 'http://localhost:8084', + }, + "iast_FULL": { + "APP_URL": 'http://localhost:8085', + }, +} + +export const options = function (variants) { + let scenarios = {}; + for (const variant of Object.keys(variants)) { + scenarios[`load--insecure-bank--${variant}--warmup`] = { + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + vus: 5, + duration: '20s', + gracefulStop: '2s', + env: { + "APP_URL": variants[variant]["APP_URL"] + } + }; + + scenarios[`load--insecure-bank--${variant}--high_load`] = { + executor: 'constant-vus', + vus: 5, + startTime: '22s', + duration: '15s', + gracefulStop: '2s', + env: { + "APP_URL": variants[variant]["APP_URL"] + } + }; + } -export const options = { - discardResponseBodies: true, - vus: 5, - iterations: 40000 -}; + return { + discardResponseBodies: true, + scenarios, + } +}(variants); export default function () { // login form - const loginResponse = http.post(`${baseUrl}/login`, { + const loginResponse = http.post(`${__ENV.APP_URL}/login`, { username: 'john', password: 'test' }, { @@ -21,11 +65,11 @@ export default function () { checkResponse(loginResponse, isRedirect); // dashboard - const dashboard = http.get(`${baseUrl}/dashboard`); + const dashboard = http.get(`${__ENV.APP_URL}/dashboard`); checkResponse(dashboard, isOk); // logout - const logout = http.get(`${baseUrl}/j_spring_security_logout`, { + const logout = http.get(`${__ENV.APP_URL}/j_spring_security_logout`, { redirects: 0 }); checkResponse(logout, isRedirect); diff --git a/benchmark/load/insecure-bank/start-servers.sh b/benchmark/load/insecure-bank/start-servers.sh new file mode 100755 index 00000000000..4cae95567f2 --- /dev/null +++ b/benchmark/load/insecure-bank/start-servers.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -e + +start_server() { + local VARIANT=$1 + local JAVA_OPTS=$2 + + if [ -n "$CI_JOB_TOKEN" ]; then + # Inside BP, so we can assume 24 CPU cores available and set CPU affinity + CPU_AFFINITY_APP=$3 + else + CPU_AFFINITY_APP="" + fi + + mkdir -p "${LOGS_DIR}/${VARIANT}" + ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log &PID=$! + echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log [PID=$PID]" +} + +start_server "no_agent" "-Dserver.port=8080" "taskset -c 47 " & +start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 46 " & +start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 45 " & +start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8083" "taskset -c 44 " & +start_server "iast_GLOBAL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL -Dserver.port=8084" "taskset -c 43 " & +start_server "iast_FULL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL -Dserver.port=8085" "taskset -c 42 " & + +wait diff --git a/benchmark/load/petclinic/benchmark.json b/benchmark/load/petclinic/benchmark.json deleted file mode 100644 index d569aeb4253..00000000000 --- a/benchmark/load/petclinic/benchmark.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "name": "load_petclinic", - "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"", - "service": "bash -c \"${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", - "run": "bash -c \"java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log\"", - "timeout": 150, - "iterations": 1, - "variants": { - "${NO_AGENT_VARIANT}": { - "env": { - "VARIANT": "${NO_AGENT_VARIANT}", - "JAVA_OPTS": "" - } - }, - "tracing": { - "env": { - "VARIANT": "tracing", - "JAVA_OPTS": "-javaagent:${TRACER}" - } - }, - "profiling": { - "env": { - "VARIANT": "profiling", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.profiling.enabled=true" - } - }, - "appsec": { - "env": { - "VARIANT": "appsec", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.appsec.enabled=true" - } - }, - "iast": { - "env": { - "VARIANT": "iast", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true" - } - }, - "code_origins": { - "env": { - "VARIANT": "code_origins", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.code.origin.for.spans.enabled=true" - } - } - } -} diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 6eb465c3683..a215ad7ea2d 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -1,17 +1,61 @@ import http from 'k6/http'; import {checkResponse, isOk} from "../../utils/k6.js"; -const baseUrl = 'http://localhost:8080'; +const variants = { + "no_agent": { + "APP_URL": 'http://localhost:8080', + }, + "tracing": { + "APP_URL": 'http://localhost:8081', + }, + "profiling": { + "APP_URL": 'http://localhost:8082', + }, + "appsec": { + "APP_URL": 'http://localhost:8083', + }, + "iast": { + "APP_URL": 'http://localhost:8084', + }, + "code_origins": { + "APP_URL": 'http://localhost:8085', + } +} + +export const options = function (variants) { + const scenarios = {}; + for (const variant of Object.keys(variants)) { + scenarios[`load--petclinic--${variant}--warmup`] = { + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + vus: 5, + duration: '20s', + gracefulStop: '2s', + env: { + "APP_URL": variants[variant]["APP_URL"] + } + }; + + scenarios[`load--petclinic--${variant}--high_load`] = { + executor: 'constant-vus', + vus: 5, + startTime: '22s', + duration: '15s', + gracefulStop: '2s', + env: { + "APP_URL": variants[variant]["APP_URL"] + } + }; + } -export const options = { - discardResponseBodies: true, - vus: 5, - iterations: 80000 -}; + return { + discardResponseBodies: true, + scenarios, + } +}(variants); export default function () { // find owner - const ownersList = http.get(`${baseUrl}/owners?lastName=`); + const ownersList = http.get(`${__ENV.APP_URL}/owners?lastName=`); checkResponse(ownersList, isOk); } diff --git a/benchmark/load/petclinic/start-servers.sh b/benchmark/load/petclinic/start-servers.sh new file mode 100755 index 00000000000..1ebbb4e0418 --- /dev/null +++ b/benchmark/load/petclinic/start-servers.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -e + +start_server() { + local VARIANT=$1 + local JAVA_OPTS=$2 + + if [ -n "$CI_JOB_TOKEN" ]; then + # Inside BP, so we can assume 24 CPU cores available and set CPU affinity + CPU_AFFINITY_APP=$3 + else + CPU_AFFINITY_APP="" + fi + + mkdir -p "${LOGS_DIR}/${VARIANT}" + ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${LOGS_DIR}/${VARIANT}/petclinic.log &PID=$! + echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${LOGS_DIR}/${VARIANT}/petclinic.log [PID=$!]" +} + +start_server "no_agent" "-Dserver.port=8080" "taskset -c 31-32 " & +start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 33-34 " & +start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 35-36 " & +start_server "appsec" "-javaagent:${TRACER} -Ddd.appsec.enabled=true -Dserver.port=8083" "taskset -c 37-38 " & +start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8084" "taskset -c 39-40 " & +start_server "code_origins" "-javaagent:${TRACER} -Ddd.code.origin.for.spans.enabled=true -Dserver.port=8085" "taskset -c 41-42 " & + +wait diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index 432c65d3fd5..d43a28383d5 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -1,5 +1,77 @@ #!/usr/bin/env bash -set -eu + +set -e + +function message() { + echo "$(date +"%T"): $1" +} + +function healthcheck() { + local url=$1 + + while true; do + if [[ $(curl -fso /dev/null -w "%{http_code}" "${url}") = 200 ]]; then + break + fi + done +} + +type=$1 + +if [ -n "$CI_JOB_TOKEN" ]; then + # Inside BP, so we can assume 24 CPU cores on the second socket available and set CPU affinity + export CPU_AFFINITY_K6="taskset -c 24-27 " +else + export CPU_AFFINITY_K6="" +fi source "${UTILS_DIR}/update-java-version.sh" 17 -"${UTILS_DIR}/run-sirun-benchmarks.sh" "$@" + +for app in *; do + if [[ ! -d "${app}" ]]; then + continue + fi + + message "${type} benchmark: ${app} started" + + export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" + mkdir -p ${OUTPUT_DIR} + + export LOGS_DIR="${ARTIFACTS_DIR}/${type}/${app}" + mkdir -p ${LOGS_DIR} + + # Using profiler variants for healthcheck as they are the slowest + if [ "${app}" == "petclinic" ]; then + HEALTHCHECK_URL=http://localhost:8082 + REPETITIONS_COUNT=5 + elif [ "${app}" == "insecure-bank" ]; then + HEALTHCHECK_URL=http://localhost:8082/login + REPETITIONS_COUNT=2 + else + echo "Unknown app ${app}" + exit 1 + fi + + for i in $(seq 1 $REPETITIONS_COUNT); do + bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" & + + echo "Waiting for serves to start..." + if [ "${app}" == "petclinic" ]; then + for port in $(seq 8080 8085); do + healthcheck http://localhost:$port + done + elif [ "${app}" == "insecure-bank" ]; then + for port in $(seq 8080 8085); do + healthcheck http://localhost:$port/login + done + fi + echo "Servers are up!" + + ( + cd ${app} && + bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh 'pkill java'" + ) + done + + message "${type} benchmark: ${app} finished" +done diff --git a/benchmark/utils/run-k6-load-test.sh b/benchmark/utils/run-k6-load-test.sh index 1d9753dc6e7..d3415f54eef 100755 --- a/benchmark/utils/run-k6-load-test.sh +++ b/benchmark/utils/run-k6-load-test.sh @@ -1,9 +1,7 @@ #!/usr/bin/env bash set -eu -url=$1 -output=$2 -command=$3 +command=$1 exit_code=0 cleanup() { @@ -14,14 +12,10 @@ cleanup() { trap cleanup EXIT ERR INT TERM -# wait for the HTTP server to be up -while true; do - if [[ $(curl -fso /dev/null -w "%{http_code}" "${url}") = 200 ]]; then - break - fi -done +echo "Starting k6 load test, logs are recorded into ${LOGS_DIR}/k6.log..." # run the k6 benchmark and store the result as JSON -k6 run k6.js --out "json=${output}/k6_$(date +%s).json" &>>"${output}/k6.log" - +k6 run k6.js --out "json=${OUTPUT_DIR}/k6_$(date +%s).json" > "${LOGS_DIR}/k6.log" 2>&1 exit_code=$? + +echo "k6 load test done !!!" diff --git a/benchmark/utils/update-java-version.sh b/benchmark/utils/update-java-version.sh index 2ea1c352841..3d76603e0ef 100755 --- a/benchmark/utils/update-java-version.sh +++ b/benchmark/utils/update-java-version.sh @@ -1,3 +1,5 @@ readonly target=$1 readonly NEW_PATH=$(echo "${PATH}" | sed -e "s@/usr/lib/jvm/[[:digit:]]\+@/usr/lib/jvm/${target}@g") export PATH="${NEW_PATH}" + +java --version