From db10cbc1f245e60e68acc41b1ddd960ed19912ab Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Wed, 11 Jun 2025 17:31:30 +0200 Subject: [PATCH 01/56] fix: Add CPU affinity to load benchmarks --- benchmark/load/petclinic/benchmark.json | 4 ++-- benchmark/load/petclinic/k6.js | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmark/load/petclinic/benchmark.json b/benchmark/load/petclinic/benchmark.json index d569aeb4253..12f05362896 100644 --- a/benchmark/load/petclinic/benchmark.json +++ b/benchmark/load/petclinic/benchmark.json @@ -1,8 +1,8 @@ { "name": "load_petclinic", "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"", - "service": "bash -c \"${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", - "run": "bash -c \"java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log\"", + "service": "bash -c \"taskset -c 24-30 ${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", + "run": "bash -c \"taskset -c 31-47 java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log\"", "timeout": 150, "iterations": 1, "variants": { diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 6eb465c3683..542824a1af1 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -4,6 +4,7 @@ import {checkResponse, isOk} from "../../utils/k6.js"; const baseUrl = 'http://localhost:8080'; export const options = { + executor: 'shared-iterations', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors discardResponseBodies: true, vus: 5, iterations: 80000 From 7245a767c0f142be5ff4d0bf1dad96209ec0de31 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Wed, 11 Jun 2025 17:37:20 +0200 Subject: [PATCH 02/56] wip: Force trigger Gitlab CI --- benchmark/load/petclinic/k6.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 542824a1af1..e41851b3698 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -4,7 +4,7 @@ import {checkResponse, isOk} from "../../utils/k6.js"; const baseUrl = 'http://localhost:8080'; export const options = { - executor: 'shared-iterations', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + executor: 'shared-iterations', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors discardResponseBodies: true, vus: 5, iterations: 80000 From 20f1d6a6a00580d9f6af735935a9c3d1c682cc4d Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Thu, 12 Jun 2025 12:34:24 +0200 Subject: [PATCH 03/56] wip: Generalize affinities, export k6 results, change executor --- benchmark/load/petclinic/benchmark.json | 28 ++++++++++++++++++------- benchmark/load/petclinic/k6.js | 4 ++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/benchmark/load/petclinic/benchmark.json b/benchmark/load/petclinic/benchmark.json index 12f05362896..39f2abdde7d 100644 --- a/benchmark/load/petclinic/benchmark.json +++ b/benchmark/load/petclinic/benchmark.json @@ -1,45 +1,57 @@ { "name": "load_petclinic", "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"", - "service": "bash -c \"taskset -c 24-30 ${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", - "run": "bash -c \"taskset -c 31-47 java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log\"", + "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${REPORTS_DIR}/${VARIANT} 'pkill java'\"", + "run": "bash -c \"${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${REPORTS_DIR}/${VARIANT}/petclinic.log\"", "timeout": 150, "iterations": 1, "variants": { "${NO_AGENT_VARIANT}": { "env": { "VARIANT": "${NO_AGENT_VARIANT}", - "JAVA_OPTS": "" + "JAVA_OPTS": "", + "CPU_AFFINITY_K6": "taskset -c 24-30 ", + "CPU_AFFINITY_APP": "taskset -c 31-32 " } }, "tracing": { "env": { "VARIANT": "tracing", - "JAVA_OPTS": "-javaagent:${TRACER}" + "JAVA_OPTS": "-javaagent:${TRACER}", + "CPU_AFFINITY_K6": "taskset -c 24-30 ", + "CPU_AFFINITY_APP": "taskset -c 33-34 " } }, "profiling": { "env": { "VARIANT": "profiling", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.profiling.enabled=true" + "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.profiling.enabled=true", + "CPU_AFFINITY_K6": "taskset -c 24-30 ", + "CPU_AFFINITY_APP": "taskset -c 35-36 " } }, "appsec": { "env": { "VARIANT": "appsec", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.appsec.enabled=true" + "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.appsec.enabled=true", + "CPU_AFFINITY_K6": "taskset -c 24-30 ", + "CPU_AFFINITY_APP": "taskset -c 37-38 " } }, "iast": { "env": { "VARIANT": "iast", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true" + "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true", + "CPU_AFFINITY_K6": "taskset -c 24-30 ", + "CPU_AFFINITY_APP": "taskset -c 39-40 " } }, "code_origins": { "env": { "VARIANT": "code_origins", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.code.origin.for.spans.enabled=true" + "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.code.origin.for.spans.enabled=true", + "CPU_AFFINITY_K6": "taskset -c 24-30 ", + "CPU_AFFINITY_APP": "taskset -c 41-42 " } } } diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index e41851b3698..d719c78db0e 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -4,10 +4,10 @@ import {checkResponse, isOk} from "../../utils/k6.js"; const baseUrl = 'http://localhost:8080'; export const options = { - executor: 'shared-iterations', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors discardResponseBodies: true, vus: 5, - iterations: 80000 + duration: '2m', }; export default function () { From 6ce4ca60cb6f35e197eaf8a4de7f3ca376b65a23 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Thu, 12 Jun 2025 13:25:33 +0200 Subject: [PATCH 04/56] wip: Save backup of logs --- .gitlab/benchmarks.yml | 1 + benchmark/load/petclinic/benchmark.json | 4 ++-- benchmark/utils/run-sirun-benchmarks.sh | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 05a5adfcbb4..e78f694c883 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -20,6 +20,7 @@ name: "reports" paths: - reports/ + - backup-reports/ expire_in: 3 months variables: UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance. diff --git a/benchmark/load/petclinic/benchmark.json b/benchmark/load/petclinic/benchmark.json index 39f2abdde7d..11a3e933652 100644 --- a/benchmark/load/petclinic/benchmark.json +++ b/benchmark/load/petclinic/benchmark.json @@ -1,8 +1,8 @@ { "name": "load_petclinic", "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"", - "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${REPORTS_DIR}/${VARIANT} 'pkill java'\"", - "run": "bash -c \"${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${REPORTS_DIR}/${VARIANT}/petclinic.log\"", + "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", + "run": "bash -c \"${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log\"", "timeout": 150, "iterations": 1, "variants": { diff --git a/benchmark/utils/run-sirun-benchmarks.sh b/benchmark/utils/run-sirun-benchmarks.sh index c0bc732dcfa..b8258978b22 100755 --- a/benchmark/utils/run-sirun-benchmarks.sh +++ b/benchmark/utils/run-sirun-benchmarks.sh @@ -28,6 +28,8 @@ run_benchmark() { message "${type} benchmark: ${app} finished" + cp -r "${REPORTS_DIR}" "${REPORTS_DIR}/../backup-reports/" + cd .. fi } From 06cf0bb49105d70609960eb8db671d4b44e31c23 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Thu, 12 Jun 2025 13:48:00 +0200 Subject: [PATCH 05/56] wip: Reduce execution time --- benchmark/load/petclinic/k6.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index d719c78db0e..e04d5a46eae 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -7,7 +7,7 @@ export const options = { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors discardResponseBodies: true, vus: 5, - duration: '2m', + duration: '20s', }; export default function () { From 5edc665f48233fd4f5d3be938c0578fc045aadf2 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Thu, 12 Jun 2025 14:19:22 +0200 Subject: [PATCH 06/56] wip: Tweak duration, add warmup stage, backup logs --- benchmark/load/insecure-bank/k6.js | 15 ++++++++++++--- benchmark/load/petclinic/k6.js | 16 ++++++++++++---- benchmark/utils/run-sirun-benchmarks.sh | 3 ++- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 993ac6639f7..5596f9f7021 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -4,9 +4,18 @@ import {checkResponse, isOk, isRedirect} from "../../utils/k6.js"; const baseUrl = 'http://localhost:8080'; export const options = { - discardResponseBodies: true, - vus: 5, - iterations: 40000 + warmup: { + discardResponseBodies: true, + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + vus: 5, + duration: '15s', + }, + high_load: { + discardResponseBodies: true, + executor: 'constant-vus', + vus: 5, + duration: '20s', + }, }; export default function () { diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index e04d5a46eae..e6b3d2e9e02 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -4,10 +4,18 @@ import {checkResponse, isOk} from "../../utils/k6.js"; const baseUrl = 'http://localhost:8080'; export const options = { - executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors - discardResponseBodies: true, - vus: 5, - duration: '20s', + warmup: { + discardResponseBodies: true, + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + vus: 5, + duration: '15s', + }, + high_load: { + discardResponseBodies: true, + executor: 'constant-vus', + vus: 5, + duration: '20s', + }, }; export default function () { diff --git a/benchmark/utils/run-sirun-benchmarks.sh b/benchmark/utils/run-sirun-benchmarks.sh index b8258978b22..216b7cf839c 100755 --- a/benchmark/utils/run-sirun-benchmarks.sh +++ b/benchmark/utils/run-sirun-benchmarks.sh @@ -28,7 +28,8 @@ run_benchmark() { message "${type} benchmark: ${app} finished" - cp -r "${REPORTS_DIR}" "${REPORTS_DIR}/../backup-reports/" + mkdir -p "${ARTIFACTS_DIR}/backup-reports/" + cp -r "${REPORTS_DIR}" "${ARTIFACTS_DIR}/backup-reports/" cd .. fi From 85a9deed54ddec283a3f966be386b9c8c1513b85 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Thu, 12 Jun 2025 14:46:33 +0200 Subject: [PATCH 07/56] fix: backup-reports location --- benchmark/utils/run-sirun-benchmarks.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/utils/run-sirun-benchmarks.sh b/benchmark/utils/run-sirun-benchmarks.sh index 216b7cf839c..0e157e160eb 100755 --- a/benchmark/utils/run-sirun-benchmarks.sh +++ b/benchmark/utils/run-sirun-benchmarks.sh @@ -28,8 +28,8 @@ run_benchmark() { message "${type} benchmark: ${app} finished" - mkdir -p "${ARTIFACTS_DIR}/backup-reports/" - cp -r "${REPORTS_DIR}" "${ARTIFACTS_DIR}/backup-reports/" + mkdir -p "${ARTIFACTS_DIR}/../backup-reports/" + cp -r "${REPORTS_DIR}" "${ARTIFACTS_DIR}/../backup-reports/" cd .. fi From 2d59094837c8007f11ae65ee248fc63ee54ca26d Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Thu, 12 Jun 2025 15:49:42 +0200 Subject: [PATCH 08/56] fix: Fix k6 config --- benchmark/load/insecure-bank/k6.js | 25 +++++++++++++------------ benchmark/load/petclinic/k6.js | 25 +++++++++++++------------ 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 5596f9f7021..9d15153c3c6 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -4,18 +4,19 @@ import {checkResponse, isOk, isRedirect} from "../../utils/k6.js"; const baseUrl = 'http://localhost:8080'; export const options = { - warmup: { - discardResponseBodies: true, - executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors - vus: 5, - duration: '15s', - }, - high_load: { - discardResponseBodies: true, - executor: 'constant-vus', - vus: 5, - duration: '20s', - }, + discardResponseBodies: true, + scenarios: { + warmup: { + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + vus: 5, + duration: '15s', + }, + high_load: { + executor: 'constant-vus', + vus: 5, + duration: '20s', + }, + } }; export default function () { diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index e6b3d2e9e02..a4ad0a88a0c 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -4,18 +4,19 @@ import {checkResponse, isOk} from "../../utils/k6.js"; const baseUrl = 'http://localhost:8080'; export const options = { - warmup: { - discardResponseBodies: true, - executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors - vus: 5, - duration: '15s', - }, - high_load: { - discardResponseBodies: true, - executor: 'constant-vus', - vus: 5, - duration: '20s', - }, + discardResponseBodies: true, + scenarios: { + warmup: { + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + vus: 5, + duration: '15s', + }, + high_load: { + executor: 'constant-vus', + vus: 5, + duration: '20s', + }, + } }; export default function () { From 76060dc921dff505b672a5f4205dc9ef9a3fdd75 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Thu, 12 Jun 2025 18:20:35 +0200 Subject: [PATCH 09/56] fix: Fix k6 scenario names --- benchmark/load/insecure-bank/k6.js | 4 ++-- benchmark/load/petclinic/k6.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 9d15153c3c6..22b5e8bf4bf 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -6,12 +6,12 @@ const baseUrl = 'http://localhost:8080'; export const options = { discardResponseBodies: true, scenarios: { - warmup: { + [`warmup--load:insecure-bank:${__ENV.VARIANT}`]: { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, duration: '15s', }, - high_load: { + [`high_load--load:insecure-bank:${__ENV.VARIANT}`]: { executor: 'constant-vus', vus: 5, duration: '20s', diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index a4ad0a88a0c..124d4416e60 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -6,12 +6,12 @@ const baseUrl = 'http://localhost:8080'; export const options = { discardResponseBodies: true, scenarios: { - warmup: { + [`warmup--load:petclinic:${__ENV.VARIANT}`]: { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, duration: '15s', }, - high_load: { + [`high_load--load:petclinic:${__ENV.VARIANT}`]: { executor: 'constant-vus', vus: 5, duration: '20s', From 02de5827bd753cdc7f347693dacda8f746b8a4b9 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Thu, 12 Jun 2025 18:41:17 +0200 Subject: [PATCH 10/56] fix: Fix k6 scenario names --- benchmark/load/insecure-bank/k6.js | 4 ++-- benchmark/load/petclinic/k6.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 22b5e8bf4bf..a643f92c013 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -6,12 +6,12 @@ const baseUrl = 'http://localhost:8080'; export const options = { discardResponseBodies: true, scenarios: { - [`warmup--load:insecure-bank:${__ENV.VARIANT}`]: { + [`warmup--load--insecure-bank--${__ENV.VARIANT}`]: { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, duration: '15s', }, - [`high_load--load:insecure-bank:${__ENV.VARIANT}`]: { + [`high_load--load--insecure-bank--${__ENV.VARIANT}`]: { executor: 'constant-vus', vus: 5, duration: '20s', diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 124d4416e60..ed1304fd0f6 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -6,12 +6,12 @@ const baseUrl = 'http://localhost:8080'; export const options = { discardResponseBodies: true, scenarios: { - [`warmup--load:petclinic:${__ENV.VARIANT}`]: { + [`warmup--load--petclinic--${__ENV.VARIANT}`]: { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, duration: '15s', }, - [`high_load--load:petclinic:${__ENV.VARIANT}`]: { + [`high_load--load--petclinic--${__ENV.VARIANT}`]: { executor: 'constant-vus', vus: 5, duration: '20s', From 391519f1221d797dc6ca620f78da1b5d15d401f6 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Fri, 13 Jun 2025 12:09:01 +0200 Subject: [PATCH 11/56] fix: Change k6 names format --- benchmark/load/insecure-bank/k6.js | 4 ++-- benchmark/load/petclinic/k6.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index a643f92c013..69ee5bd10e6 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -6,12 +6,12 @@ const baseUrl = 'http://localhost:8080'; export const options = { discardResponseBodies: true, scenarios: { - [`warmup--load--insecure-bank--${__ENV.VARIANT}`]: { + [`load--insecure-bank--${__ENV.VARIANT}--warmup`]: { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, duration: '15s', }, - [`high_load--load--insecure-bank--${__ENV.VARIANT}`]: { + [`load--insecure-bank--${__ENV.VARIANT}--high_load`]: { executor: 'constant-vus', vus: 5, duration: '20s', diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index ed1304fd0f6..4e518a51557 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -6,12 +6,12 @@ const baseUrl = 'http://localhost:8080'; export const options = { discardResponseBodies: true, scenarios: { - [`warmup--load--petclinic--${__ENV.VARIANT}`]: { + [`load--petclinic--${__ENV.VARIANT}--warmup`]: { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, duration: '15s', }, - [`high_load--load--petclinic--${__ENV.VARIANT}`]: { + [`load--petclinic--${__ENV.VARIANT}--high_load`]: { executor: 'constant-vus', vus: 5, duration: '20s', From 8ba2eee363fc745884fc246f6a16df709148e50c Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Fri, 13 Jun 2025 14:40:48 +0200 Subject: [PATCH 12/56] fix: Increase warmup time --- benchmark/load/insecure-bank/k6.js | 2 +- benchmark/load/petclinic/k6.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 69ee5bd10e6..312fb5be2ad 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -9,7 +9,7 @@ export const options = { [`load--insecure-bank--${__ENV.VARIANT}--warmup`]: { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, - duration: '15s', + duration: '25s', }, [`load--insecure-bank--${__ENV.VARIANT}--high_load`]: { executor: 'constant-vus', diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 4e518a51557..67e30b4dab6 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -9,7 +9,7 @@ export const options = { [`load--petclinic--${__ENV.VARIANT}--warmup`]: { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, - duration: '15s', + duration: '25s', }, [`load--petclinic--${__ENV.VARIANT}--high_load`]: { executor: 'constant-vus', From 3e9ec83e436f24f47fbd4a944fe0e4602fd5c284 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Fri, 13 Jun 2025 14:44:57 +0200 Subject: [PATCH 13/56] fix: Reduce gracefulStop --- benchmark/load/insecure-bank/k6.js | 2 ++ benchmark/load/petclinic/k6.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 312fb5be2ad..7d776c5bf77 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -10,11 +10,13 @@ export const options = { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, duration: '25s', + gracefulStop: '5s', }, [`load--insecure-bank--${__ENV.VARIANT}--high_load`]: { executor: 'constant-vus', vus: 5, duration: '20s', + gracefulStop: '5s', }, } }; diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 67e30b4dab6..dfae0abd164 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -10,11 +10,13 @@ export const options = { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, duration: '25s', + gracefulStop: '5s', }, [`load--petclinic--${__ENV.VARIANT}--high_load`]: { executor: 'constant-vus', vus: 5, duration: '20s', + gracefulStop: '5s', }, } }; From 987daf6c6095d5811cd383be3e2400864313bd36 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Fri, 13 Jun 2025 14:45:50 +0200 Subject: [PATCH 14/56] fix: Shift start time --- benchmark/load/insecure-bank/k6.js | 3 ++- benchmark/load/petclinic/k6.js | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 7d776c5bf77..b3348df050d 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -9,12 +9,13 @@ export const options = { [`load--insecure-bank--${__ENV.VARIANT}--warmup`]: { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, - duration: '25s', + duration: '15s', gracefulStop: '5s', }, [`load--insecure-bank--${__ENV.VARIANT}--high_load`]: { executor: 'constant-vus', vus: 5, + startTime: '15s', duration: '20s', gracefulStop: '5s', }, diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index dfae0abd164..c8ed1a15bb9 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -9,12 +9,13 @@ export const options = { [`load--petclinic--${__ENV.VARIANT}--warmup`]: { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, - duration: '25s', + duration: '15s', gracefulStop: '5s', }, [`load--petclinic--${__ENV.VARIANT}--high_load`]: { executor: 'constant-vus', vus: 5, + startTime: '15s', duration: '20s', gracefulStop: '5s', }, From 4ec4ad74396a19e0807215725e1f40e4a1b6c3de Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Fri, 13 Jun 2025 15:51:01 +0200 Subject: [PATCH 15/56] fix: Fix local runs, reduce warmup duration --- benchmark/load/insecure-bank/k6.js | 8 ++++---- benchmark/load/petclinic/benchmark.json | 24 ++++++------------------ benchmark/load/petclinic/k6.js | 8 ++++---- benchmark/load/run.sh | 10 ++++++++++ 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index b3348df050d..96ab173f90d 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -9,15 +9,15 @@ export const options = { [`load--insecure-bank--${__ENV.VARIANT}--warmup`]: { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, - duration: '15s', - gracefulStop: '5s', + duration: '10s', + gracefulStop: '2s', }, [`load--insecure-bank--${__ENV.VARIANT}--high_load`]: { executor: 'constant-vus', vus: 5, - startTime: '15s', + startTime: '12s', duration: '20s', - gracefulStop: '5s', + gracefulStop: '2s', }, } }; diff --git a/benchmark/load/petclinic/benchmark.json b/benchmark/load/petclinic/benchmark.json index 11a3e933652..a5e90f17a86 100644 --- a/benchmark/load/petclinic/benchmark.json +++ b/benchmark/load/petclinic/benchmark.json @@ -9,49 +9,37 @@ "${NO_AGENT_VARIANT}": { "env": { "VARIANT": "${NO_AGENT_VARIANT}", - "JAVA_OPTS": "", - "CPU_AFFINITY_K6": "taskset -c 24-30 ", - "CPU_AFFINITY_APP": "taskset -c 31-32 " + "JAVA_OPTS": "" } }, "tracing": { "env": { "VARIANT": "tracing", - "JAVA_OPTS": "-javaagent:${TRACER}", - "CPU_AFFINITY_K6": "taskset -c 24-30 ", - "CPU_AFFINITY_APP": "taskset -c 33-34 " + "JAVA_OPTS": "-javaagent:${TRACER}" } }, "profiling": { "env": { "VARIANT": "profiling", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.profiling.enabled=true", - "CPU_AFFINITY_K6": "taskset -c 24-30 ", - "CPU_AFFINITY_APP": "taskset -c 35-36 " + "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.profiling.enabled=true" } }, "appsec": { "env": { "VARIANT": "appsec", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.appsec.enabled=true", - "CPU_AFFINITY_K6": "taskset -c 24-30 ", - "CPU_AFFINITY_APP": "taskset -c 37-38 " + "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.appsec.enabled=true" } }, "iast": { "env": { "VARIANT": "iast", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true", - "CPU_AFFINITY_K6": "taskset -c 24-30 ", - "CPU_AFFINITY_APP": "taskset -c 39-40 " + "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true" } }, "code_origins": { "env": { "VARIANT": "code_origins", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.code.origin.for.spans.enabled=true", - "CPU_AFFINITY_K6": "taskset -c 24-30 ", - "CPU_AFFINITY_APP": "taskset -c 41-42 " + "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.code.origin.for.spans.enabled=true" } } } diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index c8ed1a15bb9..10f86ff6af1 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -9,15 +9,15 @@ export const options = { [`load--petclinic--${__ENV.VARIANT}--warmup`]: { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, - duration: '15s', - gracefulStop: '5s', + duration: '10s', + gracefulStop: '2s', }, [`load--petclinic--${__ENV.VARIANT}--high_load`]: { executor: 'constant-vus', vus: 5, - startTime: '15s', + startTime: '12s', duration: '20s', - gracefulStop: '5s', + gracefulStop: '2s', }, } }; diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index 432c65d3fd5..06283aa499a 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -1,5 +1,15 @@ #!/usr/bin/env bash set -eu +if [ -n "$CI_JOB_TOKEN" ]; then + # Inside BP, so we can assume 24 CPU cores available and set CPU affinity + export CPU_AFFINITY_K6="taskset -c 24-30 " + export CPU_AFFINITY_APP="taskset -c 31-32 " +else + export CPU_AFFINITY_K6="" + export CPU_AFFINITY_APP="" +fi + + source "${UTILS_DIR}/update-java-version.sh" 17 "${UTILS_DIR}/run-sirun-benchmarks.sh" "$@" From 6b085c9f0b5db52c80fc6bfc662d74864dbe9c6f Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Fri, 13 Jun 2025 16:58:03 +0200 Subject: [PATCH 16/56] fix: Change debug logs location --- .gitlab/benchmarks.yml | 2 +- benchmark/utils/run-sirun-benchmarks.sh | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index e78f694c883..69061859391 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -20,7 +20,7 @@ name: "reports" paths: - reports/ - - backup-reports/ + - debug-logs/ expire_in: 3 months variables: UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance. diff --git a/benchmark/utils/run-sirun-benchmarks.sh b/benchmark/utils/run-sirun-benchmarks.sh index 0e157e160eb..c0bc732dcfa 100755 --- a/benchmark/utils/run-sirun-benchmarks.sh +++ b/benchmark/utils/run-sirun-benchmarks.sh @@ -28,9 +28,6 @@ run_benchmark() { message "${type} benchmark: ${app} finished" - mkdir -p "${ARTIFACTS_DIR}/../backup-reports/" - cp -r "${REPORTS_DIR}" "${ARTIFACTS_DIR}/../backup-reports/" - cd .. fi } From 4654c1a4757203210ac3e1bf378672fc37e62b35 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Mon, 16 Jun 2025 11:20:02 +0200 Subject: [PATCH 17/56] tweak: CPU affinities to make sure bottleneck is on k6 side --- benchmark/load/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index 06283aa499a..2b84550b803 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -3,8 +3,8 @@ set -eu if [ -n "$CI_JOB_TOKEN" ]; then # Inside BP, so we can assume 24 CPU cores available and set CPU affinity - export CPU_AFFINITY_K6="taskset -c 24-30 " - export CPU_AFFINITY_APP="taskset -c 31-32 " + export CPU_AFFINITY_K6="taskset -c 25-26 " + export CPU_AFFINITY_APP="taskset -c 27-47 " else export CPU_AFFINITY_K6="" export CPU_AFFINITY_APP="" From c71da3e8ea17364d55812403a0cb162c00d6291e Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Mon, 16 Jun 2025 11:35:24 +0200 Subject: [PATCH 18/56] tweak: Output Java version --- benchmark/utils/update-java-version.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmark/utils/update-java-version.sh b/benchmark/utils/update-java-version.sh index 2ea1c352841..3d76603e0ef 100755 --- a/benchmark/utils/update-java-version.sh +++ b/benchmark/utils/update-java-version.sh @@ -1,3 +1,5 @@ readonly target=$1 readonly NEW_PATH=$(echo "${PATH}" | sed -e "s@/usr/lib/jvm/[[:digit:]]\+@/usr/lib/jvm/${target}@g") export PATH="${NEW_PATH}" + +java --version From b1378452a3e54366a340c8edc76379c052816852 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Mon, 16 Jun 2025 12:23:47 +0200 Subject: [PATCH 19/56] test: Use constant-arrival-rate executor --- benchmark/load/petclinic/k6.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 10f86ff6af1..02de58df8c1 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -7,17 +7,21 @@ export const options = { discardResponseBodies: true, scenarios: { [`load--petclinic--${__ENV.VARIANT}--warmup`]: { - executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors - vus: 5, + executor: 'constant-arrival-rate', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + preAllocatedVUs: 5, duration: '10s', gracefulStop: '2s', + timeUnit: '1s', + rate: 200, }, [`load--petclinic--${__ENV.VARIANT}--high_load`]: { - executor: 'constant-vus', - vus: 5, + executor: 'constant-arrival-rate', + preAllocatedVUs: 5, startTime: '12s', duration: '20s', gracefulStop: '2s', + timeUnit: '1s', + rate: 200, }, } }; From 4ea19a05167555f934706fcd998cadcba5475a18 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Mon, 16 Jun 2025 16:13:06 +0200 Subject: [PATCH 20/56] tweak: Increase warmup time for petclinic --- benchmark/load/petclinic/k6.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 02de58df8c1..4f6d184ecce 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -7,12 +7,10 @@ export const options = { discardResponseBodies: true, scenarios: { [`load--petclinic--${__ENV.VARIANT}--warmup`]: { - executor: 'constant-arrival-rate', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors - preAllocatedVUs: 5, - duration: '10s', + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + vus: 5, + duration: '20s', gracefulStop: '2s', - timeUnit: '1s', - rate: 200, }, [`load--petclinic--${__ENV.VARIANT}--high_load`]: { executor: 'constant-arrival-rate', From 28de22aeb6f177ae60b61c46473bfde7228f1c1e Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Mon, 16 Jun 2025 16:56:15 +0200 Subject: [PATCH 21/56] fix: Fix mistake in startTime --- benchmark/load/petclinic/k6.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 4f6d184ecce..c11db7f01b2 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -15,7 +15,7 @@ export const options = { [`load--petclinic--${__ENV.VARIANT}--high_load`]: { executor: 'constant-arrival-rate', preAllocatedVUs: 5, - startTime: '12s', + startTime: '22s', duration: '20s', gracefulStop: '2s', timeUnit: '1s', From 4cbcacc3d2c1cffbeeb0d24c23e4ee5956cb3a27 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Mon, 16 Jun 2025 17:39:22 +0200 Subject: [PATCH 22/56] tweak: Reduce RPS to compensate for loss on profiling petclinic variant --- benchmark/load/petclinic/k6.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index c11db7f01b2..bd6cbbe8f6c 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -19,7 +19,7 @@ export const options = { duration: '20s', gracefulStop: '2s', timeUnit: '1s', - rate: 200, + rate: 150, }, } }; From 238ea6aa8799cf2c51e8b8f188f0f130bbaaf69e Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Fri, 13 Jun 2025 15:46:20 +0200 Subject: [PATCH 23/56] tweak: Parallelize load tests --- benchmark/load/petclinic/benchmark.json | 39 +------------ benchmark/load/petclinic/k6.js | 71 ++++++++++++++++------- benchmark/load/petclinic/start-servers.sh | 25 ++++++++ benchmark/load/run.sh | 8 +++ 4 files changed, 85 insertions(+), 58 deletions(-) create mode 100755 benchmark/load/petclinic/start-servers.sh diff --git a/benchmark/load/petclinic/benchmark.json b/benchmark/load/petclinic/benchmark.json index a5e90f17a86..3aa83ede8db 100644 --- a/benchmark/load/petclinic/benchmark.json +++ b/benchmark/load/petclinic/benchmark.json @@ -2,45 +2,10 @@ "name": "load_petclinic", "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"", "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", - "run": "bash -c \"${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log\"", + "run": "bash -c \"${SCRIPT_DIR}/load/petclinic/start-servers.sh\"", "timeout": 150, "iterations": 1, "variants": { - "${NO_AGENT_VARIANT}": { - "env": { - "VARIANT": "${NO_AGENT_VARIANT}", - "JAVA_OPTS": "" - } - }, - "tracing": { - "env": { - "VARIANT": "tracing", - "JAVA_OPTS": "-javaagent:${TRACER}" - } - }, - "profiling": { - "env": { - "VARIANT": "profiling", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.profiling.enabled=true" - } - }, - "appsec": { - "env": { - "VARIANT": "appsec", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.appsec.enabled=true" - } - }, - "iast": { - "env": { - "VARIANT": "iast", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true" - } - }, - "code_origins": { - "env": { - "VARIANT": "code_origins", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.code.origin.for.spans.enabled=true" - } - } + "default": {} } } diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index bd6cbbe8f6c..4b7082fcdc5 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -1,32 +1,61 @@ import http from 'k6/http'; import {checkResponse, isOk} from "../../utils/k6.js"; -const baseUrl = 'http://localhost:8080'; +const variants = { + "no_agent": { + "APP_URL": 'http://localhost:8080', + }, + "tracing": { + "APP_URL": 'http://localhost:8081', + }, + "profiling": { + "APP_URL": 'http://localhost:8082', + }, + "appsec": { + "APP_URL": 'http://localhost:8083', + }, + "iast": { + "APP_URL": 'http://localhost:8084', + }, + "code_origins": { + "APP_URL": 'http://localhost:8085', + } +} + +export const options = function (variants) { + let scenarios = {}; + for (const variant of Object.keys(variants)) { + scenarios = { + ...{ + [`load--petclinic--${variant}--warmup`]: { + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + vus: 5, + duration: '10s', + gracefulStop: '2s', + env: { ...variants[variant] } + }, + [`load--petclinic--${variant}--high_load`]: { + executor: 'constant-vus', + vus: 5, + startTime: '12s', + duration: '20s', + gracefulStop: '2s', + env: { ...variants[variant] } + }, + }, + ...scenarios + }; + } -export const options = { - discardResponseBodies: true, - scenarios: { - [`load--petclinic--${__ENV.VARIANT}--warmup`]: { - executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors - vus: 5, - duration: '20s', - gracefulStop: '2s', - }, - [`load--petclinic--${__ENV.VARIANT}--high_load`]: { - executor: 'constant-arrival-rate', - preAllocatedVUs: 5, - startTime: '22s', - duration: '20s', - gracefulStop: '2s', - timeUnit: '1s', - rate: 150, - }, + return { + discardResponseBodies: true, + scenarios, } -}; +}(variants); export default function () { // find owner - const ownersList = http.get(`${baseUrl}/owners?lastName=`); + const ownersList = http.get(`${__ENV.APP_URL}/owners?lastName=`); checkResponse(ownersList, isOk); } diff --git a/benchmark/load/petclinic/start-servers.sh b/benchmark/load/petclinic/start-servers.sh new file mode 100755 index 00000000000..769a6f4671b --- /dev/null +++ b/benchmark/load/petclinic/start-servers.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -eu + +start_server() { + local VARIANT=$1 + local JAVA_OPTS=$2 + + if [ -n "$CI_JOB_TOKEN" ]; then + # Inside BP, so we can assume 24 CPU cores available and set CPU affinity + CPU_AFFINITY_APP=$3 + else + CPU_AFFINITY_APP="" + fi + + ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log +} + +start_server "${NO_AGENT_VARIANT}" "-Dserver.port=8080" "taskset -c 31-32 " & +start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 33-34 " & +start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 35-36 " & +start_server "appsec" "-javaagent:${TRACER} -Ddd.appsec.enabled=true -Dserver.port=8083" "taskset -c 37-38 " & +start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8084" "taskset -c 39-40 " & +start_server "code_origins" "-javaagent:${TRACER} -Ddd.code.origin.for.spans.enabled=true -Dserver.port=8085" "taskset -c 41-42 " & + +wait diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index 2b84550b803..f657bd64376 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -12,4 +12,12 @@ fi source "${UTILS_DIR}/update-java-version.sh" 17 + +if [ -n "$CI_JOB_TOKEN" ]; then + # Inside BP, so we can assume 24 CPU cores available and set CPU affinity + export CPU_AFFINITY_K6="taskset -c 24-30 " +else + export CPU_AFFINITY_K6="" +fi + "${UTILS_DIR}/run-sirun-benchmarks.sh" "$@" From e82f1e4713a00a0ff830efd7bfc569752e4b39df Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Fri, 13 Jun 2025 16:17:17 +0200 Subject: [PATCH 24/56] tweak: Parallelize insecure-bank, tweak logs storage --- benchmark/load/insecure-bank/benchmark.json | 35 +------- benchmark/load/insecure-bank/k6.js | 82 ++++++++++++++----- benchmark/load/insecure-bank/start-servers.sh | 28 +++++++ 3 files changed, 92 insertions(+), 53 deletions(-) create mode 100755 benchmark/load/insecure-bank/start-servers.sh diff --git a/benchmark/load/insecure-bank/benchmark.json b/benchmark/load/insecure-bank/benchmark.json index 89785199d61..0df7cba9b99 100644 --- a/benchmark/load/insecure-bank/benchmark.json +++ b/benchmark/load/insecure-bank/benchmark.json @@ -1,40 +1,11 @@ { "name": "load_insecure-bank", "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"", - "service": "bash -c \"${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080/login ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", - "run": "bash -c \"java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log\"", + "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080/login ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", + "run": "bash -c \"${SCRIPT_DIR}/load/insecure-bank/start-servers.sh\"", "timeout": 150, "iterations": 1, "variants": { - "${NO_AGENT_VARIANT}": { - "env": { - "VARIANT": "${NO_AGENT_VARIANT}", - "JAVA_OPTS": "" - } - }, - "tracing": { - "env": { - "VARIANT": "tracing", - "JAVA_OPTS": "-javaagent:${TRACER}" - } - }, - "iast": { - "env": { - "VARIANT": "iast", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true" - } - }, - "iast_GLOBAL": { - "env": { - "VARIANT": "iast_GLOBAL", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL" - } - }, - "iast_FULL": { - "env": { - "VARIANT": "iast_FULL", - "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL" - } - } + "default": {} } } diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 96ab173f90d..52a2a581c64 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -1,31 +1,71 @@ import http from 'k6/http'; import {checkResponse, isOk, isRedirect} from "../../utils/k6.js"; -const baseUrl = 'http://localhost:8080'; +const variants = { + "no_agent": { + "APP_URL": 'http://localhost:8080', + }, + "tracing": { + "APP_URL": 'http://localhost:8081', + }, + "profiling": { + "APP_URL": 'http://localhost:8082', + }, + "iast": { + "APP_URL": 'http://localhost:8083', + }, + "iast_GLOBAL": { + "APP_URL": 'http://localhost:8084', + }, + "iast_FULL": { + "APP_URL": 'http://localhost:8085', + }, + "iast_INACTIVE": { + "APP_URL": 'http://localhost:8086', + }, + "iast_TELEMETRY_OFF": { + "APP_URL": 'http://localhost:8087', + }, + "iast_HARDCODED_SECRET_DISABLED": { + "APP_URL": 'http://localhost:8088', + }, +} + +export const options = function (variants) { + let scenarios = {}; + for (const variant of Object.keys(variants)) { + scenarios = { + ...{ + [`load--insecure-bank--${variant}--warmup`]: { + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + vus: 5, + duration: '10s', + gracefulStop: '2s', + env: { ...variants[variant] } + }, + [`load--insecure-bank--${variant}--high_load`]: { + executor: 'constant-vus', + vus: 5, + startTime: '12s', + duration: '20s', + gracefulStop: '2s', + env: { ...variants[variant] } + }, + }, + ...scenarios + }; + } -export const options = { - discardResponseBodies: true, - scenarios: { - [`load--insecure-bank--${__ENV.VARIANT}--warmup`]: { - executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors - vus: 5, - duration: '10s', - gracefulStop: '2s', - }, - [`load--insecure-bank--${__ENV.VARIANT}--high_load`]: { - executor: 'constant-vus', - vus: 5, - startTime: '12s', - duration: '20s', - gracefulStop: '2s', - }, + return { + discardResponseBodies: true, + scenarios, } -}; +}(variants); export default function () { // login form - const loginResponse = http.post(`${baseUrl}/login`, { + const loginResponse = http.post(`${__ENV.APP_URL}/login`, { username: 'john', password: 'test' }, { @@ -34,11 +74,11 @@ export default function () { checkResponse(loginResponse, isRedirect); // dashboard - const dashboard = http.get(`${baseUrl}/dashboard`); + const dashboard = http.get(`${__ENV.APP_URL}/dashboard`); checkResponse(dashboard, isOk); // logout - const logout = http.get(`${baseUrl}/j_spring_security_logout`, { + const logout = http.get(`${__ENV.APP_URL}/j_spring_security_logout`, { redirects: 0 }); checkResponse(logout, isRedirect); diff --git a/benchmark/load/insecure-bank/start-servers.sh b/benchmark/load/insecure-bank/start-servers.sh new file mode 100755 index 00000000000..5d626a8e474 --- /dev/null +++ b/benchmark/load/insecure-bank/start-servers.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -eu + +start_server() { + local VARIANT=$1 + local JAVA_OPTS=$2 + + if [ -n "$CI_JOB_TOKEN" ]; then + # Inside BP, so we can assume 24 CPU cores available and set CPU affinity + CPU_AFFINITY_APP=$3 + else + CPU_AFFINITY_APP="" + fi + + ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log +} + +start_server "${NO_AGENT_VARIANT}" "-Dserver.port=8080" "taskset -c 47 " & +start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 46 " & +start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 45 " & +start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8083" "taskset -c 44 " & +start_server "iast_GLOBAL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL -Dserver.port=8084" "taskset -c 43 " & +start_server "iast_FULL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL -Dserver.port=8085" "taskset -c 42 " & +start_server "iast_INACTIVE" "-javaagent:${TRACER} -Ddd.iast.enabled=inactive -Dserver.port=8086" "taskset -c 41 " & +start_server "iast_TELEMETRY_OFF" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.telemetry.verbosity=OFF -Dserver.port=8087" "taskset -c 40 " & +start_server "iast_HARDCODED_SECRET_DISABLED" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.hardcoded-secret.enabled=false -Dserver.port=8088" "taskset -c 39 " & + +wait From 660cf7cced30692016fe73f579b06c7c42e13850 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Fri, 13 Jun 2025 16:38:03 +0200 Subject: [PATCH 25/56] fix: Fix run script location --- benchmark/load/insecure-bank/benchmark.json | 2 +- benchmark/load/petclinic/benchmark.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/load/insecure-bank/benchmark.json b/benchmark/load/insecure-bank/benchmark.json index 0df7cba9b99..b4b85d704ef 100644 --- a/benchmark/load/insecure-bank/benchmark.json +++ b/benchmark/load/insecure-bank/benchmark.json @@ -2,7 +2,7 @@ "name": "load_insecure-bank", "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"", "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080/login ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", - "run": "bash -c \"${SCRIPT_DIR}/load/insecure-bank/start-servers.sh\"", + "run": "bash -c \"${UTILS_DIR}/../load/insecure-bank/start-servers.sh\"", "timeout": 150, "iterations": 1, "variants": { diff --git a/benchmark/load/petclinic/benchmark.json b/benchmark/load/petclinic/benchmark.json index 3aa83ede8db..1f703fc2af1 100644 --- a/benchmark/load/petclinic/benchmark.json +++ b/benchmark/load/petclinic/benchmark.json @@ -2,7 +2,7 @@ "name": "load_petclinic", "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"", "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", - "run": "bash -c \"${SCRIPT_DIR}/load/petclinic/start-servers.sh\"", + "run": "bash -c \"${UTILS_DIR}/../load/petclinic/start-servers.sh\"", "timeout": 150, "iterations": 1, "variants": { From a0cb0157b7c3e51649f48ce24810360efd0c4660 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Fri, 13 Jun 2025 17:58:09 +0200 Subject: [PATCH 26/56] tweak: Increase iterations count --- benchmark/load/insecure-bank/benchmark.json | 2 +- benchmark/load/petclinic/benchmark.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/load/insecure-bank/benchmark.json b/benchmark/load/insecure-bank/benchmark.json index b4b85d704ef..d00330362f3 100644 --- a/benchmark/load/insecure-bank/benchmark.json +++ b/benchmark/load/insecure-bank/benchmark.json @@ -4,7 +4,7 @@ "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080/login ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", "run": "bash -c \"${UTILS_DIR}/../load/insecure-bank/start-servers.sh\"", "timeout": 150, - "iterations": 1, + "iterations": 2, "variants": { "default": {} } diff --git a/benchmark/load/petclinic/benchmark.json b/benchmark/load/petclinic/benchmark.json index 1f703fc2af1..adf53565d58 100644 --- a/benchmark/load/petclinic/benchmark.json +++ b/benchmark/load/petclinic/benchmark.json @@ -4,7 +4,7 @@ "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", "run": "bash -c \"${UTILS_DIR}/../load/petclinic/start-servers.sh\"", "timeout": 150, - "iterations": 1, + "iterations": 2, "variants": { "default": {} } From 4d2fbd65a48a117bf52e4872546ea9e9c624531a Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 14:15:11 +0200 Subject: [PATCH 27/56] fix: Fix paths --- benchmark/load/insecure-bank/benchmark.json | 3 +-- benchmark/load/insecure-bank/start-servers.sh | 1 + benchmark/load/petclinic/benchmark.json | 3 +-- benchmark/load/petclinic/start-servers.sh | 1 + 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/load/insecure-bank/benchmark.json b/benchmark/load/insecure-bank/benchmark.json index d00330362f3..f9cd7f09371 100644 --- a/benchmark/load/insecure-bank/benchmark.json +++ b/benchmark/load/insecure-bank/benchmark.json @@ -1,7 +1,6 @@ { "name": "load_insecure-bank", - "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"", - "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080/login ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", + "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080/login ${OUTPUT_DIR} 'pkill java'\"", "run": "bash -c \"${UTILS_DIR}/../load/insecure-bank/start-servers.sh\"", "timeout": 150, "iterations": 2, diff --git a/benchmark/load/insecure-bank/start-servers.sh b/benchmark/load/insecure-bank/start-servers.sh index 5d626a8e474..c7e389e6a61 100755 --- a/benchmark/load/insecure-bank/start-servers.sh +++ b/benchmark/load/insecure-bank/start-servers.sh @@ -12,6 +12,7 @@ start_server() { CPU_AFFINITY_APP="" fi + mkdir -p "${OUTPUT_DIR}/${VARIANT}" ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log } diff --git a/benchmark/load/petclinic/benchmark.json b/benchmark/load/petclinic/benchmark.json index adf53565d58..d364663876f 100644 --- a/benchmark/load/petclinic/benchmark.json +++ b/benchmark/load/petclinic/benchmark.json @@ -1,7 +1,6 @@ { "name": "load_petclinic", - "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"", - "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${OUTPUT_DIR}/${VARIANT} 'pkill java'\"", + "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${OUTPUT_DIR} 'pkill java'\"", "run": "bash -c \"${UTILS_DIR}/../load/petclinic/start-servers.sh\"", "timeout": 150, "iterations": 2, diff --git a/benchmark/load/petclinic/start-servers.sh b/benchmark/load/petclinic/start-servers.sh index 769a6f4671b..001db7d4da8 100755 --- a/benchmark/load/petclinic/start-servers.sh +++ b/benchmark/load/petclinic/start-servers.sh @@ -12,6 +12,7 @@ start_server() { CPU_AFFINITY_APP="" fi + mkdir -p "${OUTPUT_DIR}/${VARIANT}" ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log } From 077c92d8537437309d353f8044c90e65305fbd42 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 14:37:18 +0200 Subject: [PATCH 28/56] wip: Remove sirun, add verbosity --- benchmark/load/insecure-bank/benchmark.json | 10 ------- benchmark/load/insecure-bank/start-servers.sh | 5 ++-- benchmark/load/petclinic/benchmark.json | 10 ------- benchmark/load/petclinic/start-servers.sh | 5 ++-- benchmark/load/run.sh | 28 +++++++++++-------- 5 files changed, 23 insertions(+), 35 deletions(-) delete mode 100644 benchmark/load/insecure-bank/benchmark.json delete mode 100644 benchmark/load/petclinic/benchmark.json diff --git a/benchmark/load/insecure-bank/benchmark.json b/benchmark/load/insecure-bank/benchmark.json deleted file mode 100644 index f9cd7f09371..00000000000 --- a/benchmark/load/insecure-bank/benchmark.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "load_insecure-bank", - "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080/login ${OUTPUT_DIR} 'pkill java'\"", - "run": "bash -c \"${UTILS_DIR}/../load/insecure-bank/start-servers.sh\"", - "timeout": 150, - "iterations": 2, - "variants": { - "default": {} - } -} diff --git a/benchmark/load/insecure-bank/start-servers.sh b/benchmark/load/insecure-bank/start-servers.sh index c7e389e6a61..456c85f6989 100755 --- a/benchmark/load/insecure-bank/start-servers.sh +++ b/benchmark/load/insecure-bank/start-servers.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -set -eu + +set -exu start_server() { local VARIANT=$1 @@ -16,7 +17,7 @@ start_server() { ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log } -start_server "${NO_AGENT_VARIANT}" "-Dserver.port=8080" "taskset -c 47 " & +start_server "no_agent" "-Dserver.port=8080" "taskset -c 47 " & start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 46 " & start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 45 " & start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8083" "taskset -c 44 " & diff --git a/benchmark/load/petclinic/benchmark.json b/benchmark/load/petclinic/benchmark.json deleted file mode 100644 index d364663876f..00000000000 --- a/benchmark/load/petclinic/benchmark.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "load_petclinic", - "service": "bash -c \"${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh http://localhost:8080 ${OUTPUT_DIR} 'pkill java'\"", - "run": "bash -c \"${UTILS_DIR}/../load/petclinic/start-servers.sh\"", - "timeout": 150, - "iterations": 2, - "variants": { - "default": {} - } -} diff --git a/benchmark/load/petclinic/start-servers.sh b/benchmark/load/petclinic/start-servers.sh index 001db7d4da8..917908f3fb0 100755 --- a/benchmark/load/petclinic/start-servers.sh +++ b/benchmark/load/petclinic/start-servers.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -set -eu + +set -exu start_server() { local VARIANT=$1 @@ -16,7 +17,7 @@ start_server() { ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log } -start_server "${NO_AGENT_VARIANT}" "-Dserver.port=8080" "taskset -c 31-32 " & +start_server "no_agent" "-Dserver.port=8080" "taskset -c 31-32 " & start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 33-34 " & start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 35-36 " & start_server "appsec" "-javaagent:${TRACER} -Ddd.appsec.enabled=true -Dserver.port=8083" "taskset -c 37-38 " & diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index f657bd64376..9813968cdf5 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -1,23 +1,29 @@ #!/usr/bin/env bash -set -eu + +set -exu + +local type=$1 +local app=$2 + +export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" if [ -n "$CI_JOB_TOKEN" ]; then - # Inside BP, so we can assume 24 CPU cores available and set CPU affinity - export CPU_AFFINITY_K6="taskset -c 25-26 " - export CPU_AFFINITY_APP="taskset -c 27-47 " + # Inside BP, so we can assume 24 CPU cores on the second socket available and set CPU affinity + export CPU_AFFINITY_K6="taskset -c 24-30 " else export CPU_AFFINITY_K6="" - export CPU_AFFINITY_APP="" fi - source "${UTILS_DIR}/update-java-version.sh" 17 -if [ -n "$CI_JOB_TOKEN" ]; then - # Inside BP, so we can assume 24 CPU cores available and set CPU affinity - export CPU_AFFINITY_K6="taskset -c 24-30 " +if [ "${app}" == "petclinic" ]; then + HEALTHCHECK_URL=http://localhost:8080 +elif [ "${app}" == "insecure-bank" ]; then + HEALTHCHECK_URL=http://localhost:8080/login else - export CPU_AFFINITY_K6="" + echo "Unknown app ${app}" + exit 1 fi -"${UTILS_DIR}/run-sirun-benchmarks.sh" "$@" +bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" +bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh ${HEALTHCHECK_URL} ${OUTPUT_DIR} 'pkill java'" From c61647a51f6a3ed25753ff93974cb5c2477682a4 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 14:38:55 +0200 Subject: [PATCH 29/56] wip: Fix bash error --- benchmark/load/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index 9813968cdf5..fe02d75f58e 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -2,8 +2,8 @@ set -exu -local type=$1 -local app=$2 +type=$1 +app=$2 export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" From 82bd4f5f77735b3d6e74d33bec446e34eebefef9 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 14:39:55 +0200 Subject: [PATCH 30/56] wip: Fix bash error --- benchmark/load/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index fe02d75f58e..18819110bfc 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -exu +set -ex type=$1 app=$2 From cfa13c08d08cd6a52b2881fd52eacb53c3ddfc93 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 14:42:10 +0200 Subject: [PATCH 31/56] wip: Fix bash error --- benchmark/load/run.sh | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index 18819110bfc..2e79b663156 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -3,9 +3,6 @@ set -ex type=$1 -app=$2 - -export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" if [ -n "$CI_JOB_TOKEN" ]; then # Inside BP, so we can assume 24 CPU cores on the second socket available and set CPU affinity @@ -16,14 +13,18 @@ fi source "${UTILS_DIR}/update-java-version.sh" 17 -if [ "${app}" == "petclinic" ]; then - HEALTHCHECK_URL=http://localhost:8080 -elif [ "${app}" == "insecure-bank" ]; then - HEALTHCHECK_URL=http://localhost:8080/login -else - echo "Unknown app ${app}" - exit 1 -fi +for app in *; do + export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" + + if [ "${app}" == "petclinic" ]; then + HEALTHCHECK_URL=http://localhost:8080 + elif [ "${app}" == "insecure-bank" ]; then + HEALTHCHECK_URL=http://localhost:8080/login + else + echo "Unknown app ${app}" + exit 1 + fi -bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" -bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh ${HEALTHCHECK_URL} ${OUTPUT_DIR} 'pkill java'" + bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" + bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh ${HEALTHCHECK_URL} ${OUTPUT_DIR} 'pkill java'" +done From 86fc41f7ef378e8e95b8af3bbecd1b03ec5e432d Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 14:43:28 +0200 Subject: [PATCH 32/56] wip: Fix bash error --- benchmark/load/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index 2e79b663156..f077243011c 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -25,6 +25,6 @@ for app in *; do exit 1 fi - bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" + bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" & bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh ${HEALTHCHECK_URL} ${OUTPUT_DIR} 'pkill java'" done From 4c0c13a06656d0a5ba6fff310600f39b8f1aebde Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 14:51:23 +0200 Subject: [PATCH 33/56] wip: Fix bash error --- benchmark/load/run.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index f077243011c..4abe278ed62 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -14,7 +14,14 @@ fi source "${UTILS_DIR}/update-java-version.sh" 17 for app in *; do + if [[ ! -d "${app}" ]]; then + continue + fi + + message "${type} benchmark: ${app} started" + export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" + mkdir -p ${OUTPUT_DIR} if [ "${app}" == "petclinic" ]; then HEALTHCHECK_URL=http://localhost:8080 @@ -26,5 +33,10 @@ for app in *; do fi bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" & - bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh ${HEALTHCHECK_URL} ${OUTPUT_DIR} 'pkill java'" + ( + cd ${app} && + bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh ${HEALTHCHECK_URL} ${OUTPUT_DIR} 'pkill java'" + ) + + message "${type} benchmark: ${app} finished" done From 7fa008342116f2678d8b0853c7ecf65df274b085 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 14:59:05 +0200 Subject: [PATCH 34/56] wip: Fix bash error --- benchmark/load/run.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index 4abe278ed62..bbe42bc9440 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -2,6 +2,10 @@ set -ex +function message() { + echo "$(date +"%T"): $1" +} + type=$1 if [ -n "$CI_JOB_TOKEN" ]; then From 3dd9879b110e8599a82d05f1cc0044a883f07b6c Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 15:04:14 +0200 Subject: [PATCH 35/56] wip: Add more log lines --- benchmark/utils/run-k6-load-test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmark/utils/run-k6-load-test.sh b/benchmark/utils/run-k6-load-test.sh index 1d9753dc6e7..cc825316b50 100755 --- a/benchmark/utils/run-k6-load-test.sh +++ b/benchmark/utils/run-k6-load-test.sh @@ -14,14 +14,16 @@ cleanup() { trap cleanup EXIT ERR INT TERM -# wait for the HTTP server to be up +echo "Waiting for the HTTP server on ${url} to be up" while true; do if [[ $(curl -fso /dev/null -w "%{http_code}" "${url}") = 200 ]]; then break fi done +echo "Server is up! Starting k6 load test..." # run the k6 benchmark and store the result as JSON k6 run k6.js --out "json=${output}/k6_$(date +%s).json" &>>"${output}/k6.log" - exit_code=$? + +echo "k6 load test done!" From b5268bb312bba69430b61a7ab06c477463443d2a Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 15:11:28 +0200 Subject: [PATCH 36/56] wip: Add more log lines --- benchmark/utils/run-k6-load-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/utils/run-k6-load-test.sh b/benchmark/utils/run-k6-load-test.sh index cc825316b50..f950c5e71de 100755 --- a/benchmark/utils/run-k6-load-test.sh +++ b/benchmark/utils/run-k6-load-test.sh @@ -20,10 +20,10 @@ while true; do break fi done -echo "Server is up! Starting k6 load test..." +echo "Server is up! Starting k6 load test, log is in ${output}/k6.log..." # run the k6 benchmark and store the result as JSON -k6 run k6.js --out "json=${output}/k6_$(date +%s).json" &>>"${output}/k6.log" +k6 run k6.js --out "json=${output}/k6_$(date +%s).json" > "${output}/k6.log" 2>&1 exit_code=$? echo "k6 load test done!" From cd930aa25c64ddd615785ea644b775d477837fce Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 15:15:44 +0200 Subject: [PATCH 37/56] wip: Fix REPORTS_DIR --- benchmark/benchmarks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmarks.sh b/benchmark/benchmarks.sh index fac7c3f3fbb..55f1d1ec43e 100755 --- a/benchmark/benchmarks.sh +++ b/benchmark/benchmarks.sh @@ -3,7 +3,7 @@ set -eu readonly SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) export TRACER_DIR="${SCRIPT_DIR}/.." -export REPORTS_DIR="${SCRIPT_DIR}/reports" +export REPORTS_DIR="${ARTIFACTS_DIR}" export UTILS_DIR="${SCRIPT_DIR}/utils" export SHELL_UTILS_DIR="${UTILS_DIR}/shell" export K6_UTILS_DIR="${UTILS_DIR}/k6" From 946959996f7c5a3168af1b138b5d72656e47aa9e Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 15:21:53 +0200 Subject: [PATCH 38/56] wip: Fix k6.js --- benchmark/load/insecure-bank/k6.js | 30 ++++++++++++++---------------- benchmark/load/petclinic/k6.js | 30 ++++++++++++++---------------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 52a2a581c64..e3d0f6f0909 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -35,22 +35,20 @@ export const options = function (variants) { let scenarios = {}; for (const variant of Object.keys(variants)) { scenarios = { - ...{ - [`load--insecure-bank--${variant}--warmup`]: { - executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors - vus: 5, - duration: '10s', - gracefulStop: '2s', - env: { ...variants[variant] } - }, - [`load--insecure-bank--${variant}--high_load`]: { - executor: 'constant-vus', - vus: 5, - startTime: '12s', - duration: '20s', - gracefulStop: '2s', - env: { ...variants[variant] } - }, + [`load--insecure-bank--${variant}--warmup`]: { + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + vus: 5, + duration: '10s', + gracefulStop: '2s', + env: { ...variants[variant] } + }, + [`load--insecure-bank--${variant}--high_load`]: { + executor: 'constant-vus', + vus: 5, + startTime: '12s', + duration: '20s', + gracefulStop: '2s', + env: { ...variants[variant] } }, ...scenarios }; diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 4b7082fcdc5..2a36a5922d6 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -26,22 +26,20 @@ export const options = function (variants) { let scenarios = {}; for (const variant of Object.keys(variants)) { scenarios = { - ...{ - [`load--petclinic--${variant}--warmup`]: { - executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors - vus: 5, - duration: '10s', - gracefulStop: '2s', - env: { ...variants[variant] } - }, - [`load--petclinic--${variant}--high_load`]: { - executor: 'constant-vus', - vus: 5, - startTime: '12s', - duration: '20s', - gracefulStop: '2s', - env: { ...variants[variant] } - }, + [`load--petclinic--${variant}--warmup`]: { + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + vus: 5, + duration: '10s', + gracefulStop: '2s', + env: { ...variants[variant] } + }, + [`load--petclinic--${variant}--high_load`]: { + executor: 'constant-vus', + vus: 5, + startTime: '12s', + duration: '20s', + gracefulStop: '2s', + env: { ...variants[variant] } }, ...scenarios }; From b17c403ad737980e338447e6b8510642af9f6aa6 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 15:39:53 +0200 Subject: [PATCH 39/56] wip: Fix env vars --- benchmark/benchmarks.sh | 7 +++---- benchmark/dacapo/run.sh | 2 +- benchmark/load/run.sh | 2 +- benchmark/utils/run-sirun-benchmarks.sh | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/benchmark/benchmarks.sh b/benchmark/benchmarks.sh index 55f1d1ec43e..509b95bedab 100755 --- a/benchmark/benchmarks.sh +++ b/benchmark/benchmarks.sh @@ -3,7 +3,6 @@ set -eu readonly SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) export TRACER_DIR="${SCRIPT_DIR}/.." -export REPORTS_DIR="${ARTIFACTS_DIR}" export UTILS_DIR="${SCRIPT_DIR}/utils" export SHELL_UTILS_DIR="${UTILS_DIR}/shell" export K6_UTILS_DIR="${UTILS_DIR}/k6" @@ -33,9 +32,9 @@ if [[ ! -f "${TRACER}" ]]; then cd "${SCRIPT_DIR}" fi -# Cleanup previous reports -rm -rf "${REPORTS_DIR}" -mkdir -p "${REPORTS_DIR}" +# Cleanup previous artifacts +rm -rf "${ARTIFACTS_DIR}" +mkdir -p "${ARTIFACTS_DIR}" if [[ "$#" == '0' ]]; then for type in 'startup' 'load' 'dacapo'; do diff --git a/benchmark/dacapo/run.sh b/benchmark/dacapo/run.sh index ece44f9e5f0..074ed8a07d9 100755 --- a/benchmark/dacapo/run.sh +++ b/benchmark/dacapo/run.sh @@ -16,7 +16,7 @@ run_benchmark() { export BENCHMARK="${type}" # create output folder for the test - export OUTPUT_DIR="${REPORTS_DIR}/dacapo/${type}" + export OUTPUT_DIR="${ARTIFACTS_DIR}/dacapo/${type}" mkdir -p "${OUTPUT_DIR}" # substitute environment variables in the json file diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index bbe42bc9440..ed4c3fe2a6d 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -24,7 +24,7 @@ for app in *; do message "${type} benchmark: ${app} started" - export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" + export OUTPUT_DIR="${ARTIFACTS_DIR}/${type}/${app}" mkdir -p ${OUTPUT_DIR} if [ "${app}" == "petclinic" ]; then diff --git a/benchmark/utils/run-sirun-benchmarks.sh b/benchmark/utils/run-sirun-benchmarks.sh index c0bc732dcfa..03ed60870af 100755 --- a/benchmark/utils/run-sirun-benchmarks.sh +++ b/benchmark/utils/run-sirun-benchmarks.sh @@ -14,7 +14,7 @@ run_benchmark() { cd "${app}" # create output folder for the test - export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" + export OUTPUT_DIR="${ARTIFACTS_DIR}/${type}/${app}" mkdir -p "${OUTPUT_DIR}" # substitute environment variables in the json file From 50dbe86ecbdfff0306332668e08a1bfffd298048 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 15:45:46 +0200 Subject: [PATCH 40/56] wip: Fix k6.js --- benchmark/load/insecure-bank/k6.js | 36 +++++++++++++++------------- benchmark/load/petclinic/k6.js | 38 ++++++++++++++++-------------- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index e3d0f6f0909..8a9ade9dbae 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -34,23 +34,25 @@ const variants = { export const options = function (variants) { let scenarios = {}; for (const variant of Object.keys(variants)) { - scenarios = { - [`load--insecure-bank--${variant}--warmup`]: { - executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors - vus: 5, - duration: '10s', - gracefulStop: '2s', - env: { ...variants[variant] } - }, - [`load--insecure-bank--${variant}--high_load`]: { - executor: 'constant-vus', - vus: 5, - startTime: '12s', - duration: '20s', - gracefulStop: '2s', - env: { ...variants[variant] } - }, - ...scenarios + scenarios[`load--insecure-bank--${variant}--warmup`] = { + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + vus: 5, + duration: '10s', + gracefulStop: '2s', + env: { + "APP_URL": variants[variant]["APP_URL"] + } + }; + + scenarios[`load--insecure-bank--${variant}--high_load`] = { + executor: 'constant-vus', + vus: 5, + startTime: '12s', + duration: '20s', + gracefulStop: '2s', + env: { + "APP_URL": variants[variant]["APP_URL"] + } }; } diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 2a36a5922d6..1d6035a4bd1 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -23,25 +23,27 @@ const variants = { } export const options = function (variants) { - let scenarios = {}; + const scenarios = {}; for (const variant of Object.keys(variants)) { - scenarios = { - [`load--petclinic--${variant}--warmup`]: { - executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors - vus: 5, - duration: '10s', - gracefulStop: '2s', - env: { ...variants[variant] } - }, - [`load--petclinic--${variant}--high_load`]: { - executor: 'constant-vus', - vus: 5, - startTime: '12s', - duration: '20s', - gracefulStop: '2s', - env: { ...variants[variant] } - }, - ...scenarios + scenarios[`load--petclinic--${variant}--warmup`] = { + executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors + vus: 5, + duration: '10s', + gracefulStop: '2s', + env: { + "APP_URL": variants[variant]["APP_URL"] + } + }; + + scenario[`load--petclinic--${variant}--high_load`] = { + executor: 'constant-vus', + vus: 5, + startTime: '12s', + duration: '20s', + gracefulStop: '2s', + env: { + "APP_URL": variants[variant]["APP_URL"] + } }; } From 365255e45ebbd57e8fcd4bb1749c55b8e659ec5d Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 15:54:35 +0200 Subject: [PATCH 41/56] wip: Fix k6.js & cleanup --- benchmark/benchmarks.sh | 4 ---- benchmark/load/petclinic/k6.js | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/benchmark/benchmarks.sh b/benchmark/benchmarks.sh index 509b95bedab..943015709e2 100755 --- a/benchmark/benchmarks.sh +++ b/benchmark/benchmarks.sh @@ -32,10 +32,6 @@ if [[ ! -f "${TRACER}" ]]; then cd "${SCRIPT_DIR}" fi -# Cleanup previous artifacts -rm -rf "${ARTIFACTS_DIR}" -mkdir -p "${ARTIFACTS_DIR}" - if [[ "$#" == '0' ]]; then for type in 'startup' 'load' 'dacapo'; do run_benchmarks "$type" diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 1d6035a4bd1..539770ce690 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -35,7 +35,7 @@ export const options = function (variants) { } }; - scenario[`load--petclinic--${variant}--high_load`] = { + scenarios[`load--petclinic--${variant}--high_load`] = { executor: 'constant-vus', vus: 5, startTime: '12s', From 7cbbd5100b716733d13dd752860b8c8d60cd881b Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 16:11:18 +0200 Subject: [PATCH 42/56] wip: Reduce verbosity for load apps --- benchmark/load/insecure-bank/start-servers.sh | 5 +++-- benchmark/load/petclinic/start-servers.sh | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/benchmark/load/insecure-bank/start-servers.sh b/benchmark/load/insecure-bank/start-servers.sh index 456c85f6989..23c9e522327 100755 --- a/benchmark/load/insecure-bank/start-servers.sh +++ b/benchmark/load/insecure-bank/start-servers.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -exu +set -eu start_server() { local VARIANT=$1 @@ -14,7 +14,8 @@ start_server() { fi mkdir -p "${OUTPUT_DIR}/${VARIANT}" - ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log + ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log && PID=$! + echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log && PID=$PID" } start_server "no_agent" "-Dserver.port=8080" "taskset -c 47 " & diff --git a/benchmark/load/petclinic/start-servers.sh b/benchmark/load/petclinic/start-servers.sh index 917908f3fb0..9ccfeaa4d02 100755 --- a/benchmark/load/petclinic/start-servers.sh +++ b/benchmark/load/petclinic/start-servers.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -exu +set -eu start_server() { local VARIANT=$1 @@ -14,7 +14,8 @@ start_server() { fi mkdir -p "${OUTPUT_DIR}/${VARIANT}" - ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log + ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log && PID=$! + echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log && PID=$PID" } start_server "no_agent" "-Dserver.port=8080" "taskset -c 31-32 " & From b799d7274a554475b611bba37a1c3f807605af91 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 16:21:43 +0200 Subject: [PATCH 43/56] wip: Disable unbound var check --- benchmark/load/insecure-bank/start-servers.sh | 2 +- benchmark/load/petclinic/start-servers.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/load/insecure-bank/start-servers.sh b/benchmark/load/insecure-bank/start-servers.sh index 23c9e522327..379e397db6e 100755 --- a/benchmark/load/insecure-bank/start-servers.sh +++ b/benchmark/load/insecure-bank/start-servers.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -eu +set -e start_server() { local VARIANT=$1 diff --git a/benchmark/load/petclinic/start-servers.sh b/benchmark/load/petclinic/start-servers.sh index 9ccfeaa4d02..190b00dfb4d 100755 --- a/benchmark/load/petclinic/start-servers.sh +++ b/benchmark/load/petclinic/start-servers.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -eu +set -e start_server() { local VARIANT=$1 From 0746b0cbfd6dad8dcfd0481ab63b446a93ca8deb Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 16:29:16 +0200 Subject: [PATCH 44/56] wip: Fix pid collection --- benchmark/load/insecure-bank/start-servers.sh | 2 +- benchmark/load/petclinic/start-servers.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmark/load/insecure-bank/start-servers.sh b/benchmark/load/insecure-bank/start-servers.sh index 379e397db6e..5a8ffbd7177 100755 --- a/benchmark/load/insecure-bank/start-servers.sh +++ b/benchmark/load/insecure-bank/start-servers.sh @@ -14,7 +14,7 @@ start_server() { fi mkdir -p "${OUTPUT_DIR}/${VARIANT}" - ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log && PID=$! + ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log & && PID=$! echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log && PID=$PID" } diff --git a/benchmark/load/petclinic/start-servers.sh b/benchmark/load/petclinic/start-servers.sh index 190b00dfb4d..8d2d1880741 100755 --- a/benchmark/load/petclinic/start-servers.sh +++ b/benchmark/load/petclinic/start-servers.sh @@ -14,8 +14,8 @@ start_server() { fi mkdir -p "${OUTPUT_DIR}/${VARIANT}" - ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log && PID=$! - echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log && PID=$PID" + ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log & && PID=$! + echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log && PID=$!" } start_server "no_agent" "-Dserver.port=8080" "taskset -c 31-32 " & From df980c39cae0417b62824425e12e5520124f6cb5 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 16:33:05 +0200 Subject: [PATCH 45/56] wip: Fix pid collection --- benchmark/load/insecure-bank/start-servers.sh | 2 +- benchmark/load/petclinic/start-servers.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/load/insecure-bank/start-servers.sh b/benchmark/load/insecure-bank/start-servers.sh index 5a8ffbd7177..c35144aaafa 100755 --- a/benchmark/load/insecure-bank/start-servers.sh +++ b/benchmark/load/insecure-bank/start-servers.sh @@ -14,7 +14,7 @@ start_server() { fi mkdir -p "${OUTPUT_DIR}/${VARIANT}" - ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log & && PID=$! + ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log &PID=$! echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log && PID=$PID" } diff --git a/benchmark/load/petclinic/start-servers.sh b/benchmark/load/petclinic/start-servers.sh index 8d2d1880741..1bdae7c98a0 100755 --- a/benchmark/load/petclinic/start-servers.sh +++ b/benchmark/load/petclinic/start-servers.sh @@ -14,7 +14,7 @@ start_server() { fi mkdir -p "${OUTPUT_DIR}/${VARIANT}" - ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log & && PID=$! + ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log &PID=$! echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log && PID=$!" } From 8aa0ccaa8325e8df281778f7049a7f17f757fbbe Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 16:45:31 +0200 Subject: [PATCH 46/56] Revert "wip: Fix env vars" This reverts commit a8c774e5 --- benchmark/benchmarks.sh | 1 + benchmark/dacapo/run.sh | 2 +- benchmark/load/run.sh | 2 +- benchmark/utils/run-sirun-benchmarks.sh | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmark/benchmarks.sh b/benchmark/benchmarks.sh index 943015709e2..d15e75e126f 100755 --- a/benchmark/benchmarks.sh +++ b/benchmark/benchmarks.sh @@ -3,6 +3,7 @@ set -eu readonly SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) export TRACER_DIR="${SCRIPT_DIR}/.." +export REPORTS_DIR="${ARTIFACTS_DIR}" export UTILS_DIR="${SCRIPT_DIR}/utils" export SHELL_UTILS_DIR="${UTILS_DIR}/shell" export K6_UTILS_DIR="${UTILS_DIR}/k6" diff --git a/benchmark/dacapo/run.sh b/benchmark/dacapo/run.sh index 074ed8a07d9..ece44f9e5f0 100755 --- a/benchmark/dacapo/run.sh +++ b/benchmark/dacapo/run.sh @@ -16,7 +16,7 @@ run_benchmark() { export BENCHMARK="${type}" # create output folder for the test - export OUTPUT_DIR="${ARTIFACTS_DIR}/dacapo/${type}" + export OUTPUT_DIR="${REPORTS_DIR}/dacapo/${type}" mkdir -p "${OUTPUT_DIR}" # substitute environment variables in the json file diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index ed4c3fe2a6d..bbe42bc9440 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -24,7 +24,7 @@ for app in *; do message "${type} benchmark: ${app} started" - export OUTPUT_DIR="${ARTIFACTS_DIR}/${type}/${app}" + export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" mkdir -p ${OUTPUT_DIR} if [ "${app}" == "petclinic" ]; then diff --git a/benchmark/utils/run-sirun-benchmarks.sh b/benchmark/utils/run-sirun-benchmarks.sh index 03ed60870af..c0bc732dcfa 100755 --- a/benchmark/utils/run-sirun-benchmarks.sh +++ b/benchmark/utils/run-sirun-benchmarks.sh @@ -14,7 +14,7 @@ run_benchmark() { cd "${app}" # create output folder for the test - export OUTPUT_DIR="${ARTIFACTS_DIR}/${type}/${app}" + export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" mkdir -p "${OUTPUT_DIR}" # substitute environment variables in the json file From ab054ebcf1f017e10283d42a7383b25a456381db Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 16:51:58 +0200 Subject: [PATCH 47/56] wip: Separate LOGS_DIR --- .gitlab/benchmarks.yml | 1 - benchmark/benchmarks.sh | 2 +- benchmark/load/insecure-bank/start-servers.sh | 6 +++--- benchmark/load/petclinic/start-servers.sh | 6 +++--- benchmark/load/run.sh | 7 +++++-- benchmark/utils/run-k6-load-test.sh | 7 +++---- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 69061859391..05a5adfcbb4 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -20,7 +20,6 @@ name: "reports" paths: - reports/ - - debug-logs/ expire_in: 3 months variables: UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance. diff --git a/benchmark/benchmarks.sh b/benchmark/benchmarks.sh index d15e75e126f..0b245038afa 100755 --- a/benchmark/benchmarks.sh +++ b/benchmark/benchmarks.sh @@ -3,7 +3,7 @@ set -eu readonly SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) export TRACER_DIR="${SCRIPT_DIR}/.." -export REPORTS_DIR="${ARTIFACTS_DIR}" +export REPORTS_DIR="${SCRIPT_DIR}/reports" export UTILS_DIR="${SCRIPT_DIR}/utils" export SHELL_UTILS_DIR="${UTILS_DIR}/shell" export K6_UTILS_DIR="${UTILS_DIR}/k6" diff --git a/benchmark/load/insecure-bank/start-servers.sh b/benchmark/load/insecure-bank/start-servers.sh index c35144aaafa..ae81e64e7df 100755 --- a/benchmark/load/insecure-bank/start-servers.sh +++ b/benchmark/load/insecure-bank/start-servers.sh @@ -13,9 +13,9 @@ start_server() { CPU_AFFINITY_APP="" fi - mkdir -p "${OUTPUT_DIR}/${VARIANT}" - ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log &PID=$! - echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${OUTPUT_DIR}/${VARIANT}/insecure-bank.log && PID=$PID" + mkdir -p "${LOGS_DIR}/${VARIANT}" + ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log &PID=$! + echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log [PID=$PID]" } start_server "no_agent" "-Dserver.port=8080" "taskset -c 47 " & diff --git a/benchmark/load/petclinic/start-servers.sh b/benchmark/load/petclinic/start-servers.sh index 1bdae7c98a0..1ebbb4e0418 100755 --- a/benchmark/load/petclinic/start-servers.sh +++ b/benchmark/load/petclinic/start-servers.sh @@ -13,9 +13,9 @@ start_server() { CPU_AFFINITY_APP="" fi - mkdir -p "${OUTPUT_DIR}/${VARIANT}" - ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log &PID=$! - echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${OUTPUT_DIR}/${VARIANT}/petclinic.log && PID=$!" + mkdir -p "${LOGS_DIR}/${VARIANT}" + ${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${LOGS_DIR}/${VARIANT}/petclinic.log &PID=$! + echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${LOGS_DIR}/${VARIANT}/petclinic.log [PID=$!]" } start_server "no_agent" "-Dserver.port=8080" "taskset -c 31-32 " & diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index bbe42bc9440..e5e20efa596 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -10,7 +10,7 @@ type=$1 if [ -n "$CI_JOB_TOKEN" ]; then # Inside BP, so we can assume 24 CPU cores on the second socket available and set CPU affinity - export CPU_AFFINITY_K6="taskset -c 24-30 " + export CPU_AFFINITY_K6="taskset -c 24-27 " else export CPU_AFFINITY_K6="" fi @@ -27,6 +27,9 @@ for app in *; do export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}" mkdir -p ${OUTPUT_DIR} + export LOGS_DIR="${ARTIFACTS_DIR}/${type}/${app}" + mkdir -p ${LOGS_DIR} + if [ "${app}" == "petclinic" ]; then HEALTHCHECK_URL=http://localhost:8080 elif [ "${app}" == "insecure-bank" ]; then @@ -39,7 +42,7 @@ for app in *; do bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" & ( cd ${app} && - bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh ${HEALTHCHECK_URL} ${OUTPUT_DIR} 'pkill java'" + bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh ${HEALTHCHECK_URL} 'pkill java'" ) message "${type} benchmark: ${app} finished" diff --git a/benchmark/utils/run-k6-load-test.sh b/benchmark/utils/run-k6-load-test.sh index f950c5e71de..c897a6e2300 100755 --- a/benchmark/utils/run-k6-load-test.sh +++ b/benchmark/utils/run-k6-load-test.sh @@ -2,8 +2,7 @@ set -eu url=$1 -output=$2 -command=$3 +command=$2 exit_code=0 cleanup() { @@ -20,10 +19,10 @@ while true; do break fi done -echo "Server is up! Starting k6 load test, log is in ${output}/k6.log..." +echo "Server is up! Starting k6 load test, log is in ${LOGS_DIR}/k6.log..." # run the k6 benchmark and store the result as JSON -k6 run k6.js --out "json=${output}/k6_$(date +%s).json" > "${output}/k6.log" 2>&1 +k6 run k6.js --out "json=${OUTPUT_DIR}/k6_$(date +%s).json" > "${LOGS_DIR}/k6.log" 2>&1 exit_code=$? echo "k6 load test done!" From b23b4dc9f7be1455c8c2f1cb7f309abc359f0e96 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 17:21:45 +0200 Subject: [PATCH 48/56] wip: Reduce verbosity --- benchmark/load/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index e5e20efa596..dcfdbf03973 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -ex +set -e function message() { echo "$(date +"%T"): $1" From b137583e6b8158c79ae26b0168baf99d08816a33 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 17:28:44 +0200 Subject: [PATCH 49/56] tweak: k6 config, multiple repetitions --- benchmark/load/insecure-bank/k6.js | 6 +++--- benchmark/load/petclinic/k6.js | 6 +++--- benchmark/load/run.sh | 17 ++++++++++------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 8a9ade9dbae..c2116f8fe4f 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -37,7 +37,7 @@ export const options = function (variants) { scenarios[`load--insecure-bank--${variant}--warmup`] = { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, - duration: '10s', + duration: '5s', gracefulStop: '2s', env: { "APP_URL": variants[variant]["APP_URL"] @@ -47,8 +47,8 @@ export const options = function (variants) { scenarios[`load--insecure-bank--${variant}--high_load`] = { executor: 'constant-vus', vus: 5, - startTime: '12s', - duration: '20s', + startTime: '7s', + duration: '15s', gracefulStop: '2s', env: { "APP_URL": variants[variant]["APP_URL"] diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 539770ce690..14e110e47b6 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -28,7 +28,7 @@ export const options = function (variants) { scenarios[`load--petclinic--${variant}--warmup`] = { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, - duration: '10s', + duration: '5s', gracefulStop: '2s', env: { "APP_URL": variants[variant]["APP_URL"] @@ -38,8 +38,8 @@ export const options = function (variants) { scenarios[`load--petclinic--${variant}--high_load`] = { executor: 'constant-vus', vus: 5, - startTime: '12s', - duration: '20s', + startTime: '7s', + duration: '15s', gracefulStop: '2s', env: { "APP_URL": variants[variant]["APP_URL"] diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index dcfdbf03973..b7dae1719be 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -30,20 +30,23 @@ for app in *; do export LOGS_DIR="${ARTIFACTS_DIR}/${type}/${app}" mkdir -p ${LOGS_DIR} + # Using profiler variants for healthcheck as they are the slowest if [ "${app}" == "petclinic" ]; then - HEALTHCHECK_URL=http://localhost:8080 + HEALTHCHECK_URL=http://localhost:8082 elif [ "${app}" == "insecure-bank" ]; then - HEALTHCHECK_URL=http://localhost:8080/login + HEALTHCHECK_URL=http://localhost:8082/login else echo "Unknown app ${app}" exit 1 fi - bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" & - ( - cd ${app} && - bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh ${HEALTHCHECK_URL} 'pkill java'" - ) + for i in $(seq 1 5); do + bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" & + ( + cd ${app} && + bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh ${HEALTHCHECK_URL} 'pkill java'" + ) + done message "${type} benchmark: ${app} finished" done From 72502fbbe1f9b6c6ba1e8aa9d1b0efbba40c1028 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 17:57:35 +0200 Subject: [PATCH 50/56] tweak: multiple repetitions --- benchmark/load/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index b7dae1719be..7b1261612c8 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -40,7 +40,7 @@ for app in *; do exit 1 fi - for i in $(seq 1 5); do + for i in $(seq 1 2); do bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" & ( cd ${app} && From 1df85151afe7e7d42d4a6a487f20abf454905327 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 18:03:01 +0200 Subject: [PATCH 51/56] tweak: Update BP branch, multiple repetitions --- .gitlab/benchmarks.yml | 2 +- benchmark/load/run.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 05a5adfcbb4..8b621365070 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -15,7 +15,7 @@ script: - export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}" - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - - git clone --branch dd-trace-java/tracer-benchmarks https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform + - git clone --branch dd-trace-java/tracer-benchmarks-parallel https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform artifacts: name: "reports" paths: diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index 7b1261612c8..7a98f028287 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -33,14 +33,16 @@ for app in *; do # Using profiler variants for healthcheck as they are the slowest if [ "${app}" == "petclinic" ]; then HEALTHCHECK_URL=http://localhost:8082 + REPETITIONS_COUNT=5 elif [ "${app}" == "insecure-bank" ]; then HEALTHCHECK_URL=http://localhost:8082/login + REPETITIONS_COUNT=2 else echo "Unknown app ${app}" exit 1 fi - for i in $(seq 1 2); do + for i in $(seq 1 $REPETITIONS_COUNT); do bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" & ( cd ${app} && From db5d3e1507dff8f329391018333adf6b0f4d3615 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 18:15:10 +0200 Subject: [PATCH 52/56] tweak: Tweak RPS for petclinic --- benchmark/load/petclinic/k6.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 14e110e47b6..1ba03e16b0c 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -36,11 +36,13 @@ export const options = function (variants) { }; scenarios[`load--petclinic--${variant}--high_load`] = { - executor: 'constant-vus', - vus: 5, + executor: 'constant-arrival-rate', + preAllocatedVUs: 5, startTime: '7s', - duration: '15s', + duration: '20s', gracefulStop: '2s', + timeUnit: '1s', + rate: 150, env: { "APP_URL": variants[variant]["APP_URL"] } From e4eced28b214cade3428114af3c5ac5947af1629 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 19:08:42 +0200 Subject: [PATCH 53/56] tweak: wait for health check --- benchmark/load/run.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index 7a98f028287..11a8f899995 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -6,6 +6,16 @@ function message() { echo "$(date +"%T"): $1" } +function healthcheck() { + local url=$1 + + while true; do + if [[ $(curl -fso /dev/null -w "%{http_code}" "${url}") = 200 ]]; then + break + fi + done +} + type=$1 if [ -n "$CI_JOB_TOKEN" ]; then @@ -44,6 +54,17 @@ for app in *; do for i in $(seq 1 $REPETITIONS_COUNT); do bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" & + + if [ "${app}" == "petclinic" ]; then + for port in $(seq 8080 8085); do + healthcheck http://localhost:$port + done + elif [ "${app}" == "insecure-bank" ]; then + for port in $(seq 8080 8088); do + healthcheck http://localhost:$port/login + done + fi + ( cd ${app} && bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh ${HEALTHCHECK_URL} 'pkill java'" From 8e78e94203c86a7c4a8dc890f289023d80c66a11 Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Tue, 17 Jun 2025 19:10:39 +0200 Subject: [PATCH 54/56] tweak: Tweak RPS --- benchmark/load/insecure-bank/k6.js | 4 ++-- benchmark/load/petclinic/k6.js | 12 +++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index c2116f8fe4f..9cf62422066 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -37,7 +37,7 @@ export const options = function (variants) { scenarios[`load--insecure-bank--${variant}--warmup`] = { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, - duration: '5s', + duration: '10s', gracefulStop: '2s', env: { "APP_URL": variants[variant]["APP_URL"] @@ -47,7 +47,7 @@ export const options = function (variants) { scenarios[`load--insecure-bank--${variant}--high_load`] = { executor: 'constant-vus', vus: 5, - startTime: '7s', + startTime: '12s', duration: '15s', gracefulStop: '2s', env: { diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 1ba03e16b0c..3c97bfb2cad 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -28,7 +28,7 @@ export const options = function (variants) { scenarios[`load--petclinic--${variant}--warmup`] = { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, - duration: '5s', + duration: '10s', gracefulStop: '2s', env: { "APP_URL": variants[variant]["APP_URL"] @@ -36,13 +36,11 @@ export const options = function (variants) { }; scenarios[`load--petclinic--${variant}--high_load`] = { - executor: 'constant-arrival-rate', - preAllocatedVUs: 5, - startTime: '7s', - duration: '20s', + executor: 'constant-vus', + vus: 5, + startTime: '12s', + duration: '15s', gracefulStop: '2s', - timeUnit: '1s', - rate: 150, env: { "APP_URL": variants[variant]["APP_URL"] } From c40ca32d4ad3e624f394e62e831efcd5a540a20d Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Wed, 18 Jun 2025 11:50:16 +0200 Subject: [PATCH 55/56] tweak: Increase warmup time --- benchmark/load/insecure-bank/k6.js | 4 ++-- benchmark/load/petclinic/k6.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 9cf62422066..1bb35e7f5b6 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -37,7 +37,7 @@ export const options = function (variants) { scenarios[`load--insecure-bank--${variant}--warmup`] = { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, - duration: '10s', + duration: '20s', gracefulStop: '2s', env: { "APP_URL": variants[variant]["APP_URL"] @@ -47,7 +47,7 @@ export const options = function (variants) { scenarios[`load--insecure-bank--${variant}--high_load`] = { executor: 'constant-vus', vus: 5, - startTime: '12s', + startTime: '22s', duration: '15s', gracefulStop: '2s', env: { diff --git a/benchmark/load/petclinic/k6.js b/benchmark/load/petclinic/k6.js index 3c97bfb2cad..a215ad7ea2d 100644 --- a/benchmark/load/petclinic/k6.js +++ b/benchmark/load/petclinic/k6.js @@ -28,7 +28,7 @@ export const options = function (variants) { scenarios[`load--petclinic--${variant}--warmup`] = { executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors vus: 5, - duration: '10s', + duration: '20s', gracefulStop: '2s', env: { "APP_URL": variants[variant]["APP_URL"] @@ -38,7 +38,7 @@ export const options = function (variants) { scenarios[`load--petclinic--${variant}--high_load`] = { executor: 'constant-vus', vus: 5, - startTime: '12s', + startTime: '22s', duration: '15s', gracefulStop: '2s', env: { From e3de5081c7cdebeab1104ad024cdc2a69bfe4e1c Mon Sep 17 00:00:00 2001 From: Dmytro Yurchenko Date: Wed, 18 Jun 2025 13:48:24 +0200 Subject: [PATCH 56/56] tweak: Remove unused variants in insecure-bank, tidy k6 .sh commands --- benchmark/load/insecure-bank/k6.js | 9 --------- benchmark/load/insecure-bank/start-servers.sh | 3 --- benchmark/load/run.sh | 6 ++++-- benchmark/utils/run-k6-load-test.sh | 13 +++---------- 4 files changed, 7 insertions(+), 24 deletions(-) diff --git a/benchmark/load/insecure-bank/k6.js b/benchmark/load/insecure-bank/k6.js index 1bb35e7f5b6..2dc5ce582c6 100644 --- a/benchmark/load/insecure-bank/k6.js +++ b/benchmark/load/insecure-bank/k6.js @@ -20,15 +20,6 @@ const variants = { "iast_FULL": { "APP_URL": 'http://localhost:8085', }, - "iast_INACTIVE": { - "APP_URL": 'http://localhost:8086', - }, - "iast_TELEMETRY_OFF": { - "APP_URL": 'http://localhost:8087', - }, - "iast_HARDCODED_SECRET_DISABLED": { - "APP_URL": 'http://localhost:8088', - }, } export const options = function (variants) { diff --git a/benchmark/load/insecure-bank/start-servers.sh b/benchmark/load/insecure-bank/start-servers.sh index ae81e64e7df..4cae95567f2 100755 --- a/benchmark/load/insecure-bank/start-servers.sh +++ b/benchmark/load/insecure-bank/start-servers.sh @@ -24,8 +24,5 @@ start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dser start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8083" "taskset -c 44 " & start_server "iast_GLOBAL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL -Dserver.port=8084" "taskset -c 43 " & start_server "iast_FULL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL -Dserver.port=8085" "taskset -c 42 " & -start_server "iast_INACTIVE" "-javaagent:${TRACER} -Ddd.iast.enabled=inactive -Dserver.port=8086" "taskset -c 41 " & -start_server "iast_TELEMETRY_OFF" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.telemetry.verbosity=OFF -Dserver.port=8087" "taskset -c 40 " & -start_server "iast_HARDCODED_SECRET_DISABLED" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.hardcoded-secret.enabled=false -Dserver.port=8088" "taskset -c 39 " & wait diff --git a/benchmark/load/run.sh b/benchmark/load/run.sh index 11a8f899995..d43a28383d5 100755 --- a/benchmark/load/run.sh +++ b/benchmark/load/run.sh @@ -55,19 +55,21 @@ for app in *; do for i in $(seq 1 $REPETITIONS_COUNT); do bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" & + echo "Waiting for serves to start..." if [ "${app}" == "petclinic" ]; then for port in $(seq 8080 8085); do healthcheck http://localhost:$port done elif [ "${app}" == "insecure-bank" ]; then - for port in $(seq 8080 8088); do + for port in $(seq 8080 8085); do healthcheck http://localhost:$port/login done fi + echo "Servers are up!" ( cd ${app} && - bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh ${HEALTHCHECK_URL} 'pkill java'" + bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh 'pkill java'" ) done diff --git a/benchmark/utils/run-k6-load-test.sh b/benchmark/utils/run-k6-load-test.sh index c897a6e2300..d3415f54eef 100755 --- a/benchmark/utils/run-k6-load-test.sh +++ b/benchmark/utils/run-k6-load-test.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash set -eu -url=$1 -command=$2 +command=$1 exit_code=0 cleanup() { @@ -13,16 +12,10 @@ cleanup() { trap cleanup EXIT ERR INT TERM -echo "Waiting for the HTTP server on ${url} to be up" -while true; do - if [[ $(curl -fso /dev/null -w "%{http_code}" "${url}") = 200 ]]; then - break - fi -done -echo "Server is up! Starting k6 load test, log is in ${LOGS_DIR}/k6.log..." +echo "Starting k6 load test, logs are recorded into ${LOGS_DIR}/k6.log..." # run the k6 benchmark and store the result as JSON k6 run k6.js --out "json=${OUTPUT_DIR}/k6_$(date +%s).json" > "${LOGS_DIR}/k6.log" 2>&1 exit_code=$? -echo "k6 load test done!" +echo "k6 load test done !!!"