Skip to content

Commit ed75e06

Browse files
tobyhedetobyhede
committed
feat: pgbench with postgres, pgbouncer and proxy
1 parent a01fbab commit ed75e06

15 files changed

+455
-4
lines changed

‎.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,7 @@ rust-toolchain.toml
1818

1919
# credentials for local dev
2020
.env.proxy.docker
21+
22+
## benchmark result data
23+
tests/benchmark/results/*.csv
24+
tests/benchmark/benchmark-*.png

‎tests/benchmark/docker/pgbouncer/Dockerfile

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
FROM alpine:3.19
2+
3+
RUN apk add pgbouncer
4+
RUN adduser -D -S pgbouncer
5+
6+
ADD pgbouncer.ini /etc/pgbouncer/pgbouncer.ini
7+
ADD userlist.txt /etc/pgbouncer/userlist.txt
8+
9+
EXPOSE 6433
10+
USER pgbouncer
11+
CMD /usr/bin/pgbouncer /etc/pgbouncer/pgbouncer.ini

‎tests/benchmark/docker/pgbouncer/pgbouncer.ini

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
[databases]
2+
cipherstash = host=host.docker.internal port=5532 dbname=cipherstash user=cipherstash password=password
3+
4+
[users]
5+
;user1 = pool_size=5 pool_mode=transaction max_user_connections=10
6+
7+
[pgbouncer]
8+
listen_addr = *
9+
listen_port = 6433
10+
listen_backlog = 1000
11+
so_reuseport = 1
12+
; pool_mode = session
13+
14+
default_pool_size = 100
15+
min_pool_size = 20
16+
17+
max_db_connections = 100
18+
max_client_conn = 1000
19+
20+
syslog = 0
21+
log_connections = 0
22+
log_disconnections = 0
23+
verbose = 0
24+
25+
auth_type = scram-sha-256
26+
auth_file = /etc/pgbouncer/userlist.txt

‎tests/benchmark/docker/pgbouncer/userlist.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"cipherstash" "password"

‎tests/benchmark/mise.toml

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
[settings]
2+
3+
trusted_config_paths = ["./env/mise.passthrough.toml"]
4+
5+
[task_config]
6+
includes = ["./tasks"]
7+
8+
9+
[env]
10+
CS_DATABASE__HOST = "host.docker.internal"
11+
POSTGRES_DB = "{{env.CS_DATABASE__NAME}}"
12+
PGUSER = "{{env.CS_DATABASE__USERNAME}}"
13+
PGPASSWORD = "{{env.CS_DATABASE__PASSWORD}}"
14+
PGHOST = "{{env.CS_DATABASE__HOST}}"
15+
PGPORT = "{{env.CS_DATABASE__PORT}}"
16+
17+
# ====================================================================================================
18+
19+
[tasks."benchmark"]
20+
alias = 'b'
21+
description = "Run benchmarks"
22+
run = """
23+
set -e
24+
25+
mise run benchmark:clean
26+
27+
mise run benchmark:setup
28+
29+
# # Extended protocol with default script
30+
# mise run benchmark_service --target=postgres --protocol=extended --port=5532 --time=5
31+
# mise run benchmark_service --target=pgbouncer --protocol=extended --port=6433 --time=5
32+
# mise run benchmark_service --target=proxy --protocol=extended --port=6432 --time=5
33+
34+
# Extended protocol with plaintext script
35+
mise run benchmark_service --target=postgres --transaction=plaintext --protocol=extended --port=5532 --time=5
36+
mise run benchmark_service --target=pgbouncer --transaction=plaintext --protocol=extended --port=6433 --time=5
37+
mise run benchmark_service --target=proxy --transaction=plaintext --protocol=extended --port=6432 --time=5
38+
39+
# Extended protocol with encrypted script
40+
mise run benchmark_service --target=proxy --transaction=encrypted --protocol=extended --port=6432 --time=5
41+
42+
mise run benchmark:plot
43+
"""
44+
45+
[tasks."benchmark:setup"]
46+
run = """
47+
cat sql/benchmark-schema.sql | docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME} -f-
48+
49+
# Initialize pgbench
50+
docker compose run --rm postgres${CONTAINER_SUFFIX:-} pgbench --host=${CS_DATABASE__HOST} --port=${CS_DATABASE__PORT} --scale=1 -i --no-vacuum
51+
"""
52+
53+
[tasks."benchmark:plot"]
54+
alias = 'b'
55+
description = "Plot graphs from benchmark results"
56+
run = """
57+
python plot.py
58+
"""
59+
60+
[tasks."benchmark:up"]
61+
alias = 'u'
62+
description = "Run PostgreSQL with docker compose"
63+
run = """
64+
set -e
65+
66+
mise run postgres:up
67+
mise run pgbouncer:up
68+
mise run proxy:up
69+
70+
"""
71+
72+
[tasks."benchmark:clean"]
73+
description = "Clean old benchmark results"
74+
run = """
75+
rm -rf {{config_root}}/results/*.csv
76+
"""
77+
78+
[tasks."pgbouncer:up"]
79+
alias = 'u'
80+
description = "Run pgbouncer"
81+
run = """
82+
set -e
83+
docker compose up --build pgbouncer --detach
84+
"""
85+
86+
87+
# [tasks."down"]
88+
# alias = 'u'
89+
# description = "Tear down PostgreSQL and Proxy containers"
90+
# run = """
91+
# set -e
92+
# if [[ -z "${CS_BENCHMARK_BUILD}" ]]; then
93+
# mise run proxy:down
94+
# mise run postgres:down
95+
# fi
96+
# """
97+
98+
# [tasks."proxy:up"]
99+
# alias = 'u'
100+
# description = "Run CipherStash Proxy with docker compose"
101+
# run = """
102+
# set -e
103+
104+
# docker compose up --build proxy --detach
105+
# """
106+
107+
# [tasks."proxy:down"]
108+
# alias = 'pd'
109+
# description = "Tear down Proxy container"
110+
# run = """
111+
# set -e
112+
# docker compose down proxy
113+
# """
114+
115+
# [tasks."postgres:up"]
116+
# alias = 'pgu'
117+
# description = "Run PostgreSQL with docker compose"
118+
# run = """
119+
# set -e
120+
# docker compose up postgres --detach
121+
# """
122+
123+
# [tasks."postgres:down"]
124+
# alias = 'pgd'
125+
# description = "Tear down PostgreSQL container"
126+
# run = """
127+
# set -e
128+
# docker compose down postgres
129+
# """

‎tests/benchmark/plot.py

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
import csv
2+
import glob
3+
import matplotlib.pyplot as plt
4+
import time
5+
6+
MARKERS = [
7+
"^",
8+
"o",
9+
"s",
10+
"d",
11+
"v",
12+
"P",
13+
">",
14+
"X",
15+
"<",
16+
"*",
17+
]
18+
19+
20+
def read_csv(file_name):
21+
with open(file_name) as csv_file:
22+
csv_reader = csv.DictReader(csv_file)
23+
24+
rows = []
25+
for csv_line in csv_reader:
26+
try:
27+
row = {
28+
"clients": int(csv_line["clients"]),
29+
"latency": float(csv_line["latency"]),
30+
"init_conn_time": float(csv_line["init_conn_time"]),
31+
"tps": float(csv_line["tps"]),
32+
}
33+
rows.append(row)
34+
except:
35+
print("Unable to parse row from csv_line:", csv_line)
36+
37+
return rows
38+
39+
40+
def main():
41+
fig, ax = plt.subplots(figsize=(16, 12), layout="constrained")
42+
43+
files = [f for f in glob.glob("results/*.csv")]
44+
files = sorted(files)
45+
46+
for i, file in enumerate(files):
47+
label = file.replace(".csv", "")
48+
data = read_csv(file)
49+
50+
clients = [d["clients"] for d in data]
51+
tps = [d["tps"] for d in data]
52+
ax.plot(
53+
clients,
54+
tps,
55+
label=label,
56+
linestyle="-",
57+
marker=MARKERS[i],
58+
markeredgewidth=1,
59+
markersize=10,
60+
)
61+
62+
# Ensure the baseline starts at zero
63+
ax.set_xlim(left=0) # Set x-axis lower limit to 0
64+
ax.set_ylim(bottom=0) # Set y-axis lower limit to 0
65+
66+
67+
fig.legend(loc="outside upper left")
68+
plt.xlabel("clients")
69+
plt.ylabel("tps")
70+
plt.title("Transactions per second")
71+
72+
ts = time.strftime("%Y%m%d%H%M")
73+
file_name = "benchmark-{}.png".format(ts)
74+
plt.savefig(file_name)
75+
76+
77+
if __name__ == "__main__":
78+
main()

‎tests/benchmark/sql/benchmark-schema.sql

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
TRUNCATE TABLE cs_configuration_v1;
2+
3+
DROP TABLE IF EXISTS benchmark_plaintext;
4+
CREATE TABLE benchmark_plaintext (
5+
id serial primary key,
6+
username text,
7+
email text
8+
);
9+
10+
DROP TABLE IF EXISTS benchmark_encrypted;
11+
CREATE TABLE benchmark_encrypted (
12+
id serial primary key,
13+
username text,
14+
email cs_encrypted_v1
15+
);
16+
17+
SELECT cs_add_column_v1(
18+
'benchmark_encrypted',
19+
'email'
20+
);
21+
22+
SELECT cs_encrypt_v1();
23+
SELECT cs_activate_v1();

‎tests/benchmark/sql/transaction-default.sql

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
\set aid random(1, 100000 * :scale)
2+
\set bid random(1, 1 * :scale)
3+
\set tid random(1, 10 * :scale)
4+
\set delta random(-5000, 5000)
5+
BEGIN;
6+
UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid;
7+
SELECT abalance FROM pgbench_accounts WHERE aid = :aid;
8+
UPDATE pgbench_tellers SET tbalance = tbalance + :delta WHERE tid = :tid;
9+
UPDATE pgbench_branches SET bbalance = bbalance + :delta WHERE bid = :bid;
10+
INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP);
11+
END;

‎tests/benchmark/sql/transaction-encrypted.sql

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
2+
3+
\set a random(1, 100000)
4+
\set b random(1, 100000)
5+
\set c random(1, 100000)
6+
7+
\set username hash_fnv1a(:a)
8+
9+
\set email hash_fnv1a(:b)
10+
11+
\set email_update hash_fnv1a(:c)
12+
13+
BEGIN;
14+
15+
INSERT INTO benchmark_encrypted(username, email) VALUES (:username, :email);
16+
17+
SELECT username FROM benchmark_encrypted WHERE email = :email;
18+
19+
UPDATE benchmark_encrypted SET email = :email_update WHERE username = :username;
20+
21+
SELECT username FROM benchmark_encrypted WHERE email = :email_update;
22+
23+
END;
24+

‎tests/benchmark/sql/transaction-plaintext.sql

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
\set a random(1, 100000)
2+
\set b random(1, 100000)
3+
\set c random(1, 100000)
4+
5+
\set username hash_fnv1a(:a)
6+
7+
\set email hash_fnv1a(:b)
8+
9+
\set email_update hash_fnv1a(:c)
10+
11+
12+
BEGIN;
13+
14+
INSERT INTO benchmark_plaintext(username, email) VALUES (:username, :email);
15+
16+
SELECT username FROM benchmark_plaintext WHERE email = :email;
17+
18+
UPDATE benchmark_plaintext SET email = :email_update WHERE username = :username;
19+
20+
SELECT username FROM benchmark_plaintext WHERE email = :email_update;
21+
22+
END;
23+

‎tests/benchmark/tasks/benchmark_service.sh

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env bash
2+
#MISE description="Run pgbench against a single target"
3+
#USAGE flag "--host <host>" default="host.docker.internal" help="Host of target service"
4+
#USAGE flag "--port <port>" help="Port of target service"
5+
#USAGE flag "--time <time>" help="Time for each run"
6+
#USAGE flag "--target <target>" help="Target service" {
7+
#USAGE choices "postgres" "proxy" "pgbouncer"
8+
#USAGE }
9+
#USAGE flag "--protocol <protocol>" default="simple" help="Procol to use" {
10+
#USAGE choices "simple" "extended" "prepared"
11+
#USAGE }
12+
#USAGE flag "--transaction <transaction>" default="default" help="Procol to use" {
13+
#USAGE choices "default" "plaintext" "encrypted"
14+
#USAGE }
15+
#!/bin/bash
16+
17+
set -e
18+
19+
# clients_array=(10)
20+
clients_array=(5 10 50 75 100 125)
21+
22+
23+
# output="results/$usage_target-$(date +"%Y%m%d%H%M").csv"
24+
output="results/$usage_target-$usage_protocol-$usage_transaction.csv"
25+
26+
# CSV header
27+
echo "clients,latency,init_conn_time,tps" > $output
28+
29+
for clients in "${clients_array[@]}" ; do
30+
echo "Benchmark {clients: $clients, target: $usage_target, protocol: $usage_protocol, transaction: $usage_transaction}"
31+
mise run pgbench --host=$usage_host --port=$usage_port --transaction=$usage_transaction --protocol=$usage_protocol --clients $clients --time $usage_time --output $output
32+
sleep 2
33+
done
34+
35+

0 commit comments

Comments
 (0)