Skip to content

Commit 442b984

Browse files
jgouxjgoux
authored
Merge pull request #11 from supabase-community/chore/fly-deployment
chore - fly deployment
2 parents 720f055 + b17699b commit 442b984

File tree

16 files changed

+478
-24
lines changed

16 files changed

+478
-24
lines changed

‎apps/certbot-service/.env.example

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
CERTBOT_DOMAIN=db.postgres.new
2+
CERTBOT_EMAIL="<your-email>"
3+
CLOUDFLARE_API_TOKEN="<cloudflare-api-token>"
4+
AWS_ACCESS_KEY_ID=minioadmin
5+
AWS_ENDPOINT_URL_S3=http://minio:9000
6+
AWS_REGION=us-east-1
7+
AWS_SECRET_ACCESS_KEY=minioadmin
8+
BUCKET_NAME=test
9+
S3FS_MOUNT=/mnt/s3

‎apps/certbot-service/Dockerfile

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# syntax = docker/dockerfile:1
2+
3+
# Adjust CERTBOT_VERSION as desired
4+
ARG CERTBOT_VERSION=2.11.0
5+
FROM certbot/dns-cloudflare:v${CERTBOT_VERSION} as base
6+
7+
WORKDIR /app
8+
9+
# Build S3FS
10+
FROM base as build-s3fs
11+
12+
# Install dependencies
13+
RUN apk add --no-cache \
14+
git \
15+
build-base \
16+
automake \
17+
autoconf \
18+
libxml2-dev \
19+
fuse-dev \
20+
curl-dev
21+
22+
RUN git clone https://github.com/s3fs-fuse/s3fs-fuse.git --branch v1.94 && \
23+
cd s3fs-fuse && \
24+
./autogen.sh && \
25+
./configure && \
26+
make && \
27+
make install
28+
29+
# Final stage
30+
FROM base
31+
32+
# Install dependencies
33+
RUN apk add --no-cache \
34+
bash \
35+
curl \
36+
fuse \
37+
libxml2
38+
39+
COPY --from=build-s3fs /usr/local/bin/s3fs /usr/local/bin/s3fs
40+
COPY certbot.sh deploy-hook.sh entrypoint.sh /app/
41+
42+
RUN chmod +x certbot.sh
43+
RUN chmod +x deploy-hook.sh
44+
45+
ENTRYPOINT [ "./entrypoint.sh" ]
46+
47+
CMD [ "./certbot.sh" ]

‎apps/certbot-service/README.md

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
# Certbot
2+
3+
This service is responsible for managing the certificates for the PGLite instances.
4+
5+
It uses `fly machine run --schedule weekly` to wake up the service every week to renew the certificates if needed. Let's Encrypt certificates are valid for 90 days.
6+
7+
## Testing certbot-service locally
8+
9+
Copy `.env.example` to `.env` and set the missing environment variables.
10+
11+
Start minio to emulate the S3 service:
12+
13+
```shell
14+
docker compose up -d minio
15+
```
16+
17+
Initialize the bucket:
18+
19+
```shell
20+
docker compose up minio-init
21+
```
22+
23+
Build and run the certbot service:
24+
25+
```shell
26+
docker compose up --build certbot-service
27+
```
28+
29+
The certificates will be generated in `/mnt/s3/tls`.
30+
31+
## Deploying to fly.io
32+
33+
1. Create a new app if it doesn't exist
34+
35+
```shell
36+
flyctl apps create postgres-new-certbot
37+
```
38+
39+
2. Build and deploy the Docker image to fly.io image registry
40+
41+
```shell
42+
flyctl deploy --build-only --push -a postgres-new-certbot --image-label
43+
latest
44+
```
45+
46+
1. Set the appropriate environment variables and secrets for the app "postgres-new-certbot" (see `.env.example`) in fly.io UI.
47+
48+
2. Setup [cron-manager](https://github.com/fly-apps/cron-manager?tab=readme-ov-file#getting-started) to run the certbot service every 2 weeks with the following `schedules.json`:
49+
50+
```json
51+
[
52+
{
53+
"name": "postgres-new-certbot",
54+
"app_name": "postgres-new-certbot",
55+
"schedule": "0 0 1,15 * *",
56+
"region": "ord",
57+
"command": "./certbot.sh",
58+
"command_timeout": 120,
59+
"enabled": true,
60+
"config": {
61+
"metadata": {
62+
"fly_process_group": "cron"
63+
},
64+
"auto_destroy": true,
65+
"disable_machine_autostart": true,
66+
"guest": {
67+
"cpu_kind": "shared",
68+
"cpus": 1,
69+
"memory_mb": 256
70+
},
71+
"image": "registry.fly.io/postgres-new-certbot:latest",
72+
"restart": {
73+
"max_retries": 1,
74+
"policy": "no"
75+
}
76+
}
77+
}
78+
]
79+
```
80+
81+
5. Test running the job by SSHing into cron-manager console
82+
83+
Run this command in the cron-manager root folder:
84+
85+
```shell
86+
flyctl ssh console
87+
```
88+
89+
Once in the cron-manager instance:
90+
91+
```shell
92+
cm jobs trigger 1
93+
```
94+
95+
If you open the "postgres-new-certbot" live logs in fly.io UI, you should see the job being executed.
96+
97+
6. You can check if the certificates are present in the Tigris bucket
98+
99+
Run this command in the apps/db-instance folder:
100+
101+
```shell
102+
flyctl storage dashboard
103+
```
104+
105+
It should open the Tigris dashboard where you can check the bucket's content. The certificates should be created under `/tls`.

‎apps/certbot-service/certbot.sh

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
CONFIG_DIR="${S3FS_MOUNT}/tls/letsencrypt"
6+
CERT_PATH="${CONFIG_DIR}/live/${CERTBOT_DOMAIN}/fullchain.pem"
7+
CLOUD_FLARE_INI="/app/cloudflare.ini"
8+
DEPLOY_HOOK="/app/deploy-hook.sh"
9+
10+
renew_certificate() {
11+
echo "Certificates exist. Renewing..."
12+
certbot renew --non-interactive \
13+
--cert-name "${CERTBOT_DOMAIN}" \
14+
--dns-cloudflare \
15+
--dns-cloudflare-credentials "${CLOUD_FLARE_INI}" \
16+
--deploy-hook "${DEPLOY_HOOK}" \
17+
--config-dir "${CONFIG_DIR}"
18+
}
19+
20+
create_certificate() {
21+
echo "Certificates do not exist. Creating..."
22+
certbot certonly --non-interactive \
23+
--agree-tos \
24+
--cert-name "${CERTBOT_DOMAIN}" \
25+
--email "${CERTBOT_EMAIL}" \
26+
--dns-cloudflare \
27+
--dns-cloudflare-credentials "${CLOUD_FLARE_INI}" \
28+
--dns-cloudflare-propagation-seconds 60 \
29+
-d "*.${CERTBOT_DOMAIN}" \
30+
--deploy-hook "${DEPLOY_HOOK}" \
31+
--config-dir "${CONFIG_DIR}"
32+
}
33+
34+
main() {
35+
if [[ -f "${CERT_PATH}" ]]; then
36+
renew_certificate
37+
else
38+
create_certificate
39+
fi
40+
}
41+
42+
main "$@"

‎apps/certbot-service/deploy-hook.sh

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
SOURCE_DIR="$S3FS_MOUNT/tls/letsencrypt/live/$CERTBOT_DOMAIN"
6+
TARGET_DIR="$S3FS_MOUNT/tls"
7+
8+
# Ensure the target directory exists
9+
mkdir -p $TARGET_DIR
10+
11+
# Copy the key and cert to the target directory
12+
cp -f $SOURCE_DIR/privkey.pem $TARGET_DIR/key.pem
13+
cp -f $SOURCE_DIR/fullchain.pem $TARGET_DIR/cert.pem

‎apps/certbot-service/docker-compose.yml

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
services:
2+
certbot-service:
3+
image: certbot-service
4+
build:
5+
context: .
6+
env_file:
7+
- .env
8+
devices:
9+
- /dev/fuse
10+
cap_add:
11+
- SYS_ADMIN
12+
depends_on:
13+
minio:
14+
condition: service_healthy
15+
minio:
16+
image: minio/minio
17+
environment:
18+
MINIO_ROOT_USER: minioadmin
19+
MINIO_ROOT_PASSWORD: minioadmin
20+
ports:
21+
- 9000:9000
22+
- 9001:9001
23+
command: server /data --console-address :9001
24+
healthcheck:
25+
test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1
26+
interval: 5s
27+
timeout: 5s
28+
retries: 1
29+
minio-init:
30+
image: minio/mc
31+
entrypoint: >
32+
/bin/sh -c "
33+
mc alias set local http://minio:9000 minioadmin minioadmin;
34+
(mc ls local/test || mc mb local/test);
35+
"
36+
depends_on:
37+
minio:
38+
condition: service_healthy

‎apps/certbot-service/entrypoint.sh

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
# s3fs ################################
6+
cleanup() {
7+
echo "Unmounting s3fs..."
8+
fusermount -u $S3FS_MOUNT
9+
exit 0
10+
}
11+
12+
forward_signal() {
13+
kill -$1 "$MAIN_PID"
14+
}
15+
16+
trap 'forward_signal SIGINT' SIGINT
17+
trap 'forward_signal SIGTERM' SIGTERM
18+
trap 'cleanup' EXIT
19+
20+
# Create the mount point directory
21+
mkdir -p $S3FS_MOUNT
22+
23+
# Mount the S3 bucket
24+
s3fs $BUCKET_NAME $S3FS_MOUNT -o use_path_request_style -o url=$AWS_ENDPOINT_URL_S3 -o endpoint=$AWS_REGION
25+
26+
# Check if the mount was successful
27+
if mountpoint -q $S3FS_MOUNT; then
28+
echo "S3 bucket mounted successfully at $S3FS_MOUNT"
29+
else
30+
echo "Failed to mount S3 bucket"
31+
exit 1
32+
fi
33+
34+
# cloudflare.ini ######################
35+
echo "dns_cloudflare_api_token = $CLOUDFLARE_API_TOKEN" > /app/cloudflare.ini
36+
chmod 600 /app/cloudflare.ini
37+
38+
# Execute the original command
39+
"$@" &
40+
MAIN_PID=$!
41+
42+
wait $MAIN_PID

‎apps/certbot-service/fly.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
app = 'postgres-new-certbot'
2+
primary_region = 'yyz'

‎apps/db-service/.env.example

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,5 @@ AWS_ACCESS_KEY_ID=minioadmin
66
AWS_SECRET_ACCESS_KEY=minioadmin
77
# Only if you need to test against a bucket hosted in AWS S3
88
# AWS_SESSION_TOKEN=<aws-session-token>
9-
AWS_ENDPOINT_URL=http://minio:9000
10-
DOMAIN=*.db.example.com
9+
AWS_ENDPOINT_URL_S3=http://minio:9000
10+
WILDCARD_DOMAIN=db.example.com

‎apps/db-service/README.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,9 @@ To stop all Docker containers, run:
8383
```shell
8484
docker compose down
8585
```
86+
87+
## Deployment
88+
89+
The db-service is deployed on Fly.io.
90+
91+
A Tigris bucket is used to store the DB tarballs and the TLS certificates.

‎apps/db-service/entrypoint.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ set -o pipefail
55

66
cleanup() {
77
echo "Unmounting s3fs..."
8-
fusermount -u $S3FS_MOUNT
8+
umount $S3FS_MOUNT
99
exit 0
1010
}
1111

@@ -21,7 +21,7 @@ trap 'cleanup' EXIT
2121
mkdir -p $S3FS_MOUNT
2222

2323
# Mount the S3 bucket
24-
s3fs $BUCKET_NAME $S3FS_MOUNT -o use_path_request_style -o url=$AWS_ENDPOINT_URL -o endpoint=$AWS_REGION
24+
s3fs $BUCKET_NAME $S3FS_MOUNT -o use_path_request_style -o url=$AWS_ENDPOINT_URL_S3 -o endpoint=$AWS_REGION
2525

2626
# Check if the mount was successful
2727
if mountpoint -q $S3FS_MOUNT; then

‎apps/db-service/fly.toml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
app = 'postgres-new-dev'
22
primary_region = 'yyz'
33

4-
[[service]]
4+
[[services]]
55
internal_port = 5432
66
protocol = "tcp"
77
auto_stop_machines = true
88
auto_start_machines = true
99
min_machines_running = 0
1010

11-
[[service.ports]]
11+
[[services.ports]]
1212
port = 5432
1313

14-
[[service.concurrency]]
14+
[services.concurrency]
1515
type = "connections"
1616
hard_limit = 25
1717
soft_limit = 20
@@ -22,5 +22,5 @@ cpu_kind = 'shared'
2222
cpus = 1
2323

2424
[mounts]
25-
source = "postgres-new-data"
25+
source = "postgres_new_data"
2626
destination = "/mnt/data"

‎apps/db-service/scripts/generate-certs.sh

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ set -e
44
set -o pipefail
55

66
S3FS_MOUNT=${S3FS_MOUNT:=.}
7-
DOMAIN="${DOMAIN:=*.db.example.com}"
7+
DOMAIN="*.${WILDCARD_DOMAIN:=db.example.com}"
88
CERT_DIR="$S3FS_MOUNT/tls"
99

1010
mkdir -p $CERT_DIR
@@ -17,3 +17,9 @@ openssl genpkey -algorithm RSA -out key.pem
1717
openssl req -new -key key.pem -out csr.pem -subj "/CN=$DOMAIN"
1818

1919
openssl x509 -req -in csr.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out cert.pem -days 365
20+
21+
# create fullchain by concatenating the server certificate and the CA certificate
22+
cat cert.pem ca-cert.pem > fullchain.pem
23+
24+
# replace cert with the fullchain
25+
mv fullchain.pem cert.pem

0 commit comments

Comments
 (0)