PEP 458: Add RSTUF services in the Warehouse Infra

This commit adds the RSTUF services to the Warehouse infrastructure
for development and sets the minimum required to start RSTUF services.

It adds the RSTUF API, which is used later to integrate into Warehouse
and RSTUF Worker, which is responsible for computing the TUF metadata.

The RSTUF requires the Postgres and Redis.
Postgres stores the rstuf database used for TUF metadata computing.
Redis stores the task message queue between RSTUF API and Worker, task
backend result, and live settings between RSTUF services.

RSTUF shares the same Postgres and Redis in development environment
but has a specific setup to use its own Postgres database and Redis
database ID.

Postgresql URI
`RSTUF_SQL_SERVER=postgresql://postgres@db:5432/rstuf`

Redis DB Broker and Result is id 1
`RSTUF_BROKER_SERVER=redis://redis/1`
`RSTUF_REDIS_SERVER_DB_RESULT=1`

Redis DB for TUF repository settings is 2
`RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2`

This commit also includes TUF database creation in the Makefile
during the `make initdb`.

Signed-off-by: Kairo de Araujo <[email protected]>

Author: kairoaraujo

Makefile

@@ -101,6 +101,8 @@ initdb: .state/docker-build-base
 	docker compose run --rm web psql -h db -d postgres -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname ='warehouse';"
 	docker compose run --rm web psql -h db -d postgres -U postgres -c "DROP DATABASE IF EXISTS warehouse"
 	docker compose run --rm web psql -h db -d postgres -U postgres -c "CREATE DATABASE warehouse ENCODING 'UTF8'"
+	docker compose run --rm web psql -h db -d postgres -U postgres -c "DROP DATABASE IF EXISTS rstuf"
+	docker compose run --rm web psql -h db -d postgres -U postgres -c "CREATE DATABASE rstuf ENCODING 'UTF8'"
 	docker compose run --rm web bash -c "xz -d -f -k dev/$(DB).sql.xz --stdout | psql -h db -d warehouse -U postgres -v ON_ERROR_STOP=1 -1 -f -"
 	docker compose run --rm web psql -h db -d warehouse -U postgres -c "UPDATE users SET name='Ee Durbin' WHERE username='ewdurbin'"
 	$(MAKE) runmigrations

dev/rstuf/keys/online/192e4994afd0bc3168eac3a043c6ce6f207d8d0784eee1355a1af20e83364a5e

@@ -0,0 +1 @@
+0f6f30eb3328a1d580313b4af01ac79a@@@@100000@@@@fcc8a33399d89b2e57ab5c8f547d175bd676fd7fd24b5c0582d166911a551558@@@@3428d03305625b390d5f5fb1a3ee4bfc@@@@d80049a3b85e0aff77ddd5dc10369c00eda2e0bdb22b05d0e30a733400d3592ea2028f0122f00f4dad60a3974e89047b7df344db56159956f8b43bc64f81f9bcea72ff3d92cc57218519768b522f61a56706f84c99fb89e063ad286c7fd6b19dd43ae11f6b3f73172d149cca85ac672ba0b574f1b9a0fd603afe1a0c91fa57bb0830644ecdc01ef8161b515ee4ffd7170c5bb4b221c04a05392a80b2b0d597bea737011e6a9fbb1c0e259f8ee02e671c2a0e096a54b050d34e016fbe23e9617997d2a73d5bfc78be0351b983f960e9338c05588a7ccf040116531ed548d8030dea1c1da4da3693330317b0b5cc133940476b806715f8704e9319fdaf7669f3b590335296ebe74de97465a6f1c9c6484ab162a2d9d5ebaad8232e1818484bb2b301bf3ebfb4d89c0cc779b04338f2e1ed15858d7dcbca743eef632d865a019760a101b82835cca08201b3f47fcac54fc7

docker-compose.yml

@@ -8,15 +8,16 @@ volumes:
   policies:
   vault:
   caches:
+  rstuf-worker-data:
+  rstuf-metadata:
 
 services:
   vault:
     # NOTE: pinned for consistency with whats available in our deployment
     image: vault:1.12.3
     restart: on-failure
     entrypoint: /bin/sh
-    command: /etc/vault/entry.sh
-    stop_signal: SIGINT
+    command: ["/etc/vault/entry.sh"]
     environment:
       VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200
       VAULT_DEV_ROOT_TOKEN_ID: "an insecure vault access token"
@@ -50,7 +51,6 @@ services:
 
   localstack:
     image: localstack/localstack:1.4
-    stop_signal: SIGKILL
     environment:
       SERVICES: "sqs"
       HOSTNAME: "localstack"
@@ -112,6 +112,8 @@ services:
       # Included to support linters during development
       - ./gunicorn-prod.conf.py:/opt/warehouse/src/gunicorn-prod.conf.py:z
       - ./gunicorn-uploads.conf.py:/opt/warehouse/src/gunicorn-uploads.conf.py:z
+      - rstuf-metadata:/var/opt/warehouse/metadata
+      - ./dev/tufkeys:/opt/warehouse/src/dev/tufkeys:z
 
   web:
     image: warehouse:docker-compose
@@ -136,12 +138,12 @@ services:
     pull_policy: never
     working_dir: /var/opt/warehouse
     command: python -m http.server 9001
-    stop_signal: SIGINT
     volumes:
       - packages:/var/opt/warehouse/packages
       - packages-archive:/var/opt/warehouse/packages-archive
       - sponsorlogos:/var/opt/warehouse/sponsorlogos
       - simple:/var/opt/warehouse/simple
+      - rstuf-metadata:/var/opt/warehouse/metadata
     ports:
       - "9001:9001"
 
@@ -160,6 +162,40 @@ services:
       ARCHIVE_FILES_BACKEND: "warehouse.packaging.services.LocalArchiveFileStorage path=/var/opt/warehouse/packages-archive/ url=http://files:9001/packages-archive/{path}"
       SIMPLE_BACKEND: "warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://files:9001/simple/{path}"
 
+  rstuf-api:
+    image: ghcr.io/repository-service-tuf/repository-service-tuf-api:latest
+    ports:
+      - 8001:80
+    environment:
+      - RSTUF_BROKER_SERVER=redis://redis/1
+      - RSTUF_REDIS_SERVER=redis://redis
+      - RSTUF_REDIS_SERVER_DB_RESULT=1
+      - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2
+
+  rstuf-worker:
+    image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:latest
+    volumes:
+      - rstuf-metadata:/var/opt/repository-service-tuf/storage
+      - ./dev/rstuf/keys/online:/var/opt/repository-service-tuf/keystorage
+    environment:
+      - RSTUF_STORAGE_BACKEND=LocalStorage
+      - RSTUF_LOCAL_STORAGE_BACKEND_PATH=/var/opt/repository-service-tuf/storage
+      - RSTUF_KEYVAULT_BACKEND=LocalKeyVault
+      - RSTUF_LOCAL_KEYVAULT_PATH=/var/opt/repository-service-tuf/keystorage
+      - RSTUF_LOCAL_KEYVAULT_KEYS="192e4994afd0bc3168eac3a043c6ce6f207d8d0784eee1355a1af20e83364a5e,secret"
+      - RSTUF_BROKER_SERVER=redis://redis/1
+      - RSTUF_REDIS_SERVER=redis://redis
+      - RSTUF_REDIS_SERVER_DB_RESULT=1
+      - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2
+      - RSTUF_SQL_SERVER=postgresql://postgres@db:5432/rstuf
+    healthcheck:
+      test: "exit 0"
+    restart: always
+    tty: true
+    depends_on:
+      db:
+        condition: service_healthy
+
   static:
     build:
       context: .
@@ -169,8 +205,7 @@ services:
       - "35729:35729" # LiveReload
     environment:
       NODE_ENV: "development"
-    command: ["npm", "run", "watch"]
-    stop_signal: SIGKILL
+    command: bash -c "npm run watch"
     volumes:
       - ./warehouse:/opt/warehouse/src/warehouse:z
       - ./webpack.config.js:/opt/warehouse/src/webpack.config.js:z
@@ -190,7 +225,6 @@ services:
     image: warehouse:docker-compose
     pull_policy: never
     command: python /opt/warehouse/dev/notdatadog.py 0.0.0.0:8125
-    stop_signal: SIGINT
     environment:
       METRICS_OUTPUT: "false"
     ports:
@@ -218,7 +252,6 @@ services:
     image: warehouse:docker-compose-docs
     pull_policy: never
     command: mkdocs serve -a 0.0.0.0:8000 -f docs/mkdocs-user-docs.yml
-    stop_signal: SIGINT
     volumes:
       - ./bin:/opt/warehouse/src/bin:z
       - ./docs/mkdocs-user-docs.yml:/opt/warehouse/src/docs/mkdocs-user-docs.yml:z
@@ -230,7 +263,6 @@ services:
     image: warehouse:docker-compose-docs
     pull_policy: never
     command: mkdocs serve -a 0.0.0.0:8000 -f docs/mkdocs-blog.yml
-    stop_signal: SIGINT
     volumes:
       # we mount git because rss, thanks feed nerds
       - ./.git:/opt/warehouse/src/.git:ro