move parts of our db initialization into db service (#15549)

* move parts of our db initialization into db service

* simplify dev experience further by complicating Makefile

* remove orphans on purge

* sush encoding warning on make tests

* give more time in bin/tests for db-initialization

* pause web and workers during resetdb

Author: ewdurbin

Makefile

@@ -55,6 +55,8 @@ build:
 	docker system prune -f --filter "label=com.docker.compose.project=warehouse"
 
 serve: .state/docker-build
+	$(MAKE) .state/db-populated
+	$(MAKE) .state/search-indexed
 	docker compose up --remove-orphans
 
 debug: .state/docker-build-base
@@ -97,17 +99,30 @@ translations: .state/docker-build-base
 requirements/%.txt: requirements/%.in
 	docker compose run --rm base bin/pip-compile --generate-hashes --output-file=$@ $<
 
-initdb: .state/docker-build-base
-	docker compose run --rm web psql -h db -d postgres -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname ='warehouse';"
-	docker compose run --rm web psql -h db -d postgres -U postgres -c "DROP DATABASE IF EXISTS warehouse"
-	docker compose run --rm web psql -h db -d postgres -U postgres -c "CREATE DATABASE warehouse ENCODING 'UTF8'"
-	docker compose run --rm web psql -h db -d postgres -U postgres -c "DROP DATABASE IF EXISTS rstuf"
-	docker compose run --rm web psql -h db -d postgres -U postgres -c "CREATE DATABASE rstuf ENCODING 'UTF8'"
-	docker compose run --rm web bash -c "xz -d -f -k dev/$(DB).sql.xz --stdout | psql -h db -d warehouse -U postgres -v ON_ERROR_STOP=1 -1 -f -"
-	docker compose run --rm web psql -h db -d warehouse -U postgres -c "UPDATE users SET name='Ee Durbin' WHERE username='ewdurbin'"
-	$(MAKE) runmigrations
+resetdb: .state/docker-build-base
+	docker compose pause web worker
+	docker compose up -d db
+	docker compose exec --user postgres db /docker-entrypoint-initdb.d/init-dbs.sh
+	rm -f .state/db-populated .state/db-migrated
+	$(MAKE) initdb
+	docker compose unpause web worker
+
+.state/search-indexed: .state/db-populated
+	$(MAKE) reindex
+	mkdir -p .state && touch .state/search-indexed
+
+.state/db-populated: .state/db-migrated
 	docker compose run --rm web python -m warehouse sponsors populate-db
 	docker compose run --rm web python -m warehouse classifiers sync
+	mkdir -p .state && touch .state/db-populated
+
+.state/db-migrated: .state/docker-build-base
+	docker compose up -d db
+	docker compose exec db /usr/local/bin/wait-for-db
+	$(MAKE) runmigrations
+	mkdir -p .state && touch .state/db-migrated
+
+initdb: .state/docker-build-base .state/db-populated
 	$(MAKE) reindex
 
 runmigrations: .state/docker-build-base
@@ -126,11 +141,11 @@ clean:
 	rm -rf dev/*.sql
 
 purge: stop clean
-	rm -rf .state dev/.coverage* dev/.mypy_cache dev/.pip-cache dev/.pip-tools-cache dev/.pytest_cache
-	docker compose down -v
+	rm -rf .state dev/.coverage* dev/.mypy_cache dev/.pip-cache dev/.pip-tools-cache dev/.pytest_cache .state/db-populated .state/db-migrated
+	docker compose down -v --remove-orphans
 	docker compose rm --force
 
 stop:
 	docker compose stop
 
-.PHONY: default build serve initdb shell dbshell tests dev-docs user-docs deps clean purge debug stop compile-pot runmigrations
+.PHONY: default build serve resetdb initdb shell dbshell tests dev-docs user-docs deps clean purge debug stop compile-pot runmigrations

bin/tests

@@ -19,12 +19,12 @@ done
 
 # Test the postgres connection
 ATTEMPTS=0
-until [ $ATTEMPTS -eq 5 ] || pg_isready -t 10 -h $POSTGRES_HOST; do
+until [ $ATTEMPTS -eq 12 ] || pg_isready -t 10 -h $POSTGRES_HOST; do
   >&2 echo "Postgres is unavailable, sleeping"
   sleep $(( ATTEMPTS++ ))
 done
 
-if [ $ATTEMPTS -eq 5 ]; then
+if [ $ATTEMPTS -eq 12 ]; then
   >&2 echo "Postgres is unavailable, exiting"
   exit 1
 fi

bin/wait-for-db

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+# Click requires us to ensure we have a well configured environment to run
+# our click commands. So we'll set our environment to ensure our locale is
+# correct.
+export LC_ALL="${ENCODING:-en_US.UTF-8}"
+export LANG="${ENCODING:-en_US.UTF-8}"
+
+echo -n 'waiting for db to be prepared.'
+
+QUERY="select name from users where username='ewdurbin';"
+
+ATTEMPTS=0
+until [ $ATTEMPTS -eq 60 ] || [ "$(psql -U postgres -d warehouse -A -t -c "$QUERY" 2>&1)" == "Ee Durbin" ]; do
+  >&2 >/dev/null
+  ATTEMPTS=$((ATTEMPTS+1));
+  echo -n "."
+  sleep 1
+done
+
+if [ $ATTEMPTS -eq 60 ]; then
+  >&2 echo ""
+  echo "db failed to prepare, exiting"
+  exit 1
+fi
+
+echo ""

dev/db/docker-entrypoint-initdb.d/init-dbs.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+psql -d "$POSTGRES_DB" -U "$POSTGRES_USER" <<-EOF
+    SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname ='warehouse';
+    DROP DATABASE IF EXISTS warehouse;
+    CREATE DATABASE warehouse ENCODING 'UTF8';
+    SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname ='rstuf';
+    DROP DATABASE IF EXISTS rstuf;
+    CREATE DATABASE rstuf ENCODING 'UTF8';
+EOF
+
+xz -d -f -k /example.sql.xz --stdout | psql -d warehouse -U "$POSTGRES_USER" -v ON_ERROR_STOP=1 -1 -f -
+
+psql -d warehouse -U "$POSTGRES_USER" <<-EOF
+    UPDATE users SET name='Ee Durbin' WHERE username='ewdurbin'
+EOF

docker-compose.yml

@@ -21,6 +21,10 @@ services:
       test: ["CMD", "pg_isready", "-U", "postgres", "-d", "postgres"]
       interval: 1s
       start_period: 10s
+    volumes:
+      - ./dev/db/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:z
+      - ./dev/example.sql.xz:/example.sql.xz:z
+      - ./bin/wait-for-db:/usr/local/bin/wait-for-db:z
 
   stripe:
     image: stripe/stripe-mock:v0.162.0
@@ -117,6 +121,8 @@ services:
   tests:
     image: warehouse:docker-compose
     pull_policy: never
+    environment:
+      ENCODING: "C"
     volumes: *base_volumes
     depends_on:
       db:

docs/dev/development/getting-started.rst

@@ -209,23 +209,16 @@ Once ``make build`` has finished,  run the command:
 
 .. code-block:: console
 
-    make initdb
+    make serve
 
 This command will:
 
-* create a new Postgres database,
-* install example data to the Postgres database,
+* ensure the db is prepared,
 * run migrations,
 * load some example data from `Test PyPI`_, and
 * index all the data for the search database.
+* start up the containers needed to run Warehouse
 
-Once the ``make initdb`` command has finished, you are ready to continue:
-
-.. code-block:: console
-
-    make serve
-
-This command starts the containers that run Warehouse on your local machine.
 After the initial build process, you will only need this command each time you
 want to startup Warehouse locally.
 
@@ -250,6 +243,16 @@ or that the ``static`` container has finished compiling the static assets:
 or maybe something else.
 
 
+Resetting the development database
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: console
+
+    make resetdb
+
+This command will fully reset the development database.
+
+
 Viewing Warehouse in a browser
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^