diff --git a/Makefile b/Makefile index f409fd74f5e0..9c2b67c80130 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,8 @@ build: docker system prune -f --filter "label=com.docker.compose.project=warehouse" serve: .state/docker-build + $(MAKE) .state/db-populated + $(MAKE) .state/search-indexed docker compose up --remove-orphans debug: .state/docker-build-base @@ -97,17 +99,30 @@ translations: .state/docker-build-base requirements/%.txt: requirements/%.in docker compose run --rm base bin/pip-compile --generate-hashes --output-file=$@ $< -initdb: .state/docker-build-base - docker compose run --rm web psql -h db -d postgres -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname ='warehouse';" - docker compose run --rm web psql -h db -d postgres -U postgres -c "DROP DATABASE IF EXISTS warehouse" - docker compose run --rm web psql -h db -d postgres -U postgres -c "CREATE DATABASE warehouse ENCODING 'UTF8'" - docker compose run --rm web psql -h db -d postgres -U postgres -c "DROP DATABASE IF EXISTS rstuf" - docker compose run --rm web psql -h db -d postgres -U postgres -c "CREATE DATABASE rstuf ENCODING 'UTF8'" - docker compose run --rm web bash -c "xz -d -f -k dev/$(DB).sql.xz --stdout | psql -h db -d warehouse -U postgres -v ON_ERROR_STOP=1 -1 -f -" - docker compose run --rm web psql -h db -d warehouse -U postgres -c "UPDATE users SET name='Ee Durbin' WHERE username='ewdurbin'" - $(MAKE) runmigrations +resetdb: .state/docker-build-base + docker compose pause web worker + docker compose up -d db + docker compose exec --user postgres db /docker-entrypoint-initdb.d/init-dbs.sh + rm -f .state/db-populated .state/db-migrated + $(MAKE) initdb + docker compose unpause web worker + +.state/search-indexed: .state/db-populated + $(MAKE) reindex + mkdir -p .state && touch .state/search-indexed + +.state/db-populated: .state/db-migrated docker compose run --rm web python -m warehouse sponsors populate-db docker compose run --rm web python -m warehouse classifiers sync + mkdir -p .state && touch .state/db-populated + +.state/db-migrated: .state/docker-build-base + docker compose up -d db + docker compose exec db /usr/local/bin/wait-for-db + $(MAKE) runmigrations + mkdir -p .state && touch .state/db-migrated + +initdb: .state/docker-build-base .state/db-populated $(MAKE) reindex runmigrations: .state/docker-build-base @@ -126,11 +141,11 @@ clean: rm -rf dev/*.sql purge: stop clean - rm -rf .state dev/.coverage* dev/.mypy_cache dev/.pip-cache dev/.pip-tools-cache dev/.pytest_cache - docker compose down -v + rm -rf .state dev/.coverage* dev/.mypy_cache dev/.pip-cache dev/.pip-tools-cache dev/.pytest_cache .state/db-populated .state/db-migrated + docker compose down -v --remove-orphans docker compose rm --force stop: docker compose stop -.PHONY: default build serve initdb shell dbshell tests dev-docs user-docs deps clean purge debug stop compile-pot runmigrations +.PHONY: default build serve resetdb initdb shell dbshell tests dev-docs user-docs deps clean purge debug stop compile-pot runmigrations diff --git a/bin/tests b/bin/tests index c762888e1499..ce9533a755ab 100755 --- a/bin/tests +++ b/bin/tests @@ -19,12 +19,12 @@ done # Test the postgres connection ATTEMPTS=0 -until [ $ATTEMPTS -eq 5 ] || pg_isready -t 10 -h $POSTGRES_HOST; do +until [ $ATTEMPTS -eq 12 ] || pg_isready -t 10 -h $POSTGRES_HOST; do >&2 echo "Postgres is unavailable, sleeping" sleep $(( ATTEMPTS++ )) done -if [ $ATTEMPTS -eq 5 ]; then +if [ $ATTEMPTS -eq 12 ]; then >&2 echo "Postgres is unavailable, exiting" exit 1 fi diff --git a/bin/wait-for-db b/bin/wait-for-db new file mode 100755 index 000000000000..3acfd030fed3 --- /dev/null +++ b/bin/wait-for-db @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +# Click requires us to ensure we have a well configured environment to run +# our click commands. So we'll set our environment to ensure our locale is +# correct. +export LC_ALL="${ENCODING:-en_US.UTF-8}" +export LANG="${ENCODING:-en_US.UTF-8}" + +echo -n 'waiting for db to be prepared.' + +QUERY="select name from users where username='ewdurbin';" + +ATTEMPTS=0 +until [ $ATTEMPTS -eq 60 ] || [ "$(psql -U postgres -d warehouse -A -t -c "$QUERY" 2>&1)" == "Ee Durbin" ]; do + >&2 >/dev/null + ATTEMPTS=$((ATTEMPTS+1)); + echo -n "." + sleep 1 +done + +if [ $ATTEMPTS -eq 60 ]; then + >&2 echo "" + echo "db failed to prepare, exiting" + exit 1 +fi + +echo "" diff --git a/dev/db/docker-entrypoint-initdb.d/init-dbs.sh b/dev/db/docker-entrypoint-initdb.d/init-dbs.sh new file mode 100755 index 000000000000..28892706e3ec --- /dev/null +++ b/dev/db/docker-entrypoint-initdb.d/init-dbs.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +psql -d "$POSTGRES_DB" -U "$POSTGRES_USER" <<-EOF + SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname ='warehouse'; + DROP DATABASE IF EXISTS warehouse; + CREATE DATABASE warehouse ENCODING 'UTF8'; + SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname ='rstuf'; + DROP DATABASE IF EXISTS rstuf; + CREATE DATABASE rstuf ENCODING 'UTF8'; +EOF + +xz -d -f -k /example.sql.xz --stdout | psql -d warehouse -U "$POSTGRES_USER" -v ON_ERROR_STOP=1 -1 -f - + +psql -d warehouse -U "$POSTGRES_USER" <<-EOF + UPDATE users SET name='Ee Durbin' WHERE username='ewdurbin' +EOF diff --git a/docker-compose.yml b/docker-compose.yml index f4891d652320..eea99478898e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,10 @@ services: test: ["CMD", "pg_isready", "-U", "postgres", "-d", "postgres"] interval: 1s start_period: 10s + volumes: + - ./dev/db/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:z + - ./dev/example.sql.xz:/example.sql.xz:z + - ./bin/wait-for-db:/usr/local/bin/wait-for-db:z stripe: image: stripe/stripe-mock:v0.162.0 @@ -117,6 +121,8 @@ services: tests: image: warehouse:docker-compose pull_policy: never + environment: + ENCODING: "C" volumes: *base_volumes depends_on: db: diff --git a/docs/dev/development/getting-started.rst b/docs/dev/development/getting-started.rst index 59d4a871f668..d6cddaf02582 100644 --- a/docs/dev/development/getting-started.rst +++ b/docs/dev/development/getting-started.rst @@ -209,23 +209,16 @@ Once ``make build`` has finished, run the command: .. code-block:: console - make initdb + make serve This command will: -* create a new Postgres database, -* install example data to the Postgres database, +* ensure the db is prepared, * run migrations, * load some example data from `Test PyPI`_, and * index all the data for the search database. +* start up the containers needed to run Warehouse -Once the ``make initdb`` command has finished, you are ready to continue: - -.. code-block:: console - - make serve - -This command starts the containers that run Warehouse on your local machine. After the initial build process, you will only need this command each time you want to startup Warehouse locally. @@ -250,6 +243,16 @@ or that the ``static`` container has finished compiling the static assets: or maybe something else. +Resetting the development database +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: console + + make resetdb + +This command will fully reset the development database. + + Viewing Warehouse in a browser ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^