diff --git a/.gitignore b/.gitignore index 8a4be15f614a..f4e885597c79 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ docker-compose.override.yaml node_modules/ +dev/tufkeys/ dev/example.sql dev/prod.sql dev/prod.sql.xz @@ -29,6 +30,7 @@ warehouse/.commit warehouse/static/components warehouse/static/dist warehouse/admin/static/dist +warehouse/tuf/dist tags *.sw* diff --git a/Makefile b/Makefile index 3b92f72c3cde..acecc2d09da0 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ DB := example IPYTHON := no +WAREHOUSE_CLI := docker-compose run --rm web python -m warehouse # set environment variable WAREHOUSE_IPYTHON_SHELL=1 if IPython # needed in development environment @@ -94,6 +95,19 @@ initdb: .state/docker-build-web docker-compose run --rm web python -m warehouse classifiers sync $(MAKE) reindex +inittuf: + $(WAREHOUSE_CLI) tuf dev keypair --name root --path /opt/warehouse/src/dev/tufkeys/root + $(WAREHOUSE_CLI) tuf dev keypair --name snapshot --path /opt/warehouse/src/dev/tufkeys/snapshot + $(WAREHOUSE_CLI) tuf dev keypair --name targets --path /opt/warehouse/src/dev/tufkeys/targets1 + $(WAREHOUSE_CLI) tuf dev keypair --name targets --path /opt/warehouse/src/dev/tufkeys/targets2 + $(WAREHOUSE_CLI) tuf dev keypair --name timestamp --path /opt/warehouse/src/dev/tufkeys/timestamp + $(WAREHOUSE_CLI) tuf dev keypair --name bins --path /opt/warehouse/src/dev/tufkeys/bins + $(WAREHOUSE_CLI) tuf dev keypair --name bin-n --path /opt/warehouse/src/dev/tufkeys/bin-n + $(WAREHOUSE_CLI) tuf dev init-repo + $(WAREHOUSE_CLI) tuf dev init-delegations + $(WAREHOUSE_CLI) tuf dev add-all-packages + $(WAREHOUSE_CLI) tuf dev add-all-indexes + reindex: .state/docker-build-web docker-compose run --rm web python -m warehouse search reindex @@ -102,6 +116,7 @@ shell: .state/docker-build-web clean: rm -rf dev/*.sql + rm -rf dev/tufkeys purge: stop clean rm -rf .state diff --git a/Procfile b/Procfile index c4ccc5b214d7..a8149ff1c783 100644 --- a/Procfile +++ b/Procfile @@ -4,3 +4,4 @@ web-uploads: bin/start-web ddtrace-run python -m gunicorn.app.wsgiapp -c gunicor worker: bin/start-worker celery -A warehouse worker -Q default -l info --max-tasks-per-child 32 worker-malware: bin/start-worker celery -A warehouse worker -Q malware -l info --max-tasks-per-child 32 worker-beat: bin/start-worker celery -A warehouse beat -S redbeat.RedBeatScheduler -l info +worker-tuf: bin/start-worker celery -A warehouse worker -Q tuf -l info --max-tasks-per-child 32 diff --git a/dev/environment b/dev/environment index c48ad202e5b0..99756dfd6277 100644 --- a/dev/environment +++ b/dev/environment @@ -57,3 +57,14 @@ TWOFACTORREQUIREMENT_ENABLED=true TWOFACTORMANDATE_AVAILABLE=true TWOFACTORMANDATE_ENABLED=true OIDC_ENABLED=true + +TUF_URL="http://{request.domain}:9001/metadata" +TUF_KEY_BACKEND=warehouse.tuf.services.LocalKeyService key.path=/opt/warehouse/src/dev +TUF_STORAGE_BACKEND=warehouse.tuf.services.LocalStorageService +TUF_REPOSITORY_BACKEND=warehouse.tuf.services.RepositoryService repo.path=/var/opt/warehouse/tuf_metadata +TUF_ROOT_SECRET="an insecure private key password" +TUF_SNAPSHOT_SECRET="an insecure private key password" +TUF_TARGETS_SECRET="an insecure private key password" +TUF_TIMESTAMP_SECRET="an insecure private key password" +TUF_BINS_SECRET="an insecure private key password" +TUF_BIN_N_SECRET="an insecure private key password" diff --git a/docker-compose.yml b/docker-compose.yml index 4d7095d3bee9..2a2c91b7b99b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,7 @@ volumes: simple: packages: sponsorlogos: + tuf_metadata: vault: services: @@ -88,6 +89,7 @@ services: # The :z option fixes permission issues with SELinux by setting a # permissive security context. - ./dev:/opt/warehouse/src/dev:z + - ./dev/tufkeys:/opt/warehouse/src/dev/tufkeys:z - ./docs:/opt/warehouse/src/docs:z - ./warehouse:/opt/warehouse/src/warehouse:z - ./tests:/opt/warehouse/src/tests:z @@ -96,6 +98,7 @@ services: - packages:/var/opt/warehouse/packages - sponsorlogos:/var/opt/warehouse/sponsorlogos - simple:/var/opt/warehouse/simple + - tuf_metadata:/var/opt/warehouse/tuf_metadata - ./bin:/opt/warehouse/src/bin:z - ./requirements:/opt/warehouse/src/requirements:z ports: @@ -119,6 +122,7 @@ services: - packages:/var/opt/warehouse/packages - sponsorlogos:/var/opt/warehouse/sponsorlogos - simple:/var/opt/warehouse/simple + - tuf_metadata:/var/opt/warehouse/metadata ports: - "9001:9001" @@ -129,7 +133,9 @@ services: DEVEL: "yes" command: hupper -m celery -A warehouse worker -B -S redbeat.RedBeatScheduler -l info volumes: + - ./dev:/opt/warehouse/src/dev:z - ./warehouse:/opt/warehouse/src/warehouse:z + - tuf_metadata:/var/opt/warehouse/tuf_metadata env_file: dev/environment environment: C_FORCE_ROOT: "1" diff --git a/requirements/main.in b/requirements/main.in index eb7f0c078d40..0d4b13a3fb94 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -53,6 +53,7 @@ requests requests-aws4auth redis>=2.8.0,<5.0.0 rfc3986 +securesystemslib sentry-sdk setuptools sqlalchemy[asyncio]>=0.9,<1.5.0 # https://github.com/pypi/warehouse/pull/9228 @@ -63,6 +64,7 @@ stripe structlog transaction trove-classifiers +tuf==2.0.0 typeguard webauthn>=1.0.0,<2.0.0 whitenoise diff --git a/requirements/main.txt b/requirements/main.txt index bb3bd283f940..86913d0d1344 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1238,6 +1238,7 @@ requests==2.28.1 \ # premailer # requests-aws4auth # stripe + # tuf requests-aws4auth==1.1.2 \ --hash=sha256:23b7a054326f80f86caf87e3eaf54ea41aa27adbed4297bd3456b1fa38f06a52 \ --hash=sha256:ebde0662dccda5023546055ec4cbe4470cae017ecbfce8d368b80b5e4a94d619 @@ -1258,6 +1259,12 @@ sentry-sdk==1.9.7 \ --hash=sha256:af0987fc074ada4a166bdc7e9d99d1da7811c6107e4b3416c7052ea1adb77dfc \ --hash=sha256:d391204a2a59c54b764cd351c44c67eed17b43d51f4dbe3eba48b83b70c93db9 # via -r requirements/main.in +securesystemslib==0.22.0 \ + --hash=sha256:2f58ca1ee30fde5401300fe3b3841adcf7b4369674247fa63b258e07e1f52fd2 \ + --hash=sha256:c3fc41ac32fe8bc9744b89e6ce2ebca45f4417ca737beb766a41c6cb21935662 + # via + # -r requirements/main.in + # tuf six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -1360,6 +1367,10 @@ trove-classifiers==2022.8.31 \ --hash=sha256:0db52e6a5cbe1035f306fcfee0066f22bcf842004f19dcc6258e309e36e5eb5f \ --hash=sha256:9e32190e4ec0b7a173789ee0db20c433ef25060e98f64ff32ae4111990085526 # via -r requirements/main.in +tuf==2.0.0 \ + --hash=sha256:1524b0fbd8504245f600f121daf86b8fdcb30df74410acc9655944c4868e461c \ + --hash=sha256:76e7f2a7aced84466865fac2a7127b6085afae51d4328af896fb46f952dd3a53 + # via -r requirements/main.in typeguard==2.13.3 \ --hash=sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4 \ --hash=sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1 diff --git a/tests/conftest.py b/tests/conftest.py index bfd3af365868..23b619c21c08 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,6 +36,7 @@ from pytest_postgresql.config import get_config from pytest_postgresql.janitor import DatabaseJanitor from sqlalchemy import event +from tuf.api.metadata import StorageBackendInterface import warehouse @@ -51,6 +52,8 @@ from warehouse.organizations.interfaces import IOrganizationService from warehouse.subscriptions import services as subscription_services from warehouse.subscriptions.interfaces import IBillingService, ISubscriptionService +from warehouse.tuf.interfaces import IKeyService +from warehouse.tuf.services import RepositoryService from .common.db import Session from .common.db.accounts import EmailFactory, UserFactory @@ -255,10 +258,14 @@ def app_config(database): "sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage", "billing.backend": "warehouse.subscriptions.services.MockStripeBillingService", "mail.backend": "warehouse.email.services.SMTPEmailSender", + "tuf.storage_backend": "warehouse.tuf.services.LocalStorageService", + "tuf.key_backend": "warehouse.tuf.services.LocalKeyService", + "tuf.repository_backend": "warehouse.tuf.services.RepositoryService", "malware_check.backend": ( "warehouse.malware.services.PrinterMalwareCheckService" ), "files.url": "http://localhost:7000/", + "tuf.url": "http://localhost:7000/metadata/", "sessions.secret": "123456", "sessions.url": "redis://localhost:0/", "statuspage.url": "https://2p66nmmycsj3.statuspage.io", @@ -445,6 +452,36 @@ def xmlrpc(self, path, method, *args): return xmlrpc.client.loads(resp.body) +@pytest.fixture +def tuf_repository(db_request): + class FakeStorageBackend(StorageBackendInterface): + pass + + class FakeKeyBackend(IKeyService): + pass + + db_request.registry.settings = { + "tuf.keytype": "ed25519", + "tuf.root.threshold": 1, + "tuf.root.expiry": 31536000, + "tuf.snapshot.threshold": 1, + "tuf.snapshot.expiry": 86400, + "tuf.targets.threshold": 2, + "tuf.targets.expiry": 31536000, + "tuf.timestamp.threshold": 1, + "tuf.timestamp.expiry": 86400, + "tuf.bins.threshold": 1, + "tuf.bins.expiry": 31536000, + "tuf.bin-n.threshold": 1, + "tuf.bin-n.expiry": 604800, + } + + tuf_repo = RepositoryService( + FakeStorageBackend, FakeKeyBackend, db_request.registry.settings + ) + return tuf_repo + + @pytest.fixture def webtest(app_config): # TODO: Ensure that we have per test isolation of the database level diff --git a/tests/unit/cli/test_tuf.py b/tests/unit/cli/test_tuf.py new file mode 100644 index 000000000000..93b4c5fb561e --- /dev/null +++ b/tests/unit/cli/test_tuf.py @@ -0,0 +1,338 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pretend + +from securesystemslib.exceptions import StorageError + +from warehouse.cli.tuf import ( + add_all_indexes, + add_all_packages, + bump_bin_n_roles, + bump_snapshot, + init_delegations, + init_repo, + keypair, +) +from warehouse.tuf.tasks import ( + add_hashed_targets as _add_hashed_targets, + bump_bin_n_roles as _bump_bin_n_roles, + bump_snapshot as _bump_snapshot, + init_dev_repository as _init_dev_repository, + init_targets_delegation as _init_targets_delegation, +) + + +class TestCLITUF: + def test_keypair(self, cli, monkeypatch): + settings = {"tuf.root.secret": "test_password"} + registry = pretend.stub(settings=settings) + config = pretend.stub(registry=registry) + + generate_and_write_ed25519_keypair = pretend.call_recorder( + lambda *a, **kw: None + ) + + monkeypatch.setattr( + "warehouse.cli.tuf.generate_and_write_ed25519_keypair", + generate_and_write_ed25519_keypair, + ) + result = cli.invoke( + keypair, ["--name", "root", "--path", "def/tufkeys/root.key"], obj=config + ) + + assert result.exit_code == 0 + + def test_keypair_required_name(self, cli): + result = cli.invoke(keypair) + assert result.exit_code == 2 + assert "Missing option '--name'" in result.output + + def test_keypair_required_path(self, cli): + result = cli.invoke(keypair, ["--name", "root"]) + assert result.exit_code == 2 + assert "Missing option '--path'" in result.output, result.output + + def test_init_repo(self, cli): + + request = pretend.stub() + task = pretend.stub( + get_request=pretend.call_recorder(lambda *a, **kw: request), + run=pretend.call_recorder(lambda *a, **kw: None), + ) + config = pretend.stub(task=pretend.call_recorder(lambda *a, **kw: task)) + + result = cli.invoke(init_repo, obj=config) + + assert result.exit_code == 0 + assert "Repository Initialization finished." in result.output + assert config.task.calls == [ + pretend.call(_init_dev_repository), + pretend.call(_init_dev_repository), + ] + assert task.get_request.calls == [pretend.call()] + assert task.run.calls == [pretend.call(request)] + + def test_init_repo_raise_fileexistserror(self, cli): + + request = pretend.stub() + task = pretend.stub( + get_request=pretend.call_recorder(lambda *a, **kw: request), + run=pretend.raiser(FileExistsError("TUF Error detail")), + ) + config = pretend.stub(task=pretend.call_recorder(lambda *a, **kw: task)) + + result = cli.invoke(init_repo, obj=config) + + assert result.exit_code == 1 + assert "Error: TUF Error detail\n" == result.output + assert config.task.calls == [ + pretend.call(_init_dev_repository), + pretend.call(_init_dev_repository), + ] + assert task.get_request.calls == [pretend.call()] + + def test_bump_snapshot(self, cli): + + request = pretend.stub() + task = pretend.stub( + get_request=pretend.call_recorder(lambda *a, **kw: request), + run=pretend.call_recorder(lambda *a, **kw: None), + ) + config = pretend.stub(task=pretend.call_recorder(lambda *a, **kw: task)) + + result = cli.invoke(bump_snapshot, obj=config) + + assert result.exit_code == 0 + assert "Snapshot bump finished." in result.output + assert config.task.calls == [ + pretend.call(_bump_snapshot), + pretend.call(_bump_snapshot), + ] + assert task.get_request.calls == [pretend.call()] + assert task.run.calls == [pretend.call(request)] + + def test_bump_bin_n_roles(self, cli): + + request = pretend.stub() + task = pretend.stub( + get_request=pretend.call_recorder(lambda *a, **kw: request), + run=pretend.call_recorder(lambda *a, **kw: None), + ) + config = pretend.stub(task=pretend.call_recorder(lambda *a, **kw: task)) + + result = cli.invoke(bump_bin_n_roles, obj=config) + + assert result.exit_code == 0 + assert "BIN-N roles (hash bins) bump finished." in result.output + assert config.task.calls == [ + pretend.call(_bump_bin_n_roles), + pretend.call(_bump_bin_n_roles), + ] + assert task.get_request.calls == [pretend.call()] + assert task.run.calls == [pretend.call(request)] + + def test_init_delegations(self, cli): + request = pretend.stub() + task = pretend.stub( + get_request=pretend.call_recorder(lambda *a, **kw: request), + run=pretend.call_recorder(lambda *a, **kw: None), + ) + config = pretend.stub(task=pretend.call_recorder(lambda *a, **kw: task)) + + result = cli.invoke(init_delegations, obj=config) + + assert result.exit_code == 0 + assert "BINS and BIN-N roles targets delegation finished." in result.output + assert config.task.calls == [ + pretend.call(_init_targets_delegation), + pretend.call(_init_targets_delegation), + ] + assert task.get_request.calls == [pretend.call()] + assert task.run.calls == [pretend.call(request)] + + def test_init_delegations_raise_fileexistserror(self, cli): + request = pretend.stub() + task = pretend.stub( + get_request=pretend.call_recorder(lambda *a, **kw: request), + run=pretend.raiser(FileExistsError("SSLIB Error detail")), + ) + config = pretend.stub(task=pretend.call_recorder(lambda *a, **kw: task)) + + result = cli.invoke(init_delegations, obj=config) + + assert result.exit_code == 1 + assert "Error: SSLIB Error detail\n" == result.output + assert config.task.calls == [ + pretend.call(_init_targets_delegation), + pretend.call(_init_targets_delegation), + ] + assert task.get_request.calls == [pretend.call()] + + def test_init_delegations_raise_storageerror(self, cli): + request = pretend.stub() + task = pretend.stub( + get_request=pretend.call_recorder(lambda *a, **kw: request), + run=pretend.raiser(StorageError("TUF Error detail")), + ) + config = pretend.stub(task=pretend.call_recorder(lambda *a, **kw: task)) + + result = cli.invoke(init_delegations, obj=config) + + assert result.exit_code == 1 + assert "Error: TUF Error detail\n" == result.output + assert config.task.calls == [ + pretend.call(_init_targets_delegation), + pretend.call(_init_targets_delegation), + ] + assert task.get_request.calls == [pretend.call()] + + def test_add_all_packages(self, cli, monkeypatch): + + fake_files = [ + pretend.stub( + blake2_256_digest="01101234567890abcdef", + size=192, + path="00/11/0123456789abcdf", + ) + ] + all = pretend.stub(all=lambda: fake_files) + session = pretend.stub(query=pretend.call_recorder(lambda *a, **kw: all)) + session_cls = pretend.call_recorder(lambda bind: session) + monkeypatch.setattr("warehouse.db.Session", session_cls) + monkeypatch.setattr("warehouse.packaging.models.File", lambda: None) + + settings = {"sqlalchemy.engine": "fake_engine"} + request = pretend.stub(registry=settings) + task = pretend.stub( + get_request=pretend.call_recorder(lambda *a, **kw: request), + run=pretend.call_recorder(lambda *a, **kw: None), + ) + config = pretend.stub(task=pretend.call_recorder(lambda *a, **kw: task)) + + result = cli.invoke(add_all_packages, obj=config) + + assert result.exit_code == 0 + assert config.task.calls == [ + pretend.call(_add_hashed_targets), + pretend.call(_add_hashed_targets), + ] + assert task.get_request.calls == [pretend.call()] + + def test_add_all_indexes(self, cli, monkeypatch): + + fake_projects = [ + pretend.stub( + normalized_name="fake_project_name", + ) + ] + all = pretend.stub(all=lambda: fake_projects) + session = pretend.stub(query=pretend.call_recorder(lambda *a, **kw: all)) + session_cls = pretend.call_recorder(lambda bind: session) + monkeypatch.setattr("warehouse.db.Session", session_cls) + monkeypatch.setattr("warehouse.packaging.models.Project", lambda: None) + + settings = {"sqlalchemy.engine": "fake_engine"} + request = pretend.stub(registry=settings) + task = pretend.stub( + get_request=pretend.call_recorder(lambda *a, **kw: request), + run=pretend.call_recorder(lambda *a, **kw: None), + ) + config = pretend.stub(task=pretend.call_recorder(lambda *a, **kw: task)) + + simple_detail = {"content_hash": "fake_hash", "length": 199} + fake_render_simple_detail = pretend.call_recorder( + lambda *a, **kw: simple_detail + ) + monkeypatch.setattr( + "warehouse.cli.tuf.render_simple_detail", fake_render_simple_detail + ) + + result = cli.invoke(add_all_indexes, obj=config) + + assert result.exit_code == 0 + assert config.task.calls == [ + pretend.call(_add_hashed_targets), + pretend.call(_add_hashed_targets), + ] + assert task.get_request.calls == [pretend.call()] + + def test_add_all_indexes_content_hash_none(self, cli, monkeypatch): + + fake_projects = [ + pretend.stub( + normalized_name="fake_project_name", + ) + ] + all = pretend.stub(all=lambda: fake_projects) + session = pretend.stub(query=pretend.call_recorder(lambda *a, **kw: all)) + session_cls = pretend.call_recorder(lambda bind: session) + monkeypatch.setattr("warehouse.db.Session", session_cls) + monkeypatch.setattr("warehouse.packaging.models.Project", lambda: None) + + settings = {"sqlalchemy.engine": "fake_engine"} + request = pretend.stub(registry=settings) + task = pretend.stub( + get_request=pretend.call_recorder(lambda *a, **kw: request), + run=pretend.call_recorder(lambda *a, **kw: None), + ) + config = pretend.stub(task=pretend.call_recorder(lambda *a, **kw: task)) + + simple_detail = {"content_hash": None, "length": 199} + fake_render_simple_detail = pretend.call_recorder( + lambda *a, **kw: simple_detail + ) + monkeypatch.setattr( + "warehouse.cli.tuf.render_simple_detail", fake_render_simple_detail + ) + + result = cli.invoke(add_all_indexes, obj=config) + + assert result.exit_code == 1 + assert config.task.calls == [ + pretend.call(_add_hashed_targets), + ] + assert task.get_request.calls == [pretend.call()] + + def test_add_all_indexes_oserrors(self, cli, monkeypatch): + + fake_projects = [ + pretend.stub( + normalized_name="fake_project_name", + ) + ] + all = pretend.stub(all=lambda: fake_projects) + session = pretend.stub(query=pretend.call_recorder(lambda *a, **kw: all)) + session_cls = pretend.call_recorder(lambda bind: session) + monkeypatch.setattr("warehouse.db.Session", session_cls) + monkeypatch.setattr("warehouse.packaging.models.Project", lambda: None) + + settings = {"sqlalchemy.engine": "fake_engine"} + request = pretend.stub(registry=settings) + task = pretend.stub( + get_request=pretend.call_recorder(lambda *a, **kw: request), + run=pretend.call_recorder(lambda *a, **kw: None), + ) + config = pretend.stub(task=pretend.call_recorder(lambda *a, **kw: task)) + + fake_render_simple_detail = pretend.raiser(OSError) + monkeypatch.setattr( + "warehouse.cli.tuf.render_simple_detail", fake_render_simple_detail + ) + + result = cli.invoke(add_all_indexes, obj=config) + + assert result.exit_code == 1 + assert config.task.calls == [ + pretend.call(_add_hashed_targets), + ] + assert task.get_request.calls == [pretend.call()] diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index 2a55540c92d0..fd0c35f24bf6 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -11,6 +11,7 @@ # limitations under the License. import hashlib +import os.path import tempfile import pretend @@ -36,14 +37,14 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): **_simple_detail(project, db_request), request=db_request ).encode("utf-8") - content_hash, path = render_simple_detail(project, db_request) + result = render_simple_detail(project, db_request) assert fakeblake2b.calls == [pretend.call(digest_size=32)] assert fake_hasher.update.calls == [pretend.call(expected_content)] assert fake_hasher.hexdigest.calls == [pretend.call()] - assert content_hash == "deadbeefdeadbeefdeadbeefdeadbeef" - assert path == ( + assert result.get("content_hash") == "deadbeefdeadbeefdeadbeefdeadbeef" + assert result.get("path") == ( f"{project.normalized_name}/deadbeefdeadbeefdeadbeefdeadbeef" + f".{project.normalized_name}.html" ) @@ -75,6 +76,7 @@ def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): write=pretend.call_recorder(lambda data: None), flush=pretend.call_recorder(lambda: None), ) + monkeypatch.setattr(os.path, "getsize", lambda *a, **kw: 1234) class FakeNamedTemporaryFile: def __init__(self): @@ -93,7 +95,7 @@ def __exit__(self, type, value, traceback): **_simple_detail(project, db_request), request=db_request ).encode("utf-8") - content_hash, path = render_simple_detail(project, db_request, store=True) + result = render_simple_detail(project, db_request, store=True) assert fake_named_temporary_file.write.calls == [pretend.call(expected_content)] assert fake_named_temporary_file.flush.calls == [pretend.call()] @@ -126,8 +128,8 @@ def __exit__(self, type, value, traceback): ), ] - assert content_hash == "deadbeefdeadbeefdeadbeefdeadbeef" - assert path == ( + assert result.get("content_hash") == "deadbeefdeadbeefdeadbeefdeadbeef" + assert result.get("path") == ( f"{project.normalized_name}/deadbeefdeadbeefdeadbeefdeadbeef" + f".{project.normalized_name}.html" ) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 6a81494280bf..999e0648fd94 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -295,6 +295,7 @@ def __init__(self): "IntrospectionDebugPanel" ), ], + "tuf.development_metadata_expiry": 31536000, } ) @@ -358,6 +359,7 @@ def __init__(self): pretend.call(".organizations"), pretend.call(".subscriptions"), pretend.call(".packaging"), + pretend.call(".tuf"), pretend.call(".redirects"), pretend.call(".routes"), pretend.call(".sponsors"), @@ -410,7 +412,7 @@ def __init__(self): ), ] assert configurator_obj.add_static_view.calls == [ - pretend.call("static", "warehouse:static/dist/", cache_max_age=315360000) + pretend.call("static", "warehouse:static/dist/", cache_max_age=315360000), ] assert configurator_obj.add_cache_buster.calls == [ pretend.call("warehouse:static/dist/", cachebuster_obj) diff --git a/tests/unit/test_routes.py b/tests/unit/test_routes.py index 9b8f4aa0c80e..e772ed981bc7 100644 --- a/tests/unit/test_routes.py +++ b/tests/unit/test_routes.py @@ -26,6 +26,7 @@ def __init__(self): settings={ "docs.url": docs_route_url, "files.url": "https://files.example.com/packages/{path}", + "tuf.url": "https://files.example.com/metadata/{path}", } ) if warehouse: @@ -474,6 +475,7 @@ def add_policy(name, filename): domain=warehouse, ), pretend.call("packaging.file", "https://files.example.com/packages/{path}"), + pretend.call("tuf.metadata", "https://files.example.com/metadata/{path}"), pretend.call("ses.hook", "/_/ses-hook/", domain=warehouse), pretend.call("rss.updates", "/rss/updates.xml", domain=warehouse), pretend.call("rss.packages", "/rss/packages.xml", domain=warehouse), @@ -575,7 +577,6 @@ def add_policy(name, filename): view_kw={"has_translations": True}, ), ] - assert config.add_redirect.calls == [ pretend.call("/sponsor/", "/sponsors/", domain=warehouse), pretend.call("/u/{username}/", "/user/{username}/", domain=warehouse), @@ -591,6 +592,11 @@ def add_policy(name, filename): "https://files.example.com/packages/{path}", domain=warehouse, ), + pretend.call( + "/metadata/{path:.*}", + "https://files.example.com/metadata/{path}", + domain=warehouse, + ), ] assert config.add_pypi_action_route.calls == [ diff --git a/tests/unit/test_tasks.py b/tests/unit/test_tasks.py index 6ea032e5e425..fb146271a097 100644 --- a/tests/unit/test_tasks.py +++ b/tests/unit/test_tasks.py @@ -508,8 +508,12 @@ def test_includeme(env, ssl, broker_url, expected_url, transport_options): "task_queues": ( Queue("default", routing_key="task.#"), Queue("malware", routing_key="malware.#"), + Queue("tuf", routing_key="tuf.#"), ), - "task_routes": {"warehouse.malware.tasks.*": {"queue": "malware"}}, + "task_routes": { + "warehouse.malware.tasks.*": {"queue": "malware"}, + "warehouse.tuf.tasks.*": {"queue": "tuf"}, + }, "REDBEAT_REDIS_URL": (config.registry.settings["celery.scheduler_url"]), }.items(): assert app.conf[key] == value diff --git a/tests/unit/tuf/__init__.py b/tests/unit/tuf/__init__.py new file mode 100644 index 000000000000..164f68b09175 --- /dev/null +++ b/tests/unit/tuf/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit/tuf/test_services.py b/tests/unit/tuf/test_services.py new file mode 100644 index 000000000000..5b661c82785f --- /dev/null +++ b/tests/unit/tuf/test_services.py @@ -0,0 +1,918 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import datetime +import glob +import os +import shutil + +import pretend +import pytest + +from securesystemslib.exceptions import StorageError +from zope.interface.verify import verifyClass + +from warehouse.config import Environment +from warehouse.tuf import services +from warehouse.tuf.constants import BIN_N_COUNT, Role +from warehouse.tuf.interfaces import IKeyService, IRepositoryService, IStorageService + + +class TestLocalKeyService: + def test_verify_service(self): + assert verifyClass(IKeyService, services.LocalKeyService) + + def test_create_service(self): + request = pretend.stub( + registry=pretend.stub(settings={"tuf.key.path": "/tuf/key/path/"}) + ) + service = services.LocalKeyService.create_service(None, request) + assert service._key_path == "/tuf/key/path/" + + def test_basic_init(self, db_request): + service = services.LocalKeyService("/opt/warehouse/src/dev/tufkeys", db_request) + assert service._key_path == "/opt/warehouse/src/dev/tufkeys" + + def test_get(self, db_request, monkeypatch): + service = services.LocalKeyService("/opt/warehouse/src/dev/tufkeys", db_request) + + expected_priv_key_dict = { + "keytype": "ed25519", + "scheme": "ed25519", + "keyval": {"public": "720a9a588deefd5...4d08984e87bfc5a18f34618e438434c7"}, + "keyid": "2de4eb9afe9fb73...2235d3418bd63f4214d3ba7d23b516f23e", + "keyid_hash_algorithms": ["sha256", "sha512"], + } + db_request.registry.settings["tuf.root.secret"] = "tuf.root.secret" + monkeypatch.setattr(glob, "glob", lambda privkey_path: ["fake_root.key"]) + monkeypatch.setattr( + "warehouse.tuf.services.import_ed25519_privatekey_from_file", + lambda *a, **kw: expected_priv_key_dict, + ) + + root_keyid = service.get("root") + + assert root_keyid[0].key_dict == expected_priv_key_dict + + +class TestLocalStorageService: + def test_verify_service(self): + assert verifyClass(IStorageService, services.LocalStorageService) + + def test_create_service(self): + request = pretend.stub( + registry=pretend.stub(settings={"tuf.repo.path": "/tuf/metadata/path/"}) + ) + service = services.LocalStorageService.create_service(None, request) + assert service._repo_path == "/tuf/metadata/path/" + + def test_basic_init(self): + service = services.LocalStorageService("/opt/warehouse/src/dev/metadata") + assert service._repo_path == "/opt/warehouse/src/dev/metadata" + + def test_get(self, monkeypatch): + service = services.LocalStorageService("/opt/warehouse/src/dev/metadata") + monkeypatch.setattr(glob, "glob", lambda *a, **kw: ["1.root.json"]) + + fake_file_object = pretend.stub( + close=pretend.call_recorder(lambda: None), + read=pretend.call_recorder(lambda: b"fake_root_data"), + ) + monkeypatch.setitem( + services.__builtins__, "open", lambda *a, **kw: fake_file_object + ) + + with service.get("root") as r: + result = r.read() + + assert result == fake_file_object.read() + assert fake_file_object.close.calls == [pretend.call()] + + def test_get_max_version_raises_valueerror(self, monkeypatch): + service = services.LocalStorageService("/opt/warehouse/src/dev/metadata") + + monkeypatch.setattr(glob, "glob", lambda *a, **kw: []) + + fake_file_object = pretend.stub( + close=pretend.call_recorder(lambda: None), + read=pretend.call_recorder(lambda: b"fake_root_data"), + ) + monkeypatch.setitem( + services.__builtins__, "open", lambda *a, **kw: fake_file_object + ) + + with service.get("root") as r: + result = r.read() + + assert result == fake_file_object.read() + assert fake_file_object.close.calls == [pretend.call()] + + def test_get_oserror(self, monkeypatch): + service = services.LocalStorageService("/opt/warehouse/src/dev/metadata") + + monkeypatch.setattr(glob, "glob", lambda *a, **kw: ["1.root.json"]) + monkeypatch.setitem( + services.__builtins__, "open", pretend.raiser(PermissionError) + ) + + with pytest.raises(StorageError) as err: + with service.get("root"): + pass + + assert "Can't open /opt/warehouse/src/dev/metadata/1.root.json" in str( + err.value + ) + + def test_get_specific_version(self, monkeypatch): + service = services.LocalStorageService("/opt/warehouse/src/dev/metadata") + + monkeypatch.setattr( + glob, "glob", lambda *a, **kw: ["1.root.json", "2.root.json", "3.root.json"] + ) + + fake_file_object = pretend.stub( + close=pretend.call_recorder(lambda: None), + read=pretend.call_recorder(lambda: b"fake_data"), + ) + monkeypatch.setitem( + services.__builtins__, "open", lambda *a, **kw: fake_file_object + ) + + with service.get("root", version=2) as r: + result = r.read() + + assert result == fake_file_object.read() + assert fake_file_object.close.calls == [pretend.call()] + + def test_get_timestamp_specific(self, monkeypatch): + service = services.LocalStorageService("/opt/warehouse/src/dev/metadata") + + monkeypatch.setattr(glob, "glob", lambda *a, **kw: ["timestamp.json"]) + + fake_file_object = pretend.stub( + close=pretend.call_recorder(lambda: None), + read=pretend.call_recorder(lambda: b"fake_data"), + ) + monkeypatch.setitem( + services.__builtins__, "open", lambda *a, **kw: fake_file_object + ) + + with service.get(Role.TIMESTAMP.value) as r: + result = r.read() + + assert result == fake_file_object.read() + + def test_put(self, monkeypatch): + service = services.LocalStorageService("/opt/warehouse/src/dev/metadata") + + fake_file_object = pretend.stub( + closed=True, seek=pretend.call_recorder(lambda offset: None) + ) + + fake_destination_file = pretend.stub( + flush=pretend.call_recorder(lambda: None), + fileno=pretend.call_recorder(lambda: None), + ) + + class FakeDestinationFile: + def __init__(self, file, mode): + return None + + def __enter__(self): + return fake_destination_file + + def __exit__(self, type, value, traceback): + pass + + monkeypatch.setitem(services.__builtins__, "open", FakeDestinationFile) + monkeypatch.setattr(shutil, "copyfileobj", lambda *a, **kw: None) + monkeypatch.setattr(os, "fsync", lambda *a, **kw: None) + + result = service.put(fake_file_object, "2.snapshot.json") + + assert result is None + assert fake_file_object.seek.calls == [] + assert fake_destination_file.flush.calls == [pretend.call()] + assert fake_destination_file.fileno.calls == [pretend.call()] + + def test_put_file_object_closed(self, monkeypatch): + service = services.LocalStorageService("/opt/warehouse/src/dev/metadata") + + fake_file_object = pretend.stub( + closed=False, seek=pretend.call_recorder(lambda offset: None) + ) + + fake_destination_file = pretend.stub( + flush=pretend.call_recorder(lambda: None), + fileno=pretend.call_recorder(lambda: None), + ) + + class FakeDestinationFile: + def __init__(self, file, mode): + return None + + def __enter__(self): + return fake_destination_file + + def __exit__(self, type, value, traceback): + pass + + monkeypatch.setitem(services.__builtins__, "open", FakeDestinationFile) + monkeypatch.setattr(shutil, "copyfileobj", lambda *a, **kw: None) + monkeypatch.setattr(os, "fsync", lambda *a, **kw: None) + + result = service.put(fake_file_object, "2.snapshot.json") + + assert result is None + assert fake_file_object.seek.calls == [pretend.call(0)] + assert fake_destination_file.flush.calls == [pretend.call()] + assert fake_destination_file.fileno.calls == [pretend.call()] + + def test_put_raise_oserror(self, monkeypatch): + service = services.LocalStorageService("/opt/warehouse/src/dev/metadata") + + fake_file_object = pretend.stub( + closed=True, seek=pretend.call_recorder(lambda offset: None) + ) + + monkeypatch.setitem( + services.__builtins__, "open", pretend.raiser(PermissionError) + ) + monkeypatch.setattr(shutil, "copyfileobj", lambda *a, **kw: None) + monkeypatch.setattr(os, "fsync", lambda *a, **kw: None) + + with pytest.raises(StorageError) as err: + service.put(fake_file_object, "2.snapshot.json") + + assert "Can't write file 2.snapshot.json" in str(err.value) + assert fake_file_object.seek.calls == [] + + def test_store(self, monkeypatch): + """store is an alias for put""" + service = services.LocalStorageService("/opt/warehouse/src/dev/metadata") + + fake_file_object = pretend.stub( + closed=True, seek=pretend.call_recorder(lambda offset: None) + ) + + fake_destination_file = pretend.stub( + flush=pretend.call_recorder(lambda: None), + fileno=pretend.call_recorder(lambda: None), + ) + + class FakeDestinationFile: + def __init__(self, file, mode): + return None + + def __enter__(self): + return fake_destination_file + + def __exit__(self, type, value, traceback): + pass + + monkeypatch.setitem(services.__builtins__, "open", FakeDestinationFile) + monkeypatch.setattr(shutil, "copyfileobj", lambda *a, **kw: None) + monkeypatch.setattr(os, "fsync", lambda *a, **kw: None) + + result = service.store(fake_file_object, "2.snapshot.json") + + assert result is None + assert fake_file_object.seek.calls == [] + assert fake_destination_file.flush.calls == [pretend.call()] + assert fake_destination_file.fileno.calls == [pretend.call()] + + +class TestRepositoryService: + def test_verify_service(self): + assert verifyClass(IRepositoryService, services.RepositoryService) + + def test_basic_init(self): + service = services.RepositoryService( + "fake_storage", "fake_key_storage", "fake_request" + ) + assert service._storage_backend == "fake_storage" + assert service._key_storage_backend == "fake_key_storage" + assert service._request == "fake_request" + + def test_create_service(self): + fake_service = "Fake Service" + request = pretend.stub( + find_service=pretend.call_recorder(lambda interface: fake_service) + ) + service = services.RepositoryService.create_service(None, request) + assert service._storage_backend == fake_service + assert service._key_storage_backend == fake_service + assert service._request == request + assert request.find_service.calls == [ + pretend.call(IStorageService), + pretend.call(IKeyService), + ] + + def test__get_bit_lenght(self, db_request): + db_request.registry.settings["warehouse.env"] = Environment.development + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + response = repository_service._get_bit_length() + assert response == 8 + + def test__get_bit_lenght_production(self, db_request): + db_request.registry.settings["warehouse.env"] = Environment.production + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + response = repository_service._get_bit_length() + assert response == BIN_N_COUNT + + def test__is_initialized_true(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + repository_service._load = pretend.call_recorder(lambda *a: services.Root()) + + assert repository_service._is_initialized() is True + assert repository_service._load.calls in [ + [pretend.call(role)] for role in services.TOP_LEVEL_ROLE_NAMES + ] + + def test__is_initialized_false(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + repository_service._load = pretend.call_recorder(lambda *a: None) + + assert repository_service._is_initialized() is False + for pretend_call in repository_service._load.calls: + assert pretend_call in [ + pretend.call(role) for role in services.TOP_LEVEL_ROLE_NAMES + ] + + def test__is_initialized_false_by_exception(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + repository_service._load = pretend.raiser(services.StorageError) + + assert repository_service._is_initialized() is False + + def test__load(self, monkeypatch, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + fake_metadata = pretend.stub( + from_file=pretend.call_recorder(lambda *a: "Metadata") + ) + monkeypatch.setattr("warehouse.tuf.services.Metadata", fake_metadata) + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + result = repository_service._load("root") + + assert result == "Metadata" + assert fake_metadata.from_file.calls == [ + pretend.call("root", None, fake_storage_service) + ] + + def test__sign(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub( + get=pretend.call_recorder(lambda *a: ["signer1"]) + ) + + role = pretend.stub( + signatures=pretend.stub(clear=pretend.call_recorder(lambda: None)), + sign=pretend.call_recorder(lambda *a, **kw: None), + ) + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + result = repository_service._sign(role, "fake_role") + + assert result is None + assert fake_key_service.get.calls == [pretend.call("fake_role")] + assert role.signatures.clear.calls == [pretend.call()] + assert role.sign.calls == [pretend.call("signer1", append=True)] + + def test__persist(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + role = pretend.stub( + signed=pretend.stub(version=2), + to_file=pretend.call_recorder(lambda *a, **kw: None), + ) + + services.JSONSerializer = pretend.call_recorder(lambda: None) + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + result = repository_service._persist(role, "root") + + assert result is None + assert role.to_file.calls == [ + pretend.call("2.root.json", services.JSONSerializer(), fake_storage_service) + ] + assert services.JSONSerializer.calls == [pretend.call(), pretend.call()] + + def test__persist_timestamp(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + role = pretend.stub( + signed=pretend.stub(version=2), + to_file=pretend.call_recorder(lambda *a, **kw: None), + ) + + services.JSONSerializer = pretend.call_recorder(lambda: None) + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + result = repository_service._persist(role, Role.TIMESTAMP.value) + + assert result is None + assert role.to_file.calls == [ + pretend.call( + "timestamp.json", services.JSONSerializer(), fake_storage_service + ) + ] + assert services.JSONSerializer.calls == [pretend.call(), pretend.call()] + + def test__bump_expiry(self, monkeypatch, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + db_request.registry.settings["warehouse.env"] = Environment.production + test_tuf_config = { + "tuf.root.threshold": 1, + "tuf.root.expiry": 31536000, + "tuf.snapshot.threshold": 1, + "tuf.snapshot.expiry": 86400, + "tuf.targets.threshold": 2, + "tuf.targets.expiry": 31536000, + "tuf.timestamp.threshold": 1, + "tuf.timestamp.expiry": 86400, + } + for name, value in test_tuf_config.items(): + db_request.registry.settings[name] = value + + fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) + fake_datetime = pretend.stub(now=pretend.call_recorder(lambda: fake_time)) + monkeypatch.setattr("warehouse.tuf.services.datetime", fake_datetime) + + role = pretend.stub( + signed=pretend.stub(expires=fake_datetime), + ) + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + result = repository_service._bump_expiry(role, "root") + + assert result is None + assert role.signed.expires == datetime.datetime(2020, 6, 15, 9, 5, 1) + assert fake_datetime.now.calls == [pretend.call()] + + def test__bump_version(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + role = pretend.stub( + signed=pretend.stub(version=2), + ) + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + result = repository_service._bump_version(role) + + assert result is None + assert role.signed.version == 3 + + def test__update_timestamp(self, monkeypatch, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + + snapshot_version = 3 + fake_metafile = pretend.call_recorder(lambda *a, **kw: snapshot_version) + monkeypatch.setattr("warehouse.tuf.services.MetaFile", fake_metafile) + + mocked_timestamp = pretend.stub(signed=pretend.stub(snapshot_meta=2)) + repository_service._load = pretend.call_recorder(lambda *a: mocked_timestamp) + repository_service._bump_version = pretend.call_recorder(lambda *a: None) + repository_service._bump_expiry = pretend.call_recorder(lambda *a: None) + repository_service._sign = pretend.call_recorder(lambda *a: None) + repository_service._persist = pretend.call_recorder(lambda *a: None) + + result = repository_service._update_timestamp(snapshot_version) + + assert result is None + assert mocked_timestamp.signed.snapshot_meta == snapshot_version + assert repository_service._load.calls == [pretend.call(Role.TIMESTAMP.value)] + assert repository_service._bump_version.calls == [ + pretend.call(mocked_timestamp) + ] + assert repository_service._bump_expiry.calls == [ + pretend.call(mocked_timestamp, Role.TIMESTAMP.value) + ] + assert repository_service._sign.calls == [ + pretend.call(mocked_timestamp, Role.TIMESTAMP.value) + ] + assert repository_service._persist.calls == [ + pretend.call(mocked_timestamp, Role.TIMESTAMP.value) + ] + + def test__update_snapshot(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + snapshot_version = 3 + test_target_meta = [("bins", 3), ("f", 4)] + mocked_snapshot = pretend.stub( + signed=pretend.stub( + meta={}, + version=snapshot_version, + ) + ) + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + repository_service._load = pretend.call_recorder(lambda *a: mocked_snapshot) + repository_service._bump_version = pretend.call_recorder(lambda *a: None) + repository_service._bump_expiry = pretend.call_recorder(lambda *a: None) + repository_service._sign = pretend.call_recorder(lambda *a: None) + repository_service._persist = pretend.call_recorder(lambda *a: None) + + result = repository_service._update_snapshot(test_target_meta) + + assert result is snapshot_version + assert repository_service._load.calls == [pretend.call(Role.SNAPSHOT.value)] + assert repository_service._bump_version.calls == [pretend.call(mocked_snapshot)] + assert repository_service._bump_expiry.calls == [ + pretend.call(mocked_snapshot, Role.SNAPSHOT.value) + ] + assert repository_service._sign.calls == [ + pretend.call(mocked_snapshot, Role.SNAPSHOT.value) + ] + assert repository_service._persist.calls == [ + pretend.call(mocked_snapshot, Role.SNAPSHOT.value) + ] + + def test_init_dev_repository(self, db_request): + fake_key = { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid": ( + "6dcd53f0a90fca17700f819e939a74b133aa5cd8619f3dc03228c0c68dcc2abb" + ), + "keyid_hash_algorithms": ["sha256", "sha512"], + "keyval": { + "public": ( + "c864d93b521d5851275a7b7c79fb0ac76311c206262eabd67319eba6665b1417" + ), + "private": ( + "bbe40143bfe1a3b6a41647f590e398fb8dd38fddf6b279edefdc022cdb649cdc" + ), + }, + } + fake_signers = [ + pretend.stub( + key_dict=fake_key, + sign=pretend.call_recorder(lambda *a: "key1"), + ), + pretend.stub( + key_dict=fake_key, + sign=pretend.call_recorder(lambda *a: "key1"), + ), + ] + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub( + get=pretend.call_recorder(lambda *a: fake_signers) + ) + + db_request.registry.settings["warehouse.env"] = Environment.production + test_tuf_config = { + "tuf.root.threshold": 1, + "tuf.root.expiry": 31536000, + "tuf.snapshot.threshold": 1, + "tuf.snapshot.expiry": 86400, + "tuf.targets.threshold": 2, + "tuf.targets.expiry": 31536000, + "tuf.timestamp.threshold": 1, + "tuf.timestamp.expiry": 86400, + } + for name, value in test_tuf_config.items(): + db_request.registry.settings[name] = value + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + + repository_service._is_initialized = pretend.call_recorder(lambda: False) + repository_service._bump_expiry = pretend.call_recorder(lambda *a: None) + repository_service._sign = pretend.call_recorder(lambda *a: None) + repository_service._persist = pretend.call_recorder(lambda *a: None) + + result = repository_service.init_dev_repository() + assert result is None + + def test_init_dev_repository_already_initialized(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + repository_service._is_initialized = pretend.call_recorder(lambda: True) + with pytest.raises(FileExistsError) as err: + repository_service.init_dev_repository() + + assert "TUF Metadata Repository files already exists." in str(err) + + def test_init_targets_delegation(self, db_request): + fake_key = { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid": ( + "6dcd53f0a90fca17700f819e939a74b133aa5cd8619f3dc03228c0c68dcc2abb" + ), + "keyid_hash_algorithms": ["sha256", "sha512"], + "keyval": { + "public": ( + "c864d93b521d5851275a7b7c79fb0ac76311c206262eabd67319eba6665b1417" + ), + "private": ( + "bbe40143bfe1a3b6a41647f590e398fb8dd38fddf6b279edefdc022cdb649cdc" + ), + }, + } + fake_signers = [ + pretend.stub( + key_dict=fake_key, + sign=pretend.call_recorder(lambda *a: "key1"), + ), + pretend.stub( + key_dict=fake_key, + sign=pretend.call_recorder(lambda *a: "key1"), + ), + ] + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub( + get=pretend.call_recorder(lambda *a: fake_signers) + ) + + db_request.registry.settings["warehouse.env"] = Environment.development + + test_tuf_config = { + "tuf.root.threshold": 1, + "tuf.root.expiry": 31536000, + "tuf.snapshot.threshold": 1, + "tuf.snapshot.expiry": 86400, + "tuf.targets.threshold": 2, + "tuf.targets.expiry": 31536000, + "tuf.timestamp.threshold": 1, + "tuf.timestamp.expiry": 86400, + "tuf.bins.threshold": 1, + "tuf.bins.expiry": 31536000, + "tuf.bin-n.threshold": 1, + "tuf.bin-n.expiry": 604800, + } + for name, value in test_tuf_config.items(): + db_request.registry.settings[name] = value + + fake_targets = pretend.stub( + signed=pretend.stub( + delegations=None, + roles={}, + add_key=pretend.call_recorder(lambda *a: None), + version=3, + ) + ) + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + repository_service._load = pretend.call_recorder(lambda *a: fake_targets) + repository_service._bump_version = pretend.call_recorder(lambda *a: None) + repository_service._bump_expiry = pretend.call_recorder(lambda *a: None) + repository_service._sign = pretend.call_recorder(lambda *a: None) + repository_service._persist = pretend.call_recorder(lambda *a: None) + repository_service._update_timestamp = pretend.call_recorder(lambda *a: None) + repository_service._update_snapshot = pretend.call_recorder(lambda *a: 3) + + result = repository_service.init_targets_delegation() + + assert result is None + assert repository_service._load.calls == [pretend.call("targets")] + assert repository_service._bump_version.calls == [pretend.call(fake_targets)] + assert repository_service._update_snapshot.calls == [ + pretend.call([("targets", 3), ("bins", 1)]) + ] + assert repository_service._update_timestamp.calls == [pretend.call(3)] + + def test_add_hashed_targets(self, db_request): + fake_key = { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid": ( + "6dcd53f0a90fca17700f819e939a74b133aa5cd8619f3dc03228c0c68dcc2abb" + ), + "keyid_hash_algorithms": ["sha256", "sha512"], + "keyval": { + "public": ( + "c864d93b521d5851275a7b7c79fb0ac76311c206262eabd67319eba6665b1417" + ), + "private": ( + "bbe40143bfe1a3b6a41647f590e398fb8dd38fddf6b279edefdc022cdb649cdc" + ), + }, + } + fake_signers = [ + pretend.stub( + key_dict=fake_key, + sign=pretend.call_recorder(lambda *a: "key1"), + ), + pretend.stub( + key_dict=fake_key, + sign=pretend.call_recorder(lambda *a: "key1"), + ), + ] + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub( + get=pretend.call_recorder(lambda *a: fake_signers) + ) + + db_request.registry.settings["warehouse.env"] = Environment.development + + test_tuf_config = { + "tuf.bin-n.threshold": 1, + "tuf.bin-n.expiry": 604800, + } + for name, value in test_tuf_config.items(): + db_request.registry.settings[name] = value + + fake_bins = pretend.stub( + signed=pretend.stub( + delegations=pretend.stub( + succinct_roles=pretend.stub( + get_role_for_target=pretend.call_recorder(lambda *a: "bin-n-3d") + ) + ), + ) + ) + fake_bin_n = pretend.stub(signed=pretend.stub(targets={}, version=4)) + + def mocked_load(role): + if role == "bins": + return fake_bins + else: + return fake_bin_n + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + + repository_service._load = pretend.call_recorder(lambda r: mocked_load(r)) + repository_service._bump_version = pretend.call_recorder(lambda *a: None) + repository_service._bump_expiry = pretend.call_recorder(lambda *a: None) + repository_service._sign = pretend.call_recorder(lambda *a: None) + repository_service._persist = pretend.call_recorder(lambda *a: None) + repository_service._update_timestamp = pretend.call_recorder(lambda *a: None) + repository_service._update_snapshot = pretend.call_recorder(lambda *a: 3) + + targets = [ + services.TargetFile( + 1024, + {"blake2b-256": "fake_hash_0123456789abcdef"}, + "/xy/some_package.tar.gz", + {"backsigned": True}, + ), + services.TargetFile( + 1024, + {"blake2b-256": "fake_hash_0123456789abcdef"}, + "/xy/some_package.tar.gz", + {"backsigned": True}, + ), + ] + result = repository_service.add_hashed_targets(targets) + + assert result is None + assert repository_service._load.calls == [ + pretend.call("bins"), + pretend.call("bin-n-3d"), + ] + assert repository_service._bump_version.calls == [pretend.call(fake_bin_n)] + assert repository_service._bump_expiry.calls == [ + pretend.call(fake_bin_n, "bin-n") + ] + assert repository_service._sign.calls == [pretend.call(fake_bin_n, "bin-n")] + assert repository_service._sign.calls == [pretend.call(fake_bin_n, "bin-n")] + assert repository_service._update_snapshot.calls == [ + pretend.call([("bin-n-3d", 4)]) + ] + assert repository_service._update_timestamp.calls == [pretend.call(3)] + + def test_bump_bin_n_roles(self, db_request): + + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + fake_bins = pretend.stub( + signed=pretend.stub( + delegations=pretend.stub( + succinct_roles=pretend.stub( + get_roles=pretend.call_recorder(lambda: ["bin-0", "bin-f"]) + ) + ), + ) + ) + fake_bin_n = pretend.stub(signed=pretend.stub(targets={}, version=5)) + + def mocked_load(role): + if role == "bins": + return fake_bins + else: + return fake_bin_n + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + repository_service._load = pretend.call_recorder(lambda r: mocked_load(r)) + repository_service._bump_version = pretend.call_recorder(lambda *a: None) + repository_service._bump_expiry = pretend.call_recorder(lambda *a: None) + repository_service._sign = pretend.call_recorder(lambda *a: None) + repository_service._persist = pretend.call_recorder(lambda *a: None) + repository_service._update_timestamp = pretend.call_recorder(lambda *a: None) + repository_service._update_snapshot = pretend.call_recorder(lambda *a: 6) + + result = repository_service.bump_bin_n_roles() + + assert result is None + assert repository_service._load.calls == [ + pretend.call("bins"), + pretend.call("bin-0"), + pretend.call("bin-f"), + ] + assert repository_service._bump_version.calls == [ + pretend.call(fake_bin_n), + pretend.call(fake_bin_n), + ] + assert repository_service._bump_expiry.calls == [ + pretend.call(fake_bin_n, "bin-n"), + pretend.call(fake_bin_n, "bin-n"), + ] + assert repository_service._sign.calls == [ + pretend.call(fake_bin_n, "bin-n"), + pretend.call(fake_bin_n, "bin-n"), + ] + assert repository_service._sign.calls == [ + pretend.call(fake_bin_n, "bin-n"), + pretend.call(fake_bin_n, "bin-n"), + ] + assert repository_service._update_snapshot.calls == [ + pretend.call([("bin-0", 5), ("bin-f", 5)]) + ] + assert repository_service._update_timestamp.calls == [pretend.call(6)] + + def test_bump_snapshot(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + repository_service._update_snapshot = pretend.call_recorder(lambda *a: 41) + repository_service._update_timestamp = pretend.call_recorder(lambda *a: None) + + result = repository_service.bump_snapshot() + + assert result is None + assert repository_service._update_snapshot.calls == [pretend.call([])] + assert repository_service._update_timestamp.calls == [pretend.call(41)] diff --git a/tests/unit/tuf/test_tasks.py b/tests/unit/tuf/test_tasks.py new file mode 100644 index 000000000000..35e756d63fd6 --- /dev/null +++ b/tests/unit/tuf/test_tasks.py @@ -0,0 +1,181 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pretend + +from warehouse.tuf import tasks +from warehouse.tuf.interfaces import IRepositoryService +from warehouse.tuf.services import TargetFile + + +class TestBumpSnapshot: + def test_success(self, db_request, monkeypatch): + + fake_irepository = pretend.stub() + fake_irepository.bump_snapshot = pretend.call_recorder(lambda: None) + + db_request.registry.settings["celery.scheduler_url"] = "fake_schedule" + db_request.find_service = pretend.call_recorder( + lambda interface: fake_irepository + ) + + class FakeRedisLock: + def __init__(self): + return None + + def __enter__(self): + return None + + def __exit__(self, type, value, traceback): + pass + + mocked_redis = pretend.stub(lock=lambda *a: FakeRedisLock()) + monkeypatch.setattr( + "warehouse.tuf.tasks.redis.StrictRedis.from_url", + lambda *a, **kw: mocked_redis, + ) + + task = pretend.stub() + tasks.bump_snapshot(task, db_request) + + assert db_request.find_service.calls == [pretend.call(IRepositoryService)] + assert fake_irepository.bump_snapshot.calls == [pretend.call()] + + +class TestBumpBinNRoles: + def test_success(self, db_request, monkeypatch): + + fake_irepository = pretend.stub() + fake_irepository.bump_bin_n_roles = pretend.call_recorder(lambda: None) + + db_request.registry.settings["celery.scheduler_url"] = "fake_schedule" + db_request.find_service = pretend.call_recorder( + lambda interface: fake_irepository + ) + + class FakeRedisLock: + def __init__(self): + return None + + def __enter__(self): + return None + + def __exit__(self, type, value, traceback): + pass + + mocked_redis = pretend.stub(lock=lambda *a: FakeRedisLock()) + monkeypatch.setattr( + "warehouse.tuf.tasks.redis.StrictRedis.from_url", + lambda *a, **kw: mocked_redis, + ) + + task = pretend.stub() + tasks.bump_bin_n_roles(task, db_request) + + assert db_request.find_service.calls == [pretend.call(IRepositoryService)] + assert fake_irepository.bump_bin_n_roles.calls == [pretend.call()] + + +class TestInitRepository: + def test_success(self, db_request): + + fake_irepository = pretend.stub() + fake_irepository.init_dev_repository = pretend.call_recorder(lambda: None) + + db_request.registry.settings["celery.scheduler_url"] = "fake_schedule" + db_request.find_service = pretend.call_recorder( + lambda interface: fake_irepository + ) + + task = pretend.stub() + tasks.init_dev_repository(task, db_request) + + assert fake_irepository.init_dev_repository.calls == [pretend.call()] + assert db_request.find_service.calls == [pretend.call(IRepositoryService)] + + +class TestInitTargetsDelegation: + def test_success(self, db_request, monkeypatch): + + fake_irepository = pretend.stub() + fake_irepository.init_targets_delegation = pretend.call_recorder(lambda: None) + + db_request.registry.settings["celery.scheduler_url"] = "fake_schedule" + db_request.find_service = pretend.call_recorder( + lambda interface: fake_irepository + ) + + class FakeRedisLock: + def __init__(self): + return None + + def __enter__(self): + return None + + def __exit__(self, type, value, traceback): + pass + + mocked_redis = pretend.stub(lock=lambda *a: FakeRedisLock()) + monkeypatch.setattr( + "warehouse.tuf.tasks.redis.StrictRedis.from_url", + lambda *a, **kw: mocked_redis, + ) + + task = pretend.stub() + tasks.init_targets_delegation(task, db_request) + + assert fake_irepository.init_targets_delegation.calls == [pretend.call()] + assert db_request.find_service.calls == [pretend.call(IRepositoryService)] + + +class TestAddHashedTargets: + def test_success(self, db_request, monkeypatch): + + fake_irepository = pretend.stub() + fake_irepository.add_hashed_targets = pretend.call_recorder( + lambda *a, **kw: None + ) + + db_request.registry.settings["celery.scheduler_url"] = "fake_schedule" + db_request.find_service = pretend.call_recorder( + lambda interface: fake_irepository + ) + + class FakeRedisLock: + def __init__(self): + return None + + def __enter__(self): + return None + + def __exit__(self, type, value, traceback): + pass + + mocked_redis = pretend.stub(lock=lambda *a: FakeRedisLock()) + monkeypatch.setattr( + "warehouse.tuf.tasks.redis.StrictRedis.from_url", + lambda *a, **kw: mocked_redis, + ) + + fake_fileinfo = { + "hashes": {"blake2b-256": "dlskjflkdjflsdjfsdfdfsdfsdfs"}, + "length": 1025, + "custom": {"backsigned": True}, + } + + targets = TargetFile.from_dict(fake_fileinfo, "file/path") + + task = pretend.stub() + tasks.add_hashed_targets(task, db_request, targets) + + fake_irepository.add_hashed_targets.calls == [pretend.call()] + assert db_request.find_service.calls == [pretend.call(IRepositoryService)] diff --git a/warehouse/cli/tuf.py b/warehouse/cli/tuf.py new file mode 100644 index 000000000000..7edf21be84c5 --- /dev/null +++ b/warehouse/cli/tuf.py @@ -0,0 +1,178 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click + +from securesystemslib.exceptions import StorageError # type: ignore +from securesystemslib.interface import ( # type: ignore + generate_and_write_ed25519_keypair, +) + +from warehouse.cli import warehouse +from warehouse.packaging.utils import render_simple_detail +from warehouse.tuf.tasks import ( + TargetFile, + add_hashed_targets as _add_hashed_targets, + bump_bin_n_roles as _bump_bin_n_roles, + bump_snapshot as _bump_snapshot, + init_dev_repository as _init_dev_repository, + init_targets_delegation as _init_targets_delegation, +) + + +@warehouse.group() # pragma: no-branch +def tuf(): + """ + Manage Warehouse's TUF state. + """ + + +@tuf.group() +def dev(): + """ + TUF Development purposes commands + """ + + +@dev.command() +@click.pass_obj +@click.option( + "--name", "name_", required=True, help="The name of the TUF role for this keypair" +) +@click.option( + "--path", + "path_", + required=True, + help="The basename of the Ed25519 keypair to generate", +) +def keypair(config, name_, path_): + """ + Generate a new TUF keypair. + """ + password = config.registry.settings[f"tuf.{name_}.secret"] + generate_and_write_ed25519_keypair(password, filepath=path_) + + +@dev.command() +@click.pass_obj +def init_repo(config): + """ + Initialize a new TUF repository if it does not exist. + """ + try: + request = config.task(_init_dev_repository).get_request() + config.task(_init_dev_repository).run(request) + except FileExistsError as err: + raise click.ClickException(str(err)) + + click.echo("Repository Initialization finished.") + + +@dev.command() +@click.pass_obj +def bump_snapshot(config): + """ + Bump Snapshot metadata. + """ + request = config.task(_bump_snapshot).get_request() + config.task(_bump_snapshot).run(request) + click.echo("Snapshot bump finished.") + + +@dev.command() +@click.pass_obj +def bump_bin_n_roles(config): + """ + Bump delegated targets roles (BIN-N). + """ + request = config.task(_bump_bin_n_roles).get_request() + config.task(_bump_bin_n_roles).run(request) + click.echo("BIN-N roles (hash bins) bump finished.") + + +@dev.command() +@click.pass_obj +def init_delegations(config): + """ + Create delegated targets roles (BINS and BIN-N). + + Given an initialized (but empty) TUF repository, create the delegated + targets role (bins) and its hashed bin delegations (each bin-n). + """ + request = config.task(_init_targets_delegation).get_request() + try: + config.task(_init_targets_delegation).run(request) + except (FileExistsError, StorageError) as err: + raise click.ClickException(str(err)) + + click.echo("BINS and BIN-N roles targets delegation finished.") + + +@dev.command() +@click.pass_obj +def add_all_packages(config): + """ + Collect every PyPI package and add as targets. + + Collect the "paths" for every PyPI package and add as targets.These are + packages already in existence, so we'll add some additional data to their targets to + indicate that we're back-signing them. + """ + + from warehouse.db import Session + from warehouse.packaging.models import File + + request = config.task(_add_hashed_targets).get_request() + db = Session(bind=request.registry["sqlalchemy.engine"]) + targets = list() + for file in db.query(File).all(): + hashes = {"blake2b-256": file.blake2_256_digest} + targets.append( + TargetFile( + length=file.size, + hashes=hashes, + path=file.path, + unrecognized_fields={"backsigned": True}, + ) + ) + + config.task(_add_hashed_targets).run(request, targets) + + +@dev.command() +@click.pass_obj +def add_all_indexes(config): + """ + Collect every PyPI project Index and add as targets. + """ + from warehouse.db import Session + from warehouse.packaging.models import Project + + request = config.task(_add_hashed_targets).get_request() + request.db = Session(bind=request.registry["sqlalchemy.engine"]) + + targets = list() + for project in request.db.query(Project).all(): + try: + simple_detail = render_simple_detail(project, request, store=True) + except OSError as err: + click.ClickException(str(err)) + hashes = {"blake2b-256": simple_detail.get("content_hash")} + targets.append( + TargetFile( + length=simple_detail.get("length"), + hashes=hashes, + path=f"{project.normalized_name}/{project.normalized_name}.html", + ) + ) + + config.task(_add_hashed_targets).run(request, targets) diff --git a/warehouse/config.py b/warehouse/config.py index 08a7ad3f56fc..6d3120b2e344 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -228,6 +228,13 @@ def configure(settings=None): default=21600, # 6 hours ) maybe_set_compound(settings, "billing", "backend", "BILLING_BACKEND") + maybe_set(settings, "tuf.url", "TUF_URL") + maybe_set(settings, "tuf.root.secret", "TUF_ROOT_SECRET") + maybe_set(settings, "tuf.snapshot.secret", "TUF_SNAPSHOT_SECRET") + maybe_set(settings, "tuf.targets.secret", "TUF_TARGETS_SECRET") + maybe_set(settings, "tuf.timestamp.secret", "TUF_TIMESTAMP_SECRET") + maybe_set(settings, "tuf.bins.secret", "TUF_BINS_SECRET") + maybe_set(settings, "tuf.bin-n.secret", "TUF_BIN_N_SECRET") maybe_set_compound(settings, "files", "backend", "FILES_BACKEND") maybe_set_compound(settings, "simple", "backend", "SIMPLE_BACKEND") maybe_set_compound(settings, "docs", "backend", "DOCS_BACKEND") @@ -237,6 +244,9 @@ def configure(settings=None): maybe_set_compound(settings, "metrics", "backend", "METRICS_BACKEND") maybe_set_compound(settings, "breached_passwords", "backend", "BREACHED_PASSWORDS") maybe_set_compound(settings, "malware_check", "backend", "MALWARE_CHECK_BACKEND") + maybe_set_compound(settings, "tuf", "key_backend", "TUF_KEY_BACKEND") + maybe_set_compound(settings, "tuf", "storage_backend", "TUF_STORAGE_BACKEND") + maybe_set_compound(settings, "tuf", "repository_backend", "TUF_REPOSITORY_BACKEND") # Pythondotorg integration settings maybe_set(settings, "pythondotorg.host", "PYTHONDOTORG_HOST", default="python.org") @@ -358,6 +368,10 @@ def configure(settings=None): ], ) + # For development only: this artificially prolongs the expirations of any + # Warehouse-generated TUF metadata by approximately one year. + settings.setdefault("tuf.development_metadata_expiry", 31536000) + # Actually setup our Pyramid Configurator with the values pulled in from # the environment as well as the ones passed in to the configure function. config = Configurator(settings=settings) @@ -565,6 +579,9 @@ def configure(settings=None): # Allow the packaging app to register any services it has. config.include(".packaging") + # Register TUF support for package integrity + config.include(".tuf") + # Configure redirection support config.include(".redirects") diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 2ce26022001b..14573956f240 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -84,12 +84,12 @@ def render_simple_detail(project, request, store=False): f"{project.normalized_name}/{content_hash}.{project.normalized_name}.html" ) + length = None if store: storage = request.find_service(ISimpleStorage) with tempfile.NamedTemporaryFile() as f: f.write(content.encode("utf-8")) f.flush() - storage.store( simple_detail_path, f.name, @@ -99,6 +99,7 @@ def render_simple_detail(project, request, store=False): "hash": content_hash, }, ) + length = os.path.getsize(f.name) storage.store( os.path.join(project.normalized_name, "index.html"), f.name, @@ -109,4 +110,4 @@ def render_simple_detail(project, request, store=False): }, ) - return (content_hash, simple_detail_path) + return {"content_hash": content_hash, "path": simple_detail_path, "length": length} diff --git a/warehouse/routes.py b/warehouse/routes.py index 95cffa522f59..89fad88055c0 100644 --- a/warehouse/routes.py +++ b/warehouse/routes.py @@ -17,7 +17,7 @@ def includeme(config): # Forklift is properly split out into it's own project. warehouse = config.get_settings().get("warehouse.domain") files_url = config.get_settings()["files.url"] - + metadata_url = config.get_settings()["tuf.url"] # Simple Route for health checks. config.add_route("health", "/_health/") @@ -463,6 +463,7 @@ def includeme(config): domain=warehouse, ) config.add_route("packaging.file", files_url) + config.add_route("tuf.metadata", metadata_url) # SES Webhooks config.add_route("ses.hook", "/_/ses-hook/", domain=warehouse) @@ -598,6 +599,7 @@ def includeme(config): ) config.add_redirect("/pypi/", "/", domain=warehouse) config.add_redirect("/packages/{path:.*}", files_url, domain=warehouse) + config.add_redirect("/metadata/{path:.*}", metadata_url, domain=warehouse) # Legacy Action Redirects config.add_pypi_action_redirect("rss", "/rss/updates.xml", domain=warehouse) diff --git a/warehouse/tasks.py b/warehouse/tasks.py index b49b07c63856..c2362a1ea28d 100644 --- a/warehouse/tasks.py +++ b/warehouse/tasks.py @@ -206,8 +206,12 @@ def includeme(config): task_queues=( Queue("default", routing_key="task.#"), Queue("malware", routing_key="malware.#"), + Queue("tuf", routing_key="tuf.#"), ), - task_routes={"warehouse.malware.tasks.*": {"queue": "malware"}}, + task_routes={ + "warehouse.malware.tasks.*": {"queue": "malware"}, + "warehouse.tuf.tasks.*": {"queue": "tuf"}, + }, task_serializer="json", worker_disable_rate_limits=True, REDBEAT_REDIS_URL=s["celery.scheduler_url"], diff --git a/warehouse/tuf/README.md b/warehouse/tuf/README.md new file mode 100644 index 000000000000..9a1c10b15686 --- /dev/null +++ b/warehouse/tuf/README.md @@ -0,0 +1,242 @@ +# General TUF Warehouse implementation Notes + +## Current Warehouse and tools (pip, twine, WebUI) flow investigation + +### twine + 1. post resquest ``/simple/{project}`` and file name to Warehouse + 2. Warehouse proceed with validations + 3. Warehouse uses the ``forklift.legacy.file_upload()`` and writes in the + ``db`` and ``Storage[/packages/{blakeb_256/XX/YY}/{filename}]`` + + +### PyPI WebUI + + 1. from the ``manage.views.ManageProjectRelease()`` request to remove the a + release version using the ``utils.project.remove_project()`` + 2. The file is deleted from ``db``, but not from the `Storage[/packages]` + +### PIP +Using diferent commands ``pip `` + +#### index + 1. Request ``/simple/{project}`` + 2. Warehouse render dynamically the index + ``legacy.api.simple_detail()`` -> ``packaging.utils.simple_details()`` + if the project exists. + +#### download + 1. Call ``pip index`` + 2. Look for the latest version in the simpleindex and request from + ``Storage[/packages/{blakeb_256/XX/YY}/{filename}]`` + +#### install + 1. Call ``pip index`` + 2. Call ``pip install`` + 3. Look into the dependencies + 4. loop for the dependencies + ```mermaid + sequenceDiagram + participant pip + participant warehouse + pip->>warehouse: simple/ + warehouse-->>pip: 404, simple index not found + warehouse->>pip: 200, simple index + loop look in the index + pip->>pip: Get latest version or specific version + end + pip->>warehouse: Get specific version /packages/{blake2b_256/XX/YY}/ + warehouse-->pip: 404, not found + warehouse->>pip: 200, + pip->>pip: Looking for dependencies dependencies + ``` + + +## General flows on Warehouse +```mermaid + flowchart TD + + subgraph pip["pip "] + download + index + install + end + PyPI[PyPI WebUI] + twine + + subgraph warehouse + request["request /simple/{project} dynamic (transversal)"] + subgraph forklift + legacy.file_upload["legacy.file_upload()"] + end + subgraph legacy + api.simple_detail["api.simple_detail()"] + end + subgraph manage + views.ManageProjectRelease + end + subgraph utils + project.remove_project["project.remove_project()"] + end + subgraph packaging + utils._simple_detail["utils._simple_detail()"] + utils.render_simple_index["utils.render_simple_index()"] + end + end + + db[(Database)] + simple[("[local, SaaS]\n/simple/{project}/index.html\n/simple//.html")] + packages[("[local, SaaS]\n/packages/{blake2b_256/XX/YY}/")] + + + download--1-->request + download--2-->packages + install--1-->request + install--2-->packages + index-->request + twine-->request + PyPI-->views.ManageProjectRelease + request-->legacy.file_upload + views.ManageProjectRelease-->project.remove_project + legacy.file_upload--->db + legacy.file_upload--->packages + project.remove_project-->db + request-->api.simple_detail + api.simple_detail-->utils._simple_detail + utils.render_simple_index-.->simple + + + linkStyle 0,2,4,12,13 stroke:blue; + linkStyle 1,3 stroke:green; + linkStyle 5,7,9,10 stroke:yellow; + linkStyle 6,8,11 stroke:red; + style utils.render_simple_index fill:purple + style db fill:black,stroke:grey + style packages fill:black,stroke:grey + style simple fill:purple,stroke:grey +``` + +- Recently was merge [PR 458](https://github.com/pypa/warehouse/pull/8586), that +enables the persistent index for Simple Details. + +## TUF WIP + +This work refactors the [Draft PR](https://github.com/pypa/warehouse/pull/7488) by @ +woodruffw, to build a new repository tool on top of the Python-TUF Metadata API, and +use it instead of the Python-TUF repository tool that was deprecated in v1.0.0. + +**Note to reviewer** + +The current implementation has some development-only components, and lacks a few services for full PEP458 compliance as well as extensive tests. However, it should qualify for a review of the overall architecture and flow (see details in 'Overview' below). Components and functionality that are planned for subsequent PRs are listed in 'Next steps' below. + +### Overview + + ```mermaid + classDiagram + direction LR + class MetadataRepository { + <> + +storage_backend + +key_backend + initialize() + load_role() + bump_role_version() + timestamp_bump_version() + snapshot_bump_version() + snapshot_update_meta() + delegate_targets_roles() + add_targets() + } + class `tuf.interfaces` { + zope.interface.Interface + IKeyService(Interface) + IStorageService(Interface) + IRepositoryService(Interface) + } + class `tuf.services` { + IKeyService + IRepositoryService + IStorageService + LocalKeyService(IKeyService) + LocalStorageService(IStorageService) + RepositoryService(IRepositoryService) + } + class `tuf.tasks` { + init_repository + init_targets_delegation + bump_snapshot + bump_bin_n_roles + add_hashed_targets + } + + class `cli.tuf`{ + dev keypairs + dev init-repo + dev init-delegations + dev add-all-packages + dev add-all-indexes + dev bump-snapshot + dev bump-bin-n-roles + } + + + `tuf.services` <|-- `tuf.interfaces` + `tuf.services` --* MetadataRepository + `tuf.tasks` -- `tuf.services` + `cli.tuf` -- `tuf.tasks` + warehouse -- `cli.tuf` + warehouse -- `tuf.tasks` + ``` + +#### warehouse.tuf.repository + +- ``MetadataRepository`` implements a custom TUF metadata repository tool on top of +the new Python-TUF Metadata API to create and maintain (update, sign, sync with storage) TUF metadata for Warehouse. + + +#### warehouse.tuf.services + +- ``LocalKeyService`` provides a local file storage backend for TUF role keys used by the repository tool (development only!!). +- ``LocalStorageService`` provides a local file storage backend for TUF role metadata used by the repository tool. +- ``RepositoryService`` provides methods for common Warehouse-TUF tasks, using the repository tool. + +#### warehouse.tuf.tasks + +Defines common Warehouse-TUF tasks that use the `RepositoryService` for +- bootstrapping a metadata repository (`init_repository`, `init_targets_delegation`), +- updating metadata upon package upload (`add_hashed_targets`) +- scheduled metadata updates (`bump_bin_n_roles`, `bump_snapshot`) + +#### warehouse.cli.tuf + +Defines development commands for bootstrapping a TUF metadata repository (`keypair`, `init_repo`, `init_delegations`), backsigning existing packages and simple index pages (`add_all_packages`, `add_all_indexes`), and for manually triggering scheduled tasks (`bump_bin_n_roles`, `bump_snapshot`). CLI calls go through `warehouse.cli.tasks`, to take advantage of the Celery/Redis queue. + + +### Next steps: + +- [ ] Polish the new Warehouse metadata repository tool based on review feedback +- [ ] PRs to implement TUF in the Warehouse request flow + - upload target file + - delete target file + - tasks for refreshing indexes/projects +- [ ] Tests + + +## Using the Warehouse development environment for TUF + +Follow the official Warehouse until [``make initdb``](https://warehouse.pypa.io/development/getting-started.html#) + +```shell +$ make inittuf +``` + +The metadata is available at http://localhost:9001/metadata/ + +You can also upload a file using the Warehouse and add the targets using CLI +- Create a user [using Web UI](https://warehouse.pypa.io/development/getting-started.html#viewing-warehouse-in-a-browser) +- Validate the [email](https://warehouse.pypa.io/development/email.html) +- Upload file using ``twine`` + +```shell +docker-compose run --rm web python -m warehouse tuf dev add-all-packages +docker-compose run --rm web python -m warehouse tuf dev add-all-indexes +``` diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py new file mode 100644 index 000000000000..19dd21ccb1c7 --- /dev/null +++ b/warehouse/tuf/__init__.py @@ -0,0 +1,72 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from celery.schedules import crontab + +from warehouse.tuf.interfaces import IKeyService, IRepositoryService, IStorageService +from warehouse.tuf.services import SPEC_VERSION +from warehouse.tuf.tasks import bump_bin_n_roles, bump_snapshot + + +def includeme(config): + config.add_settings( + { + "tuf.keytype": "ed25519", + "tuf.root.threshold": 1, + "tuf.root.expiry": 31536000, + "tuf.snapshot.threshold": 1, + "tuf.snapshot.expiry": 86400, + "tuf.targets.threshold": 2, + "tuf.targets.expiry": 31536000, + "tuf.timestamp.threshold": 1, + "tuf.timestamp.expiry": 86400, + "tuf.bins.threshold": 1, + "tuf.bins.expiry": 31536000, + "tuf.bin-n.threshold": 1, + # NOTE: This is a deviation from PEP 458, as published: the PEP + # stipulates that bin-n metadata expires every 24 hours, which is + # both burdensome for mirrors and requires a large number of redundant + # signing operations even when the targets themselves do not change. + # An amended version of the PEP should be published, at which point + # this note can be removed. + "tuf.bin-n.expiry": 604800, + "tuf.spec_version": SPEC_VERSION, + } + ) + + key_service_class = config.maybe_dotted(config.registry.settings["tuf.key_backend"]) + config.register_service_factory(key_service_class.create_service, IKeyService) + + storage_service_class = config.maybe_dotted( + config.registry.settings["tuf.storage_backend"] + ) + config.register_service_factory( + storage_service_class.create_service, IStorageService + ) + + repository_service_class = config.maybe_dotted( + config.registry.settings["tuf.repository_backend"] + ) + config.register_service_factory( + repository_service_class.create_service, IRepositoryService + ) + + # Per PEP 458: The snapshot and timestamp metadata expire every 24 hours. + # We conservatively bump them every 6 hours. + # Note that bumping the snapshot causes us to bump the timestamp, so we + # only need to explicitly bump the former. + # NOTE: PEP 458 currently specifies that each bin-n role expires every 24 hours, + # but Warehouse sets them to expire every 7 days instead. See the corresponding + # note in tuf/__init__.py. + # We conservatively bump all delegated bins at least once daily. + config.add_periodic_task(crontab(minute=0, hour="*/6"), bump_snapshot) + config.add_periodic_task(crontab(minute=0, hour=0), bump_bin_n_roles) diff --git a/warehouse/tuf/constants.py b/warehouse/tuf/constants.py new file mode 100644 index 000000000000..7e37876f769c --- /dev/null +++ b/warehouse/tuf/constants.py @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import enum + + +@enum.unique +class Role(enum.Enum): + ROOT: str = "root" + SNAPSHOT: str = "snapshot" + TARGETS: str = "targets" + TIMESTAMP: str = "timestamp" + BINS: str = "bins" + BIN_N: str = "bin-n" + + +HASH_ALGORITHM = "blake2b" + +TUF_REPO_LOCK = "tuf-repo" + +BIN_N_COUNT = 32 diff --git a/warehouse/tuf/interfaces.py b/warehouse/tuf/interfaces.py new file mode 100644 index 000000000000..529f447c8ce9 --- /dev/null +++ b/warehouse/tuf/interfaces.py @@ -0,0 +1,105 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zope.interface import Interface + + +class IKeyService(Interface): + def create_service(context, request): + """ + Create the service, given the context and request for which it is being + created. + """ + + def get(rolename): + """Return a key from specific rolename""" + + +class IStorageService(Interface): + def create_service(context, request): + """ + Create the service, given the context and request for which it is being + created. + """ + + def get(rolename, version): + """ + Return metadata from specific role name, optionally specific version. + """ + + def put(file_object, filename): + """ + Stores file object with a specific filename. + + An alias to store() to be compatible with + ``tuf.api.metadata.StorageBackendInterface`` + """ + + def store(file_object, filename): + """ + Stores file object with a specific filename. + """ + + +class IRepositoryService(Interface): + def create_service(context, request): + """ + Create the service, given the context and request for which it is being + created. + """ + + def init_dev_repository(): + """ + Initializes a Metadata Repository from scratch, including a new root. + """ + + def init_targets_delegation(): + """ + Delegate targets role bins further delegates to the bin-n roles, + which sign for all distribution files belonging to registered PyPI + projects. + """ + + def bump_snapshot(): + """ + Bump the Snapshot Metadata Role + """ + + def bump_bin_n_roles(): + """ + Bump all BIN-N delegate roles Metadata + """ + + def add_hashed_targets(targets): + """ + Add hashed Targets + + Args: + targets: list of dictionary with file ``info`` and ``path``. + + ``info`` contains a dict with ``length``, ``hashes`` optionally + ``custom`` nested dictionary. + ``path`` file path + + Example: + ``` + [ + { + "info": { + "hashes": {"blake2b-256": file.blake2_256_digest}, + "length": 256, + "custom": {"key": "value}, + }, + "path": "/xx/yy/file.tar.gz" + } + ] + """ diff --git a/warehouse/tuf/services.py b/warehouse/tuf/services.py new file mode 100644 index 000000000000..753d3fada153 --- /dev/null +++ b/warehouse/tuf/services.py @@ -0,0 +1,493 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import glob +import os.path +import shutil +import warnings + +from contextlib import contextmanager +from datetime import datetime, timedelta +from typing import Dict, List, Tuple + +from securesystemslib.exceptions import StorageError # type: ignore +from securesystemslib.interface import ( # type: ignore + import_ed25519_privatekey_from_file, +) +from securesystemslib.signer import SSlibSigner # type: ignore +from tuf.api.metadata import ( + SPECIFICATION_VERSION, + TOP_LEVEL_ROLE_NAMES, + DelegatedRole, + Delegations, + Key, + Metadata, + MetaFile, + Role, + Root, + Snapshot, + SuccinctRoles, + TargetFile, + Targets, + Timestamp, +) +from tuf.api.serialization.json import JSONSerializer +from zope.interface import implementer + +from warehouse.config import Environment +from warehouse.tuf.constants import BIN_N_COUNT, Role as RoleType +from warehouse.tuf.interfaces import IKeyService, IRepositoryService, IStorageService + +SPEC_VERSION: str = ".".join(SPECIFICATION_VERSION) + + +class InsecureKeyWarning(UserWarning): + pass + + +@implementer(IKeyService) +class LocalKeyService: + """ + A service to read private TUF role keys as local files for development. + + NOTE: Do not use in production! + """ + + def __init__(self, key_path, request): + warnings.warn( + "LocalKeyService is intended only for use in development, you " + "should not use it in production to avoid unnecessary key exposure.", + InsecureKeyWarning, + ) + + self._key_path = key_path + self._request = request + + @classmethod + def create_service(cls, context, request): + return cls(request.registry.settings["tuf.key.path"], request) + + def get(self, rolename): + """ + Returns a list of ``securesystemslib.signer.Signer`` objects for passed + TUF role name from configured TUF key path. + """ + privkey_path = os.path.join(self._key_path, "tufkeys", f"{rolename}*") + role_keys = glob.glob(privkey_path) + keys_sslib = [ + SSlibSigner( + import_ed25519_privatekey_from_file( + key, self._request.registry.settings[f"tuf.{rolename}.secret"] + ) + ) + for key in role_keys + if "pub" not in key + ] + + return keys_sslib + + +@implementer(IStorageService) +class LocalStorageService: + """ + A storage service with methods to read and write TUF role metadata as local files. + """ + + def __init__(self, repo_path): + self._repo_path = repo_path + + @classmethod + def create_service(cls, context, request): + return cls( + request.registry.settings["tuf.repo.path"], + ) + + @contextmanager + def get(self, role, version=None): + """ + Yields TUF role metadata file object for the passed role name, from the + configured TUF repo path, optionally at the passed version (latest if None). + """ + + if role == RoleType.TIMESTAMP.value: + filename = os.path.join(self._repo_path, f"{role}.json") + else: + if version is None: + filenames = glob.glob(os.path.join(self._repo_path, f"*.{role}.json")) + versions = [ + int(name.split("/")[-1].split(".", 1)[0]) for name in filenames + ] + try: + version = max(versions) + except ValueError: + version = 1 + + filename = os.path.join(self._repo_path, f"{version}.{role}.json") + + file_object = None + try: + file_object = open(filename, "rb") + yield file_object + except OSError: + raise StorageError(f"Can't open {filename}") + finally: + if file_object is not None: + file_object.close() + + def put(self, file_object, filename): + """ + Writes passed file object to configured TUF repo path using the passed filename. + """ + file_path = os.path.join(self._repo_path, filename) + if not file_object.closed: + file_object.seek(0) + + try: + with open(file_path, "wb") as destination_file: + shutil.copyfileobj(file_object, destination_file) + destination_file.flush() + os.fsync(destination_file.fileno()) + except OSError: + raise StorageError(f"Can't write file {filename}") + + def store(self, file_object, filename): + self.put(file_object, filename) + + +@implementer(IRepositoryService) +class RepositoryService: + """ + A repository service to create and maintain TUF role metadata. + """ + + def __init__(self, storage_service, key_service, request): + self._storage_backend = storage_service + self._key_storage_backend = key_service + self._request = request + + @classmethod + def create_service(cls, context, request): + """ + Creates a new repository service object configuring services to read and write + TUF role metadata (``IStorageService``) and to read private keys + (``IKeyService``). + """ + storage_service = request.find_service(IStorageService) + key_service = request.find_service(IKeyService) + return cls(storage_service, key_service, request) + + def _get_bit_length(self): + """ + Returns a 'hash bin delegation' management object. + """ + if self._request.registry.settings["warehouse.env"] == Environment.development: + bit_length = 8 + else: + bit_length = BIN_N_COUNT + + return bit_length + + def _is_initialized(self) -> bool: + """ + Returns True if any top-level role metadata exists, False otherwise. + """ + try: + if any(role for role in TOP_LEVEL_ROLE_NAMES if self._load(role)): + return True + except StorageError: + pass + + return False + + def _load(self, role_name: str) -> Metadata: + """ + Loads latest version of metadata for rolename using configured storage backend. + + NOTE: The storage backend is expected to translate rolenames to filenames and + figure out the latest version. + """ + return Metadata.from_file(role_name, None, self._storage_backend) + + def _sign(self, role: Metadata, role_name: str) -> None: + """ + Re-signs metadata with role-specific key from global key store. + + The metadata role type is used as default key id. This is only allowed for + top-level roles. + """ + role.signatures.clear() + for signer in self._key_storage_backend.get(role_name): + role.sign(signer, append=True) + + def _persist(self, role: Metadata, role_name: str) -> None: + """ + Persists metadata using the configured storage backend. + + The metadata role type is used as default role name. This is only allowed for + top-level roles. All names but 'timestamp' are prefixed with a version number. + """ + filename = f"{role_name}.json" + + if role_name != RoleType.TIMESTAMP.value: + filename = f"{role.signed.version}.{filename}" + + role.to_file(filename, JSONSerializer(), self._storage_backend) + + def _bump_expiry(self, role: Metadata, expiry_id: str) -> None: + """ + Bumps metadata expiration date by role-specific interval. + + The metadata role type is used as default expiry id. This is only allowed for + top-level roles. + """ + # FIXME: Review calls to _bump_expiry. Currently, it is called in every + # update-sign-persist cycle. + # PEP 458 is unspecific about when to bump expiration, e.g. in the course of a + # consistent snapshot only 'timestamp' is bumped: + # https://www.python.org/dev/peps/pep-0458/#producing-consistent-snapshots + role.signed.expires = datetime.now().replace(microsecond=0) + timedelta( + seconds=self._request.registry.settings[f"tuf.{expiry_id}.expiry"] + ) + + def _bump_version(self, role: Metadata) -> None: + """ + Bumps metadata version by 1. + """ + role.signed.version += 1 + + def _update_timestamp(self, snapshot_version: int) -> None: + """ + Loads 'timestamp', updates meta info about passed 'snapshot' metadata, + bumps version and expiration, signs and persists. + """ + timestamp = self._load(Timestamp.type) + timestamp.signed.snapshot_meta = MetaFile(version=snapshot_version) + + self._bump_version(timestamp) + self._bump_expiry(timestamp, RoleType.TIMESTAMP.value) + self._sign(timestamp, RoleType.TIMESTAMP.value) + self._persist(timestamp, RoleType.TIMESTAMP.value) + + def _update_snapshot(self, targets_meta: List[Tuple[str, int]]) -> int: + """ + Loads 'snapshot', updates meta info about passed 'targets' metadata, bumps + version and expiration, signs and persists. Returns new snapshot version, e.g. + to update 'timestamp'. + """ + snapshot = self._load(Snapshot.type) + + for name, version in targets_meta: + snapshot.signed.meta[f"{name}.json"] = MetaFile(version=version) + + self._bump_expiry(snapshot, RoleType.SNAPSHOT.value) + self._bump_version(snapshot) + self._sign(snapshot, RoleType.SNAPSHOT.value) + self._persist(snapshot, RoleType.SNAPSHOT.value) + + return snapshot.signed.version + + def init_dev_repository(self) -> None: + """ + Creates development TUF top-level role metadata (root, targets, snapshot, + timestamp). + + FIXME: In production 'root' and 'targets' roles require offline singing keys, + which may not be available at the time of initializing this metadata. + """ + # FIXME: Is this a meaningful check? It is rather superficial. + if self._is_initialized(): + raise FileExistsError("TUF Metadata Repository files already exists.") + + # Bootstrap default top-level metadata to be updated below if necessary + targets = Targets() + snapshot = Snapshot() + timestamp = Timestamp() + root = Root() + + # Populate public key store, and define trusted signing keys and required + # signature thresholds for each top-level role in 'root'. + for role_name in TOP_LEVEL_ROLE_NAMES: + threshold = self._request.registry.settings[f"tuf.{role_name}.threshold"] + signers = self._key_storage_backend.get(role_name) + + # FIXME: Is this a meaningful check? Should we check more than just the + # threshold? And maybe in a different place, e.g. independently of + # bootstrapping the metadata, because in production we do not have access to + # all top-level role signing keys at the time of bootstrapping the metadata. + assert len(signers) >= threshold, ( + f"not enough keys ({len(signers)}) for " + f"signing threshold '{threshold}'" + ) + + root.roles[role_name] = Role([], threshold) # type: ignore + for signer in signers: + root.add_key(Key.from_securesystemslib_key(signer.key_dict), role_name) + + # Add signature wrapper, bump expiration, and sign and persist + for role in [targets, snapshot, timestamp, root]: + metadata = Metadata(role) # type: ignore + self._bump_expiry(metadata, role.type) + self._sign(metadata, role.type) + self._persist(metadata, role.type) + + def init_targets_delegation(self) -> None: + """ + Creates TUF metadata for hash bin delegated targets roles (bins, bin-n). + + Metadata is created for one 'bins' role and a configured number of 'bin-n' + roles. It is populated with configured expiration times, signature thresholds + and verification keys, and signed and persisted using the configured key and + storage services. + + FIXME: In production the 'bins' role requires an offline singing key, which may + not be available at the time of initializing this metadata. + + FIXME: Consider combining 'init_dev_repository' and 'init_targets_delegation' + to create and persist all initial metadata at once, at version 1. + + """ + # Track names and versions of new and updated targets for 'snapshot' update + targets_meta = [] + + # Update top-level 'targets' role, to delegate trust for all target files to + # 'bins' role, defining target path patterns, trusted signing keys and required + # signature thresholds. + targets = self._load(Targets.type) + targets.signed.delegations = Delegations(keys={}, roles={}) + targets.signed.delegations.roles[ # type: ignore + RoleType.BINS.value + ] = DelegatedRole( + name=RoleType.BINS.value, + keyids=[], + threshold=self._request.registry.settings[ + f"tuf.{RoleType.BINS.value}.threshold" + ], + terminating=False, + paths=["*/*", "*/*/*/*"], + ) + + for signer in self._key_storage_backend.get(RoleType.BINS.value): + targets.signed.add_key( + Key.from_securesystemslib_key(signer.key_dict), RoleType.BINS.value + ) + + # Bump version and expiration, and sign and persist updated 'targets'. + self._bump_version(targets) + self._bump_expiry(targets, RoleType.TARGETS.value) + self._sign(targets, RoleType.TARGETS.value) + self._persist(targets, RoleType.TARGETS.value) + + targets_meta.append((RoleType.TARGETS.value, targets.signed.version)) + + succinct_roles = SuccinctRoles( + [], 1, self._get_bit_length(), RoleType.BIN_N.value + ) + # Create new 'bins' role and delegate trust from 'bins' for all target files to + # 'bin-n' roles based on file path hash prefixes, a.k.a hash bin delegation. + bins = Metadata(Targets()) + bins.signed.delegations = Delegations(keys={}, succinct_roles=succinct_roles) + for delegated_name in succinct_roles.get_roles(): + for signer in self._key_storage_backend.get(RoleType.BIN_N.value): + bins.signed.add_key( + Key.from_securesystemslib_key(signer.key_dict), delegated_name + ) + bin_n = Metadata(Targets()) + self._bump_expiry(bin_n, RoleType.BIN_N.value) + self._sign(bin_n, RoleType.BIN_N.value) + self._persist(bin_n, delegated_name) + + # Bump expiration, and sign and persist new 'bins' role. + self._bump_expiry(bins, RoleType.BINS.value) + self._sign(bins, RoleType.BINS.value) + self._persist(bins, RoleType.BINS.value) + + targets_meta.append((RoleType.BINS.value, bins.signed.version)) + + self._update_timestamp(self._update_snapshot(targets_meta)) + + def add_hashed_targets(self, targets: List[TargetFile]) -> None: + """ + Updates 'bin-n' roles metadata, assigning each passed target to the correct bin. + + Assignment is based on the hash prefix of the target file path. All metadata is + signed and persisted using the configured key and storage services. + + Updating 'bin-n' also updates 'snapshot' and 'timestamp'. + """ + # Group target files by responsible 'bin-n' roles + bin_n = self._load(RoleType.BINS.value) + bin_n_succinct_roles = bin_n.signed.delegations.succinct_roles + bin_n_target_groups: Dict[str, List[TargetFile]] = {} + + for target in targets: + bin_n_name = bin_n_succinct_roles.get_role_for_target(target.path) + + if bin_n_name not in bin_n_target_groups: + bin_n_target_groups[bin_n_name] = [] + + bin_n_target_groups[bin_n_name].append(target) + + # Update target file info in responsible 'bin-n' roles, bump version and expiry + # and sign and persist + targets_meta = [] + for bin_n_name, target_files in bin_n_target_groups.items(): + bin_n = self._load(bin_n_name) + + for target_file in target_files: + bin_n.signed.targets[target_file.path] = target_file + + self._bump_expiry(bin_n, RoleType.BIN_N.value) + self._bump_version(bin_n) + self._sign(bin_n, RoleType.BIN_N.value) + self._persist(bin_n, bin_n_name) + + targets_meta.append((bin_n_name, bin_n.signed.version)) + + self._update_timestamp(self._update_snapshot(targets_meta)) + + def bump_bin_n_roles(self) -> None: + """ + Bumps version and expiration date of 'bin-n' role metadata (multiple). + + The version numbers are incremented by one, the expiration dates are renewed + using a configured expiration interval, and the metadata is signed and persisted + using the configured key and storage services. + + Updating 'bin-n' also updates 'snapshot' and 'timestamp'. + """ + bin_n = self._load(RoleType.BINS.value) + bin_n_succinct_roles = bin_n.signed.delegations.succinct_roles + targets_meta = [] + for bin_n_name in bin_n_succinct_roles.get_roles(): + bin_n = self._load(bin_n_name) + + self._bump_expiry(bin_n, RoleType.BIN_N.value) + self._bump_version(bin_n) + self._sign(bin_n, RoleType.BIN_N.value) + self._persist(bin_n, bin_n_name) + + targets_meta.append((bin_n_name, bin_n.signed.version)) + + self._update_timestamp(self._update_snapshot(targets_meta)) + + def bump_snapshot(self) -> None: + """ + Bumps version and expiration date of TUF 'snapshot' role metadata. + + The version number is incremented by one, the expiration date renewed using a + configured expiration interval, and the metadata is signed and persisted using + the configured key and storage services. + + Updating 'snapshot' also updates 'timestamp'. + """ + self._update_timestamp(self._update_snapshot([])) diff --git a/warehouse/tuf/tasks.py b/warehouse/tuf/tasks.py new file mode 100644 index 000000000000..91a33cdfffb9 --- /dev/null +++ b/warehouse/tuf/tasks.py @@ -0,0 +1,58 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import redis + +from warehouse.tasks import task +from warehouse.tuf.constants import TUF_REPO_LOCK +from warehouse.tuf.services import IRepositoryService, TargetFile # noqa F401 + + +@task(bind=True, ignore_result=True, acks_late=True) +def bump_snapshot(task, request): + r = redis.StrictRedis.from_url(request.registry.settings["celery.scheduler_url"]) + + with r.lock(TUF_REPO_LOCK): + repository_service = request.find_service(IRepositoryService) + repository_service.bump_snapshot() + + +@task(bind=True, ignore_result=True, acks_late=True) +def bump_bin_n_roles(task, request): + r = redis.StrictRedis.from_url(request.registry.settings["celery.scheduler_url"]) + + with r.lock(TUF_REPO_LOCK): + repository_service = request.find_service(IRepositoryService) + repository_service.bump_bin_n_roles() + + +@task(bind=True, ignore_result=True, acks_late=True) +def init_dev_repository(task, request): + repository_service = request.find_service(IRepositoryService) + repository_service.init_dev_repository() + + +@task(bind=True, ignore_result=True, acks_late=True) +def init_targets_delegation(task, request): + r = redis.StrictRedis.from_url(request.registry.settings["celery.scheduler_url"]) + + with r.lock(TUF_REPO_LOCK): + repository_service = request.find_service(IRepositoryService) + repository_service.init_targets_delegation() + + +@task(bind=True, ignore_result=True, acks_late=True) +def add_hashed_targets(task, request, targets): + r = redis.StrictRedis.from_url(request.registry.settings["celery.scheduler_url"]) + + with r.lock(TUF_REPO_LOCK): + repository_service = request.find_service(IRepositoryService) + repository_service.add_hashed_targets(targets)