From cb6285c11fd233b9bd57a8abc6ebf0d4f86e7f17 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 23 Sep 2024 13:15:00 -0400 Subject: [PATCH 01/14] remove Provenance.provenance_digest The latest PEP 740 language doesn't include this, so we don't need it. Signed-off-by: William Woodruff --- requirements/main.in | 1 - requirements/main.txt | 4 +- tests/unit/attestations/test_services.py | 6 --- warehouse/attestations/models.py | 8 +--- warehouse/attestations/services.py | 8 +--- ...0dd_remove_provenance_provenance_digest.py | 42 +++++++++++++++++++ 6 files changed, 45 insertions(+), 24 deletions(-) create mode 100644 warehouse/migrations/versions/2af8015830dd_remove_provenance_provenance_digest.py diff --git a/requirements/main.in b/requirements/main.in index c94115b8aa6a..521bbf70fb93 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -61,7 +61,6 @@ requests requests-aws4auth redis>=2.8.0,<6.0.0 rfc3986 -rfc8785 sentry-sdk setuptools sigstore~=3.3.0 diff --git a/requirements/main.txt b/requirements/main.txt index 83e25bf5a27e..4a1fab03f7fe 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1993,9 +1993,7 @@ rfc3986==2.0.0 \ rfc8785==0.1.3 \ --hash=sha256:167efe3b5cdd09dded9d0cfc8fec1f48f5cd9f8f13b580ada4efcac138925048 \ --hash=sha256:6116062831c62e7ac5d027973a1fe07b601ccd854bca4a2b401938a00a20b0c0 - # via - # -r requirements/main.in - # sigstore + # via sigstore rich==13.8.1 \ --hash=sha256:1760a3c0848469b97b558fc61c85233e3dafb69c7a071b4d60c38099d3cd4c06 \ --hash=sha256:8260cda28e3db6bf04d2d1ef4dbc03ba80a824c88b0e7668a0f23126a424844a diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index cc54d1564aa6..c2185885817d 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -9,12 +9,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import hashlib import json import pretend import pytest -import rfc8785 from pydantic import TypeAdapter from pypi_attestations import ( @@ -52,10 +50,6 @@ def test_build_provenance(self, db_request, dummy_attestation): provenance = service.build_provenance(db_request, file, [dummy_attestation]) assert isinstance(provenance, DatabaseProvenance) - assert ( - provenance.provenance_digest - == hashlib.sha256(rfc8785.dumps(provenance.provenance)).hexdigest() - ) assert provenance.file == file assert file.provenance == provenance diff --git a/warehouse/attestations/models.py b/warehouse/attestations/models.py index f210735a4cf8..5f09f37596be 100644 --- a/warehouse/attestations/models.py +++ b/warehouse/attestations/models.py @@ -16,7 +16,7 @@ from uuid import UUID from sqlalchemy import ForeignKey, orm -from sqlalchemy.dialects.postgresql import CITEXT, JSONB +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, mapped_column from warehouse import db @@ -44,9 +44,3 @@ class Provenance(db.Model): # This JSONB has the structure of a PEP 740 provenance object. provenance: Mapped[dict] = mapped_column(JSONB, nullable=False, deferred=True) - - # The SHA-2/256 digest of the provenance object stored in this row. - # Postgres uses a compact binary representation under the hood and is - # unlikely to provide a permanently stable serialization, so this is the - # hash of the RFC 8785 serialization. - provenance_digest: Mapped[str] = mapped_column(CITEXT) diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index 29161555a4a9..0f2a2065b6db 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -12,11 +12,9 @@ from __future__ import annotations -import hashlib import typing import warnings -import rfc8785 import sentry_sdk from pydantic import TypeAdapter, ValidationError @@ -93,11 +91,7 @@ def _build_provenance( mode="json" ) - db_provenance = DatabaseProvenance( - file=file, - provenance=provenance, - provenance_digest=hashlib.sha256(rfc8785.dumps(provenance)).hexdigest(), - ) + db_provenance = DatabaseProvenance(file=file, provenance=provenance) return db_provenance diff --git a/warehouse/migrations/versions/2af8015830dd_remove_provenance_provenance_digest.py b/warehouse/migrations/versions/2af8015830dd_remove_provenance_provenance_digest.py new file mode 100644 index 000000000000..264bd332ab4b --- /dev/null +++ b/warehouse/migrations/versions/2af8015830dd_remove_provenance_provenance_digest.py @@ -0,0 +1,42 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +remove Provenance.provenance_digest + +Revision ID: 2af8015830dd +Revises: a8050411bc65 +Create Date: 2024-09-23 17:09:16.199384 +""" + +import sqlalchemy as sa + +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "2af8015830dd" +down_revision = "a8050411bc65" + + +def upgrade(): + op.drop_column("provenance", "provenance_digest") + + +def downgrade(): + op.add_column( + "provenance", + sa.Column( + "provenance_digest", + postgresql.CITEXT(), + autoincrement=False, + nullable=False, + ), + ) From 6d4b8cb8c768feb890510b17eaa0956d0a0025d2 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 23 Sep 2024 14:01:42 -0400 Subject: [PATCH 02/14] WIP route for provenance retrieval Signed-off-by: William Woodruff --- warehouse/attestations/views.py | 38 +++++++++++++++++++++++++++++++++ warehouse/packaging/models.py | 19 +++++++++++++++++ warehouse/routes.py | 9 ++++++++ 3 files changed, 66 insertions(+) create mode 100644 warehouse/attestations/views.py diff --git a/warehouse/attestations/views.py b/warehouse/attestations/views.py new file mode 100644 index 000000000000..b73bd61c7f7c --- /dev/null +++ b/warehouse/attestations/views.py @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pyramid.httpexceptions import HTTPForbidden, HTTPNotFound +from pyramid.request import Request +from pyramid.view import view_config + +from warehouse.admin.flags import AdminFlagValue +from warehouse.packaging.models import File + + +@view_config( + route_name="attestations.provenance", + context=File, + require_methods=["GET"], + renderer="json", + require_csrf=False, + has_translations=False, +) +def provenance_for_release_file(file: File, request: Request): + if request.flags.enabled(AdminFlagValue.DISABLE_PEP740): + return HTTPForbidden(json={"message": "Attestations temporarily disabled"}) + + if not file.provenance: + return HTTPNotFound( + json={"message": f"No provenance available for {file.filename}"} + ) + + return file.provenance.provenance diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 949635f6e7c9..76299f95db15 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -163,6 +163,25 @@ def __contains__(self, project): return True +class FileFactory: + def __init__(self, request): + self.request = request + + def __getitem__(self, filename): + try: + return self.request.db.query(File).filter(File.filename == filename).one() + except NoResultFound: + raise KeyError from None + + def __contains__(self, filename): + try: + self[filename] + except KeyError: + return False + else: + return True + + class LifecycleStatus(enum.StrEnum): QuarantineEnter = "quarantine-enter" QuarantineExit = "quarantine-exit" diff --git a/warehouse/routes.py b/warehouse/routes.py index 9cb6e76969d6..60642ef436c6 100644 --- a/warehouse/routes.py +++ b/warehouse/routes.py @@ -576,6 +576,15 @@ def includeme(config): domain=warehouse, ) + # PEP 740 URLs + config.add_route( + "attestations.provenance", + "/_/provenance/{filename}/", + factory="warehouse.packaging.models:FileFactory", + traverse="/{filename}", + domain=warehouse, + ) + # Mock URLs config.add_route( "mock.billing.checkout-session", From fb69073bfed28809ee6e28b065e186e809cb6ea5 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 23 Sep 2024 14:47:22 -0400 Subject: [PATCH 03/14] connect persistence, get tests passing Signed-off-by: William Woodruff --- tests/conftest.py | 10 + ...leproject-3.0.0.tar.gz.publish.attestation | 49 +++ tests/functional/forklift/test_legacy.py | 80 ++++- tests/unit/forklift/test_legacy.py | 340 ++---------------- tests/unit/test_config.py | 1 + tests/unit/test_routes.py | 8 + warehouse/config.py | 17 +- warehouse/forklift/legacy.py | 124 ++----- 8 files changed, 223 insertions(+), 406 deletions(-) create mode 100644 tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation diff --git a/tests/conftest.py b/tests/conftest.py index b66e02672934..8a773b0ad9ad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,6 +45,8 @@ from warehouse.accounts import services as account_services from warehouse.accounts.interfaces import ITokenService, IUserService from warehouse.admin.flags import AdminFlag, AdminFlagValue +from warehouse.attestations import services as attestations_services +from warehouse.attestations.interfaces import IIntegrityService from warehouse.email import services as email_services from warehouse.email.interfaces import IEmailSender from warehouse.helpdesk import services as helpdesk_services @@ -174,6 +176,7 @@ def pyramid_services( project_service, github_oidc_service, activestate_oidc_service, + integrity_service, macaroon_service, helpdesk_service, ): @@ -195,6 +198,7 @@ def pyramid_services( services.register_service( activestate_oidc_service, IOIDCPublisherService, None, name="activestate" ) + services.register_service(integrity_service, IIntegrityService, None) services.register_service(macaroon_service, IMacaroonService, None, name="") services.register_service(helpdesk_service, IHelpDeskService, None) @@ -326,6 +330,7 @@ def get_app_config(database, nondefaults=None): "docs.backend": "warehouse.packaging.services.LocalDocsStorage", "sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage", "billing.backend": "warehouse.subscriptions.services.MockStripeBillingService", + "integrity.backend": "warehouse.attestations.services.NullIntegrityService", "billing.api_base": "http://stripe:12111", "billing.api_version": "2020-08-27", "mail.backend": "warehouse.email.services.SMTPEmailSender", @@ -557,6 +562,11 @@ def dummy_attestation(): ) +@pytest.fixture +def integrity_service(db_session): + return attestations_services.NullIntegrityService(db_session) + + @pytest.fixture def macaroon_service(db_session): return macaroon_services.DatabaseMacaroonService(db_session) diff --git a/tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation b/tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation new file mode 100644 index 000000000000..76ecef058bf5 --- /dev/null +++ b/tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation @@ -0,0 +1,49 @@ +{ + "version": 1, + "verification_material": { + "certificate": "MIIC6zCCAnGgAwIBAgIUFgmhIYx8gvBGePCTacG/4kbBdRwwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjQwODI5MTcwOTM5WhcNMjQwODI5MTcxOTM5WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEtGrMPml4OtsRJ3Z6qRahs0kHCZxP4n9fvrJE957WVxgAGg4k6a1PbRJY9nT9wKpRrZmKV++AgA9ndhdruXXaAKOCAZAwggGMMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQUosNvhYEuTPfgyU/dZfu93lFGRNswHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjpKFWixi4YZD8wQAYDVR0RAQH/BDYwNIEyOTE5NDM2MTU4MjM2LWNvbXB1dGVAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20wKQYKKwYBBAGDvzABAQQbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMCsGCisGAQQBg78wAQgEHQwbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGRnx0/aQAABAMARzBFAiBogvcKHIIR9FcX1vQgDhGtAl0XQoMRiEB3OdUWO94P1gIhANdJlyISdtvVrHes25dWKTLepy+IzQmzfQU/S7cxWHmOMAoGCCqGSM49BAMDA2gAMGUCMGe2xTiuenbjdt1d2e4IaCiwRh2G4KAtyujRESSSUbpuGme/o9ouiApeONBv2CvvGAIxAOEkAGFO3aALE3IPNosxqaz9MbqJOdmYhB1Cz1D7xbFc/m243VxJWxaC/uOFEpyiYQ==", + "transparency_entries": [ + { + "logIndex": "125970014", + "logId": { + "keyId": "wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0=" + }, + "kindVersion": { + "kind": "dsse", + "version": "0.0.1" + }, + "integratedTime": "1724951379", + "inclusionPromise": { + "signedEntryTimestamp": "MEUCIQCHrKFTeXNY432S0bUSBS69S8d5JnNcDXa41q6OEvxEwgIgaZstc5Jpm0IgwFC7RDTXYEAKk+3aG/MkRkaPdJdyn8U=" + }, + "inclusionProof": { + "logIndex": "4065752", + "rootHash": "7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=", + "treeSize": "4065754", + "hashes": [ + "NwJgWJoxjearbnEIT9bnWXpzo0LGNrR1cpWId0g66rE=", + "kLjpW3Eh7pQJNOvyntghzF57tcfqk2IzX7cqiBDgGf8=", + "FW8y9LQ1i3q+MnbeGJipKGl4VfX1zRBOD7TmhbEw7uI=", + "mKcbGJDJ/+buNbXy9Eyv94nVoAyUauuIlN3cJg3qSBY=", + "5VytqqAHhfRkRWMrY43UXWCnRBb7JwElMlKpY5JueBc=", + "mZJnD39LTKdis2wUTz1OOMx3r7HwgJh9rnb2VwiPzts=", + "MXZOQFJFiOjREF0xwMOCXu29HwTchjTtl/BeFoI51wY=", + "g8zCkHnLwO3LojK7g5AnqE8ezSNRnCSz9nCL5GD3a8A=", + "RrZsD/RSxNoujlvq/MsCEvLSkKZfv0jmQM9Kp7qbJec=", + "QxmVWsbTp4cClxuAkuT51UH2EY7peHMVGKq7+b+cGwQ=", + "Q2LAtNzOUh+3PfwfMyNxYb06fTQmF3VeTT6Fr6Upvfc=", + "ftwAu6v62WFDoDmcZ1JKfrRPrvuiIw5v3BvRsgQj7N8=" + ], + "checkpoint": { + "envelope": "rekor.sigstore.dev - 1193050959916656506\n4065754\n7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=\n\n— rekor.sigstore.dev wNI9ajBGAiEAhMomhZHOTNB5CVPO98CMXCv01ZlIF+C+CgzraAB01r8CIQCEuXbv6aqguUpB/ig5eXRIbarvxLXkg3nX48DzambktQ==\n" + } + }, + "canonicalizedBody": "eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiZHNzZSIsInNwZWMiOnsiZW52ZWxvcGVIYXNoIjp7ImFsZ29yaXRobSI6InNoYTI1NiIsInZhbHVlIjoiOWRiNGJjMzE3MTgyZWI3NzljNDIyY2Q0NGI2ZDdlYTk5ZWM1M2Q3M2JiY2ZjZWVmZTIyNWVlYjQ3NTQyMjc4OCJ9LCJwYXlsb2FkSGFzaCI6eyJhbGdvcml0aG0iOiJzaGEyNTYiLCJ2YWx1ZSI6IjlkYjY0MjlhOTkzZGFiYTI4NzAwODk2ZTY2MzNjNzkxYWE0MDM3ODQ4NjJiYzY2MDBkM2E4NjYwMGQzYjA1NjMifSwic2lnbmF0dXJlcyI6W3sic2lnbmF0dXJlIjoiTUVVQ0lCaGlOL25NR0w3aHpZQk9QQjlUTGtuaEdTZEtuQ0Q0ekI3TDV5ZXc0QmJ3QWlFQXJzOHl6MCtCT2NnSEtzS0JzTXVOeVlhREdaRTBVV0JuMEdwNVpGMzUvU2M9IiwidmVyaWZpZXIiOiJMUzB0TFMxQ1JVZEpUaUJEUlZKVVNVWkpRMEZVUlMwdExTMHRDazFKU1VNMmVrTkRRVzVIWjBGM1NVSkJaMGxWUm1kdGFFbFplRGhuZGtKSFpWQkRWR0ZqUnk4MGEySkNaRkozZDBObldVbExiMXBKZW1vd1JVRjNUWGNLVG5wRlZrMUNUVWRCTVZWRlEyaE5UV015Ykc1ak0xSjJZMjFWZFZwSFZqSk5ValIzU0VGWlJGWlJVVVJGZUZaNllWZGtlbVJIT1hsYVV6RndZbTVTYkFwamJURnNXa2RzYUdSSFZYZElhR05PVFdwUmQwOUVTVFZOVkdOM1QxUk5OVmRvWTA1TmFsRjNUMFJKTlUxVVkzaFBWRTAxVjJwQlFVMUdhM2RGZDFsSUNrdHZXa2w2YWpCRFFWRlpTVXR2V2tsNmFqQkVRVkZqUkZGblFVVjBSM0pOVUcxc05FOTBjMUpLTTFvMmNWSmhhSE13YTBoRFduaFFORzQ1Wm5aeVNrVUtPVFUzVjFaNFowRkhaelJyTm1FeFVHSlNTbGs1YmxRNWQwdHdVbkphYlV0V0t5dEJaMEU1Ym1Sb1pISjFXRmhoUVV0UFEwRmFRWGRuWjBkTlRVRTBSd3BCTVZWa1JIZEZRaTkzVVVWQmQwbElaMFJCVkVKblRsWklVMVZGUkVSQlMwSm5aM0pDWjBWR1FsRmpSRUY2UVdSQ1owNVdTRkUwUlVablVWVnZjMDUyQ21oWlJYVlVVR1puZVZVdlpGcG1kVGt6YkVaSFVrNXpkMGgzV1VSV1VqQnFRa0puZDBadlFWVXpPVkJ3ZWpGWmEwVmFZalZ4VG1wd1MwWlhhWGhwTkZrS1drUTRkMUZCV1VSV1VqQlNRVkZJTDBKRVdYZE9TVVY1VDFSRk5VNUVUVEpOVkZVMFRXcE5Na3hYVG5aaVdFSXhaRWRXUVZwSFZqSmFWM2gyWTBkV2VRcE1iV1I2V2xoS01tRlhUbXhaVjA1cVlqTldkV1JETldwaU1qQjNTMUZaUzB0M1dVSkNRVWRFZG5wQlFrRlJVV0poU0ZJd1kwaE5Oa3g1T1doWk1rNTJDbVJYTlRCamVUVnVZakk1Ym1KSFZYVlpNamwwVFVOelIwTnBjMGRCVVZGQ1p6YzRkMEZSWjBWSVVYZGlZVWhTTUdOSVRUWk1lVGxvV1RKT2RtUlhOVEFLWTNrMWJtSXlPVzVpUjFWMVdUSTVkRTFKUjB0Q1oyOXlRbWRGUlVGa1dqVkJaMUZEUWtoM1JXVm5RalJCU0ZsQk0xUXdkMkZ6WWtoRlZFcHFSMUkwWXdwdFYyTXpRWEZLUzFoeWFtVlFTek12YURSd2VXZERPSEEzYnpSQlFVRkhVbTU0TUM5aFVVRkJRa0ZOUVZKNlFrWkJhVUp2WjNaalMwaEpTVkk1Um1OWUNqRjJVV2RFYUVkMFFXd3dXRkZ2VFZKcFJVSXpUMlJWVjA4NU5GQXhaMGxvUVU1a1NteDVTVk5rZEhaV2NraGxjekkxWkZkTFZFeGxjSGtyU1hwUmJYb0tabEZWTDFNM1kzaFhTRzFQVFVGdlIwTkRjVWRUVFRRNVFrRk5SRUV5WjBGTlIxVkRUVWRsTW5oVWFYVmxibUpxWkhReFpESmxORWxoUTJsM1VtZ3lSd28wUzBGMGVYVnFVa1ZUVTFOVlluQjFSMjFsTDI4NWIzVnBRWEJsVDA1Q2RqSkRkblpIUVVsNFFVOUZhMEZIUms4ellVRk1SVE5KVUU1dmMzaHhZWG81Q2sxaWNVcFBaRzFaYUVJeFEzb3hSRGQ0WWtaakwyMHlORE5XZUVwWGVHRkRMM1ZQUmtWd2VXbFpVVDA5Q2kwdExTMHRSVTVFSUVORlVsUkpSa2xEUVZSRkxTMHRMUzBLIn1dfX0=" + } + ] + }, + "envelope": { + "statement": "eyJfdHlwZSI6Imh0dHBzOi8vaW4tdG90by5pby9TdGF0ZW1lbnQvdjEiLCJzdWJqZWN0IjpbeyJuYW1lIjoic2FtcGxlcHJvamVjdC0zLjAuMC50YXIuZ3oiLCJkaWdlc3QiOnsic2hhMjU2IjoiMTE3ZWQ4OGU1ZGIwNzNiYjkyOTY5YTc1NDU3NDVmZDk3N2VlODViNzAxOTcwNmRkMjU2YTY0MDU4ZjcwOTYzZCJ9fV0sInByZWRpY2F0ZVR5cGUiOiJodHRwczovL2RvY3MucHlwaS5vcmcvYXR0ZXN0YXRpb25zL3B1Ymxpc2gvdjEiLCJwcmVkaWNhdGUiOm51bGx9", + "signature": "MEUCIBhiN/nMGL7hzYBOPB9TLknhGSdKnCD4zB7L5yew4BbwAiEArs8yz0+BOcgHKsKBsMuNyYaDGZE0UWBn0Gp5ZF35/Sc=" + } +} diff --git a/tests/functional/forklift/test_legacy.py b/tests/functional/forklift/test_legacy.py index ddbadc5139a5..f6f216469981 100644 --- a/tests/functional/forklift/test_legacy.py +++ b/tests/functional/forklift/test_legacy.py @@ -13,17 +13,24 @@ import base64 from http import HTTPStatus +from pathlib import Path import pymacaroons import pytest from webob.multidict import MultiDict +from tests.common.db.oidc import GitHubPublisherFactory +from tests.common.db.packaging import ProjectFactory, RoleFactory from warehouse.macaroons import caveats -from ...common.db.accounts import UserFactory +from ...common.db.accounts import EmailFactory, UserFactory from ...common.db.macaroons import MacaroonFactory +_HERE = Path(__file__).parent +_ASSETS = _HERE.parent / "_fixtures" +assert _ASSETS.is_dir() + def test_incorrect_post_redirect(webtest): """ @@ -270,3 +277,74 @@ def test_invalid_classifier_upload_error(webtest): status=HTTPStatus.BAD_REQUEST, ) assert "'This :: Is :: Invalid' is not a valid classifier" in resp.body.decode() + + +def test_provenance_available_after_upload(webtest): + user = UserFactory.create( + password=( # 'password' + "$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ" + "HOJaqfBroT0JCieHug281c" + ) + ) + EmailFactory.create(user=user, verified=True) + project = ProjectFactory.create(name="sampleproject") + RoleFactory.create(user=user, project=project, role_name="Owner") + publisher = GitHubPublisherFactory.create(projects=[project]) + + # Construct the macaroon. This needs to be based on a Trusted Publisher, which is + # required to upload attestations + dm = MacaroonFactory.create( + oidc_publisher_id=publisher.id, + caveats=[ + caveats.OIDCPublisher(oidc_publisher_id=str(publisher.id)), + caveats.ProjectID(project_ids=[str(p.id) for p in publisher.projects]), + ], + additional={"oidc": {"ref": "someref", "sha": "somesha"}}, + ) + + m = pymacaroons.Macaroon( + location="localhost", + identifier=str(dm.id), + key=dm.key, + version=pymacaroons.MACAROON_V2, + ) + for caveat in dm.caveats: + m.add_first_party_caveat(caveats.serialize(caveat)) + serialized_macaroon = f"pypi-{m.serialize()}" + + with open(_ASSETS / "sampleproject-3.0.0.tar.gz", "rb") as f: + content = f.read() + + with open( + _ASSETS / "sampleproject-3.0.0.tar.gz.publish.attestation", + ) as f: + attestation = f.read() + + webtest.set_authorization(("Basic", ("__token__", serialized_macaroon))) + webtest.post( + "/legacy/?:action=file_upload", + params={ + "name": "sampleproject", + "sha256_digest": ( + "117ed88e5db073bb92969a7545745fd977ee85b7019706dd256a64058f70963d" + ), + "filetype": "sdist", + "metadata_version": "2.1", + "version": "3.0.0", + "attestations": f"[{attestation}]", + }, + upload_files=[("content", "sampleproject-3.0.0.tar.gz", content)], + status=HTTPStatus.OK, + ) + + assert len(project.releases) == 1 + assert project.releases[0].files.count() == 1 + assert project.releases[0].files[0].provenance is not None + + # While we needed to be authenticated to upload a project, this is no longer + # required to view it. + webtest.authorization = None + expected_filename = "sampleproject-3.0.0.tar.gz" + + response = webtest.get(f"/_/provenance/{expected_filename}/", status=HTTPStatus.OK) + assert response.json == project.releases[0].files[0].provenance.provenance diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 4264b503767d..6b9475f980ff 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -13,6 +13,7 @@ import base64 import hashlib import io +import json import re import tarfile import tempfile @@ -24,15 +25,8 @@ import pretend import pytest -from pypi_attestations import ( - Attestation, - Distribution, - Envelope, - VerificationError, - VerificationMaterial, -) +from pypi_attestations import Attestation, Envelope, VerificationMaterial from pyramid.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPTooManyRequests -from sigstore.verify import Verifier from sqlalchemy import and_, exists from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import joinedload @@ -43,6 +37,7 @@ from warehouse.accounts.utils import UserContext from warehouse.admin.flags import AdminFlag, AdminFlagValue +from warehouse.attestations.interfaces import IIntegrityService from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy, metadata from warehouse.macaroons import IMacaroonService, caveats, security_policy @@ -2458,6 +2453,7 @@ def test_upload_attestation_fails_without_oidc_publisher( metrics, project_service, macaroon_service, + integrity_service, ): project = ProjectFactory.create() owner = UserFactory.create() @@ -2516,6 +2512,7 @@ def test_upload_attestation_fails_without_oidc_publisher( IMacaroonService: macaroon_service, IMetricsService: metrics, IProjectService: project_service, + IIntegrityService: integrity_service, }.get(svc) db_request.user_agent = "warehouse-tests/6.6.6" @@ -2526,8 +2523,7 @@ def test_upload_attestation_fails_without_oidc_publisher( assert resp.status_code == 400 assert resp.status == ( - "400 Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions." + "400 Attestations are only supported when using Trusted Publishing" ) @pytest.mark.parametrize( @@ -3439,12 +3435,13 @@ def test_upload_succeeds_creates_release( ), ] - def test_upload_with_valid_attestation_succeeds( + def test_upload_succeeds_with_valid_attestation( self, monkeypatch, pyramid_config, db_request, metrics, + integrity_service, ): from warehouse.events.models import HasEvents @@ -3494,9 +3491,9 @@ def test_upload_with_valid_attestation_succeeds( ), } ) - storage_service = pretend.stub(store=lambda path, filepath, meta: None) db_request.find_service = lambda svc, name=None, context=None: { + IIntegrityService: integrity_service, IFileStorage: storage_service, IMetricsService: metrics, }.get(svc) @@ -3505,286 +3502,49 @@ def test_upload_with_valid_attestation_succeeds( lambda self, *, tag, request=None, additional: None ) monkeypatch.setattr(HasEvents, "record_event", record_event) - - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: ( - "https://docs.pypi.org/attestations/publish/v1", - None, - ) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - resp = legacy.file_upload(db_request) assert resp.status_code == 200 - assert len(verify.calls) == 1 - verified_distribution = verify.calls[0].args[3] - assert verified_distribution == Distribution( - name=filename, digest=_TAR_GZ_PKG_SHA256 - ) - - def test_upload_with_invalid_attestation_predicate_type_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None - ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - invalid_predicate_type = "Unsupported predicate type" - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: (invalid_predicate_type, None) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - f"400 Attestation with unsupported predicate type: {invalid_predicate_type}" - ) - - def test_upload_with_multiple_attestations_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}," - f" {attestation.model_dump_json()}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None - ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: ( - "https://docs.pypi.org/attestations/publish/v1", - None, - ) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - "400 Only a single attestation per-file is supported at the moment." - ) - - def test_upload_with_malformed_attestation_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": "[{'a_malformed_attestation': 3}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None + assert ( + pretend.call("warehouse.upload.attestations.ok") in metrics.increment.calls ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - "400 Error while decoding the included attestation:" - ) + # The file was created and has an associated provenance object. + file = db_request.db.query(File).filter(File.filename == filename).one() + assert file.provenance is not None @pytest.mark.parametrize( - ("verify_exception", "expected_msg"), + "invalid_attestations", [ - ( - VerificationError, - "400 Could not verify the uploaded artifact using the included " - "attestation", - ), - ( - ValueError, - "400 Unknown error while trying to verify included attestations", - ), + # Bad top-level types. + "", + {}, + 1, + # Empty attestation sets not permitted. + [], + # Wrong version number. + [ + { + "version": 2, + "verification_material": { + "certificate": "somebase64string", + "transparency_entries": [{}], + }, + "envelope": { + "statement": "somebase64string", + "signature": "somebase64string", + }, + }, + ], ], ) - def test_upload_with_failing_attestation_verification( + def test_upload_fails_attestation_error( self, monkeypatch, pyramid_config, db_request, - metrics, - verify_exception, - expected_msg, + invalid_attestations, ): from warehouse.events.models import HasEvents @@ -3804,16 +3564,6 @@ def test_upload_with_failing_attestation_verification( db_request.db.add(Classifier(classifier="Programming Language :: Python")) filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) pyramid_config.testing_securitypolicy(identity=identity) db_request.user = None @@ -3822,7 +3572,7 @@ def test_upload_with_failing_attestation_verification( { "metadata_version": "1.2", "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", + "attestations": json.dumps(invalid_attestations), "version": version, "summary": "This is my summary!", "filetype": "sdist", @@ -3835,30 +3585,18 @@ def test_upload_with_failing_attestation_verification( } ) - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - record_event = pretend.call_recorder( lambda self, *, tag, request=None, additional: None ) monkeypatch.setattr(HasEvents, "record_event", record_event) - def failing_verify(_self, _verifier, _policy, _dist): - raise verify_exception("error") - - monkeypatch.setattr(Attestation, "verify", failing_verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - with pytest.raises(HTTPBadRequest) as excinfo: legacy.file_upload(db_request) resp = excinfo.value assert resp.status_code == 400 - assert resp.status.startswith(expected_msg) + assert resp.status.startswith("400 Malformed attestations") @pytest.mark.parametrize( ("url", "expected"), diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index ba312034fefb..0c174501b8a4 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -434,6 +434,7 @@ def __init__(self): pretend.call(".accounts"), pretend.call(".macaroons"), pretend.call(".oidc"), + pretend.call(".attestations"), pretend.call(".manage"), pretend.call(".organizations"), pretend.call(".subscriptions"), diff --git a/tests/unit/test_routes.py b/tests/unit/test_routes.py index 58f1d909f265..a4040f4df5f5 100644 --- a/tests/unit/test_routes.py +++ b/tests/unit/test_routes.py @@ -563,6 +563,14 @@ def add_redirect_rule(*args, **kwargs): traverse="/{name}", domain=warehouse, ), + # PEP 740 URLs + pretend.call( + "attestations.provenance", + "/_/provenance/{filename}/", + factory="warehouse.packaging.models:FileFactory", + traverse="/{filename}", + domain=warehouse, + ), # Mock URLs pretend.call( "mock.billing.checkout-session", diff --git a/warehouse/config.py b/warehouse/config.py index 32065f72933c..db83c5a37aff 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -519,12 +519,14 @@ def configure(settings=None): "VERIFY_EMAIL_RATELIMIT_STRING", default="3 per 6 hours", ) - maybe_set( - settings, - "warehouse.account.accounts_search_ratelimit_string", - "ACCOUNTS_SEARCH_RATELIMIT_STRING", - default="100 per hour", - ), + ( + maybe_set( + settings, + "warehouse.account.accounts_search_ratelimit_string", + "ACCOUNTS_SEARCH_RATELIMIT_STRING", + default="100 per hour", + ), + ) maybe_set( settings, "warehouse.account.password_reset_ratelimit_string", @@ -805,6 +807,9 @@ def configure(settings=None): # Register support for OIDC based authentication config.include(".oidc") + # Register support for attestations + config.include(".attestations") + # Register logged-in views config.include(".manage") diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 44521a6d20b0..b96d9e40702c 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -29,13 +29,7 @@ import wtforms import wtforms.validators -from pydantic import TypeAdapter, ValidationError -from pypi_attestations import ( - Attestation, - AttestationType, - Distribution, - VerificationError, -) +from pypi_attestations import Attestation, Distribution from pyramid.httpexceptions import ( HTTPBadRequest, HTTPException, @@ -47,11 +41,12 @@ ) from pyramid.request import Request from pyramid.view import view_config -from sigstore.verify import Verifier from sqlalchemy import and_, exists, func, orm from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue +from warehouse.attestations.errors import AttestationUploadError +from warehouse.attestations.interfaces import IIntegrityService from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB @@ -361,88 +356,6 @@ def _is_duplicate_file(db_session, filename, hashes): return None -def _process_attestations(request, distribution: Distribution): - """ - Process any attestations included in a file upload request - - Attestations, if present, will be parsed and verified against the uploaded - artifact. Attestations are only allowed when uploading via a Trusted - Publisher, because a Trusted Publisher provides the identity that will be - used to verify the attestations. - Currently, only GitHub Actions Trusted Publishers are supported, and - attestations are discarded after verification. - """ - - metrics = request.find_service(IMetricsService, context=None) - - publisher = request.oidc_publisher - if not publisher or not publisher.publisher_name == "GitHub": - raise _exc_with_message( - HTTPBadRequest, - "Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions.", - ) - try: - attestations = TypeAdapter(list[Attestation]).validate_json( - request.POST["attestations"] - ) - except ValidationError as e: - # Log invalid (malformed) attestation upload - metrics.increment("warehouse.upload.attestations.malformed") - raise _exc_with_message( - HTTPBadRequest, - f"Error while decoding the included attestation: {e}", - ) - - if len(attestations) > 1: - metrics.increment("warehouse.upload.attestations.failed_multiple_attestations") - raise _exc_with_message( - HTTPBadRequest, - "Only a single attestation per-file is supported at the moment.", - ) - - verification_policy = publisher.publisher_verification_policy(request.oidc_claims) - for attestation_model in attestations: - try: - # For now, attestations are not stored, just verified - predicate_type, _ = attestation_model.verify( - Verifier.production(), - verification_policy, - distribution, - ) - except VerificationError as e: - # Log invalid (failed verification) attestation upload - metrics.increment("warehouse.upload.attestations.failed_verify") - raise _exc_with_message( - HTTPBadRequest, - f"Could not verify the uploaded artifact using the included " - f"attestation: {e}", - ) - except Exception as e: - with sentry_sdk.new_scope() as scope: - scope.fingerprint = [e] - sentry_sdk.capture_message( - f"Unexpected error while verifying attestation: {e}" - ) - - raise _exc_with_message( - HTTPBadRequest, - f"Unknown error while trying to verify included attestations: {e}", - ) - - if predicate_type != AttestationType.PYPI_PUBLISH_V1: - metrics.increment( - "warehouse.upload.attestations.failed_unsupported_predicate_type" - ) - raise _exc_with_message( - HTTPBadRequest, - f"Attestation with unsupported predicate type: {predicate_type}", - ) - - # Log successful attestation upload - metrics.increment("warehouse.upload.attestations.ok") - - def _sort_releases(request: Request, project: Project): releases = ( request.db.query(Release) @@ -1252,14 +1165,6 @@ def file_upload(request): k: h.hexdigest().lower() for k, h in metadata_file_hashes.items() } - if "attestations" in request.POST and not request.flags.enabled( - AdminFlagValue.DISABLE_PEP740 - ): - _process_attestations( - request=request, - distribution=Distribution(name=filename, digest=file_hashes["sha256"]), - ) - # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. @@ -1328,6 +1233,29 @@ def file_upload(request): ) ) + # If the user provided attestations, verify and store them + if "attestations" in request.POST and not request.flags.enabled( + AdminFlagValue.DISABLE_PEP740 + ): + integrity_service: IIntegrityService = request.find_service( + IIntegrityService, context=None + ) + + try: + attestations: list[Attestation] = integrity_service.parse_attestations( + request, + Distribution(name=filename, digest=file_hashes["sha256"]), + ) + integrity_service.build_provenance(request, file_, attestations) + except AttestationUploadError as e: + raise _exc_with_message( + HTTPBadRequest, + str(e), + ) + + # Log successful attestation upload + metrics.increment("warehouse.upload.attestations.ok") + # TODO: We need a better answer about how to make this transactional so # this won't take affect until after a commit has happened, for # now we'll just ignore it and save it before the transaction is From 62b05455f19d5fcdf1e7446b34c37d0b18a353e6 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 23 Sep 2024 15:06:01 -0400 Subject: [PATCH 04/14] bring coverage up Signed-off-by: William Woodruff --- tests/unit/attestations/test_views.py | 36 ++++++++++++++++++++ tests/unit/packaging/test_models.py | 48 +++++++++++++++++++++++++++ warehouse/attestations/views.py | 2 +- 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/unit/attestations/test_views.py diff --git a/tests/unit/attestations/test_views.py b/tests/unit/attestations/test_views.py new file mode 100644 index 000000000000..ade93bff0fb7 --- /dev/null +++ b/tests/unit/attestations/test_views.py @@ -0,0 +1,36 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pretend + +from warehouse.attestations import views + + +def test_provenance_for_file_not_enabled(): + request = pretend.stub( + flags=pretend.stub(enabled=lambda *a: True), + ) + + response = views.provenance_for_file(pretend.stub(), request) + assert response.status_code == 403 + assert response.json == {"message": "Attestations temporarily disabled"} + + +def test_provenance_for_file_not_present(): + request = pretend.stub( + flags=pretend.stub(enabled=lambda *a: False), + ) + file = pretend.stub(provenance=None, filename="fake-1.2.3.tar.gz") + + response = views.provenance_for_file(file, request) + assert response.status_code == 404 + assert response.json == {"message": "No provenance available for fake-1.2.3.tar.gz"} diff --git a/tests/unit/packaging/test_models.py b/tests/unit/packaging/test_models.py index 3658d733b503..e2d7ce595c3d 100644 --- a/tests/unit/packaging/test_models.py +++ b/tests/unit/packaging/test_models.py @@ -25,6 +25,7 @@ from warehouse.organizations.models import TeamProjectRoleType from warehouse.packaging.models import ( File, + FileFactory, Project, ProjectFactory, ProjectMacaroonWarningAssociation, @@ -86,6 +87,53 @@ def test_contains(self, db_request): assert "bar" not in root +class TestFileFactory: + def test_traversal_finds(self, db_request): + project = DBProjectFactory.create(name="fakeproject") + release = DBReleaseFactory.create(project=project) + rfile_1 = DBFileFactory.create( + release=release, + filename=f"{release.project.name}-{release.version}.tar.gz", + python_version="source", + packagetype="sdist", + ) + rfile_2 = DBFileFactory.create( + release=release, + filename=f"{release.project.name}-{release.version}.whl", + python_version="bdist_wheel", + packagetype="bdist_wheel", + ) + + root = FileFactory(db_request) + assert root[rfile_1.filename] == rfile_1 + assert root[rfile_2.filename] == rfile_2 + + def test_travel_cant_find(self, db_request): + project = DBProjectFactory.create(name="fakeproject") + release = DBReleaseFactory.create(project=project) + + root = FileFactory(db_request) + + # Project and release exist, but no file exists. + with pytest.raises(KeyError): + root[f"{release.project.name}-{release.version}.tar.gz"] + + def test_contains(self, db_request): + project = DBProjectFactory.create(name="fakeproject") + release = DBReleaseFactory.create(project=project) + rfile_1 = DBFileFactory.create( + release=release, + filename=f"{release.project.name}-{release.version}.tar.gz", + python_version="source", + packagetype="sdist", + ) + + root = FileFactory(db_request) + + assert rfile_1.filename in root + assert (rfile_1.filename + ".invalid") not in root + + class TestProject: def test_traversal_finds(self, db_request): project = DBProjectFactory.create() diff --git a/warehouse/attestations/views.py b/warehouse/attestations/views.py index b73bd61c7f7c..b75ef7e7fc4e 100644 --- a/warehouse/attestations/views.py +++ b/warehouse/attestations/views.py @@ -26,7 +26,7 @@ require_csrf=False, has_translations=False, ) -def provenance_for_release_file(file: File, request: Request): +def provenance_for_file(file: File, request: Request): if request.flags.enabled(AdminFlagValue.DISABLE_PEP740): return HTTPForbidden(json={"message": "Attestations temporarily disabled"}) From 68204fc6a7988d8a143effc0ba1438c6e9f53b8d Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 23 Sep 2024 17:42:08 -0400 Subject: [PATCH 05/14] config: fix wonky formatting Signed-off-by: William Woodruff --- warehouse/config.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/warehouse/config.py b/warehouse/config.py index db83c5a37aff..6f4d43fa7991 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -519,13 +519,11 @@ def configure(settings=None): "VERIFY_EMAIL_RATELIMIT_STRING", default="3 per 6 hours", ) - ( - maybe_set( - settings, - "warehouse.account.accounts_search_ratelimit_string", - "ACCOUNTS_SEARCH_RATELIMIT_STRING", - default="100 per hour", - ), + maybe_set( + settings, + "warehouse.account.accounts_search_ratelimit_string", + "ACCOUNTS_SEARCH_RATELIMIT_STRING", + default="100 per hour", ) maybe_set( settings, From e4bfdbcb7d7c991a5640a84b1ffe31cb31a9fe9e Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 24 Sep 2024 16:07:10 -0400 Subject: [PATCH 06/14] tests, warehouse: switch to a better route Signed-off-by: William Woodruff --- tests/functional/forklift/test_legacy.py | 5 +- tests/unit/packaging/test_models.py | 83 ++++++++++-------------- tests/unit/test_routes.py | 6 +- warehouse/packaging/models.py | 34 +++++----- warehouse/routes.py | 6 +- 5 files changed, 60 insertions(+), 74 deletions(-) diff --git a/tests/functional/forklift/test_legacy.py b/tests/functional/forklift/test_legacy.py index f6f216469981..dc313cd9807a 100644 --- a/tests/functional/forklift/test_legacy.py +++ b/tests/functional/forklift/test_legacy.py @@ -346,5 +346,8 @@ def test_provenance_available_after_upload(webtest): webtest.authorization = None expected_filename = "sampleproject-3.0.0.tar.gz" - response = webtest.get(f"/_/provenance/{expected_filename}/", status=HTTPStatus.OK) + response = webtest.get( + f"/metadata/{project.name}/3.0.0/{expected_filename}/provenance", + status=HTTPStatus.OK, + ) assert response.json == project.releases[0].files[0].provenance.provenance diff --git a/tests/unit/packaging/test_models.py b/tests/unit/packaging/test_models.py index e2d7ce595c3d..aac3cc37433b 100644 --- a/tests/unit/packaging/test_models.py +++ b/tests/unit/packaging/test_models.py @@ -25,7 +25,6 @@ from warehouse.organizations.models import TeamProjectRoleType from warehouse.packaging.models import ( File, - FileFactory, Project, ProjectFactory, ProjectMacaroonWarningAssociation, @@ -87,53 +86,6 @@ def test_contains(self, db_request): assert "bar" not in root -class TestFileFactory: - def test_traversal_finds(self, db_request): - project = DBProjectFactory.create(name="fakeproject") - release = DBReleaseFactory.create(project=project) - rfile_1 = DBFileFactory.create( - release=release, - filename=f"{release.project.name}-{release.version}.tar.gz", - python_version="source", - packagetype="sdist", - ) - rfile_2 = DBFileFactory.create( - release=release, - filename=f"{release.project.name}-{release.version}.whl", - python_version="bdist_wheel", - packagetype="bdist_wheel", - ) - - root = FileFactory(db_request) - assert root[rfile_1.filename] == rfile_1 - assert root[rfile_2.filename] == rfile_2 - - def test_travel_cant_find(self, db_request): - project = DBProjectFactory.create(name="fakeproject") - release = DBReleaseFactory.create(project=project) - - root = FileFactory(db_request) - - # Project and release exist, but no file exists. - with pytest.raises(KeyError): - root[f"{release.project.name}-{release.version}.tar.gz"] - - def test_contains(self, db_request): - project = DBProjectFactory.create(name="fakeproject") - release = DBReleaseFactory.create(project=project) - rfile_1 = DBFileFactory.create( - release=release, - filename=f"{release.project.name}-{release.version}.tar.gz", - python_version="source", - packagetype="sdist", - ) - - root = FileFactory(db_request) - - assert rfile_1.filename in root - assert (rfile_1.filename + ".invalid") not in root - - class TestProject: def test_traversal_finds(self, db_request): project = DBProjectFactory.create() @@ -546,6 +498,41 @@ def test_repr(self, db_session): class TestRelease: + def test_getattr(self, db_session): + project = DBProjectFactory.create() + release = DBReleaseFactory.create(project=project) + file = DBFileFactory.create( + release=release, + filename=f"{release.project.name}-{release.version}.tar.gz", + python_version="source", + ) + + assert release[file.filename] == file + + def test_getattr_invalid_file(self, db_session): + project = DBProjectFactory.create() + release = DBReleaseFactory.create(project=project) + + with pytest.raises(KeyError): + # Well-formed filename, but the File doesn't actually exist. + release[f"{release.project.name}-{release.version}.tar.gz"] + + def test_getattr_wrong_file_for_release(self, db_session): + project = DBProjectFactory.create() + release1 = DBReleaseFactory.create(project=project) + release2 = DBReleaseFactory.create(project=project) + file = DBFileFactory.create( + release=release1, + filename=f"{release1.project.name}-{release1.version}.tar.gz", + python_version="source", + ) + + assert release1[file.filename] == file + + # Accessing a file through a different release does not work. + with pytest.raises(KeyError): + release2[file.filename] + def test_has_meta_true_with_keywords(self, db_session): release = DBReleaseFactory.create(keywords="foo, bar") assert release.has_meta diff --git a/tests/unit/test_routes.py b/tests/unit/test_routes.py index a4040f4df5f5..c414734b7469 100644 --- a/tests/unit/test_routes.py +++ b/tests/unit/test_routes.py @@ -566,9 +566,9 @@ def add_redirect_rule(*args, **kwargs): # PEP 740 URLs pretend.call( "attestations.provenance", - "/_/provenance/{filename}/", - factory="warehouse.packaging.models:FileFactory", - traverse="/{filename}", + "/metadata/{project_name}/{release}/{filename}/provenance", + factory="warehouse.packaging.models:ProjectFactory", + traverse="/{project_name}/{release}/{filename}", domain=warehouse, ), # Mock URLs diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 76299f95db15..2e4eb05fb2f7 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -163,25 +163,6 @@ def __contains__(self, project): return True -class FileFactory: - def __init__(self, request): - self.request = request - - def __getitem__(self, filename): - try: - return self.request.db.query(File).filter(File.filename == filename).one() - except NoResultFound: - raise KeyError from None - - def __contains__(self, filename): - try: - self[filename] - except KeyError: - return False - else: - return True - - class LifecycleStatus(enum.StrEnum): QuarantineEnter = "quarantine-enter" QuarantineExit = "quarantine-exit" @@ -713,6 +694,18 @@ def __table_args__(cls): # noqa uploader: Mapped[User] = orm.relationship(User) uploaded_via: Mapped[str | None] + def __getitem__(self, filename: str) -> File: + session: orm.Session = orm.object_session(self) # type: ignore[assignment] + + try: + return ( + session.query(File) + .filter(File.release == self, File.filename == filename) + .one() + ) + except NoResultFound: + raise KeyError from None + @property def urls(self): _urls = OrderedDict() @@ -866,6 +859,9 @@ def __table_args__(cls): # noqa Index("release_files_cached_idx", "cached"), ) + __parent__ = dotted_navigator("release") + __name__ = dotted_navigator("filename") + release_id: Mapped[UUID] = mapped_column( ForeignKey("releases.id", onupdate="CASCADE", ondelete="CASCADE"), ) diff --git a/warehouse/routes.py b/warehouse/routes.py index 60642ef436c6..826eb4575e5e 100644 --- a/warehouse/routes.py +++ b/warehouse/routes.py @@ -579,9 +579,9 @@ def includeme(config): # PEP 740 URLs config.add_route( "attestations.provenance", - "/_/provenance/{filename}/", - factory="warehouse.packaging.models:FileFactory", - traverse="/{filename}", + "/metadata/{project_name}/{release}/{filename}/provenance", + factory="warehouse.packaging.models:ProjectFactory", + traverse="/{project_name}/{release}/{filename}", domain=warehouse, ) From dff7eb3946b3c4cfbbe3f5a454ccbd7a829fb062 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 26 Sep 2024 13:39:39 -0400 Subject: [PATCH 07/14] rename route, content negotiation Signed-off-by: William Woodruff --- tests/functional/forklift/test_legacy.py | 2 +- tests/unit/attestations/test_views.py | 35 +++++++++++++++------- tests/unit/test_routes.py | 2 +- warehouse/attestations/views.py | 37 +++++++++++++++++++++++- warehouse/routes.py | 2 +- 5 files changed, 64 insertions(+), 14 deletions(-) diff --git a/tests/functional/forklift/test_legacy.py b/tests/functional/forklift/test_legacy.py index dc313cd9807a..ba76b4f888e1 100644 --- a/tests/functional/forklift/test_legacy.py +++ b/tests/functional/forklift/test_legacy.py @@ -347,7 +347,7 @@ def test_provenance_available_after_upload(webtest): expected_filename = "sampleproject-3.0.0.tar.gz" response = webtest.get( - f"/metadata/{project.name}/3.0.0/{expected_filename}/provenance", + f"/integrity/{project.name}/3.0.0/{expected_filename}/provenance", status=HTTPStatus.OK, ) assert response.json == project.releases[0].files[0].provenance.provenance diff --git a/tests/unit/attestations/test_views.py b/tests/unit/attestations/test_views.py index ade93bff0fb7..172833820b99 100644 --- a/tests/unit/attestations/test_views.py +++ b/tests/unit/attestations/test_views.py @@ -11,26 +11,41 @@ # limitations under the License. import pretend +import pytest from warehouse.attestations import views -def test_provenance_for_file_not_enabled(): - request = pretend.stub( - flags=pretend.stub(enabled=lambda *a: True), - ) +def test_select_content_type(db_request): + db_request.accept = "application/json" - response = views.provenance_for_file(pretend.stub(), request) + assert views._select_content_type(db_request) == views.MIME_PYPI_INTEGRITY_V1_JSON + + +# Backstop; can be removed/changed once this view supports HTML. +@pytest.mark.parametrize( + "content_type", + [views.MIME_TEXT_HTML, views.MIME_PYPI_INTEGRITY_V1_HTML], +) +def test_provenance_for_file_bad_accept(db_request, content_type): + db_request.accept = content_type + response = views.provenance_for_file(pretend.stub(), db_request) + assert response.status_code == 406 + assert response.json == {"message": "Request not acceptable"} + + +def test_provenance_for_file_not_enabled(db_request, monkeypatch): + monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: True)) + + response = views.provenance_for_file(pretend.stub(), db_request) assert response.status_code == 403 assert response.json == {"message": "Attestations temporarily disabled"} -def test_provenance_for_file_not_present(): - request = pretend.stub( - flags=pretend.stub(enabled=lambda *a: False), - ) +def test_provenance_for_file_not_present(db_request, monkeypatch): + monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: False)) file = pretend.stub(provenance=None, filename="fake-1.2.3.tar.gz") - response = views.provenance_for_file(file, request) + response = views.provenance_for_file(file, db_request) assert response.status_code == 404 assert response.json == {"message": "No provenance available for fake-1.2.3.tar.gz"} diff --git a/tests/unit/test_routes.py b/tests/unit/test_routes.py index c414734b7469..be756d3086e0 100644 --- a/tests/unit/test_routes.py +++ b/tests/unit/test_routes.py @@ -566,7 +566,7 @@ def add_redirect_rule(*args, **kwargs): # PEP 740 URLs pretend.call( "attestations.provenance", - "/metadata/{project_name}/{release}/{filename}/provenance", + "/integrity/{project_name}/{release}/{filename}/provenance", factory="warehouse.packaging.models:ProjectFactory", traverse="/{project_name}/{release}/{filename}", domain=warehouse, diff --git a/warehouse/attestations/views.py b/warehouse/attestations/views.py index b75ef7e7fc4e..dc6d70404da7 100644 --- a/warehouse/attestations/views.py +++ b/warehouse/attestations/views.py @@ -10,12 +10,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pyramid.httpexceptions import HTTPForbidden, HTTPNotFound +from pyramid.httpexceptions import HTTPForbidden, HTTPNotAcceptable, HTTPNotFound from pyramid.request import Request from pyramid.view import view_config from warehouse.admin.flags import AdminFlagValue +from warehouse.cache.http import add_vary from warehouse.packaging.models import File +from warehouse.utils.cors import _CORS_HEADERS + +MIME_TEXT_HTML = "text/html" +MIME_PYPI_INTEGRITY_V1_HTML = "application/vnd.pypi.integrity.v1+html" +MIME_PYPI_INTEGRITY_V1_JSON = "application/vnd.pypi.integrity.v1+json" + + +def _select_content_type(request: Request) -> str: + offers = request.accept.acceptable_offers( + [ + # JSON currently has the highest priority. + MIME_PYPI_INTEGRITY_V1_JSON, + MIME_TEXT_HTML, + MIME_PYPI_INTEGRITY_V1_HTML, + ] + ) + + # Default case: JSON. + if not offers: + return MIME_PYPI_INTEGRITY_V1_JSON + else: + return offers[0][0] @view_config( @@ -25,8 +48,17 @@ renderer="json", require_csrf=False, has_translations=False, + decorator=[ + add_vary("Accept"), + ], ) def provenance_for_file(file: File, request: Request): + # Determine our response content-type. For the time being, only the JSON + # type is accepted. + request.response.content_type = _select_content_type(request) + if request.response.content_type != MIME_PYPI_INTEGRITY_V1_JSON: + return HTTPNotAcceptable(json={"message": "Request not acceptable"}) + if request.flags.enabled(AdminFlagValue.DISABLE_PEP740): return HTTPForbidden(json={"message": "Attestations temporarily disabled"}) @@ -35,4 +67,7 @@ def provenance_for_file(file: File, request: Request): json={"message": f"No provenance available for {file.filename}"} ) + # Apply CORS headers. + request.response.headers.update(_CORS_HEADERS) + return file.provenance.provenance diff --git a/warehouse/routes.py b/warehouse/routes.py index 826eb4575e5e..ec0a2b73a8b4 100644 --- a/warehouse/routes.py +++ b/warehouse/routes.py @@ -579,7 +579,7 @@ def includeme(config): # PEP 740 URLs config.add_route( "attestations.provenance", - "/metadata/{project_name}/{release}/{filename}/provenance", + "/integrity/{project_name}/{release}/{filename}/provenance", factory="warehouse.packaging.models:ProjectFactory", traverse="/{project_name}/{release}/{filename}", domain=warehouse, From 2b4cf3a0c772460d72d0e8ea9323d99760fcf2f6 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 26 Sep 2024 13:57:12 -0400 Subject: [PATCH 08/14] attestations.views -> api.integrity Signed-off-by: William Woodruff --- .../test_views.py => api/test_integrity.py} | 15 +++++++++------ .../{attestations/views.py => api/integrity.py} | 0 2 files changed, 9 insertions(+), 6 deletions(-) rename tests/unit/{attestations/test_views.py => api/test_integrity.py} (78%) rename warehouse/{attestations/views.py => api/integrity.py} (100%) diff --git a/tests/unit/attestations/test_views.py b/tests/unit/api/test_integrity.py similarity index 78% rename from tests/unit/attestations/test_views.py rename to tests/unit/api/test_integrity.py index 172833820b99..10251e6c2ac4 100644 --- a/tests/unit/attestations/test_views.py +++ b/tests/unit/api/test_integrity.py @@ -13,23 +13,26 @@ import pretend import pytest -from warehouse.attestations import views +from warehouse.api import integrity def test_select_content_type(db_request): db_request.accept = "application/json" - assert views._select_content_type(db_request) == views.MIME_PYPI_INTEGRITY_V1_JSON + assert ( + integrity._select_content_type(db_request) + == integrity.MIME_PYPI_INTEGRITY_V1_JSON + ) # Backstop; can be removed/changed once this view supports HTML. @pytest.mark.parametrize( "content_type", - [views.MIME_TEXT_HTML, views.MIME_PYPI_INTEGRITY_V1_HTML], + [integrity.MIME_TEXT_HTML, integrity.MIME_PYPI_INTEGRITY_V1_HTML], ) def test_provenance_for_file_bad_accept(db_request, content_type): db_request.accept = content_type - response = views.provenance_for_file(pretend.stub(), db_request) + response = integrity.provenance_for_file(pretend.stub(), db_request) assert response.status_code == 406 assert response.json == {"message": "Request not acceptable"} @@ -37,7 +40,7 @@ def test_provenance_for_file_bad_accept(db_request, content_type): def test_provenance_for_file_not_enabled(db_request, monkeypatch): monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: True)) - response = views.provenance_for_file(pretend.stub(), db_request) + response = integrity.provenance_for_file(pretend.stub(), db_request) assert response.status_code == 403 assert response.json == {"message": "Attestations temporarily disabled"} @@ -46,6 +49,6 @@ def test_provenance_for_file_not_present(db_request, monkeypatch): monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: False)) file = pretend.stub(provenance=None, filename="fake-1.2.3.tar.gz") - response = views.provenance_for_file(file, db_request) + response = integrity.provenance_for_file(file, db_request) assert response.status_code == 404 assert response.json == {"message": "No provenance available for fake-1.2.3.tar.gz"} diff --git a/warehouse/attestations/views.py b/warehouse/api/integrity.py similarity index 100% rename from warehouse/attestations/views.py rename to warehouse/api/integrity.py From 784d7731f9b0c717eb2ec94a6c72288912e5b12b Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 26 Sep 2024 14:06:01 -0400 Subject: [PATCH 09/14] api/integrity: use the same caching headers as simple Signed-off-by: William Woodruff --- warehouse/api/integrity.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/warehouse/api/integrity.py b/warehouse/api/integrity.py index dc6d70404da7..966c6992c5af 100644 --- a/warehouse/api/integrity.py +++ b/warehouse/api/integrity.py @@ -15,7 +15,8 @@ from pyramid.view import view_config from warehouse.admin.flags import AdminFlagValue -from warehouse.cache.http import add_vary +from warehouse.cache.http import add_vary, cache_control +from warehouse.cache.origin import origin_cache from warehouse.packaging.models import File from warehouse.utils.cors import _CORS_HEADERS @@ -50,6 +51,12 @@ def _select_content_type(request: Request) -> str: has_translations=False, decorator=[ add_vary("Accept"), + cache_control(10 * 60), # 10 minutes + origin_cache( + 1 * 24 * 60 * 60, # 1 day + stale_while_revalidate=5 * 60, # 5 minutes + stale_if_error=1 * 24 * 60 * 60, # 1 day + ), ], ) def provenance_for_file(file: File, request: Request): From 335d75a19329a63bb11dea5e5ad5860046676801 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Thu, 26 Sep 2024 23:39:00 +0000 Subject: [PATCH 10/14] Split up functional test --- tests/common/db/packaging.py | 9 ++++ tests/functional/api/test_integrity.py | 58 ++++++++++++++++++++++++ tests/functional/forklift/test_legacy.py | 34 +++++++------- 3 files changed, 85 insertions(+), 16 deletions(-) create mode 100644 tests/functional/api/test_integrity.py diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index f948d6cbe899..3b97b20cbd92 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -28,6 +28,7 @@ JournalEntry, ProhibitedProjectName, Project, + Provenance, Release, Role, RoleInvitation, @@ -142,6 +143,14 @@ class Meta: ) +class ProvenanceFactory(WarehouseFactory): + class Meta: + model = Provenance + + file = factory.SubFactory(FileFactory) + provenance = factory.Faker("json") + + class FileEventFactory(WarehouseFactory): class Meta: model = File.Event diff --git a/tests/functional/api/test_integrity.py b/tests/functional/api/test_integrity.py new file mode 100644 index 000000000000..e6f5f36e801c --- /dev/null +++ b/tests/functional/api/test_integrity.py @@ -0,0 +1,58 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from http import HTTPStatus +from pathlib import Path + +from ...common.db.packaging import ( + FileFactory, + ProjectFactory, + ProvenanceFactory, + ReleaseFactory, +) + +_HERE = Path(__file__).parent +_ASSETS = _HERE.parent / "_fixtures" +assert _ASSETS.is_dir() + + +def test_provenance_available(webtest): + with open( + _ASSETS / "sampleproject-3.0.0.tar.gz.publish.attestation", + ) as f: + attestation_contents = f.read() + attestation_json = json.loads(attestation_contents) + + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project) + file_ = FileFactory.create(release=release, packagetype="sdist") + ProvenanceFactory.create( + file=file_, + provenance={"attestation_bundles": [{"attestations": [attestation_json]}]}, + ) + + response = webtest.get( + f"/integrity/{project.name}/{release.version}/{file_.filename}/provenance", + status=HTTPStatus.OK, + ) + assert response.json + assert "attestation_bundles" in response.json + attestation_bundles = response.json["attestation_bundles"] + assert len(attestation_bundles) == 1 + attestation_bundle = attestation_bundles[0] + assert "attestations" in attestation_bundle + attestations = attestation_bundle["attestations"] + assert len(attestations) == 1 + attestation = attestations[0] + assert attestation == attestation_json diff --git a/tests/functional/forklift/test_legacy.py b/tests/functional/forklift/test_legacy.py index ba76b4f888e1..ccf29be4e4b1 100644 --- a/tests/functional/forklift/test_legacy.py +++ b/tests/functional/forklift/test_legacy.py @@ -11,6 +11,7 @@ # limitations under the License. import base64 +import json from http import HTTPStatus from pathlib import Path @@ -279,7 +280,7 @@ def test_invalid_classifier_upload_error(webtest): assert "'This :: Is :: Invalid' is not a valid classifier" in resp.body.decode() -def test_provenance_available_after_upload(webtest): +def test_provenance_upload(webtest): user = UserFactory.create( password=( # 'password' "$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ" @@ -318,7 +319,7 @@ def test_provenance_available_after_upload(webtest): with open( _ASSETS / "sampleproject-3.0.0.tar.gz.publish.attestation", ) as f: - attestation = f.read() + attestation_contents = f.read() webtest.set_authorization(("Basic", ("__token__", serialized_macaroon))) webtest.post( @@ -331,23 +332,24 @@ def test_provenance_available_after_upload(webtest): "filetype": "sdist", "metadata_version": "2.1", "version": "3.0.0", - "attestations": f"[{attestation}]", + "attestations": f"[{attestation_contents}]", }, upload_files=[("content", "sampleproject-3.0.0.tar.gz", content)], status=HTTPStatus.OK, ) assert len(project.releases) == 1 - assert project.releases[0].files.count() == 1 - assert project.releases[0].files[0].provenance is not None - - # While we needed to be authenticated to upload a project, this is no longer - # required to view it. - webtest.authorization = None - expected_filename = "sampleproject-3.0.0.tar.gz" - - response = webtest.get( - f"/integrity/{project.name}/3.0.0/{expected_filename}/provenance", - status=HTTPStatus.OK, - ) - assert response.json == project.releases[0].files[0].provenance.provenance + release = project.releases[0] + assert release.files.count() == 1 + file_ = project.releases[0].files[0] + assert file_.provenance is not None + provenance = file_.provenance.provenance + assert "attestation_bundles" in provenance + attestation_bundles = provenance["attestation_bundles"] + assert len(attestation_bundles) == 1 + bundle = provenance["attestation_bundles"][0] + assert "attestations" in bundle + attestations = bundle["attestations"] + assert len(attestations) == 1 + attestation = attestations[0] + assert attestation == json.loads(attestation_contents) From 9cc69d219ce01a49d3c4a0227b074417ada0cc0f Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Thu, 26 Sep 2024 23:39:27 +0000 Subject: [PATCH 11/14] Explicity add the Provenance object to the database session --- warehouse/forklift/legacy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index b96d9e40702c..979826c5764a 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -1246,7 +1246,9 @@ def file_upload(request): request, Distribution(name=filename, digest=file_hashes["sha256"]), ) - integrity_service.build_provenance(request, file_, attestations) + request.db.add( + integrity_service.build_provenance(request, file_, attestations) + ) except AttestationUploadError as e: raise _exc_with_message( HTTPBadRequest, From 813f65f3e6224473222dffb74eb50caafd33dac3 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 3 Oct 2024 09:23:17 -0400 Subject: [PATCH 12/14] Apply suggestions from code review Co-authored-by: Mike Fiedler --- tests/functional/forklift/test_legacy.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/functional/forklift/test_legacy.py b/tests/functional/forklift/test_legacy.py index ccf29be4e4b1..efc96c2d51f2 100644 --- a/tests/functional/forklift/test_legacy.py +++ b/tests/functional/forklift/test_legacy.py @@ -281,13 +281,7 @@ def test_invalid_classifier_upload_error(webtest): def test_provenance_upload(webtest): - user = UserFactory.create( - password=( # 'password' - "$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ" - "HOJaqfBroT0JCieHug281c" - ) - ) - EmailFactory.create(user=user, verified=True) + user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password") project = ProjectFactory.create(name="sampleproject") RoleFactory.create(user=user, project=project, role_name="Owner") publisher = GitHubPublisherFactory.create(projects=[project]) From 6bab1114100961a4f4db94ce80d5d29c10b02dd6 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 3 Oct 2024 09:27:56 -0400 Subject: [PATCH 13/14] tests: more UserFactory simplification Signed-off-by: William Woodruff --- tests/functional/forklift/test_legacy.py | 26 ++++-------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/tests/functional/forklift/test_legacy.py b/tests/functional/forklift/test_legacy.py index efc96c2d51f2..ed79477a52ec 100644 --- a/tests/functional/forklift/test_legacy.py +++ b/tests/functional/forklift/test_legacy.py @@ -25,7 +25,7 @@ from tests.common.db.packaging import ProjectFactory, RoleFactory from warehouse.macaroons import caveats -from ...common.db.accounts import EmailFactory, UserFactory +from ...common.db.accounts import UserFactory from ...common.db.macaroons import MacaroonFactory _HERE = Path(__file__).parent @@ -76,13 +76,7 @@ def test_remove_doc_upload(webtest): ], ) def test_file_upload(webtest, upload_url, additional_data): - user = UserFactory.create( - with_verified_primary_email=True, - password=( # 'password' - "$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ" - "HOJaqfBroT0JCieHug281c" - ), - ) + user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password") # Construct the macaroon dm = MacaroonFactory.create( @@ -143,13 +137,7 @@ def test_file_upload(webtest, upload_url, additional_data): def test_duplicate_file_upload_error(webtest): - user = UserFactory.create( - with_verified_primary_email=True, - password=( # 'password' - "$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ" - "HOJaqfBroT0JCieHug281c" - ), - ) + user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password") # Construct the macaroon dm = MacaroonFactory.create( @@ -223,13 +211,7 @@ def test_duplicate_file_upload_error(webtest): def test_invalid_classifier_upload_error(webtest): - user = UserFactory.create( - with_verified_primary_email=True, - password=( # 'password' - "$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ" - "HOJaqfBroT0JCieHug281c" - ), - ) + user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password") # Construct the macaroon dm = MacaroonFactory.create( From 9fea457913337a2dcd0f9f7ae26310f983fcfbbd Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 3 Oct 2024 09:29:38 -0400 Subject: [PATCH 14/14] routes: rename attestations -> integrity Signed-off-by: William Woodruff --- tests/unit/test_routes.py | 2 +- warehouse/api/integrity.py | 2 +- warehouse/routes.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_routes.py b/tests/unit/test_routes.py index be756d3086e0..d4e59ea0a79e 100644 --- a/tests/unit/test_routes.py +++ b/tests/unit/test_routes.py @@ -565,7 +565,7 @@ def add_redirect_rule(*args, **kwargs): ), # PEP 740 URLs pretend.call( - "attestations.provenance", + "integrity.provenance", "/integrity/{project_name}/{release}/{filename}/provenance", factory="warehouse.packaging.models:ProjectFactory", traverse="/{project_name}/{release}/{filename}", diff --git a/warehouse/api/integrity.py b/warehouse/api/integrity.py index 966c6992c5af..cecf3bc33d0f 100644 --- a/warehouse/api/integrity.py +++ b/warehouse/api/integrity.py @@ -43,7 +43,7 @@ def _select_content_type(request: Request) -> str: @view_config( - route_name="attestations.provenance", + route_name="integrity.provenance", context=File, require_methods=["GET"], renderer="json", diff --git a/warehouse/routes.py b/warehouse/routes.py index ec0a2b73a8b4..acd97bee69b1 100644 --- a/warehouse/routes.py +++ b/warehouse/routes.py @@ -578,7 +578,7 @@ def includeme(config): # PEP 740 URLs config.add_route( - "attestations.provenance", + "integrity.provenance", "/integrity/{project_name}/{release}/{filename}/provenance", factory="warehouse.packaging.models:ProjectFactory", traverse="/{project_name}/{release}/{filename}",