feat(demo-mode) also sync ProjectDebugFile and ProguardArtifactRelease (#92731)

Closes
https://linear.app/getsentry/issue/TET-528/extend-demo-mode-sync-task-to-handle-debug-files-and-proguard

Author: constantinius

src/sentry/conf/server.py

@@ -1307,8 +1307,8 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
         # Run every 1 minute
         "schedule": crontab(minute="*/1"),
     },
-    "demo_mode_sync_artifact_bundles": {
-        "task": "sentry.demo_mode.tasks.sync_artifact_bundles",
+    "demo_mode_sync_debug_artifacts": {
+        "task": "sentry.demo_mode.tasks.sync_debug_artifacts",
         # Run every hour
         "schedule": crontab(minute="0", hour="*/1"),
     },

src/sentry/demo_mode/tasks.py

@@ -13,6 +13,7 @@
     ProjectArtifactBundle,
     ReleaseArtifactBundle,
 )
+from sentry.models.debugfile import ProguardArtifactRelease, ProjectDebugFile
 from sentry.models.files import FileBlobOwner
 from sentry.models.organization import Organization
 from sentry.models.project import Project
@@ -23,11 +24,11 @@
 
 
 @instrumented_task(
-    name="sentry.demo_mode.tasks.sync_artifact_bundles",
+    name="sentry.demo_mode.tasks.sync_debug_artifacts",
     queue="demo_mode",
     taskworker_config=TaskworkerConfig(namespace=demomode_tasks),
 )
-def sync_artifact_bundles():
+def sync_debug_artifacts():
 
     if (
         not options.get("sentry.demo_mode.sync_artifact_bundles.enable")
@@ -43,6 +44,8 @@ def sync_artifact_bundles():
     lookback_days = options.get("sentry.demo_mode.sync_artifact_bundles.lookback_days")
 
     _sync_artifact_bundles(source_org, target_org, lookback_days)
+    _sync_project_debug_files(source_org, target_org, lookback_days)
+    _sync_proguard_artifact_releases(source_org, target_org, lookback_days)
 
 
 def _sync_artifact_bundles(source_org: Organization, target_org: Organization, lookback_days=1):
@@ -67,6 +70,62 @@ def _sync_artifact_bundles(source_org: Organization, target_org: Organization, l
         _sync_artifact_bundle(source_artifact_bundle, target_org)
 
 
+def _sync_project_debug_files(source_org: Organization, target_org: Organization, lookback_days=1):
+    if not source_org or not target_org:
+        return
+
+    source_project_ids = Project.objects.filter(
+        organization_id=source_org.id,
+    ).values_list("id", flat=True)
+
+    target_project_ids = Project.objects.filter(
+        organization_id=target_org.id,
+    ).values_list("id", flat=True)
+
+    source_project_debug_files = ProjectDebugFile.objects.filter(
+        project_id__in=source_project_ids,
+    )
+
+    target_project_debug_files = ProjectDebugFile.objects.filter(
+        project_id__in=target_project_ids,
+    )
+
+    different_project_debug_files = source_project_debug_files.exclude(
+        debug_id__in=target_project_debug_files.values_list("debug_id", flat=True)
+    )
+
+    for source_project_debug_file in different_project_debug_files:
+        _sync_project_debug_file(source_project_debug_file, target_org)
+
+
+def _sync_proguard_artifact_releases(
+    source_org: Organization, target_org: Organization, lookback_days=1
+):
+    if not source_org or not target_org:
+        return
+
+    cutoff_date = timezone.now() - timedelta(days=lookback_days)
+
+    proguard_artifact_releases = ProguardArtifactRelease.objects.filter(
+        Q(organization_id=source_org.id) | Q(organization_id=target_org.id),
+        date_added__gte=cutoff_date,
+    )
+
+    source_proguard_artifact_releases = proguard_artifact_releases.filter(
+        organization_id=source_org.id,
+    )
+    target_proguard_artifact_releases = proguard_artifact_releases.filter(
+        organization_id=target_org.id,
+    )
+
+    different_proguard_artifact_releases = source_proguard_artifact_releases.exclude(
+        proguard_uuid__in=target_proguard_artifact_releases.values_list("proguard_uuid", flat=True)
+    )
+
+    for source_proguard_artifact_release in different_proguard_artifact_releases:
+        _sync_proguard_artifact_release(source_proguard_artifact_release, target_org)
+
+
 def _sync_artifact_bundle(source_artifact_bundle: ArtifactBundle, target_org: Organization):
     try:
         with atomic_transaction(
@@ -144,6 +203,75 @@ def _sync_release_artifact_bundle(
     )
 
 
+def _sync_project_debug_file(
+    source_project_debug_file: ProjectDebugFile, target_org: Organization
+) -> ProjectDebugFile | None:
+    try:
+        with atomic_transaction(using=(router.db_for_write(ProjectDebugFile))):
+            target_project = _find_matching_project(
+                source_project_debug_file.project_id,
+                target_org.id,
+            )
+
+            if not target_project:
+                return None
+
+            return ProjectDebugFile.objects.create(
+                project_id=target_project.id,
+                file=source_project_debug_file.file,
+                checksum=source_project_debug_file.checksum,
+                object_name=source_project_debug_file.object_name,
+                cpu_name=source_project_debug_file.cpu_name,
+                debug_id=source_project_debug_file.debug_id,
+                code_id=source_project_debug_file.code_id,
+                data=source_project_debug_file.data,
+                date_accessed=source_project_debug_file.date_accessed,
+            )
+    except IntegrityError as e:
+        sentry_sdk.capture_exception(e)
+        return None
+
+
+def _sync_proguard_artifact_release(
+    source_proguard_artifact_release: ProguardArtifactRelease, target_org: Organization
+):
+    try:
+        with atomic_transaction(using=(router.db_for_write(ProguardArtifactRelease))):
+            target_project = _find_matching_project(
+                source_proguard_artifact_release.project_id,
+                target_org.id,
+            )
+
+            if not target_project:
+                return
+
+            # project_debug_file _should_ already be synced, but we'll make sure it is
+            project_debug_file = ProjectDebugFile.objects.filter(
+                project_id=target_project.id,
+                debug_id=source_proguard_artifact_release.project_debug_file.debug_id,
+            ).first()
+
+            if not project_debug_file:
+                project_debug_file = _sync_project_debug_file(
+                    source_proguard_artifact_release.project_debug_file, target_org
+                )
+
+            if not project_debug_file:
+                # we require a project debug file
+                return
+
+            ProguardArtifactRelease.objects.create(
+                organization_id=target_org.id,
+                project_id=target_project.id,
+                release_name=source_proguard_artifact_release.release_name,
+                proguard_uuid=source_proguard_artifact_release.proguard_uuid,
+                project_debug_file=project_debug_file,
+                date_added=source_proguard_artifact_release.date_added,
+            )
+    except IntegrityError as e:
+        sentry_sdk.capture_exception(e)
+
+
 def _find_matching_project(project_id, organization_id):
     try:
         source_project = Project.objects.get(id=project_id)

tests/sentry/demo_mode/test_tasks.py

@@ -1,15 +1,21 @@
 from datetime import datetime, timedelta
 from unittest import mock
+from uuid import uuid1
 
 from django.db.utils import IntegrityError
 from django.utils import timezone
 
-from sentry.demo_mode.tasks import _sync_artifact_bundles
+from sentry.demo_mode.tasks import (
+    _sync_artifact_bundles,
+    _sync_proguard_artifact_releases,
+    _sync_project_debug_files,
+)
 from sentry.models.artifactbundle import (
     ArtifactBundle,
     ProjectArtifactBundle,
     ReleaseArtifactBundle,
 )
+from sentry.models.debugfile import ProguardArtifactRelease, ProjectDebugFile
 from sentry.models.organization import Organization
 from sentry.models.project import Project
 from sentry.testutils.cases import TestCase
@@ -52,6 +58,23 @@ def set_up_artifact_bundle(
 
         return artifact_bundle, project_artifact_bundle, release_artifact_bundle
 
+    def set_up_proguard_artifact_release(
+        self,
+        organization: Organization,
+        project: Project,
+        date_added: datetime | None = None,
+    ):
+        date_added = date_added or timezone.now()
+        proguard_artifact_release = ProguardArtifactRelease.objects.create(
+            organization_id=organization.id,
+            project_id=project.id,
+            release_name="release",
+            proguard_uuid=uuid1(),
+            project_debug_file=self.create_dif_file(project),
+            date_added=date_added,
+        )
+        return proguard_artifact_release
+
     def test_sync_artifact_bundles_no_bundles(self):
 
         _sync_artifact_bundles(source_org=self.source_org, target_org=self.target_org)
@@ -145,3 +168,87 @@ def test_sync_artifact_bundles_rolls_back_on_error(self, _):
         assert not ArtifactBundle.objects.filter(organization_id=self.target_org.id).exists()
         assert not ProjectArtifactBundle.objects.filter(organization_id=self.target_org.id).exists()
         assert not ReleaseArtifactBundle.objects.filter(organization_id=self.target_org.id).exists()
+
+    def test_sync_project_debug_files(self):
+        source_project_debug_file = self.create_dif_file(self.source_proj_foo)
+
+        assert not ProjectDebugFile.objects.filter(
+            project_id=self.target_proj_foo.id,
+            debug_id=source_project_debug_file.debug_id,
+        ).exists()
+
+        _sync_project_debug_files(source_org=self.source_org, target_org=self.target_org)
+
+        target_project_debug_file = ProjectDebugFile.objects.get(
+            project_id=self.target_proj_foo.id,
+            debug_id=source_project_debug_file.debug_id,
+        )
+
+        assert target_project_debug_file.debug_id == source_project_debug_file.debug_id
+        assert target_project_debug_file.code_id == source_project_debug_file.code_id
+        assert target_project_debug_file.cpu_name == source_project_debug_file.cpu_name
+
+    def test_sync_project_debug_files_with_old_uploads(self):
+        source_project_debug_file = self.create_dif_file(
+            self.source_proj_foo,
+            date_accessed=timezone.now() - timedelta(days=2),
+        )
+
+        assert not ProjectDebugFile.objects.filter(
+            project_id=self.target_proj_foo.id,
+            debug_id=source_project_debug_file.debug_id,
+        ).exists()
+
+        _sync_project_debug_files(source_org=self.source_org, target_org=self.target_org)
+
+        assert ProjectDebugFile.objects.filter(
+            project_id=self.target_proj_foo.id,
+            debug_id=source_project_debug_file.debug_id,
+        ).exists()
+
+    def test_sync_proguard_artifact_releases(self):
+        source_proguard_artifact_release = self.set_up_proguard_artifact_release(
+            self.source_org,
+            self.source_proj_foo,
+        )
+
+        assert not ProguardArtifactRelease.objects.filter(
+            organization_id=self.target_org.id,
+            proguard_uuid=source_proguard_artifact_release.proguard_uuid,
+        ).exists()
+
+        _sync_proguard_artifact_releases(source_org=self.source_org, target_org=self.target_org)
+
+        target_proguard_artifact_release = ProguardArtifactRelease.objects.get(
+            organization_id=self.target_org.id,
+            proguard_uuid=source_proguard_artifact_release.proguard_uuid,
+        )
+
+        assert (
+            target_proguard_artifact_release.release_name
+            == source_proguard_artifact_release.release_name
+        )
+        assert (
+            target_proguard_artifact_release.proguard_uuid
+            == source_proguard_artifact_release.proguard_uuid
+        )
+        assert target_proguard_artifact_release.project_id == self.target_proj_foo.id
+
+    def test_sync_proguard_artifact_releases_with_old_uploads(self):
+        source_proguard_artifact_release = self.set_up_proguard_artifact_release(
+            self.source_org,
+            self.source_proj_foo,
+            date_added=timezone.now() - timedelta(days=2),
+        )
+
+        assert not ProguardArtifactRelease.objects.filter(
+            organization_id=self.target_org.id,
+            proguard_uuid=source_proguard_artifact_release.proguard_uuid,
+        ).exists()
+
+        _sync_artifact_bundles(source_org=self.source_org, target_org=self.target_org)
+
+        assert not ProguardArtifactRelease.objects.filter(
+            organization_id=self.target_org.id,
+            proguard_uuid=source_proguard_artifact_release.proguard_uuid,
+        ).exists()