diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 5438cc2a3b3d62..a1580000b744a7 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -1305,8 +1305,8 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str: # Run every 1 minute "schedule": crontab(minute="*/1"), }, - "demo_mode_sync_artifact_bundles": { - "task": "sentry.demo_mode.tasks.sync_artifact_bundles", + "demo_mode_sync_debug_artifacts": { + "task": "sentry.demo_mode.tasks.sync_debug_artifacts", # Run every hour "schedule": crontab(minute="0", hour="*/1"), }, diff --git a/src/sentry/demo_mode/tasks.py b/src/sentry/demo_mode/tasks.py index 9b42b61c68c0ee..3b9546827a5e81 100644 --- a/src/sentry/demo_mode/tasks.py +++ b/src/sentry/demo_mode/tasks.py @@ -13,6 +13,7 @@ ProjectArtifactBundle, ReleaseArtifactBundle, ) +from sentry.models.debugfile import ProguardArtifactRelease, ProjectDebugFile from sentry.models.files import FileBlobOwner from sentry.models.organization import Organization from sentry.models.project import Project @@ -23,11 +24,11 @@ @instrumented_task( - name="sentry.demo_mode.tasks.sync_artifact_bundles", + name="sentry.demo_mode.tasks.sync_debug_artifacts", queue="demo_mode", taskworker_config=TaskworkerConfig(namespace=demomode_tasks), ) -def sync_artifact_bundles(): +def sync_debug_artifacts(): if ( not options.get("sentry.demo_mode.sync_artifact_bundles.enable") @@ -43,6 +44,8 @@ def sync_artifact_bundles(): lookback_days = options.get("sentry.demo_mode.sync_artifact_bundles.lookback_days") _sync_artifact_bundles(source_org, target_org, lookback_days) + _sync_project_debug_files(source_org, target_org, lookback_days) + _sync_proguard_artifact_releases(source_org, target_org, lookback_days) def _sync_artifact_bundles(source_org: Organization, target_org: Organization, lookback_days=1): @@ -67,6 +70,62 @@ def _sync_artifact_bundles(source_org: Organization, target_org: Organization, l _sync_artifact_bundle(source_artifact_bundle, target_org) +def _sync_project_debug_files(source_org: Organization, target_org: Organization, lookback_days=1): + if not source_org or not target_org: + return + + source_project_ids = Project.objects.filter( + organization_id=source_org.id, + ).values_list("id", flat=True) + + target_project_ids = Project.objects.filter( + organization_id=target_org.id, + ).values_list("id", flat=True) + + source_project_debug_files = ProjectDebugFile.objects.filter( + project_id__in=source_project_ids, + ) + + target_project_debug_files = ProjectDebugFile.objects.filter( + project_id__in=target_project_ids, + ) + + different_project_debug_files = source_project_debug_files.exclude( + debug_id__in=target_project_debug_files.values_list("debug_id", flat=True) + ) + + for source_project_debug_file in different_project_debug_files: + _sync_project_debug_file(source_project_debug_file, target_org) + + +def _sync_proguard_artifact_releases( + source_org: Organization, target_org: Organization, lookback_days=1 +): + if not source_org or not target_org: + return + + cutoff_date = timezone.now() - timedelta(days=lookback_days) + + proguard_artifact_releases = ProguardArtifactRelease.objects.filter( + Q(organization_id=source_org.id) | Q(organization_id=target_org.id), + date_added__gte=cutoff_date, + ) + + source_proguard_artifact_releases = proguard_artifact_releases.filter( + organization_id=source_org.id, + ) + target_proguard_artifact_releases = proguard_artifact_releases.filter( + organization_id=target_org.id, + ) + + different_proguard_artifact_releases = source_proguard_artifact_releases.exclude( + proguard_uuid__in=target_proguard_artifact_releases.values_list("proguard_uuid", flat=True) + ) + + for source_proguard_artifact_release in different_proguard_artifact_releases: + _sync_proguard_artifact_release(source_proguard_artifact_release, target_org) + + def _sync_artifact_bundle(source_artifact_bundle: ArtifactBundle, target_org: Organization): try: with atomic_transaction( @@ -144,6 +203,75 @@ def _sync_release_artifact_bundle( ) +def _sync_project_debug_file( + source_project_debug_file: ProjectDebugFile, target_org: Organization +) -> ProjectDebugFile | None: + try: + with atomic_transaction(using=(router.db_for_write(ProjectDebugFile))): + target_project = _find_matching_project( + source_project_debug_file.project_id, + target_org.id, + ) + + if not target_project: + return None + + return ProjectDebugFile.objects.create( + project_id=target_project.id, + file=source_project_debug_file.file, + checksum=source_project_debug_file.checksum, + object_name=source_project_debug_file.object_name, + cpu_name=source_project_debug_file.cpu_name, + debug_id=source_project_debug_file.debug_id, + code_id=source_project_debug_file.code_id, + data=source_project_debug_file.data, + date_accessed=source_project_debug_file.date_accessed, + ) + except IntegrityError as e: + sentry_sdk.capture_exception(e) + return None + + +def _sync_proguard_artifact_release( + source_proguard_artifact_release: ProguardArtifactRelease, target_org: Organization +): + try: + with atomic_transaction(using=(router.db_for_write(ProguardArtifactRelease))): + target_project = _find_matching_project( + source_proguard_artifact_release.project_id, + target_org.id, + ) + + if not target_project: + return + + # project_debug_file _should_ already be synced, but we'll make sure it is + project_debug_file = ProjectDebugFile.objects.filter( + project_id=target_project.id, + debug_id=source_proguard_artifact_release.project_debug_file.debug_id, + ).first() + + if not project_debug_file: + project_debug_file = _sync_project_debug_file( + source_proguard_artifact_release.project_debug_file, target_org + ) + + if not project_debug_file: + # we require a project debug file + return + + ProguardArtifactRelease.objects.create( + organization_id=target_org.id, + project_id=target_project.id, + release_name=source_proguard_artifact_release.release_name, + proguard_uuid=source_proguard_artifact_release.proguard_uuid, + project_debug_file=project_debug_file, + date_added=source_proguard_artifact_release.date_added, + ) + except IntegrityError as e: + sentry_sdk.capture_exception(e) + + def _find_matching_project(project_id, organization_id): try: source_project = Project.objects.get(id=project_id) diff --git a/tests/sentry/demo_mode/test_tasks.py b/tests/sentry/demo_mode/test_tasks.py index e1eafe72917fc4..6ea8a4d4c99e06 100644 --- a/tests/sentry/demo_mode/test_tasks.py +++ b/tests/sentry/demo_mode/test_tasks.py @@ -1,15 +1,21 @@ from datetime import datetime, timedelta from unittest import mock +from uuid import uuid1 from django.db.utils import IntegrityError from django.utils import timezone -from sentry.demo_mode.tasks import _sync_artifact_bundles +from sentry.demo_mode.tasks import ( + _sync_artifact_bundles, + _sync_proguard_artifact_releases, + _sync_project_debug_files, +) from sentry.models.artifactbundle import ( ArtifactBundle, ProjectArtifactBundle, ReleaseArtifactBundle, ) +from sentry.models.debugfile import ProguardArtifactRelease, ProjectDebugFile from sentry.models.organization import Organization from sentry.models.project import Project from sentry.testutils.cases import TestCase @@ -52,6 +58,23 @@ def set_up_artifact_bundle( return artifact_bundle, project_artifact_bundle, release_artifact_bundle + def set_up_proguard_artifact_release( + self, + organization: Organization, + project: Project, + date_added: datetime | None = None, + ): + date_added = date_added or timezone.now() + proguard_artifact_release = ProguardArtifactRelease.objects.create( + organization_id=organization.id, + project_id=project.id, + release_name="release", + proguard_uuid=uuid1(), + project_debug_file=self.create_dif_file(project), + date_added=date_added, + ) + return proguard_artifact_release + def test_sync_artifact_bundles_no_bundles(self): _sync_artifact_bundles(source_org=self.source_org, target_org=self.target_org) @@ -145,3 +168,87 @@ def test_sync_artifact_bundles_rolls_back_on_error(self, _): assert not ArtifactBundle.objects.filter(organization_id=self.target_org.id).exists() assert not ProjectArtifactBundle.objects.filter(organization_id=self.target_org.id).exists() assert not ReleaseArtifactBundle.objects.filter(organization_id=self.target_org.id).exists() + + def test_sync_project_debug_files(self): + source_project_debug_file = self.create_dif_file(self.source_proj_foo) + + assert not ProjectDebugFile.objects.filter( + project_id=self.target_proj_foo.id, + debug_id=source_project_debug_file.debug_id, + ).exists() + + _sync_project_debug_files(source_org=self.source_org, target_org=self.target_org) + + target_project_debug_file = ProjectDebugFile.objects.get( + project_id=self.target_proj_foo.id, + debug_id=source_project_debug_file.debug_id, + ) + + assert target_project_debug_file.debug_id == source_project_debug_file.debug_id + assert target_project_debug_file.code_id == source_project_debug_file.code_id + assert target_project_debug_file.cpu_name == source_project_debug_file.cpu_name + + def test_sync_project_debug_files_with_old_uploads(self): + source_project_debug_file = self.create_dif_file( + self.source_proj_foo, + date_accessed=timezone.now() - timedelta(days=2), + ) + + assert not ProjectDebugFile.objects.filter( + project_id=self.target_proj_foo.id, + debug_id=source_project_debug_file.debug_id, + ).exists() + + _sync_project_debug_files(source_org=self.source_org, target_org=self.target_org) + + assert ProjectDebugFile.objects.filter( + project_id=self.target_proj_foo.id, + debug_id=source_project_debug_file.debug_id, + ).exists() + + def test_sync_proguard_artifact_releases(self): + source_proguard_artifact_release = self.set_up_proguard_artifact_release( + self.source_org, + self.source_proj_foo, + ) + + assert not ProguardArtifactRelease.objects.filter( + organization_id=self.target_org.id, + proguard_uuid=source_proguard_artifact_release.proguard_uuid, + ).exists() + + _sync_proguard_artifact_releases(source_org=self.source_org, target_org=self.target_org) + + target_proguard_artifact_release = ProguardArtifactRelease.objects.get( + organization_id=self.target_org.id, + proguard_uuid=source_proguard_artifact_release.proguard_uuid, + ) + + assert ( + target_proguard_artifact_release.release_name + == source_proguard_artifact_release.release_name + ) + assert ( + target_proguard_artifact_release.proguard_uuid + == source_proguard_artifact_release.proguard_uuid + ) + assert target_proguard_artifact_release.project_id == self.target_proj_foo.id + + def test_sync_proguard_artifact_releases_with_old_uploads(self): + source_proguard_artifact_release = self.set_up_proguard_artifact_release( + self.source_org, + self.source_proj_foo, + date_added=timezone.now() - timedelta(days=2), + ) + + assert not ProguardArtifactRelease.objects.filter( + organization_id=self.target_org.id, + proguard_uuid=source_proguard_artifact_release.proguard_uuid, + ).exists() + + _sync_artifact_bundles(source_org=self.source_org, target_org=self.target_org) + + assert not ProguardArtifactRelease.objects.filter( + organization_id=self.target_org.id, + proguard_uuid=source_proguard_artifact_release.proguard_uuid, + ).exists()