chore(seer): add seer scanner rate limit (#93652)

Adds a rate limit for starting seer scanner tasks (default 1000 per hour
per project) to prevent surprise bills.

Author: roaga

src/sentry/autofix/utils.py

@@ -7,7 +7,9 @@
 from django.conf import settings
 from pydantic import BaseModel
 
+from sentry import features, options, ratelimits
 from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
+from sentry.models.organization import Organization
 from sentry.models.project import Project
 from sentry.models.repository import Repository
 from sentry.seer.signed_seer_api import sign_with_seer_secret
@@ -157,3 +159,53 @@ class SeerAutomationSource(enum.Enum):
     ISSUE_DETAILS = "issue_details"
     ALERT = "alert"
     POST_PROCESS = "post_process"
+
+
+def is_seer_scanner_rate_limited(
+    project: Project, organization: Organization
+) -> tuple[bool, int, int]:
+    """
+    Check if Seer Scanner automation is rate limited for a given project and organization.
+
+    Returns:
+        tuple[bool, int, int]:
+            - is_rate_limited: Whether the seer scanner is rate limited.
+            - current: The current number of seer scanner runs.
+            - limit: The limit for seer scanner runs.
+    """
+    if features.has("organizations:unlimited-auto-triggered-autofix-runs", organization):
+        return False, 0, 0
+
+    limit = options.get("seer.max_num_scanner_autotriggered_per_hour", 1000)
+    is_rate_limited, current, _ = ratelimits.backend.is_limited_with_value(
+        project=project,
+        key="seer.scanner.auto_triggered",
+        limit=limit,
+        window=60 * 60,  # 1 hour
+    )
+    return is_rate_limited, current, limit
+
+
+def is_seer_autotriggered_autofix_rate_limited(
+    project: Project, organization: Organization
+) -> tuple[bool, int, int]:
+    """
+    Check if Seer Autofix automation is rate limited for a given project and organization.
+
+    Returns:
+        tuple[bool, int, int]:
+            - is_rate_limited: Whether Autofix is rate limited.
+            - current: The current number of Autofix runs.
+            - limit: The limit for Autofix runs.
+    """
+    if features.has("organizations:unlimited-auto-triggered-autofix-runs", organization):
+        return False, 0, 0
+
+    limit = options.get("seer.max_num_autofix_autotriggered_per_hour", 20)
+    is_rate_limited, current, _ = ratelimits.backend.is_limited_with_value(
+        project=project,
+        key="autofix.auto_triggered",
+        limit=limit,
+        window=60 * 60,  # 1 hour
+    )
+    return is_rate_limited, current, limit

src/sentry/integrations/utils/issue_summary_for_alerts.py

@@ -5,7 +5,7 @@
 import sentry_sdk
 
 from sentry import features, options
-from sentry.autofix.utils import SeerAutomationSource
+from sentry.autofix.utils import SeerAutomationSource, is_seer_scanner_rate_limited
 from sentry.issues.grouptype import GroupCategory
 from sentry.models.group import Group
 from sentry.seer.issue_summary import get_issue_summary
@@ -31,6 +31,17 @@ def fetch_issue_summary(group: Group) -> dict[str, Any] | None:
     if not get_seer_org_acknowledgement(org_id=group.organization.id):
         return None
 
+    is_rate_limited, current, limit = is_seer_scanner_rate_limited(project, group.organization)
+    if is_rate_limited:
+        sentry_sdk.set_tags(
+            {
+                "scanner_run_count": current,
+                "scanner_run_limit": limit,
+            }
+        )
+        logger.error("Seer scanner auto-trigger rate limit hit")
+        return None
+
     from sentry import quotas
     from sentry.constants import DataCategory
 

src/sentry/options/defaults.py

@@ -601,6 +601,10 @@
 register("alerts.issue_summary_timeout", default=5, flags=FLAG_AUTOMATOR_MODIFIABLE)
 # Issue Summary Auto-trigger rate (max number of autofix runs auto-triggered per project per hour)
 register("seer.max_num_autofix_autotriggered_per_hour", default=20, flags=FLAG_AUTOMATOR_MODIFIABLE)
+# Seer Scanner Auto-trigger rate (max number of scans auto-triggered per project per hour)
+register(
+    "seer.max_num_scanner_autotriggered_per_hour", default=1000, flags=FLAG_AUTOMATOR_MODIFIABLE
+)
 
 # Codecov Integration
 register("codecov.client-secret", flags=FLAG_CREDENTIAL | FLAG_PRIORITIZE_DISK)

src/sentry/seer/issue_summary.py

@@ -10,10 +10,14 @@
 from django.conf import settings
 from django.contrib.auth.models import AnonymousUser
 
-from sentry import eventstore, features, options, quotas, ratelimits
+from sentry import eventstore, features, quotas
 from sentry.api.serializers import EventSerializer, serialize
 from sentry.api.serializers.rest_framework.base import convert_dict_key_case, snake_to_camel_case
-from sentry.autofix.utils import SeerAutomationSource, get_autofix_state
+from sentry.autofix.utils import (
+    SeerAutomationSource,
+    get_autofix_state,
+    is_seer_autotriggered_autofix_rate_limited,
+)
 from sentry.constants import DataCategory, ObjectStatus
 from sentry.eventstore.models import Event, GroupEvent
 from sentry.locks import locks
@@ -307,24 +311,18 @@ def _run_automation(
 
     if _is_issue_fixable(group, issue_summary.scores.fixability_score):
 
-        # Rate limit auto-triggered autofix runs to prevent giant bills.
-        if not features.has("organizations:unlimited-auto-triggered-autofix-runs", organization):
-            limit = options.get("seer.max_num_autofix_autotriggered_per_hour") or 20
-            is_rate_limited, current, _ = ratelimits.backend.is_limited_with_value(
-                project=project,
-                key="autofix.auto_triggered",
-                limit=limit,
-                window=60 * 60,  # 1 hour
+        is_rate_limited, current, limit = is_seer_autotriggered_autofix_rate_limited(
+            project, organization
+        )
+        if is_rate_limited:
+            sentry_sdk.set_tags(
+                {
+                    "auto_run_count": current,
+                    "auto_run_limit": limit,
+                }
             )
-            if is_rate_limited:
-                sentry_sdk.set_tags(
-                    {
-                        "auto_run_count": current,
-                        "auto_run_limit": limit,
-                    }
-                )
-                logger.error("Autofix auto-trigger rate limit hit")
-                return
+            logger.error("Autofix auto-trigger rate limit hit")
+            return
 
         has_budget: bool = quotas.backend.has_available_reserved_budget(
             org_id=group.organization.id,

src/sentry/tasks/post_process.py

@@ -1594,6 +1594,19 @@ def kick_off_seer_automation(job: PostProcessJob) -> None:
     if not seer_enabled:
         return
 
+    from sentry.autofix.utils import is_seer_scanner_rate_limited
+
+    is_rate_limited, current, limit = is_seer_scanner_rate_limited(project, group.organization)
+    if is_rate_limited:
+        sentry_sdk.set_tags(
+            {
+                "scanner_run_count": current,
+                "scanner_run_limit": limit,
+            }
+        )
+        logger.error("Seer scanner auto-trigger rate limit hit")
+        return
+
     from sentry import quotas
     from sentry.constants import DataCategory
 

tests/sentry/autofix/test_utils.py

@@ -10,8 +10,11 @@
     get_autofix_repos_from_project_code_mappings,
     get_autofix_state,
     get_autofix_state_from_pr_id,
+    is_seer_autotriggered_autofix_rate_limited,
+    is_seer_scanner_rate_limited,
 )
 from sentry.testutils.cases import TestCase
+from sentry.testutils.helpers import with_feature
 from sentry.utils import json
 
 
@@ -183,3 +186,63 @@ def test_get_autofix_state_http_error(self, mock_post):
 
         # Assertions
         assert "HTTP Error" in str(context.value)
+
+
+class TestAutomationRateLimiting(TestCase):
+    @with_feature("organizations:unlimited-auto-triggered-autofix-runs")
+    def test_scanner_rate_limited_with_unlimited_flag(self):
+        """Test scanner rate limiting bypassed with unlimited feature flag"""
+        project = self.create_project()
+        organization = project.organization
+
+        is_rate_limited, current, limit = is_seer_scanner_rate_limited(project, organization)
+
+        assert is_rate_limited is False
+        assert current == 0
+        assert limit == 0
+
+    @patch("sentry.autofix.utils.ratelimits.backend.is_limited_with_value")
+    def test_scanner_rate_limited_logic(self, mock_is_limited):
+        """Test scanner rate limiting logic"""
+        project = self.create_project()
+        organization = project.organization
+
+        mock_is_limited.return_value = (True, 950, None)
+
+        with self.options({"seer.max_num_scanner_autotriggered_per_hour": 1000}):
+            is_rate_limited, current, limit = is_seer_scanner_rate_limited(project, organization)
+
+        assert is_rate_limited is True
+        assert current == 950
+        assert limit == 1000
+
+    @with_feature("organizations:unlimited-auto-triggered-autofix-runs")
+    def test_autofix_rate_limited_with_unlimited_flag(self):
+        """Test autofix rate limiting bypassed with unlimited feature flag"""
+        project = self.create_project()
+        organization = project.organization
+
+        is_rate_limited, current, limit = is_seer_autotriggered_autofix_rate_limited(
+            project, organization
+        )
+
+        assert is_rate_limited is False
+        assert current == 0
+        assert limit == 0
+
+    @patch("sentry.autofix.utils.ratelimits.backend.is_limited_with_value")
+    def test_autofix_rate_limited_logic(self, mock_is_limited):
+        """Test autofix rate limiting logic"""
+        project = self.create_project()
+        organization = project.organization
+
+        mock_is_limited.return_value = (True, 19, None)
+
+        with self.options({"seer.max_num_autofix_autotriggered_per_hour": 20}):
+            is_rate_limited, current, limit = is_seer_autotriggered_autofix_rate_limited(
+                project, organization
+            )
+
+        assert is_rate_limited is True
+        assert current == 19
+        assert limit == 20