feat(flags): add Unleash feature flagging integration (#3888)

Adds an integration for tracking flag evaluations from [Unleash](https://www.getunleash.io/) customers. 

Implementation
Unleash has no native support for evaluation hooks/listeners, unless the user opts in for each flag. Therefore we decided to patch the `is_enabled` and `get_variant` methods on the `UnleashClient` class. The methods are wrapped and the only side effect is writing to Sentry scope, so users shouldn't see any change in behavior. 

We patch one `UnleashClient` instance instead of the whole class. The reasons for this are described in
- #3895

It's also safer to not modify the unleash import.

References
- https://develop.sentry.dev/sdk/expected-features/#feature-flags
- https://docs.getunleash.io/reference/sdks/python for methods we're patching/wrapping

---------

Co-authored-by: Anton Pirker <[email protected]>
Co-authored-by: Colton Allen <[email protected]>

Author: 3 people

.github/workflows/test-integrations-misc.yml

@@ -79,6 +79,10 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-typer-latest"
+      - name: Test unleash latest
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh "py${{ matrix.python-version }}-unleash-latest"
       - name: Generate coverage XML (Python 3.6)
         if: ${{ !cancelled() && matrix.python-version == '3.6' }}
         run: |
@@ -163,6 +167,10 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-typer"
+      - name: Test unleash pinned
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-unleash"
       - name: Generate coverage XML (Python 3.6)
         if: ${{ !cancelled() && matrix.python-version == '3.6' }}
         run: |

requirements-linting.txt

@@ -17,4 +17,5 @@ pre-commit # local linting
 httpcore
 openfeature-sdk
 launchdarkly-server-sdk
+UnleashClient
 typer

scripts/split_tox_gh_actions/split_tox_gh_actions.py

@@ -133,6 +133,7 @@
         "pure_eval",
         "trytond",
         "typer",
+        "unleash",
     ],
 }
 

sentry_sdk/integrations/unleash.py

@@ -0,0 +1,55 @@
+from functools import wraps
+from typing import Any
+
+import sentry_sdk
+from sentry_sdk.flag_utils import flag_error_processor
+from sentry_sdk.integrations import Integration, DidNotEnable
+
+try:
+    from UnleashClient import UnleashClient
+except ImportError:
+    raise DidNotEnable("UnleashClient is not installed")
+
+
+class UnleashIntegration(Integration):
+    identifier = "unleash"
+
+    @staticmethod
+    def setup_once():
+        # type: () -> None
+        # Wrap and patch evaluation methods (instance methods)
+        old_is_enabled = UnleashClient.is_enabled
+        old_get_variant = UnleashClient.get_variant
+
+        @wraps(old_is_enabled)
+        def sentry_is_enabled(self, feature, *args, **kwargs):
+            # type: (UnleashClient, str, *Any, **Any) -> Any
+            enabled = old_is_enabled(self, feature, *args, **kwargs)
+
+            # We have no way of knowing what type of unleash feature this is, so we have to treat
+            # it as a boolean / toggle feature.
+            flags = sentry_sdk.get_current_scope().flags
+            flags.set(feature, enabled)
+
+            return enabled
+
+        @wraps(old_get_variant)
+        def sentry_get_variant(self, feature, *args, **kwargs):
+            # type: (UnleashClient, str, *Any, **Any) -> Any
+            variant = old_get_variant(self, feature, *args, **kwargs)
+            enabled = variant.get("enabled", False)
+
+            # Payloads are not always used as the feature's value for application logic. They
+            # may be used for metrics or debugging context instead. Therefore, we treat every
+            # variant as a boolean toggle, using the `enabled` field.
+            flags = sentry_sdk.get_current_scope().flags
+            flags.set(feature, enabled)
+
+            return variant
+
+        UnleashClient.is_enabled = sentry_is_enabled  # type: ignore
+        UnleashClient.get_variant = sentry_get_variant  # type: ignore
+
+        # Error processor
+        scope = sentry_sdk.get_current_scope()
+        scope.add_error_processor(flag_error_processor)

setup.py

@@ -80,6 +80,7 @@ def get_file_text(file_name):
         "starlette": ["starlette>=0.19.1"],
         "starlite": ["starlite>=1.48"],
         "tornado": ["tornado>=6"],
+        "unleash": ["UnleashClient>=6.0.1"],
     },
     entry_points={
         "opentelemetry_propagator": [

tests/conftest.py

@@ -10,6 +10,7 @@
 import pytest
 import jsonschema
 
+
 try:
     import gevent
 except ImportError:

tests/integrations/unleash/__init__.py

@@ -0,0 +1,3 @@
+import pytest
+
+pytest.importorskip("UnleashClient")

tests/integrations/unleash/test_unleash.py

@@ -0,0 +1,308 @@
+import concurrent.futures as cf
+import sys
+from random import random
+from unittest import mock
+from UnleashClient import UnleashClient
+
+import pytest
+
+import sentry_sdk
+from sentry_sdk.integrations.unleash import UnleashIntegration
+from tests.integrations.unleash.testutils import mock_unleash_client
+
+
+def test_is_enabled(sentry_init, capture_events, uninstall_integration):
+    uninstall_integration(UnleashIntegration.identifier)
+
+    with mock_unleash_client():
+        client = UnleashClient()
+        sentry_init(integrations=[UnleashIntegration()])
+        client.is_enabled("hello")
+        client.is_enabled("world")
+        client.is_enabled("other")
+
+    events = capture_events()
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 1
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+            {"flag": "world", "result": False},
+            {"flag": "other", "result": False},
+        ]
+    }
+
+
+def test_get_variant(sentry_init, capture_events, uninstall_integration):
+    uninstall_integration(UnleashIntegration.identifier)
+
+    with mock_unleash_client():
+        client = UnleashClient()
+        sentry_init(integrations=[UnleashIntegration()])  # type: ignore
+        client.get_variant("no_payload_feature")
+        client.get_variant("string_feature")
+        client.get_variant("json_feature")
+        client.get_variant("csv_feature")
+        client.get_variant("number_feature")
+        client.get_variant("unknown_feature")
+
+    events = capture_events()
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 1
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "no_payload_feature", "result": True},
+            {"flag": "string_feature", "result": True},
+            {"flag": "json_feature", "result": True},
+            {"flag": "csv_feature", "result": True},
+            {"flag": "number_feature", "result": True},
+            {"flag": "unknown_feature", "result": False},
+        ]
+    }
+
+
+def test_is_enabled_threaded(sentry_init, capture_events, uninstall_integration):
+    uninstall_integration(UnleashIntegration.identifier)
+
+    with mock_unleash_client():
+        client = UnleashClient()
+        sentry_init(integrations=[UnleashIntegration()])  # type: ignore
+        events = capture_events()
+
+        def task(flag_key):
+            # Creates a new isolation scope for the thread.
+            # This means the evaluations in each task are captured separately.
+            with sentry_sdk.isolation_scope():
+                client.is_enabled(flag_key)
+                # use a tag to identify to identify events later on
+                sentry_sdk.set_tag("task_id", flag_key)
+                sentry_sdk.capture_exception(Exception("something wrong!"))
+
+        # Capture an eval before we split isolation scopes.
+        client.is_enabled("hello")
+
+        with cf.ThreadPoolExecutor(max_workers=2) as pool:
+            pool.map(task, ["world", "other"])
+
+    # Capture error in original scope
+    sentry_sdk.set_tag("task_id", "0")
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 3
+    events.sort(key=lambda e: e["tags"]["task_id"])
+
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+        ]
+    }
+    assert events[1]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+            {"flag": "other", "result": False},
+        ]
+    }
+    assert events[2]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+            {"flag": "world", "result": False},
+        ]
+    }
+
+
+def test_get_variant_threaded(sentry_init, capture_events, uninstall_integration):
+    uninstall_integration(UnleashIntegration.identifier)
+
+    with mock_unleash_client():
+        client = UnleashClient()
+        sentry_init(integrations=[UnleashIntegration()])  # type: ignore
+        events = capture_events()
+
+        def task(flag_key):
+            # Creates a new isolation scope for the thread.
+            # This means the evaluations in each task are captured separately.
+            with sentry_sdk.isolation_scope():
+                client.get_variant(flag_key)
+                # use a tag to identify to identify events later on
+                sentry_sdk.set_tag("task_id", flag_key)
+                sentry_sdk.capture_exception(Exception("something wrong!"))
+
+        # Capture an eval before we split isolation scopes.
+        client.get_variant("hello")
+
+        with cf.ThreadPoolExecutor(max_workers=2) as pool:
+            pool.map(task, ["no_payload_feature", "other"])
+
+    # Capture error in original scope
+    sentry_sdk.set_tag("task_id", "0")
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 3
+    events.sort(key=lambda e: e["tags"]["task_id"])
+
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+        ]
+    }
+    assert events[1]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "no_payload_feature", "result": True},
+        ]
+    }
+    assert events[2]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "other", "result": False},
+        ]
+    }
+
+
+@pytest.mark.skipif(sys.version_info < (3, 7), reason="requires python3.7 or higher")
+def test_is_enabled_asyncio(sentry_init, capture_events, uninstall_integration):
+    asyncio = pytest.importorskip("asyncio")
+    uninstall_integration(UnleashIntegration.identifier)
+
+    with mock_unleash_client():
+        client = UnleashClient()
+        sentry_init(integrations=[UnleashIntegration()])  # type: ignore
+        events = capture_events()
+
+        async def task(flag_key):
+            with sentry_sdk.isolation_scope():
+                client.is_enabled(flag_key)
+                # use a tag to identify to identify events later on
+                sentry_sdk.set_tag("task_id", flag_key)
+                sentry_sdk.capture_exception(Exception("something wrong!"))
+
+        async def runner():
+            return asyncio.gather(task("world"), task("other"))
+
+        # Capture an eval before we split isolation scopes.
+        client.is_enabled("hello")
+
+        asyncio.run(runner())
+
+    # Capture error in original scope
+    sentry_sdk.set_tag("task_id", "0")
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 3
+    events.sort(key=lambda e: e["tags"]["task_id"])
+
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+        ]
+    }
+    assert events[1]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+            {"flag": "other", "result": False},
+        ]
+    }
+    assert events[2]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+            {"flag": "world", "result": False},
+        ]
+    }
+
+
+@pytest.mark.skipif(sys.version_info < (3, 7), reason="requires python3.7 or higher")
+def test_get_variant_asyncio(sentry_init, capture_events, uninstall_integration):
+    asyncio = pytest.importorskip("asyncio")
+
+    uninstall_integration(UnleashIntegration.identifier)
+
+    with mock_unleash_client():
+        client = UnleashClient()
+        sentry_init(integrations=[UnleashIntegration()])  # type: ignore
+        events = capture_events()
+
+        async def task(flag_key):
+            with sentry_sdk.isolation_scope():
+                client.get_variant(flag_key)
+                # use a tag to identify to identify events later on
+                sentry_sdk.set_tag("task_id", flag_key)
+                sentry_sdk.capture_exception(Exception("something wrong!"))
+
+        async def runner():
+            return asyncio.gather(task("no_payload_feature"), task("other"))
+
+        # Capture an eval before we split isolation scopes.
+        client.get_variant("hello")
+
+        asyncio.run(runner())
+
+    # Capture error in original scope
+    sentry_sdk.set_tag("task_id", "0")
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 3
+    events.sort(key=lambda e: e["tags"]["task_id"])
+
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+        ]
+    }
+    assert events[1]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "no_payload_feature", "result": True},
+        ]
+    }
+    assert events[2]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "other", "result": False},
+        ]
+    }
+
+
+def test_wraps_original(sentry_init, uninstall_integration):
+    with mock_unleash_client():
+        client = UnleashClient()
+
+        mock_is_enabled = mock.Mock(return_value=random() < 0.5)
+        mock_get_variant = mock.Mock(return_value={"enabled": random() < 0.5})
+        client.is_enabled = mock_is_enabled
+        client.get_variant = mock_get_variant
+
+        uninstall_integration(UnleashIntegration.identifier)
+        sentry_init(integrations=[UnleashIntegration()])  # type: ignore
+
+    res = client.is_enabled("test-flag", "arg", kwarg=1)
+    assert res == mock_is_enabled.return_value
+    assert mock_is_enabled.call_args == (
+        ("test-flag", "arg"),
+        {"kwarg": 1},
+    )
+
+    res = client.get_variant("test-flag", "arg", kwarg=1)
+    assert res == mock_get_variant.return_value
+    assert mock_get_variant.call_args == (
+        ("test-flag", "arg"),
+        {"kwarg": 1},
+    )
+
+
+def test_wrapper_attributes(sentry_init, uninstall_integration):
+    with mock_unleash_client():
+        client = UnleashClient()  # <- Returns a MockUnleashClient
+
+        original_is_enabled = client.is_enabled
+        original_get_variant = client.get_variant
+
+        uninstall_integration(UnleashIntegration.identifier)
+        sentry_init(integrations=[UnleashIntegration()])  # type: ignore
+
+        # Mock clients methods have not lost their qualified names after decoration.
+        assert client.is_enabled.__name__ == "is_enabled"
+        assert client.is_enabled.__qualname__ == original_is_enabled.__qualname__
+        assert client.get_variant.__name__ == "get_variant"
+        assert client.get_variant.__qualname__ == original_get_variant.__qualname__

tests/integrations/unleash/testutils.py

@@ -0,0 +1,77 @@
+from contextlib import contextmanager
+from UnleashClient import UnleashClient
+
+
+@contextmanager
+def mock_unleash_client():
+    """
+    Temporarily replaces UnleashClient's methods with mock implementations
+    for testing.
+
+    This context manager swaps out UnleashClient's __init__, is_enabled,
+    and get_variant methods with mock versions from MockUnleashClient.
+    Original methods are restored when exiting the context.
+
+    After mocking the client class the integration can be initialized.
+    The methods on the mock client class are overridden by the
+    integration and flag tracking proceeds as expected.
+
+    Example:
+        with mock_unleash_client():
+            client = UnleashClient()  # Uses mock implementation
+            sentry_init(integrations=[UnleashIntegration()])
+    """
+    old_init = UnleashClient.__init__
+    old_is_enabled = UnleashClient.is_enabled
+    old_get_variant = UnleashClient.get_variant
+
+    UnleashClient.__init__ = MockUnleashClient.__init__
+    UnleashClient.is_enabled = MockUnleashClient.is_enabled
+    UnleashClient.get_variant = MockUnleashClient.get_variant
+
+    yield
+
+    UnleashClient.__init__ = old_init
+    UnleashClient.is_enabled = old_is_enabled
+    UnleashClient.get_variant = old_get_variant
+
+
+class MockUnleashClient:
+
+    def __init__(self, *a, **kw):
+        self.features = {
+            "hello": True,
+            "world": False,
+        }
+
+        self.feature_to_variant = {
+            "string_feature": {
+                "name": "variant1",
+                "enabled": True,
+                "payload": {"type": "string", "value": "val1"},
+            },
+            "json_feature": {
+                "name": "variant1",
+                "enabled": True,
+                "payload": {"type": "json", "value": '{"key1": 0.53}'},
+            },
+            "number_feature": {
+                "name": "variant1",
+                "enabled": True,
+                "payload": {"type": "number", "value": "134.5"},
+            },
+            "csv_feature": {
+                "name": "variant1",
+                "enabled": True,
+                "payload": {"type": "csv", "value": "abc 123\ncsbq 94"},
+            },
+            "no_payload_feature": {"name": "variant1", "enabled": True},
+        }
+
+        self.disabled_variant = {"name": "disabled", "enabled": False}
+
+    def is_enabled(self, feature, *a, **kw):
+        return self.features.get(feature, False)
+
+    def get_variant(self, feature, *a, **kw):
+        return self.feature_to_variant.get(feature, self.disabled_variant)

tox.ini

@@ -168,6 +168,10 @@ envlist =
     {py3.9,py3.11,py3.12}-langchain-latest
     {py3.9,py3.11,py3.12}-langchain-notiktoken
 
+    # LaunchDarkly
+    {py3.8,py3.12,py3.13}-launchdarkly-v9.8.0
+    {py3.8,py3.12,py3.13}-launchdarkly-latest
+
     # Litestar
     {py3.8,py3.11}-litestar-v{2.0}
     {py3.8,py3.11,py3.12}-litestar-v{2.6}
@@ -189,10 +193,6 @@ envlist =
     {py3.8,py3.12,py3.13}-openfeature-v0.7
     {py3.8,py3.12,py3.13}-openfeature-latest
 
-    # LaunchDarkly
-    {py3.8,py3.12,py3.13}-launchdarkly-v9.8.0
-    {py3.8,py3.12,py3.13}-launchdarkly-latest
-
     # OpenTelemetry (OTel)
     {py3.7,py3.9,py3.12,py3.13}-opentelemetry
 
@@ -290,6 +290,10 @@ envlist =
     {py3.7,py3.12,py3.13}-typer-v{0.15}
     {py3.7,py3.12,py3.13}-typer-latest
 
+    # Unleash
+    {py3.8,py3.12,py3.13}-unleash-v6.0.1
+    {py3.8,py3.12,py3.13}-unleash-latest
+
 [testenv]
 deps =
     # if you change requirements-testing.txt and your change is not being reflected
@@ -571,6 +575,10 @@ deps =
     launchdarkly-v9.8.0: launchdarkly-server-sdk~=9.8.0
     launchdarkly-latest: launchdarkly-server-sdk
 
+    # Unleash
+    unleash-v6.0.1: UnleashClient~=6.0.1
+    unleash-latest: UnleashClient
+
     # OpenTelemetry (OTel)
     opentelemetry: opentelemetry-distro
 
@@ -793,6 +801,7 @@ setenv =
     tornado: TESTPATH=tests/integrations/tornado
     trytond: TESTPATH=tests/integrations/trytond
     typer: TESTPATH=tests/integrations/typer
+    unleash: TESTPATH=tests/integrations/unleash
     socket: TESTPATH=tests/integrations/socket
 
 passenv =