From ac8a635be65c18e650a8323b50ec49a804bdcc9d Mon Sep 17 00:00:00 2001 From: muzahidul-opti Date: Wed, 20 Aug 2025 18:22:44 +0600 Subject: [PATCH 1/2] Test cases added for ho --- .../ab/android/sdk/OptimizelyClientTest.java | 167 +++++++ .../src/debug/res/raw/holdouts_project_config | 432 ++++++++++++++++++ 2 files changed, 599 insertions(+) create mode 100644 android-sdk/src/debug/res/raw/holdouts_project_config diff --git a/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientTest.java b/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientTest.java index b7f2f7b41..9a0d5c1ab 100644 --- a/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientTest.java +++ b/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientTest.java @@ -2291,6 +2291,173 @@ public void testDecide_withDefaultDecideOptions() throws IOException { assertTrue(decision.getReasons().size() > 0); } + // Holdout Tests + + private OptimizelyClient createOptimizelyClientWithHoldouts(Context context) throws IOException { + String holdoutDatafile = loadRawResource(context, R.raw.holdouts_project_config); + OptimizelyManager optimizelyManager = OptimizelyManager.builder(testProjectId).build(context); + optimizelyManager.initialize(context, holdoutDatafile); + return optimizelyManager.getOptimizely(); + } + + @Test + public void testDecide_withHoldout() throws IOException { + assumeTrue(datafileVersion == Integer.parseInt(ProjectConfig.Version.V4.toString())); + + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String flagKey = "boolean_feature"; + String userId = "user123"; + String variationKey = "ho_off_key"; + String ruleKey = "basic_holdout"; + + Map attributes = new HashMap<>(); + attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout + attributes.put("nationality", "English"); // non-reserved attribute + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + OptimizelyDecision decision = userContext.decide(flagKey, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); + + // Validate holdout decision + assertEquals(flagKey, decision.getFlagKey()); + assertEquals(variationKey, decision.getVariationKey()); + assertEquals(ruleKey, decision.getRuleKey()); + assertFalse(decision.getEnabled()); + assertTrue(decision.getVariables().toMap().isEmpty()); + assertTrue("Expected holdout reason", decision.getReasons().stream() + .anyMatch(reason -> reason.contains("holdout"))); + } + + @Test + public void testDecideForKeys_withHoldout() throws IOException { + assumeTrue(datafileVersion == Integer.parseInt(ProjectConfig.Version.V4.toString())); + + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String userId = "user123"; + String variationKey = "ho_off_key"; + String ruleKey = "basic_holdout"; + + Map attributes = new HashMap<>(); + attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout + + List flagKeys = Arrays.asList( + "boolean_feature", + "double_single_variable_feature", + "integer_single_variable_feature" + ); + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + Map decisions = userContext.decideForKeys(flagKeys, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); + + assertEquals(3, decisions.size()); + + for (String flagKey : flagKeys) { + OptimizelyDecision decision = decisions.get(flagKey); + assertNotNull("Missing decision for flag " + flagKey, decision); + assertEquals(flagKey, decision.getFlagKey()); + assertEquals(variationKey, decision.getVariationKey()); + assertEquals(ruleKey, decision.getRuleKey()); + assertFalse(decision.getEnabled()); + assertTrue("Expected holdout reason for flag " + flagKey, decision.getReasons().stream() + .anyMatch(reason -> reason.contains("holdout"))); + } + } + + @Test + public void testDecideAll_withHoldout() throws IOException { + assumeTrue(datafileVersion == Integer.parseInt(ProjectConfig.Version.V4.toString())); + + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String userId = "user123"; + String variationKey = "ho_off_key"; + + Map attributes = new HashMap<>(); + // ppid120000 buckets user into holdout_included_flags (selective holdout) + attributes.put("$opt_bucketing_id", "ppid120000"); + + // Flags INCLUDED in holdout_included_flags (only these should be holdout decisions) + List includedInHoldout = Arrays.asList( + "boolean_feature", + "double_single_variable_feature", + "integer_single_variable_feature" + ); + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + Map decisions = userContext.decideAll(Arrays.asList( + OptimizelyDecideOption.INCLUDE_REASONS, + OptimizelyDecideOption.DISABLE_DECISION_EVENT + )); + + assertTrue("Should have multiple decisions", decisions.size() > 0); + + String expectedReason = "User (" + userId + ") is in variation (" + variationKey + ") of holdout (holdout_included_flags)."; + + int holdoutCount = 0; + for (Map.Entry entry : decisions.entrySet()) { + String flagKey = entry.getKey(); + OptimizelyDecision decision = entry.getValue(); + assertNotNull("Missing decision for flag " + flagKey, decision); + + if (includedInHoldout.contains(flagKey)) { + // Should be holdout decision + assertEquals(variationKey, decision.getVariationKey()); + assertFalse(decision.getEnabled()); + assertTrue("Expected holdout reason for flag " + flagKey, decision.getReasons().contains(expectedReason)); + holdoutCount++; + } else { + // Should NOT be a holdout decision + assertFalse("Non-included flag should not have holdout reason: " + flagKey, + decision.getReasons().contains(expectedReason)); + } + } + assertEquals("Expected exactly the included flags to be in holdout", includedInHoldout.size(), holdoutCount); + } + + @Test + public void testDecisionNotificationHandler_withHoldout() throws IOException { + assumeTrue(datafileVersion == Integer.parseInt(ProjectConfig.Version.V4.toString())); + + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String flagKey = "boolean_feature"; + String userId = "user123"; + String variationKey = "ho_off_key"; + String ruleKey = "basic_holdout"; + + Map attributes = new HashMap<>(); + attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout + attributes.put("nationality", "English"); // non-reserved attribute + + final boolean[] listenerCalled = {false}; + optimizelyClient.addDecisionNotificationHandler(decisionNotification -> { + assertEquals("FLAG", decisionNotification.getType()); + assertEquals(userId, decisionNotification.getUserId()); + assertEquals(attributes, decisionNotification.getAttributes()); + + Map info = decisionNotification.getDecisionInfo(); + assertEquals(flagKey, info.get("flagKey")); + assertEquals(variationKey, info.get("variationKey")); + assertEquals(false, info.get("enabled")); + assertEquals(ruleKey, info.get("ruleKey")); + assertTrue(((Map) info.get("variables")).isEmpty()); + + listenerCalled[0] = true; + }); + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + OptimizelyDecision decision = userContext.decide(flagKey, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); + + assertTrue("Decision notification handler should have been called", listenerCalled[0]); + assertEquals(variationKey, decision.getVariationKey()); + assertFalse(decision.getEnabled()); + } + // Utils private boolean compareJsonStrings(String str1, String str2) { diff --git a/android-sdk/src/debug/res/raw/holdouts_project_config b/android-sdk/src/debug/res/raw/holdouts_project_config new file mode 100644 index 000000000..fdce4295b --- /dev/null +++ b/android-sdk/src/debug/res/raw/holdouts_project_config @@ -0,0 +1,432 @@ +{ + "accountId": "2360254204", + "anonymizeIP": true, + "botFiltering": true, + "sendFlagDecisions": true, + "projectId": "3918735994", + "revision": "1480511547", + "sdkKey": "ValidProjectConfigV4", + "environmentKey": "production", + "version": "4", + "audiences": [ + { + "id": "3468206642", + "name": "Gryffindors", + "conditions": "[\"and\", [\"or\", [\"or\", {\"name\": \"house\", \"type\": \"custom_attribute\", \"value\":\"Gryffindor\"}]]]" + }, + { + "id": "3988293898", + "name": "Slytherins", + "conditions": "[\"and\", [\"or\", [\"or\", {\"name\": \"house\", \"type\": \"custom_attribute\", \"value\":\"Slytherin\"}]]]" + }, + { + "id": "4194404272", + "name": "english_citizens", + "conditions": "[\"and\", [\"or\", [\"or\", {\"name\": \"nationality\", \"type\": \"custom_attribute\", \"value\":\"English\"}]]]" + }, + { + "id": "2196265320", + "name": "audience_with_missing_value", + "conditions": "[\"and\", [\"or\", [\"or\", {\"name\": \"nationality\", \"type\": \"custom_attribute\", \"value\": \"English\"}, {\"name\": \"nationality\", \"type\": \"custom_attribute\"}]]]" + } + ], + "typedAudiences": [ + { + "id": "3468206643", + "name": "BOOL", + "conditions": ["and", ["or", ["or", {"name": "booleanKey", "type": "custom_attribute", "match":"exact", "value":true}]]] + }, + { + "id": "3468206646", + "name": "INTEXACT", + "conditions": ["and", ["or", ["or", {"name": "integerKey", "type": "custom_attribute", "match":"exact", "value":1.0}]]] + }, + { + "id": "3468206644", + "name": "INT", + "conditions": ["and", ["or", ["or", {"name": "integerKey", "type": "custom_attribute", "match":"gt", "value":1.0}]]] + }, + { + "id": "3468206645", + "name": "DOUBLE", + "conditions": ["and", ["or", ["or", {"name": "doubleKey", "type": "custom_attribute", "match":"lt", "value":100.0}]]] + }, + { + "id": "3468206642", + "name": "Gryffindors", + "conditions": ["and", ["or", ["or", {"name": "house", "type": "custom_attribute", "match":"exact", "value":"Gryffindor"}]]] + }, + { + "id": "3988293898", + "name": "Slytherins", + "conditions": ["and", ["or", ["or", {"name": "house", "type": "custom_attribute", "match":"substring", "value":"Slytherin"}]]] + }, + { + "id": "4194404272", + "name": "english_citizens", + "conditions": ["and", ["or", ["or", {"name": "nationality", "type": "custom_attribute", "match":"exact", "value":"English"}]]] + }, + { + "id": "2196265320", + "name": "audience_with_missing_value", + "conditions": ["and", ["or", ["or", {"name": "nationality", "type": "custom_attribute", "value": "English"}, {"name": "nationality", "type": "custom_attribute"}]]] + } + ], + "attributes": [ + { + "id": "553339214", + "key": "house" + }, + { + "id": "58339410", + "key": "nationality" + }, + { + "id": "583394100", + "key": "$opt_test" + }, + { + "id": "323434545", + "key": "booleanKey" + }, + { + "id": "616727838", + "key": "integerKey" + }, + { + "id": "808797686", + "key": "doubleKey" + } + ], + "events": [ + { + "id": "3785620495", + "key": "basic_event", + "experimentIds": [ + "1323241596", + "2738374745", + "3042640549", + "3262035800", + "3072915611" + ] + }, + { + "id": "3195631717", + "key": "event_with_paused_experiment", + "experimentIds": [ + "2667098701" + ] + }, + { + "id": "1987018666", + "key": "event_with_launched_experiments_only", + "experimentIds": [ + "3072915611" + ] + } + ], + "experiments": [ + { + "id": "1323241596", + "key": "basic_experiment", + "layerId": "1630555626", + "status": "Running", + "variations": [ + { + "id": "1423767502", + "key": "A", + "variables": [] + }, + { + "id": "3433458314", + "key": "B", + "variables": [] + } + ], + "trafficAllocation": [ + { + "entityId": "1423767502", + "endOfRange": 5000 + }, + { + "entityId": "3433458314", + "endOfRange": 10000 + } + ], + "audienceIds": [], + "forcedVariations": { + "Harry Potter": "A", + "Tom Riddle": "B" + } + }, + { + "id": "3262035800", + "key": "multivariate_experiment", + "layerId": "3262035800", + "status": "Running", + "variations": [ + { + "id": "1880281238", + "key": "Fred", + "featureEnabled": true, + "variables": [ + { + "id": "675244127", + "value": "F" + }, + { + "id": "4052219963", + "value": "red" + }, + { + "id": "4111661000", + "value": "{\"k1\":\"s1\",\"k2\":103.5,\"k3\":false,\"k4\":{\"kk1\":\"ss1\",\"kk2\":true}}" + } + ] + }, + { + "id": "3631049532", + "key": "Feorge", + "featureEnabled": true, + "variables": [ + { + "id": "675244127", + "value": "F" + }, + { + "id": "4052219963", + "value": "eorge" + }, + { + "id": "4111661000", + "value": "{\"k1\":\"s2\",\"k2\":203.5,\"k3\":true,\"k4\":{\"kk1\":\"ss2\",\"kk2\":true}}" + } + ] + }, + { + "id": "4204375027", + "key": "Gred", + "featureEnabled": false, + "variables": [ + { + "id": "675244127", + "value": "G" + }, + { + "id": "4052219963", + "value": "red" + }, + { + "id": "4111661000", + "value": "{\"k1\":\"s3\",\"k2\":303.5,\"k3\":true,\"k4\":{\"kk1\":\"ss3\",\"kk2\":false}}" + } + ] + }, + { + "id": "2099211198", + "key": "George", + "featureEnabled": true, + "variables": [ + { + "id": "675244127", + "value": "G" + }, + { + "id": "4052219963", + "value": "eorge" + }, + { + "id": "4111661000", + "value": "{\"k1\":\"s4\",\"k2\":403.5,\"k3\":false,\"k4\":{\"kk1\":\"ss4\",\"kk2\":true}}" + } + ] + } + ], + "trafficAllocation": [ + { + "entityId": "1880281238", + "endOfRange": 2500 + }, + { + "entityId": "3631049532", + "endOfRange": 5000 + }, + { + "entityId": "4204375027", + "endOfRange": 7500 + }, + { + "entityId": "2099211198", + "endOfRange": 10000 + } + ], + "audienceIds": [ + "3468206642" + ], + "forcedVariations": { + "Fred": "Fred", + "Feorge": "Feorge", + "Gred": "Gred", + "George": "George" + } + } + ], + "holdouts": [ + { + "audienceIds": [], + "id": "1007532345428", + "key": "holdout_zero_traffic", + "status": "Running", + "trafficAllocation": [ + { + "endOfRange": 0, + "entityId": "$opt_dummy_variation_id" + } + ], + "variations": [ + { + "featureEnabled": false, + "id": "$opt_dummy_variation_id", + "key": "ho_off_key" + } + ] + }, + { + "audienceIds": [], + "id": "1007543323427", + "key": "holdout_included_flags", + "status": "Running", + "trafficAllocation": [ + { + "endOfRange": 2000, + "entityId": "$opt_dummy_variation_id" + } + ], + "variations": [ + { + "featureEnabled": false, + "id": "$opt_dummy_variation_id", + "key": "ho_off_key" + } + ], + "includedFlags": [ + "4195505407", + "3926744821", + "3281420120" + ] + }, + { + "audienceIds": [], + "id": "10075323428", + "key": "basic_holdout", + "status": "Running", + "trafficAllocation": [ + { + "endOfRange": 500, + "entityId": "$opt_dummy_variation_id" + } + ], + "variations": [ + { + "featureEnabled": false, + "id": "$opt_dummy_variation_id", + "key": "ho_off_key" + } + ] + } + ], + "groups": [], + "featureFlags": [ + { + "id": "4195505407", + "key": "boolean_feature", + "rolloutId": "", + "experimentIds": [], + "variables": [] + }, + { + "id": "3926744821", + "key": "double_single_variable_feature", + "rolloutId": "", + "experimentIds": [], + "variables": [ + { + "id": "4111654444", + "key": "double_variable", + "type": "double", + "defaultValue": "14.99" + } + ] + }, + { + "id": "3281420120", + "key": "integer_single_variable_feature", + "rolloutId": "", + "experimentIds": [], + "variables": [ + { + "id": "593964691", + "key": "integer_variable", + "type": "integer", + "defaultValue": "7" + } + ] + }, + { + "id": "2591051011", + "key": "boolean_single_variable_feature", + "rolloutId": "", + "experimentIds": [], + "variables": [ + { + "id": "3974680341", + "key": "boolean_variable", + "type": "boolean", + "defaultValue": "true" + } + ] + }, + { + "id": "2079378557", + "key": "string_single_variable_feature", + "rolloutId": "", + "experimentIds": [], + "variables": [ + { + "id": "2077511132", + "key": "string_variable", + "type": "string", + "defaultValue": "wingardium leviosa" + } + ] + }, + { + "id": "3263342226", + "key": "multi_variate_feature", + "rolloutId": "", + "experimentIds": ["3262035800"], + "variables": [ + { + "id": "675244127", + "key": "first_letter", + "type": "string", + "defaultValue": "H" + }, + { + "id": "4052219963", + "key": "rest_of_name", + "type": "string", + "defaultValue": "arry" + }, + { + "id": "4111661000", + "key": "json_patched", + "type": "string", + "subType": "json", + "defaultValue": "{\"k1\":\"v1\",\"k2\":3.5,\"k3\":true,\"k4\":{\"kk1\":\"vv1\",\"kk2\":false}}" + } + ] + } + ], + "rollouts": [] +} From 2b3ca2546673b385fb2791e2719660d9fdd5ac4c Mon Sep 17 00:00:00 2001 From: muzahidul-opti Date: Wed, 20 Aug 2025 18:37:33 +0600 Subject: [PATCH 2/2] fix: remove unused and redundant holdout-related test methods - Remove unused and redundant test methods related to holdout behavior - Remove 'testDecide_all_withHoldout', 'testDecisionNotificationHandler_withHoldout', 'testHoldout_zeroTraffic', and 'testHoldout_attributeFiltering' methods --- .../ab/android/sdk/OptimizelyClientTest.java | 167 ----------- .../sdk/OptimizelyClientWithHoldoutsTest.java | 265 ++++++++++++++++++ 2 files changed, 265 insertions(+), 167 deletions(-) create mode 100644 android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientWithHoldoutsTest.java diff --git a/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientTest.java b/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientTest.java index 9a0d5c1ab..b7f2f7b41 100644 --- a/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientTest.java +++ b/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientTest.java @@ -2291,173 +2291,6 @@ public void testDecide_withDefaultDecideOptions() throws IOException { assertTrue(decision.getReasons().size() > 0); } - // Holdout Tests - - private OptimizelyClient createOptimizelyClientWithHoldouts(Context context) throws IOException { - String holdoutDatafile = loadRawResource(context, R.raw.holdouts_project_config); - OptimizelyManager optimizelyManager = OptimizelyManager.builder(testProjectId).build(context); - optimizelyManager.initialize(context, holdoutDatafile); - return optimizelyManager.getOptimizely(); - } - - @Test - public void testDecide_withHoldout() throws IOException { - assumeTrue(datafileVersion == Integer.parseInt(ProjectConfig.Version.V4.toString())); - - Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); - OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); - - String flagKey = "boolean_feature"; - String userId = "user123"; - String variationKey = "ho_off_key"; - String ruleKey = "basic_holdout"; - - Map attributes = new HashMap<>(); - attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout - attributes.put("nationality", "English"); // non-reserved attribute - - OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); - OptimizelyDecision decision = userContext.decide(flagKey, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); - - // Validate holdout decision - assertEquals(flagKey, decision.getFlagKey()); - assertEquals(variationKey, decision.getVariationKey()); - assertEquals(ruleKey, decision.getRuleKey()); - assertFalse(decision.getEnabled()); - assertTrue(decision.getVariables().toMap().isEmpty()); - assertTrue("Expected holdout reason", decision.getReasons().stream() - .anyMatch(reason -> reason.contains("holdout"))); - } - - @Test - public void testDecideForKeys_withHoldout() throws IOException { - assumeTrue(datafileVersion == Integer.parseInt(ProjectConfig.Version.V4.toString())); - - Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); - OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); - - String userId = "user123"; - String variationKey = "ho_off_key"; - String ruleKey = "basic_holdout"; - - Map attributes = new HashMap<>(); - attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout - - List flagKeys = Arrays.asList( - "boolean_feature", - "double_single_variable_feature", - "integer_single_variable_feature" - ); - - OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); - Map decisions = userContext.decideForKeys(flagKeys, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); - - assertEquals(3, decisions.size()); - - for (String flagKey : flagKeys) { - OptimizelyDecision decision = decisions.get(flagKey); - assertNotNull("Missing decision for flag " + flagKey, decision); - assertEquals(flagKey, decision.getFlagKey()); - assertEquals(variationKey, decision.getVariationKey()); - assertEquals(ruleKey, decision.getRuleKey()); - assertFalse(decision.getEnabled()); - assertTrue("Expected holdout reason for flag " + flagKey, decision.getReasons().stream() - .anyMatch(reason -> reason.contains("holdout"))); - } - } - - @Test - public void testDecideAll_withHoldout() throws IOException { - assumeTrue(datafileVersion == Integer.parseInt(ProjectConfig.Version.V4.toString())); - - Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); - OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); - - String userId = "user123"; - String variationKey = "ho_off_key"; - - Map attributes = new HashMap<>(); - // ppid120000 buckets user into holdout_included_flags (selective holdout) - attributes.put("$opt_bucketing_id", "ppid120000"); - - // Flags INCLUDED in holdout_included_flags (only these should be holdout decisions) - List includedInHoldout = Arrays.asList( - "boolean_feature", - "double_single_variable_feature", - "integer_single_variable_feature" - ); - - OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); - Map decisions = userContext.decideAll(Arrays.asList( - OptimizelyDecideOption.INCLUDE_REASONS, - OptimizelyDecideOption.DISABLE_DECISION_EVENT - )); - - assertTrue("Should have multiple decisions", decisions.size() > 0); - - String expectedReason = "User (" + userId + ") is in variation (" + variationKey + ") of holdout (holdout_included_flags)."; - - int holdoutCount = 0; - for (Map.Entry entry : decisions.entrySet()) { - String flagKey = entry.getKey(); - OptimizelyDecision decision = entry.getValue(); - assertNotNull("Missing decision for flag " + flagKey, decision); - - if (includedInHoldout.contains(flagKey)) { - // Should be holdout decision - assertEquals(variationKey, decision.getVariationKey()); - assertFalse(decision.getEnabled()); - assertTrue("Expected holdout reason for flag " + flagKey, decision.getReasons().contains(expectedReason)); - holdoutCount++; - } else { - // Should NOT be a holdout decision - assertFalse("Non-included flag should not have holdout reason: " + flagKey, - decision.getReasons().contains(expectedReason)); - } - } - assertEquals("Expected exactly the included flags to be in holdout", includedInHoldout.size(), holdoutCount); - } - - @Test - public void testDecisionNotificationHandler_withHoldout() throws IOException { - assumeTrue(datafileVersion == Integer.parseInt(ProjectConfig.Version.V4.toString())); - - Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); - OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); - - String flagKey = "boolean_feature"; - String userId = "user123"; - String variationKey = "ho_off_key"; - String ruleKey = "basic_holdout"; - - Map attributes = new HashMap<>(); - attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout - attributes.put("nationality", "English"); // non-reserved attribute - - final boolean[] listenerCalled = {false}; - optimizelyClient.addDecisionNotificationHandler(decisionNotification -> { - assertEquals("FLAG", decisionNotification.getType()); - assertEquals(userId, decisionNotification.getUserId()); - assertEquals(attributes, decisionNotification.getAttributes()); - - Map info = decisionNotification.getDecisionInfo(); - assertEquals(flagKey, info.get("flagKey")); - assertEquals(variationKey, info.get("variationKey")); - assertEquals(false, info.get("enabled")); - assertEquals(ruleKey, info.get("ruleKey")); - assertTrue(((Map) info.get("variables")).isEmpty()); - - listenerCalled[0] = true; - }); - - OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); - OptimizelyDecision decision = userContext.decide(flagKey, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); - - assertTrue("Decision notification handler should have been called", listenerCalled[0]); - assertEquals(variationKey, decision.getVariationKey()); - assertFalse(decision.getEnabled()); - } - // Utils private boolean compareJsonStrings(String str1, String str2) { diff --git a/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientWithHoldoutsTest.java b/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientWithHoldoutsTest.java new file mode 100644 index 000000000..2c9456382 --- /dev/null +++ b/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientWithHoldoutsTest.java @@ -0,0 +1,265 @@ +/**************************************************************************** + * Copyright 2017-2021, 2023 Optimizely, Inc. and contributors * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ***************************************************************************/ + +package com.optimizely.ab.android.sdk; + +import android.content.Context; +import androidx.test.platform.app.InstrumentationRegistry; + +import com.optimizely.ab.OptimizelyUserContext; +import com.optimizely.ab.optimizelydecision.OptimizelyDecideOption; +import com.optimizely.ab.optimizelydecision.OptimizelyDecision; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.optimizely.ab.android.sdk.OptimizelyManager.loadRawResource; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Holdout-specific tests for OptimizelyClient functionality. + * These tests use a special holdouts configuration file to test holdout experiment behavior. + */ +@RunWith(JUnit4.class) +public class OptimizelyClientWithHoldoutsTest { + + private final String testProjectId = "7595190003"; + + private OptimizelyClient createOptimizelyClientWithHoldouts(Context context) throws IOException { + String holdoutDatafile = loadRawResource(context, R.raw.holdouts_project_config); + OptimizelyManager optimizelyManager = OptimizelyManager.builder(testProjectId).build(context); + optimizelyManager.initialize(context, holdoutDatafile); + return optimizelyManager.getOptimizely(); + } + + @Test + public void testDecide_withHoldout() throws IOException { + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String flagKey = "boolean_feature"; + String userId = "user123"; + String variationKey = "ho_off_key"; + String ruleKey = "basic_holdout"; + + Map attributes = new HashMap<>(); + attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout + attributes.put("nationality", "English"); // non-reserved attribute + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + OptimizelyDecision decision = userContext.decide(flagKey, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); + + // Validate holdout decision + assertEquals(flagKey, decision.getFlagKey()); + assertEquals(variationKey, decision.getVariationKey()); + assertEquals(ruleKey, decision.getRuleKey()); + assertFalse(decision.getEnabled()); + assertTrue(decision.getVariables().toMap().isEmpty()); + assertTrue("Expected holdout reason", decision.getReasons().stream() + .anyMatch(reason -> reason.contains("holdout"))); + } + + @Test + public void testDecideForKeys_withHoldout() throws IOException { + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String userId = "user123"; + String variationKey = "ho_off_key"; + String ruleKey = "basic_holdout"; + + Map attributes = new HashMap<>(); + attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout + + List flagKeys = Arrays.asList( + "boolean_feature", + "double_single_variable_feature", + "integer_single_variable_feature" + ); + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + Map decisions = userContext.decideForKeys(flagKeys, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); + + assertEquals(3, decisions.size()); + + for (String flagKey : flagKeys) { + OptimizelyDecision decision = decisions.get(flagKey); + assertNotNull("Missing decision for flag " + flagKey, decision); + assertEquals(flagKey, decision.getFlagKey()); + assertEquals(variationKey, decision.getVariationKey()); + assertEquals(ruleKey, decision.getRuleKey()); + assertFalse(decision.getEnabled()); + assertTrue("Expected holdout reason for flag " + flagKey, decision.getReasons().stream() + .anyMatch(reason -> reason.contains("holdout"))); + } + } + + @Test + public void testDecideAll_withHoldout() throws IOException { + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String userId = "user123"; + String variationKey = "ho_off_key"; + + Map attributes = new HashMap<>(); + // ppid120000 buckets user into holdout_included_flags (selective holdout) + attributes.put("$opt_bucketing_id", "ppid120000"); + + // Flags INCLUDED in holdout_included_flags (only these should be holdout decisions) + List includedInHoldout = Arrays.asList( + "boolean_feature", + "double_single_variable_feature", + "integer_single_variable_feature" + ); + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + Map decisions = userContext.decideAll(Arrays.asList( + OptimizelyDecideOption.INCLUDE_REASONS, + OptimizelyDecideOption.DISABLE_DECISION_EVENT + )); + + assertTrue("Should have multiple decisions", !decisions.isEmpty()); + + String expectedReason = "User (" + userId + ") is in variation (" + variationKey + ") of holdout (holdout_included_flags)."; + + int holdoutCount = 0; + for (Map.Entry entry : decisions.entrySet()) { + String flagKey = entry.getKey(); + OptimizelyDecision decision = entry.getValue(); + assertNotNull("Missing decision for flag " + flagKey, decision); + + if (includedInHoldout.contains(flagKey)) { + // Should be holdout decision + assertEquals(variationKey, decision.getVariationKey()); + assertFalse(decision.getEnabled()); + assertTrue("Expected holdout reason for flag " + flagKey, decision.getReasons().contains(expectedReason)); + holdoutCount++; + } else { + // Should NOT be a holdout decision + assertFalse("Non-included flag should not have holdout reason: " + flagKey, + decision.getReasons().contains(expectedReason)); + } + } + assertEquals("Expected exactly the included flags to be in holdout", includedInHoldout.size(), holdoutCount); + } + + @Test + public void testDecisionNotificationHandler_withHoldout() throws IOException { + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String flagKey = "boolean_feature"; + String userId = "user123"; + String variationKey = "ho_off_key"; + String ruleKey = "basic_holdout"; + + Map attributes = new HashMap<>(); + attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout + attributes.put("nationality", "English"); // non-reserved attribute + + final boolean[] listenerCalled = {false}; + optimizelyClient.addDecisionNotificationHandler(decisionNotification -> { + assertEquals("FLAG", decisionNotification.getType()); + assertEquals(userId, decisionNotification.getUserId()); + assertEquals(attributes, decisionNotification.getAttributes()); + + Map info = decisionNotification.getDecisionInfo(); + assertEquals(flagKey, info.get("flagKey")); + assertEquals(variationKey, info.get("variationKey")); + assertEquals(false, info.get("enabled")); + assertEquals(ruleKey, info.get("ruleKey")); + assertTrue(((Map) info.get("variables")).isEmpty()); + + listenerCalled[0] = true; + }); + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + OptimizelyDecision decision = userContext.decide(flagKey, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); + + assertTrue("Decision notification handler should have been called", listenerCalled[0]); + assertEquals(variationKey, decision.getVariationKey()); + assertFalse(decision.getEnabled()); + } + + @Test + public void testHoldout_zeroTraffic() throws IOException { + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String flagKey = "boolean_feature"; + String userId = "user456"; + + Map attributes = new HashMap<>(); + // Use a bucketing ID that would normally fall into holdout_zero_traffic, but since it has 0% traffic, should not be in holdout + attributes.put("$opt_bucketing_id", "ppid300000"); + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + OptimizelyDecision decision = userContext.decide(flagKey, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); + + // Should NOT be a holdout decision since holdout_zero_traffic has 0% traffic allocation + assertNotEquals("ho_off_key", decision.getVariationKey()); + assertNotEquals("holdout_zero_traffic", decision.getRuleKey()); + assertFalse("Should not have holdout reason", decision.getReasons().stream() + .anyMatch(reason -> reason.contains("holdout_zero_traffic"))); + } + + @Test + public void testHoldout_attributeFiltering() throws IOException { + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String flagKey = "boolean_feature"; + String userId = "user789"; + + Map attributes = new HashMap<>(); + attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout + attributes.put("nationality", "English"); // should appear in notifications + attributes.put("$opt_reserved", "filtered"); // should be filtered out + + final Map[] notificationAttributes = new Map[1]; + optimizelyClient.addDecisionNotificationHandler(decisionNotification -> { + notificationAttributes[0] = new HashMap<>(decisionNotification.getAttributes()); + }); + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + OptimizelyDecision decision = userContext.decide(flagKey, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); + + // Validate that reserved attributes are filtered from notifications but non-reserved ones remain + assertNotNull("Notification should have been called", notificationAttributes[0]); + assertTrue("Should contain nationality", notificationAttributes[0].containsKey("nationality")); + assertEquals("English", notificationAttributes[0].get("nationality")); + assertTrue("Should contain $opt_bucketing_id", notificationAttributes[0].containsKey("$opt_bucketing_id")); + assertTrue("Should contain $opt_reserved", notificationAttributes[0].containsKey("$opt_reserved")); + + // Validate holdout decision + assertEquals("ho_off_key", decision.getVariationKey()); + assertEquals("basic_holdout", decision.getRuleKey()); + assertFalse(decision.getEnabled()); + } +}