diff --git a/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientWithHoldoutsTest.java b/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientWithHoldoutsTest.java new file mode 100644 index 000000000..2c9456382 --- /dev/null +++ b/android-sdk/src/androidTest/java/com/optimizely/ab/android/sdk/OptimizelyClientWithHoldoutsTest.java @@ -0,0 +1,265 @@ +/**************************************************************************** + * Copyright 2017-2021, 2023 Optimizely, Inc. and contributors * + * * + * Licensed under the Apache License, Version 2.0 (the "License"); * + * you may not use this file except in compliance with the License. * + * You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, software * + * distributed under the License is distributed on an "AS IS" BASIS, * + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * + * See the License for the specific language governing permissions and * + * limitations under the License. * + ***************************************************************************/ + +package com.optimizely.ab.android.sdk; + +import android.content.Context; +import androidx.test.platform.app.InstrumentationRegistry; + +import com.optimizely.ab.OptimizelyUserContext; +import com.optimizely.ab.optimizelydecision.OptimizelyDecideOption; +import com.optimizely.ab.optimizelydecision.OptimizelyDecision; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.optimizely.ab.android.sdk.OptimizelyManager.loadRawResource; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Holdout-specific tests for OptimizelyClient functionality. + * These tests use a special holdouts configuration file to test holdout experiment behavior. + */ +@RunWith(JUnit4.class) +public class OptimizelyClientWithHoldoutsTest { + + private final String testProjectId = "7595190003"; + + private OptimizelyClient createOptimizelyClientWithHoldouts(Context context) throws IOException { + String holdoutDatafile = loadRawResource(context, R.raw.holdouts_project_config); + OptimizelyManager optimizelyManager = OptimizelyManager.builder(testProjectId).build(context); + optimizelyManager.initialize(context, holdoutDatafile); + return optimizelyManager.getOptimizely(); + } + + @Test + public void testDecide_withHoldout() throws IOException { + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String flagKey = "boolean_feature"; + String userId = "user123"; + String variationKey = "ho_off_key"; + String ruleKey = "basic_holdout"; + + Map attributes = new HashMap<>(); + attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout + attributes.put("nationality", "English"); // non-reserved attribute + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + OptimizelyDecision decision = userContext.decide(flagKey, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); + + // Validate holdout decision + assertEquals(flagKey, decision.getFlagKey()); + assertEquals(variationKey, decision.getVariationKey()); + assertEquals(ruleKey, decision.getRuleKey()); + assertFalse(decision.getEnabled()); + assertTrue(decision.getVariables().toMap().isEmpty()); + assertTrue("Expected holdout reason", decision.getReasons().stream() + .anyMatch(reason -> reason.contains("holdout"))); + } + + @Test + public void testDecideForKeys_withHoldout() throws IOException { + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String userId = "user123"; + String variationKey = "ho_off_key"; + String ruleKey = "basic_holdout"; + + Map attributes = new HashMap<>(); + attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout + + List flagKeys = Arrays.asList( + "boolean_feature", + "double_single_variable_feature", + "integer_single_variable_feature" + ); + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + Map decisions = userContext.decideForKeys(flagKeys, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); + + assertEquals(3, decisions.size()); + + for (String flagKey : flagKeys) { + OptimizelyDecision decision = decisions.get(flagKey); + assertNotNull("Missing decision for flag " + flagKey, decision); + assertEquals(flagKey, decision.getFlagKey()); + assertEquals(variationKey, decision.getVariationKey()); + assertEquals(ruleKey, decision.getRuleKey()); + assertFalse(decision.getEnabled()); + assertTrue("Expected holdout reason for flag " + flagKey, decision.getReasons().stream() + .anyMatch(reason -> reason.contains("holdout"))); + } + } + + @Test + public void testDecideAll_withHoldout() throws IOException { + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String userId = "user123"; + String variationKey = "ho_off_key"; + + Map attributes = new HashMap<>(); + // ppid120000 buckets user into holdout_included_flags (selective holdout) + attributes.put("$opt_bucketing_id", "ppid120000"); + + // Flags INCLUDED in holdout_included_flags (only these should be holdout decisions) + List includedInHoldout = Arrays.asList( + "boolean_feature", + "double_single_variable_feature", + "integer_single_variable_feature" + ); + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + Map decisions = userContext.decideAll(Arrays.asList( + OptimizelyDecideOption.INCLUDE_REASONS, + OptimizelyDecideOption.DISABLE_DECISION_EVENT + )); + + assertTrue("Should have multiple decisions", !decisions.isEmpty()); + + String expectedReason = "User (" + userId + ") is in variation (" + variationKey + ") of holdout (holdout_included_flags)."; + + int holdoutCount = 0; + for (Map.Entry entry : decisions.entrySet()) { + String flagKey = entry.getKey(); + OptimizelyDecision decision = entry.getValue(); + assertNotNull("Missing decision for flag " + flagKey, decision); + + if (includedInHoldout.contains(flagKey)) { + // Should be holdout decision + assertEquals(variationKey, decision.getVariationKey()); + assertFalse(decision.getEnabled()); + assertTrue("Expected holdout reason for flag " + flagKey, decision.getReasons().contains(expectedReason)); + holdoutCount++; + } else { + // Should NOT be a holdout decision + assertFalse("Non-included flag should not have holdout reason: " + flagKey, + decision.getReasons().contains(expectedReason)); + } + } + assertEquals("Expected exactly the included flags to be in holdout", includedInHoldout.size(), holdoutCount); + } + + @Test + public void testDecisionNotificationHandler_withHoldout() throws IOException { + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String flagKey = "boolean_feature"; + String userId = "user123"; + String variationKey = "ho_off_key"; + String ruleKey = "basic_holdout"; + + Map attributes = new HashMap<>(); + attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout + attributes.put("nationality", "English"); // non-reserved attribute + + final boolean[] listenerCalled = {false}; + optimizelyClient.addDecisionNotificationHandler(decisionNotification -> { + assertEquals("FLAG", decisionNotification.getType()); + assertEquals(userId, decisionNotification.getUserId()); + assertEquals(attributes, decisionNotification.getAttributes()); + + Map info = decisionNotification.getDecisionInfo(); + assertEquals(flagKey, info.get("flagKey")); + assertEquals(variationKey, info.get("variationKey")); + assertEquals(false, info.get("enabled")); + assertEquals(ruleKey, info.get("ruleKey")); + assertTrue(((Map) info.get("variables")).isEmpty()); + + listenerCalled[0] = true; + }); + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + OptimizelyDecision decision = userContext.decide(flagKey, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); + + assertTrue("Decision notification handler should have been called", listenerCalled[0]); + assertEquals(variationKey, decision.getVariationKey()); + assertFalse(decision.getEnabled()); + } + + @Test + public void testHoldout_zeroTraffic() throws IOException { + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String flagKey = "boolean_feature"; + String userId = "user456"; + + Map attributes = new HashMap<>(); + // Use a bucketing ID that would normally fall into holdout_zero_traffic, but since it has 0% traffic, should not be in holdout + attributes.put("$opt_bucketing_id", "ppid300000"); + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + OptimizelyDecision decision = userContext.decide(flagKey, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); + + // Should NOT be a holdout decision since holdout_zero_traffic has 0% traffic allocation + assertNotEquals("ho_off_key", decision.getVariationKey()); + assertNotEquals("holdout_zero_traffic", decision.getRuleKey()); + assertFalse("Should not have holdout reason", decision.getReasons().stream() + .anyMatch(reason -> reason.contains("holdout_zero_traffic"))); + } + + @Test + public void testHoldout_attributeFiltering() throws IOException { + Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); + OptimizelyClient optimizelyClient = createOptimizelyClientWithHoldouts(context); + + String flagKey = "boolean_feature"; + String userId = "user789"; + + Map attributes = new HashMap<>(); + attributes.put("$opt_bucketing_id", "ppid160000"); // deterministic bucketing into basic_holdout + attributes.put("nationality", "English"); // should appear in notifications + attributes.put("$opt_reserved", "filtered"); // should be filtered out + + final Map[] notificationAttributes = new Map[1]; + optimizelyClient.addDecisionNotificationHandler(decisionNotification -> { + notificationAttributes[0] = new HashMap<>(decisionNotification.getAttributes()); + }); + + OptimizelyUserContext userContext = optimizelyClient.createUserContext(userId, attributes); + OptimizelyDecision decision = userContext.decide(flagKey, Collections.singletonList(OptimizelyDecideOption.INCLUDE_REASONS)); + + // Validate that reserved attributes are filtered from notifications but non-reserved ones remain + assertNotNull("Notification should have been called", notificationAttributes[0]); + assertTrue("Should contain nationality", notificationAttributes[0].containsKey("nationality")); + assertEquals("English", notificationAttributes[0].get("nationality")); + assertTrue("Should contain $opt_bucketing_id", notificationAttributes[0].containsKey("$opt_bucketing_id")); + assertTrue("Should contain $opt_reserved", notificationAttributes[0].containsKey("$opt_reserved")); + + // Validate holdout decision + assertEquals("ho_off_key", decision.getVariationKey()); + assertEquals("basic_holdout", decision.getRuleKey()); + assertFalse(decision.getEnabled()); + } +} diff --git a/android-sdk/src/debug/res/raw/holdouts_project_config b/android-sdk/src/debug/res/raw/holdouts_project_config new file mode 100644 index 000000000..fdce4295b --- /dev/null +++ b/android-sdk/src/debug/res/raw/holdouts_project_config @@ -0,0 +1,432 @@ +{ + "accountId": "2360254204", + "anonymizeIP": true, + "botFiltering": true, + "sendFlagDecisions": true, + "projectId": "3918735994", + "revision": "1480511547", + "sdkKey": "ValidProjectConfigV4", + "environmentKey": "production", + "version": "4", + "audiences": [ + { + "id": "3468206642", + "name": "Gryffindors", + "conditions": "[\"and\", [\"or\", [\"or\", {\"name\": \"house\", \"type\": \"custom_attribute\", \"value\":\"Gryffindor\"}]]]" + }, + { + "id": "3988293898", + "name": "Slytherins", + "conditions": "[\"and\", [\"or\", [\"or\", {\"name\": \"house\", \"type\": \"custom_attribute\", \"value\":\"Slytherin\"}]]]" + }, + { + "id": "4194404272", + "name": "english_citizens", + "conditions": "[\"and\", [\"or\", [\"or\", {\"name\": \"nationality\", \"type\": \"custom_attribute\", \"value\":\"English\"}]]]" + }, + { + "id": "2196265320", + "name": "audience_with_missing_value", + "conditions": "[\"and\", [\"or\", [\"or\", {\"name\": \"nationality\", \"type\": \"custom_attribute\", \"value\": \"English\"}, {\"name\": \"nationality\", \"type\": \"custom_attribute\"}]]]" + } + ], + "typedAudiences": [ + { + "id": "3468206643", + "name": "BOOL", + "conditions": ["and", ["or", ["or", {"name": "booleanKey", "type": "custom_attribute", "match":"exact", "value":true}]]] + }, + { + "id": "3468206646", + "name": "INTEXACT", + "conditions": ["and", ["or", ["or", {"name": "integerKey", "type": "custom_attribute", "match":"exact", "value":1.0}]]] + }, + { + "id": "3468206644", + "name": "INT", + "conditions": ["and", ["or", ["or", {"name": "integerKey", "type": "custom_attribute", "match":"gt", "value":1.0}]]] + }, + { + "id": "3468206645", + "name": "DOUBLE", + "conditions": ["and", ["or", ["or", {"name": "doubleKey", "type": "custom_attribute", "match":"lt", "value":100.0}]]] + }, + { + "id": "3468206642", + "name": "Gryffindors", + "conditions": ["and", ["or", ["or", {"name": "house", "type": "custom_attribute", "match":"exact", "value":"Gryffindor"}]]] + }, + { + "id": "3988293898", + "name": "Slytherins", + "conditions": ["and", ["or", ["or", {"name": "house", "type": "custom_attribute", "match":"substring", "value":"Slytherin"}]]] + }, + { + "id": "4194404272", + "name": "english_citizens", + "conditions": ["and", ["or", ["or", {"name": "nationality", "type": "custom_attribute", "match":"exact", "value":"English"}]]] + }, + { + "id": "2196265320", + "name": "audience_with_missing_value", + "conditions": ["and", ["or", ["or", {"name": "nationality", "type": "custom_attribute", "value": "English"}, {"name": "nationality", "type": "custom_attribute"}]]] + } + ], + "attributes": [ + { + "id": "553339214", + "key": "house" + }, + { + "id": "58339410", + "key": "nationality" + }, + { + "id": "583394100", + "key": "$opt_test" + }, + { + "id": "323434545", + "key": "booleanKey" + }, + { + "id": "616727838", + "key": "integerKey" + }, + { + "id": "808797686", + "key": "doubleKey" + } + ], + "events": [ + { + "id": "3785620495", + "key": "basic_event", + "experimentIds": [ + "1323241596", + "2738374745", + "3042640549", + "3262035800", + "3072915611" + ] + }, + { + "id": "3195631717", + "key": "event_with_paused_experiment", + "experimentIds": [ + "2667098701" + ] + }, + { + "id": "1987018666", + "key": "event_with_launched_experiments_only", + "experimentIds": [ + "3072915611" + ] + } + ], + "experiments": [ + { + "id": "1323241596", + "key": "basic_experiment", + "layerId": "1630555626", + "status": "Running", + "variations": [ + { + "id": "1423767502", + "key": "A", + "variables": [] + }, + { + "id": "3433458314", + "key": "B", + "variables": [] + } + ], + "trafficAllocation": [ + { + "entityId": "1423767502", + "endOfRange": 5000 + }, + { + "entityId": "3433458314", + "endOfRange": 10000 + } + ], + "audienceIds": [], + "forcedVariations": { + "Harry Potter": "A", + "Tom Riddle": "B" + } + }, + { + "id": "3262035800", + "key": "multivariate_experiment", + "layerId": "3262035800", + "status": "Running", + "variations": [ + { + "id": "1880281238", + "key": "Fred", + "featureEnabled": true, + "variables": [ + { + "id": "675244127", + "value": "F" + }, + { + "id": "4052219963", + "value": "red" + }, + { + "id": "4111661000", + "value": "{\"k1\":\"s1\",\"k2\":103.5,\"k3\":false,\"k4\":{\"kk1\":\"ss1\",\"kk2\":true}}" + } + ] + }, + { + "id": "3631049532", + "key": "Feorge", + "featureEnabled": true, + "variables": [ + { + "id": "675244127", + "value": "F" + }, + { + "id": "4052219963", + "value": "eorge" + }, + { + "id": "4111661000", + "value": "{\"k1\":\"s2\",\"k2\":203.5,\"k3\":true,\"k4\":{\"kk1\":\"ss2\",\"kk2\":true}}" + } + ] + }, + { + "id": "4204375027", + "key": "Gred", + "featureEnabled": false, + "variables": [ + { + "id": "675244127", + "value": "G" + }, + { + "id": "4052219963", + "value": "red" + }, + { + "id": "4111661000", + "value": "{\"k1\":\"s3\",\"k2\":303.5,\"k3\":true,\"k4\":{\"kk1\":\"ss3\",\"kk2\":false}}" + } + ] + }, + { + "id": "2099211198", + "key": "George", + "featureEnabled": true, + "variables": [ + { + "id": "675244127", + "value": "G" + }, + { + "id": "4052219963", + "value": "eorge" + }, + { + "id": "4111661000", + "value": "{\"k1\":\"s4\",\"k2\":403.5,\"k3\":false,\"k4\":{\"kk1\":\"ss4\",\"kk2\":true}}" + } + ] + } + ], + "trafficAllocation": [ + { + "entityId": "1880281238", + "endOfRange": 2500 + }, + { + "entityId": "3631049532", + "endOfRange": 5000 + }, + { + "entityId": "4204375027", + "endOfRange": 7500 + }, + { + "entityId": "2099211198", + "endOfRange": 10000 + } + ], + "audienceIds": [ + "3468206642" + ], + "forcedVariations": { + "Fred": "Fred", + "Feorge": "Feorge", + "Gred": "Gred", + "George": "George" + } + } + ], + "holdouts": [ + { + "audienceIds": [], + "id": "1007532345428", + "key": "holdout_zero_traffic", + "status": "Running", + "trafficAllocation": [ + { + "endOfRange": 0, + "entityId": "$opt_dummy_variation_id" + } + ], + "variations": [ + { + "featureEnabled": false, + "id": "$opt_dummy_variation_id", + "key": "ho_off_key" + } + ] + }, + { + "audienceIds": [], + "id": "1007543323427", + "key": "holdout_included_flags", + "status": "Running", + "trafficAllocation": [ + { + "endOfRange": 2000, + "entityId": "$opt_dummy_variation_id" + } + ], + "variations": [ + { + "featureEnabled": false, + "id": "$opt_dummy_variation_id", + "key": "ho_off_key" + } + ], + "includedFlags": [ + "4195505407", + "3926744821", + "3281420120" + ] + }, + { + "audienceIds": [], + "id": "10075323428", + "key": "basic_holdout", + "status": "Running", + "trafficAllocation": [ + { + "endOfRange": 500, + "entityId": "$opt_dummy_variation_id" + } + ], + "variations": [ + { + "featureEnabled": false, + "id": "$opt_dummy_variation_id", + "key": "ho_off_key" + } + ] + } + ], + "groups": [], + "featureFlags": [ + { + "id": "4195505407", + "key": "boolean_feature", + "rolloutId": "", + "experimentIds": [], + "variables": [] + }, + { + "id": "3926744821", + "key": "double_single_variable_feature", + "rolloutId": "", + "experimentIds": [], + "variables": [ + { + "id": "4111654444", + "key": "double_variable", + "type": "double", + "defaultValue": "14.99" + } + ] + }, + { + "id": "3281420120", + "key": "integer_single_variable_feature", + "rolloutId": "", + "experimentIds": [], + "variables": [ + { + "id": "593964691", + "key": "integer_variable", + "type": "integer", + "defaultValue": "7" + } + ] + }, + { + "id": "2591051011", + "key": "boolean_single_variable_feature", + "rolloutId": "", + "experimentIds": [], + "variables": [ + { + "id": "3974680341", + "key": "boolean_variable", + "type": "boolean", + "defaultValue": "true" + } + ] + }, + { + "id": "2079378557", + "key": "string_single_variable_feature", + "rolloutId": "", + "experimentIds": [], + "variables": [ + { + "id": "2077511132", + "key": "string_variable", + "type": "string", + "defaultValue": "wingardium leviosa" + } + ] + }, + { + "id": "3263342226", + "key": "multi_variate_feature", + "rolloutId": "", + "experimentIds": ["3262035800"], + "variables": [ + { + "id": "675244127", + "key": "first_letter", + "type": "string", + "defaultValue": "H" + }, + { + "id": "4052219963", + "key": "rest_of_name", + "type": "string", + "defaultValue": "arry" + }, + { + "id": "4111661000", + "key": "json_patched", + "type": "string", + "subType": "json", + "defaultValue": "{\"k1\":\"v1\",\"k2\":3.5,\"k3\":true,\"k4\":{\"kk1\":\"vv1\",\"kk2\":false}}" + } + ] + } + ], + "rollouts": [] +}