From a6e33005989ed583af18629b22b728b3f7b10fe7 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 3 Apr 2024 11:37:48 -0700 Subject: [PATCH 01/72] WIP --- .../AzureAppConfigurationProvider.cs | 15 +++++++++------ ...ons.Configuration.AzureAppConfiguration.csproj | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 4f34cffa..2b9cff11 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -31,7 +31,7 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private readonly IConfigurationClientManager _configClientManager; private AzureAppConfigurationOptions _options; private Dictionary _mappedData; - private Dictionary _watchedSettings = new Dictionary(); + private Dictionary> _watchedSettings = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); @@ -247,7 +247,7 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary watchedSettings = null; + Dictionary> watchedSettings = null; List keyValueChanges = null; List changedKeyValuesCollection = null; Dictionary data = null; @@ -271,16 +271,19 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => string watchedKey = changeWatcher.Key; string watchedLabel = changeWatcher.Label; - KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); + SettingSelector watchedSettingSelector = new SettingSelector() { + KeyFilter = watchedKey, + LabelFilter = watchedLabel + }; KeyValueChange change = default; // // Find if there is a change associated with watcher - if (_watchedSettings.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) + if (_watchedSettings.TryGetValue(watchedSettingSelector, out IEnumerable matchConditions)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + async () => change = await client.GetKeyValueChange(watchedSettingSelector, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } else { @@ -348,7 +351,7 @@ await CallWithRequestTracing( if (!refreshAll) { - watchedSettings = new Dictionary(_watchedSettings); + watchedSettings = new Dictionary>(_watchedSettings); foreach (KeyValueWatcher changeWatcher in cacheExpiredWatchers.Concat(cacheExpiredMultiKeyWatchers)) { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index 121e3618..eec4e01e 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -14,7 +14,7 @@ - + From 15c53feeb0d53a0b089280c0bf555a184b34a4ef Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 4 Apr 2024 13:55:30 -0700 Subject: [PATCH 02/72] WIP testing out client extensions methods --- .../ConfigurationClientExtensions.cs | 55 +++++++++++++++++-- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 0e378f44..cf08cb4f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -16,21 +16,29 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions { internal static class ConfigurationClientExtensions { - public static async Task GetKeyValueChange(this ConfigurationClient client, ConfigurationSetting setting, CancellationToken cancellationToken) + public static async Task GetKeyValueChange(this ConfigurationClient client, SettingSelector selector, MatchConditions matchConditions, CancellationToken cancellationToken) { - if (setting == null) + if (selector == null) { - throw new ArgumentNullException(nameof(setting)); + throw new ArgumentNullException(nameof(selector)); } - if (string.IsNullOrEmpty(setting.Key)) + if (matchConditions == null) { - throw new ArgumentNullException($"{nameof(setting)}.{nameof(setting.Key)}"); + throw new ArgumentNullException(nameof(matchConditions)); } + if (!matchConditions.IfNoneMatch.HasValue) + { + throw new ArgumentException("Must have valid IfNoneMatch header.", nameof(matchConditions)); + } + + ConfigurationSetting setting = new ConfigurationSetting(selector.KeyFilter, null, selector.LabelFilter, matchConditions.IfNoneMatch.Value); + try { Response response = await client.GetConfigurationSettingAsync(setting, onlyIfChanged: true, cancellationToken).ConfigureAwait(false); + if (response.GetRawResponse().Status == (int)HttpStatusCode.OK) { return new KeyValueChange @@ -42,7 +50,7 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } } - catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound && setting.ETag != default) + catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound && matchConditions.IfNoneMatch.Value != default) { return new KeyValueChange { @@ -62,6 +70,41 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } + public static async Task IsAnyKeyValueChanged(this ConfigurationClient client, SettingSelector selector, IEnumerable matchConditions, CancellationToken cancellationToken) + { + if (selector == null) + { + throw new ArgumentNullException(nameof(selector)); + } + + if (matchConditions == null) + { + throw new ArgumentNullException(nameof(matchConditions)); + } + + if (!matchConditions.Any()) + { + throw new ArgumentException("Requires at least one MatchConditions value.", nameof(matchConditions)); + } + + foreach (MatchConditions condition in matchConditions) + { + selector.MatchConditions.Add(condition); + } + + await foreach (Page page in client.GetConfigurationSettingsAsync(selector, cancellationToken).AsPages().ConfigureAwait(false)) + { + Response response = page.GetRawResponse(); + + if (response.Status == (int)HttpStatusCode.OK) + { + return true; + } + } + + return false; + } + public static async Task> GetKeyValueChangeCollection( this ConfigurationClient client, IEnumerable keyValues, From 75a01ebeea33555858d67932d8e8d4612b8ae30e Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Fri, 5 Apr 2024 14:58:16 -0700 Subject: [PATCH 03/72] WIP added selectors to multikeywatchers --- .../AzureAppConfigurationOptions.cs | 22 +++++++++++++++++- .../AzureAppConfigurationProvider.cs | 23 +++++++++++++++++++ .../AzureAppConfigurationRefreshOptions.cs | 15 ++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index d79838da..062dfc9a 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -28,6 +28,7 @@ public class AzureAppConfigurationOptions private List _adapters; private List>> _mappers = new List>>(); private List _kvSelectors = new List(); + private List _featureFlagSelectors = new List(); private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); // The following set is sorted in descending order. @@ -39,6 +40,11 @@ public class AzureAppConfigurationOptions /// public bool ReplicaDiscoveryEnabled { get; set; } = true; + /// + /// Flag to indicate whether has been called. + /// + internal bool RegisterAllEnabled { get; private set; } = false; + /// /// The list of connection strings used to connect to an Azure App Configuration store and its replicas. /// @@ -61,6 +67,16 @@ public class AzureAppConfigurationOptions /// internal IEnumerable KeyValueSelectors => _kvSelectors; + /// + /// The configured options for feature flags. + /// + internal FeatureFlagOptions FeatureFlagOptions { get; private set; } = new FeatureFlagOptions(); + + /// + /// The configured options for refresh. + /// + internal AzureAppConfigurationRefreshOptions RefreshOptions { get; private set; } = new AzureAppConfigurationRefreshOptions(); + /// /// A collection of . /// @@ -69,7 +85,9 @@ public class AzureAppConfigurationOptions /// /// A collection of . /// - internal IEnumerable MultiKeyWatchers => _multiKeyWatchers; + internal List MultiKeyWatchers => _multiKeyWatchers; + + internal TimeSpan FeatureFlagCacheExpirationInterval { get; private set; } = RefreshConstants.DefaultFeatureFlagsCacheExpirationInterval; /// /// A collection of . @@ -226,6 +244,8 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c throw new InvalidOperationException($"Please select feature flags by either the {nameof(options.Select)} method or by setting the {nameof(options.Label)} property, not both."); } + _featureFlagSelectors = options.FeatureFlagSelectors; + if (options.FeatureFlagSelectors.Count() == 0) { // Select clause is not present diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 2b9cff11..abdce2d5 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -107,6 +107,29 @@ public AzureAppConfigurationProvider(IConfigurationClientManager configClientMan _options = options ?? throw new ArgumentNullException(nameof(options)); _optional = optional; + if (_options.RegisterAllEnabled) + { + foreach (KeyValueSelector kvSelector in _options.KeyValueSelectors) + { + options.MultiKeyWatchers.AppendUnique(new KeyValueWatcher + { + Key = kvSelector.KeyFilter, + Label = kvSelector.LabelFilter, + CacheExpirationInterval = options.RefreshOptions.CacheExpirationInterval + }); + } + + foreach (KeyValueSelector featureFlagSelector in _options.FeatureFlagOptions.FeatureFlagSelectors) + { + options.MultiKeyWatchers.AppendUnique(new KeyValueWatcher + { + Key = featureFlagSelector.KeyFilter, + Label = featureFlagSelector.LabelFilter, + CacheExpirationInterval = options.FeatureFlagOptions.CacheExpirationInterval + }); + } + } + IEnumerable watchers = options.ChangeWatchers.Union(options.MultiKeyWatchers); if (watchers.Any()) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs index 5297507c..ebb80bc5 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs @@ -1,7 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. // +using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; +using Microsoft.Extensions.Options; using System; using System.Collections.Generic; @@ -14,6 +17,7 @@ public class AzureAppConfigurationRefreshOptions { internal TimeSpan CacheExpirationInterval { get; private set; } = RefreshConstants.DefaultCacheExpirationInterval; internal ISet RefreshRegistrations = new HashSet(); + internal bool RegisterAllEnabled { get; private set; } = false; /// /// Register the specified individual key-value to be refreshed when the configuration provider's triggers a refresh. @@ -50,6 +54,17 @@ public AzureAppConfigurationRefreshOptions Register(string key, string label = L return this; } + /// + /// Register all key-values loaded outside of to be refreshed when the configuration provider's triggers a refresh. + /// The instance can be obtained by calling . + /// + public AzureAppConfigurationRefreshOptions RegisterAll() + { + RegisterAllEnabled = true; + + return this; + } + /// /// Sets the cache expiration time for the key-values registered for refresh. Default value is 30 seconds. Must be greater than 1 second. /// Any refresh operation triggered using will not update the value for a key until the cached value for that key has expired. From f1b07b43a95df29adf2d3d1e5271c418959fb2c3 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Fri, 5 Apr 2024 15:00:40 -0700 Subject: [PATCH 04/72] remove unused property --- .../AzureAppConfigurationOptions.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 062dfc9a..f4b4197a 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -87,8 +87,6 @@ public class AzureAppConfigurationOptions /// internal List MultiKeyWatchers => _multiKeyWatchers; - internal TimeSpan FeatureFlagCacheExpirationInterval { get; private set; } = RefreshConstants.DefaultFeatureFlagsCacheExpirationInterval; - /// /// A collection of . /// From 25c262cba59497c58245aad738e93016374a5097 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Tue, 9 Apr 2024 11:43:29 -0700 Subject: [PATCH 05/72] WIP check for registerall changes to change refreshall --- .../AzureAppConfigurationProvider.cs | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index abdce2d5..85efc44e 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -352,22 +352,48 @@ await CallWithRequestTracing( } } + if (!refreshAll && _options.RegisterAllEnabled) + { + foreach (KeyValueWatcher multiKeyWatcher in cacheExpiredMultiKeyWatchers) + { + SettingSelector watchedSettingSelector = new SettingSelector() + { + KeyFilter = multiKeyWatcher.Key, + LabelFilter = multiKeyWatcher.Label + }; + + _watchedSettings.TryGetValue(watchedSettingSelector, out IEnumerable matchConditions) { + + await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, + async () => refreshAll = await client.IsAnyKeyValueChanged(watchedSettingSelector, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + + if (refreshAll) + { + break; + } + } + } + else + { + changedKeyValuesCollection = await GetRefreshedKeyValueCollections(cacheExpiredMultiKeyWatchers, client, logDebugBuilder, logInfoBuilder, endpoint, cancellationToken).ConfigureAwait(false); + + if (!changedKeyValuesCollection.Any()) + { + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); + } + } + if (refreshAll) { // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true + // or if RegisterAll was called and any loaded key-value changed data = await LoadSelectedKeyValues(client, cancellationToken).ConfigureAwait(false); watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); return; } - - changedKeyValuesCollection = await GetRefreshedKeyValueCollections(cacheExpiredMultiKeyWatchers, client, logDebugBuilder, logInfoBuilder, endpoint, cancellationToken).ConfigureAwait(false); - - if (!changedKeyValuesCollection.Any()) - { - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); - } }, cancellationToken) .ConfigureAwait(false); From 302140b2be2be5a31aeeede42816ba5c27b21535 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Tue, 9 Apr 2024 13:05:15 -0700 Subject: [PATCH 06/72] WIP --- .../AzureAppConfigurationProvider.cs | 24 ++++++++++++------- .../ConfigurationClientExtensions.cs | 15 ++++++++---- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 85efc44e..de2ec908 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -362,8 +362,8 @@ await CallWithRequestTracing( LabelFilter = multiKeyWatcher.Label }; - _watchedSettings.TryGetValue(watchedSettingSelector, out IEnumerable matchConditions) { - + if (_watchedSettings.TryGetValue(watchedSettingSelector, out IEnumerable matchConditions)) + { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, async () => refreshAll = await client.IsAnyKeyValueChanged(watchedSettingSelector, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } @@ -752,7 +752,7 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { Dictionary data = null; - Dictionary watchedSettings = null; + Dictionary> watchedSettings = null; await ExecuteWithFailOverPolicyAsync( clients, @@ -870,21 +870,27 @@ await CallWithRequestTracing(async () => return serverData; } - private async Task> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) + private async Task>> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) { - Dictionary watchedSettings = new Dictionary(); + Dictionary> watchedSettings = new Dictionary>(); foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) { string watchedKey = changeWatcher.Key; string watchedLabel = changeWatcher.Label; - KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); + + SettingSelector watchedSettingSelector = new SettingSelector() + { + KeyFilter = watchedKey, + LabelFilter = watchedLabel + }; // Skip the loading for the key-value in case it has already been loaded if (existingSettings.TryGetValue(watchedKey, out ConfigurationSetting loadedKv) - && watchedKeyLabel.Equals(new KeyValueIdentifier(loadedKv.Key, loadedKv.Label))) + && watchedSettingSelector.KeyFilter == loadedKv.Key + && watchedSettingSelector.LabelFilter == loadedKv.Label) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); + watchedSettings[watchedSettingSelector] = new List() { new MatchConditions() { IfNoneMatch = loadedKv.ETag } }; continue; } @@ -902,7 +908,7 @@ private async Task> LoadKey // If the key-value was found, store it for updating the settings if (watchedKv != null) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); + watchedSettings[watchedSettingSelector] = new List() { new MatchConditions() { IfNoneMatch = loadedKv.ETag } }; existingSettings[watchedKey] = watchedKv; } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index cf08cb4f..1450af9c 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -16,7 +16,7 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions { internal static class ConfigurationClientExtensions { - public static async Task GetKeyValueChange(this ConfigurationClient client, SettingSelector selector, MatchConditions matchConditions, CancellationToken cancellationToken) + public static async Task GetKeyValueChange(this ConfigurationClient client, SettingSelector selector, IEnumerable matchConditions, CancellationToken cancellationToken) { if (selector == null) { @@ -28,12 +28,19 @@ public static async Task GetKeyValueChange(this ConfigurationCli throw new ArgumentNullException(nameof(matchConditions)); } - if (!matchConditions.IfNoneMatch.HasValue) + if (matchConditions.Count() != 1) + { + throw new ArgumentException("Requires exactly one MatchConditions value.", nameof(matchConditions)); + } + + MatchConditions condition = matchConditions.First(); + + if (condition.IfNoneMatch.HasValue) { throw new ArgumentException("Must have valid IfNoneMatch header.", nameof(matchConditions)); } - ConfigurationSetting setting = new ConfigurationSetting(selector.KeyFilter, null, selector.LabelFilter, matchConditions.IfNoneMatch.Value); + ConfigurationSetting setting = new ConfigurationSetting(selector.KeyFilter, null, selector.LabelFilter, condition.IfNoneMatch.Value); try { @@ -50,7 +57,7 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } } - catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound && matchConditions.IfNoneMatch.Value != default) + catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound && condition.IfNoneMatch.Value != default) { return new KeyValueChange { From cc1bc585f28fcb78ab69f830936284e9b74ee4a4 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 10 Apr 2024 14:58:43 -0700 Subject: [PATCH 07/72] WIP fixing types and reslving errors --- .../AzureAppConfigurationOptions.cs | 5 - .../AzureAppConfigurationProvider.cs | 158 +++++++++++------- .../ConfigurationClientExtensions.cs | 2 +- 3 files changed, 100 insertions(+), 65 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index f4b4197a..2c760a5c 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -67,11 +67,6 @@ public class AzureAppConfigurationOptions /// internal IEnumerable KeyValueSelectors => _kvSelectors; - /// - /// The configured options for feature flags. - /// - internal FeatureFlagOptions FeatureFlagOptions { get; private set; } = new FeatureFlagOptions(); - /// /// The configured options for refresh. /// diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index de2ec908..3fb0d0e5 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -107,29 +107,6 @@ public AzureAppConfigurationProvider(IConfigurationClientManager configClientMan _options = options ?? throw new ArgumentNullException(nameof(options)); _optional = optional; - if (_options.RegisterAllEnabled) - { - foreach (KeyValueSelector kvSelector in _options.KeyValueSelectors) - { - options.MultiKeyWatchers.AppendUnique(new KeyValueWatcher - { - Key = kvSelector.KeyFilter, - Label = kvSelector.LabelFilter, - CacheExpirationInterval = options.RefreshOptions.CacheExpirationInterval - }); - } - - foreach (KeyValueSelector featureFlagSelector in _options.FeatureFlagOptions.FeatureFlagSelectors) - { - options.MultiKeyWatchers.AppendUnique(new KeyValueWatcher - { - Key = featureFlagSelector.KeyFilter, - Label = featureFlagSelector.LabelFilter, - CacheExpirationInterval = options.FeatureFlagOptions.CacheExpirationInterval - }); - } - } - IEnumerable watchers = options.ChangeWatchers.Union(options.MultiKeyWatchers); if (watchers.Any()) @@ -312,6 +289,8 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa { // Load the key-value in case the previous load attempts had failed + ConfigurationSetting watchedKv = null; + try { await CallWithRequestTracing( @@ -362,7 +341,7 @@ await CallWithRequestTracing( LabelFilter = multiKeyWatcher.Label }; - if (_watchedSettings.TryGetValue(watchedSettingSelector, out IEnumerable matchConditions)) + if (_watchedSettings.TryGetValue(watchedSettingSelector, out IEnumerable matchConditions)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, async () => refreshAll = await client.IsAnyKeyValueChanged(watchedSettingSelector, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); @@ -388,8 +367,8 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa { // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true // or if RegisterAll was called and any loaded key-value changed - data = await LoadSelectedKeyValues(client, cancellationToken).ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); + (data, watchedSettings) = await LoadSelectedKeyValues(client, cancellationToken).ConfigureAwait(false); + watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, watchedSettings, cancellationToken).ConfigureAwait(false); watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); return; @@ -409,12 +388,16 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa foreach (KeyValueChange change in keyValueChanges.Concat(changedKeyValuesCollection)) { - KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); + SettingSelector changeSelector = new SettingSelector() + { + KeyFilter = change.Key, + LabelFilter = change.Label + }; + if (change.ChangeType == KeyValueChangeType.Modified) { ConfigurationSetting setting = change.Current; - ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); - watchedSettings[changeIdentifier] = settingCopy; + watchedSettings[changeSelector] = new List() { new MatchConditions() { IfNoneMatch = setting.ETag } }; foreach (Func> func in _options.Mappers) { @@ -433,7 +416,7 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa else if (change.ChangeType == KeyValueChangeType.Deleted) { _mappedData.Remove(change.Key); - watchedSettings.Remove(changeIdentifier); + watchedSettings.Remove(changeSelector); } // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting @@ -758,7 +741,7 @@ await ExecuteWithFailOverPolicyAsync( clients, async (client) => { - data = await LoadSelectedKeyValues( + (data, watchedSettings) = await LoadSelectedKeyValues( client, cancellationToken) .ConfigureAwait(false); @@ -766,6 +749,7 @@ await ExecuteWithFailOverPolicyAsync( watchedSettings = await LoadKeyValuesRegisteredForRefresh( client, data, + watchedSettings, cancellationToken) .ConfigureAwait(false); @@ -795,9 +779,10 @@ await ExecuteWithFailOverPolicyAsync( } } - private async Task> LoadSelectedKeyValues(ConfigurationClient client, CancellationToken cancellationToken) + private async Task<(Dictionary, Dictionary>)> LoadSelectedKeyValues(ConfigurationClient client, CancellationToken cancellationToken) { var serverData = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary> watchedSettings = new Dictionary>(); // Use default query if there are no key-values specified for use other than the feature flags bool useDefaultQuery = !_options.KeyValueSelectors.Any(selector => selector.KeyFilter == null || @@ -812,28 +797,80 @@ private async Task> LoadSelectedKeyValu LabelFilter = LabelFilter.Null }; + List matchConditions = new List(); + await CallWithRequestTracing(async () => { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) + AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); + + await foreach (Page page in pageableSettings.AsPages().ConfigureAwait(false)) { - serverData[setting.Key] = setting; + foreach (ConfigurationSetting setting in page.Values) + { + serverData[setting.Key] = setting; + } + + if (_options.RegisterAllEnabled) + { + matchConditions.Add(new MatchConditions { IfNoneMatch = page.GetRawResponse().Headers.ETag }); + } } }).ConfigureAwait(false); + + if (_options.RegisterAllEnabled) + { + watchedSettings[selector] = matchConditions; + + _options.MultiKeyWatchers.AppendUnique(new KeyValueWatcher() + { + Key = selector.KeyFilter, + Label = selector.LabelFilter, + CacheExpirationInterval = _options.RefreshOptions.CacheExpirationInterval + }); + } } foreach (KeyValueSelector loadOption in _options.KeyValueSelectors) { - IAsyncEnumerable settingsEnumerable; - if (string.IsNullOrEmpty(loadOption.SnapshotName)) { - settingsEnumerable = client.GetConfigurationSettingsAsync( - new SettingSelector + var selector = new SettingSelector() + { + KeyFilter = loadOption.KeyFilter, + LabelFilter = loadOption.LabelFilter + }; + + List matchConditions = new List(); + + await CallWithRequestTracing(async () => + { + AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); + + await foreach (Page page in pageableSettings.AsPages().ConfigureAwait(false)) { - KeyFilter = loadOption.KeyFilter, - LabelFilter = loadOption.LabelFilter - }, - cancellationToken); + foreach (ConfigurationSetting setting in page.Values) + { + serverData[setting.Key] = setting; + } + + if (_options.RegisterAllEnabled) + { + matchConditions.Add(new MatchConditions { IfNoneMatch = page.GetRawResponse().Headers.ETag }); + } + } + }).ConfigureAwait(false); + + if (_options.RegisterAllEnabled) + { + watchedSettings[selector] = matchConditions; + + _options.MultiKeyWatchers.AppendUnique(new KeyValueWatcher() + { + Key = selector.KeyFilter, + Label = selector.LabelFilter, + CacheExpirationInterval = _options.RefreshOptions.CacheExpirationInterval + }); + } } else { @@ -853,26 +890,26 @@ await CallWithRequestTracing(async () => throw new InvalidOperationException($"{nameof(snapshot.SnapshotComposition)} for the selected snapshot with name '{snapshot.Name}' must be 'key', found '{snapshot.SnapshotComposition}'."); } - settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( + IAsyncEnumerable settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( loadOption.SnapshotName, cancellationToken); - } - await CallWithRequestTracing(async () => - { - await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) + await CallWithRequestTracing(async () => { - serverData[setting.Key] = setting; - } - }).ConfigureAwait(false); + await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) + { + serverData[setting.Key] = setting; + } + }).ConfigureAwait(false); + } } - return serverData; + return (serverData, watchedSettings); } - private async Task>> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) + private async Task>> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, Dictionary> existingWatchedSettings, CancellationToken cancellationToken) { - Dictionary> watchedSettings = new Dictionary>(); + Dictionary> watchedSettings = new Dictionary>(existingWatchedSettings); foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) { @@ -916,15 +953,18 @@ private async Task>> Lo return watchedSettings; } - private Dictionary UpdateWatchedKeyValueCollections(Dictionary watchedSettings, IDictionary existingSettings) + private Dictionary> UpdateWatchedKeyValueCollections(Dictionary> watchedSettings, IDictionary existingSettings) { - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + if (!_options.RegisterAllEnabled) { - IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, existingSettings.Values); - - foreach (ConfigurationSetting setting in currentKeyValues) + foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) { - watchedSettings[new KeyValueIdentifier(setting.Key, setting.Label)] = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, existingSettings.Values); + + foreach (ConfigurationSetting setting in currentKeyValues) + { + watchedSettings[new KeyValueIdentifier(setting.Key, setting.Label)] = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 1450af9c..9c075a83 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -71,7 +71,7 @@ public static async Task GetKeyValueChange(this ConfigurationCli return new KeyValueChange { ChangeType = KeyValueChangeType.None, - Current = setting, + Current = null, Key = setting.Key, Label = setting.Label }; From 28ecc054b3f9c51f1ca09234be6cd53571f70320 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Fri, 12 Apr 2024 12:46:07 -0700 Subject: [PATCH 08/72] WIP fixing client extensions class --- .../AzureAppConfigurationProvider.cs | 62 +++++++++---------- .../ConfigurationClientExtensions.cs | 2 +- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 3fb0d0e5..ab306741 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -774,6 +774,20 @@ await ExecuteWithFailOverPolicyAsync( Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); + + if (_options.RegisterAllEnabled) + { + foreach (SettingSelector selector in watchedSettings.Keys) + { + _options.MultiKeyWatchers.AppendUnique(new KeyValueWatcher() + { + Key = selector.KeyFilter, + Label = selector.LabelFilter, + CacheExpirationInterval = _options.RefreshOptions.CacheExpirationInterval + }); + } + } + _watchedSettings = watchedSettings; _mappedData = mappedData; } @@ -817,17 +831,7 @@ await CallWithRequestTracing(async () => } }).ConfigureAwait(false); - if (_options.RegisterAllEnabled) - { - watchedSettings[selector] = matchConditions; - - _options.MultiKeyWatchers.AppendUnique(new KeyValueWatcher() - { - Key = selector.KeyFilter, - Label = selector.LabelFilter, - CacheExpirationInterval = _options.RefreshOptions.CacheExpirationInterval - }); - } + watchedSettings[selector] = matchConditions; } foreach (KeyValueSelector loadOption in _options.KeyValueSelectors) @@ -860,17 +864,7 @@ await CallWithRequestTracing(async () => } }).ConfigureAwait(false); - if (_options.RegisterAllEnabled) - { - watchedSettings[selector] = matchConditions; - - _options.MultiKeyWatchers.AppendUnique(new KeyValueWatcher() - { - Key = selector.KeyFilter, - Label = selector.LabelFilter, - CacheExpirationInterval = _options.RefreshOptions.CacheExpirationInterval - }); - } + watchedSettings[selector] = matchConditions; } else { @@ -953,22 +947,28 @@ private async Task>> Lo return watchedSettings; } - private Dictionary> UpdateWatchedKeyValueCollections(Dictionary> watchedSettings, IDictionary existingSettings) + private Dictionary> UpdateWatchedKeyValueCollections(Dictionary> watchedSettings) { + Dictionary> watchedSettingsCopy = new Dictionary>(); + if (!_options.RegisterAllEnabled) { foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) { - IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, existingSettings.Values); - - foreach (ConfigurationSetting setting in currentKeyValues) + var selector = new SettingSelector() { - watchedSettings[new KeyValueIdentifier(setting.Key, setting.Label)] = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); - } + KeyFilter = changeWatcher.Key, + LabelFilter = changeWatcher.Label + }; + + watchedSettingsCopy[selector] = watchedSettings[selector]; } + } else + { + return watchedSettings; } - return watchedSettings; + return watchedSettingsCopy; } private async Task> GetRefreshedKeyValueCollections( @@ -983,11 +983,9 @@ private async Task> GetRefreshedKeyValueCollections( foreach (KeyValueWatcher changeWatcher in multiKeyWatchers) { - IEnumerable currentKeyValues = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, _watchedSettings.Values); - keyValueChanges.AddRange( await client.GetKeyValueChangeCollection( - currentKeyValues, + _watchedSettings, new GetKeyValueChangeCollectionOptions { KeyFilter = changeWatcher.Key, diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 9c075a83..1a54bcd3 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -114,7 +114,7 @@ public static async Task IsAnyKeyValueChanged(this ConfigurationClient cli public static async Task> GetKeyValueChangeCollection( this ConfigurationClient client, - IEnumerable keyValues, + Dictionary> watchedSettings, GetKeyValueChangeCollectionOptions options, StringBuilder logDebugBuilder, StringBuilder logInfoBuilder, From 38ba76828a0a9191be22401fab3fddc90e4715f6 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Mon, 15 Apr 2024 15:39:07 -0700 Subject: [PATCH 09/72] WIP --- .../AzureAppConfigurationProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index ab306741..ba1f6801 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -985,7 +985,7 @@ private async Task> GetRefreshedKeyValueCollections( { keyValueChanges.AddRange( await client.GetKeyValueChangeCollection( - _watchedSettings, + watchedSettings, new GetKeyValueChangeCollectionOptions { KeyFilter = changeWatcher.Key, From 9f4c1de0de19e61c0e20051373aa888a7f6e096c Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 24 Apr 2024 12:09:50 -0700 Subject: [PATCH 10/72] WIP update feature flag logic --- .../AzureAppConfigurationOptions.cs | 2 +- .../AzureAppConfigurationProvider.cs | 13 +++++++------ .../Extensions/ConfigurationClientExtensions.cs | 16 ++++++++-------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 07e38f92..a84a18d9 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -223,7 +223,7 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) /// /// Configures options for Azure App Configuration feature flags that will be parsed and transformed into feature management configuration. /// If no filtering is specified via the then all feature flags with no label are loaded. - /// All loaded feature flags will be automatically registered for refresh on an individual flag level. + /// All feature flags loaded by the same key and label filters will be automatically registered for refresh as a collection. /// /// A callback used to configure feature flag options. public AzureAppConfigurationOptions UseFeatureFlags(Action configure = null) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 3aac5018..a5b8d21b 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -334,7 +334,7 @@ await CallWithRequestTracing( if (!refreshAll && _options.RegisterAllEnabled) { - foreach (KeyValueWatcher multiKeyWatcher in cacheExpiredMultiKeyWatchers) + foreach (KeyValueWatcher multiKeyWatcher in refreshableMultiKeyWatchers) { SettingSelector watchedSettingSelector = new SettingSelector() { @@ -345,7 +345,7 @@ await CallWithRequestTracing( if (_watchedSettings.TryGetValue(watchedSettingSelector, out IEnumerable matchConditions)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => refreshAll = await client.IsAnyKeyValueChanged(watchedSettingSelector, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + async () => refreshAll = await client.HasAnyKeyValueChanged(watchedSettingSelector, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } if (refreshAll) @@ -370,7 +370,7 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa // or if RegisterAll was called and any loaded key-value changed (data, watchedSettings) = await LoadSelectedKeyValues(client, cancellationToken).ConfigureAwait(false); watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, watchedSettings, cancellationToken).ConfigureAwait(false); - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); + watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings); logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); return; } @@ -754,7 +754,7 @@ await ExecuteWithFailOverPolicyAsync( cancellationToken) .ConfigureAwait(false); - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); + watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings); }, cancellationToken) .ConfigureAwait(false); @@ -784,7 +784,7 @@ await ExecuteWithFailOverPolicyAsync( { Key = selector.KeyFilter, Label = selector.LabelFilter, - CacheExpirationInterval = _options.RefreshOptions.CacheExpirationInterval + RefreshInterval = _options.RefreshOptions.RefreshInterval }); } } @@ -964,7 +964,8 @@ private Dictionary> UpdateWatchedK watchedSettingsCopy[selector] = watchedSettings[selector]; } - } else + } + else { return watchedSettings; } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 1a54bcd3..0b127cbe 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -77,7 +77,7 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } - public static async Task IsAnyKeyValueChanged(this ConfigurationClient client, SettingSelector selector, IEnumerable matchConditions, CancellationToken cancellationToken) + public static async Task HasAnyKeyValueChanged(this ConfigurationClient client, SettingSelector selector, IEnumerable matchConditions, CancellationToken cancellationToken) { if (selector == null) { @@ -126,9 +126,9 @@ public static async Task> GetKeyValueChangeCollectio throw new ArgumentNullException(nameof(options)); } - if (keyValues == null) + if (watchedSettings == null) { - keyValues = Enumerable.Empty(); + watchedSettings = new Dictionary>(); } if (options.KeyFilter == null) @@ -136,17 +136,17 @@ public static async Task> GetKeyValueChangeCollectio options.KeyFilter = string.Empty; } - if (keyValues.Any(k => string.IsNullOrEmpty(k.Key))) + if (watchedSettings.Any(kvp => string.IsNullOrEmpty(kvp.Key.KeyFilter))) { - throw new ArgumentNullException($"{nameof(keyValues)}[].{nameof(ConfigurationSetting.Key)}"); + throw new ArgumentNullException($"{nameof(watchedSettings)}[].{nameof(ConfigurationSetting.Key)}"); } - if (keyValues.Any(k => !string.Equals(k.Label.NormalizeNull(), options.Label.NormalizeNull()))) + if (watchedSettings.Any(kvp => !string.Equals(kvp.Key.LabelFilter.NormalizeNull(), options.Label.NormalizeNull()))) { - throw new ArgumentException("All key-values registered for refresh must use the same label.", $"{nameof(keyValues)}[].{nameof(ConfigurationSetting.Label)}"); + throw new ArgumentException("All key-values registered for refresh must use the same label.", $"{nameof(watchedSettings)}[].{nameof(ConfigurationSetting.Label)}"); } - if (keyValues.Any(k => k.Label != null && k.Label.Contains("*"))) + if (watchedSettings.Any(kvp => kvp.Key.LabelFilter != null && kvp.Key.LabelFilter.Contains("*"))) { throw new ArgumentException("The label filter cannot contain '*'", $"{nameof(options)}.{nameof(options.Label)}"); } From 937e012fcaf441df2babc6b5d3cb9f270528430d Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 25 Apr 2024 16:32:33 -0700 Subject: [PATCH 11/72] WIP client extensions --- .../AzureAppConfigurationProvider.cs | 14 ++- .../ConfigurationClientExtensions.cs | 119 ++++++------------ .../GetKeyValueChangeCollectionOptions.cs | 8 +- 3 files changed, 53 insertions(+), 88 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index a5b8d21b..97f97cf4 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -356,7 +356,7 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa } else { - changedKeyValuesCollection = await GetRefreshedKeyValueCollections(refreshableMultiKeyWatchers, client, logDebugBuilder, logInfoBuilder, endpoint, cancellationToken).ConfigureAwait(false); + changedKeyValuesCollection = await GetRefreshedKeyValueCollections(refreshableMultiKeyWatchers, watchedSettings, client, logDebugBuilder, logInfoBuilder, endpoint, cancellationToken).ConfigureAwait(false); if (!changedKeyValuesCollection.Any()) { @@ -975,6 +975,7 @@ private Dictionary> UpdateWatchedK private async Task> GetRefreshedKeyValueCollections( IEnumerable multiKeyWatchers, + Dictionary> watchedSettings, ConfigurationClient client, StringBuilder logDebugBuilder, StringBuilder logInfoBuilder, @@ -985,13 +986,18 @@ private async Task> GetRefreshedKeyValueCollections( foreach (KeyValueWatcher changeWatcher in multiKeyWatchers) { + SettingSelector selector = new SettingSelector() + { + KeyFilter = changeWatcher.Key, + LabelFilter = changeWatcher.Label.NormalizeNull() + }; + keyValueChanges.AddRange( await client.GetKeyValueChangeCollection( - watchedSettings, new GetKeyValueChangeCollectionOptions { - KeyFilter = changeWatcher.Key, - Label = changeWatcher.Label.NormalizeNull(), + Selector = selector, + MatchConditions = watchedSettings[selector], RequestTracingEnabled = _requestTracingEnabled, RequestTracingOptions = _requestTracingOptions }, diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 0b127cbe..0e3208af 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -114,7 +114,6 @@ public static async Task HasAnyKeyValueChanged(this ConfigurationClient cl public static async Task> GetKeyValueChangeCollection( this ConfigurationClient client, - Dictionary> watchedSettings, GetKeyValueChangeCollectionOptions options, StringBuilder logDebugBuilder, StringBuilder logInfoBuilder, @@ -126,114 +125,70 @@ public static async Task> GetKeyValueChangeCollectio throw new ArgumentNullException(nameof(options)); } - if (watchedSettings == null) + if (options.Selector == null) { - watchedSettings = new Dictionary>(); + throw new ArgumentNullException($"{nameof(options)}.{nameof(options.Selector)}"); } - if (options.KeyFilter == null) + if (string.IsNullOrEmpty(options.Selector.KeyFilter)) { - options.KeyFilter = string.Empty; + throw new ArgumentNullException($"{nameof(options)}.{nameof(options.Selector)}.{nameof(SettingSelector.KeyFilter)}"); } - if (watchedSettings.Any(kvp => string.IsNullOrEmpty(kvp.Key.KeyFilter))) + if (options.Selector.LabelFilter != null && options.Selector.LabelFilter.Contains("*")) { - throw new ArgumentNullException($"{nameof(watchedSettings)}[].{nameof(ConfigurationSetting.Key)}"); + throw new ArgumentException("The label filter cannot contain '*'", $"{nameof(options)}.{nameof(options.Selector)}.{nameof(options.Selector.LabelFilter)}"); } - if (watchedSettings.Any(kvp => !string.Equals(kvp.Key.LabelFilter.NormalizeNull(), options.Label.NormalizeNull()))) - { - throw new ArgumentException("All key-values registered for refresh must use the same label.", $"{nameof(watchedSettings)}[].{nameof(ConfigurationSetting.Label)}"); - } + bool hasKeyValueCollectionChanged = false; - if (watchedSettings.Any(kvp => kvp.Key.LabelFilter != null && kvp.Key.LabelFilter.Contains("*"))) - { - throw new ArgumentException("The label filter cannot contain '*'", $"{nameof(options)}.{nameof(options.Label)}"); - } + await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, + async () => hasKeyValueCollectionChanged = await client.HasAnyKeyValueChanged(options.Selector, options.MatchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - var hasKeyValueCollectionChanged = false; - var selector = new SettingSelector - { - KeyFilter = options.KeyFilter, - LabelFilter = string.IsNullOrEmpty(options.Label) ? LabelFilter.Null : options.Label, - Fields = SettingFields.ETag | SettingFields.Key - }; + var changes = new List(); - // Dictionary of eTags that we write to and use for comparison - var eTagMap = keyValues.ToDictionary(kv => kv.Key, kv => kv.ETag); + if (!hasKeyValueCollectionChanged) + { + return changes; + } - // Fetch e-tags for prefixed key-values that can be used to detect changes await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, async () => { - await foreach(ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) + await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(options.Selector, cancellationToken).ConfigureAwait(false)) { if (!eTagMap.TryGetValue(setting.Key, out ETag etag) || !etag.Equals(setting.ETag)) { - hasKeyValueCollectionChanged = true; - break; + changes.Add(new KeyValueChange + { + ChangeType = KeyValueChangeType.Modified, + Key = setting.Key, + Label = options.Label.NormalizeNull(), + Current = setting + }); + string key = setting.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); + logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); } eTagMap.Remove(setting.Key); } }).ConfigureAwait(false); - // Check for any deletions - if (eTagMap.Any()) + foreach (var kvp in eTagMap) { - hasKeyValueCollectionChanged = true; - } - - var changes = new List(); - - // If changes have been observed, refresh prefixed key-values - if (hasKeyValueCollectionChanged) - { - selector = new SettingSelector - { - KeyFilter = options.KeyFilter, - LabelFilter = string.IsNullOrEmpty(options.Label) ? LabelFilter.Null : options.Label - }; - - eTagMap = keyValues.ToDictionary(kv => kv.Key, kv => kv.ETag); - await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, - async () => - { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) - { - if (!eTagMap.TryGetValue(setting.Key, out ETag etag) || !etag.Equals(setting.ETag)) - { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Modified, - Key = setting.Key, - Label = options.Label.NormalizeNull(), - Current = setting - }); - string key = setting.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); - } - - eTagMap.Remove(setting.Key); - } - }).ConfigureAwait(false); - - foreach (var kvp in eTagMap) + changes.Add(new KeyValueChange { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Deleted, - Key = kvp.Key, - Label = options.Label.NormalizeNull(), - Current = null - }); - string key = kvp.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); - } - } - + ChangeType = KeyValueChangeType.Deleted, + Key = kvp.Key, + Label = options.Label.NormalizeNull(), + Current = null + }); + string key = kvp.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); + logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); + } + return changes; } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs index 5cb9b83d..89d5e3f8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs @@ -1,12 +1,16 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. // +using Azure; +using Azure.Data.AppConfiguration; +using System.Collections.Generic; + namespace Microsoft.Extensions.Configuration.AzureAppConfiguration { internal class GetKeyValueChangeCollectionOptions { - public string KeyFilter { get; set; } - public string Label { get; set; } + public SettingSelector Selector { get; set; } + public IEnumerable MatchConditions { get; set; } public bool RequestTracingEnabled { get; set; } public RequestTracingOptions RequestTracingOptions { get; set; } } From 7e178705c42ec4fc24d930ba317fa2c8795e5402 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Fri, 26 Apr 2024 14:11:54 -0700 Subject: [PATCH 12/72] WIP reload all flags on change --- .../AzureAppConfigurationProvider.cs | 51 ++++++++++--------- .../ConfigurationClientExtensions.cs | 29 +++-------- 2 files changed, 34 insertions(+), 46 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 97f97cf4..613c133b 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -32,7 +32,8 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private Uri _lastSuccessfulEndpoint; private AzureAppConfigurationOptions _options; private Dictionary _mappedData; - private Dictionary> _watchedSettings = new Dictionary>(); + private Dictionary _watchedSettings = new Dictionary(); + private Dictionary> _watchedCollections = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); @@ -248,7 +249,8 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary> watchedSettings = null; + Dictionary> watchedCollections = null; + Dictionary watchedSettings = null; List keyValueChanges = null; List changedKeyValuesCollection = null; Dictionary data = null; @@ -260,6 +262,7 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => { data = null; watchedSettings = null; + watchedCollections = null; keyValueChanges = new List(); changedKeyValuesCollection = null; refreshAll = false; @@ -272,26 +275,21 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => string watchedKey = changeWatcher.Key; string watchedLabel = changeWatcher.Label; - SettingSelector watchedSettingSelector = new SettingSelector() { - KeyFilter = watchedKey, - LabelFilter = watchedLabel - }; + KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); KeyValueChange change = default; // // Find if there is a change associated with watcher - if (_watchedSettings.TryGetValue(watchedSettingSelector, out IEnumerable matchConditions)) + if (_watchedSettings.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => change = await client.GetKeyValueChange(watchedSettingSelector, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } else { // Load the key-value in case the previous load attempts had failed - ConfigurationSetting watchedKv = null; - try { await CallWithRequestTracing( @@ -332,7 +330,9 @@ await CallWithRequestTracing( } } - if (!refreshAll && _options.RegisterAllEnabled) + bool hasWatchedCollectionsChanged = false; + + if (!refreshAll) { foreach (KeyValueWatcher multiKeyWatcher in refreshableMultiKeyWatchers) { @@ -342,35 +342,38 @@ await CallWithRequestTracing( LabelFilter = multiKeyWatcher.Label }; - if (_watchedSettings.TryGetValue(watchedSettingSelector, out IEnumerable matchConditions)) + if (_watchedCollections.TryGetValue(watchedSettingSelector, out IEnumerable matchConditions)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => refreshAll = await client.HasAnyKeyValueChanged(watchedSettingSelector, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + async () => hasWatchedCollectionsChanged = await client.HasAnyKeyValueChanged(watchedSettingSelector, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } - if (refreshAll) + if (hasWatchedCollectionsChanged) { break; } } - } - else - { - changedKeyValuesCollection = await GetRefreshedKeyValueCollections(refreshableMultiKeyWatchers, watchedSettings, client, logDebugBuilder, logInfoBuilder, endpoint, cancellationToken).ConfigureAwait(false); - if (!changedKeyValuesCollection.Any()) + if (hasWatchedCollectionsChanged) + { + if (!_options.RegisterAllEnabled) + { + // TODO reload all flags + changedKeyValuesCollection = await GetRefreshedKeyValueCollections(refreshableMultiKeyWatchers, watchedSettings, client, logDebugBuilder, logInfoBuilder, endpoint, cancellationToken).ConfigureAwait(false); + } + } + else { logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); } } - - if (refreshAll) + else if (hasWatchedCollectionsChanged && _options.RegisterAllEnabled) { // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true // or if RegisterAll was called and any loaded key-value changed - (data, watchedSettings) = await LoadSelectedKeyValues(client, cancellationToken).ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, watchedSettings, cancellationToken).ConfigureAwait(false); - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings); + (data, watchedCollections) = await LoadSelectedKeyValues(client, cancellationToken).ConfigureAwait(false); + watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); + watchedCollections = UpdateWatchedKeyValueCollections(watchedCollections); logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); return; } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 0e3208af..b21c4725 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -16,36 +16,21 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions { internal static class ConfigurationClientExtensions { - public static async Task GetKeyValueChange(this ConfigurationClient client, SettingSelector selector, IEnumerable matchConditions, CancellationToken cancellationToken) + public static async Task GetKeyValueChange(this ConfigurationClient client, ConfigurationSetting setting, CancellationToken cancellationToken) { - if (selector == null) + if (setting == null) { - throw new ArgumentNullException(nameof(selector)); - } - - if (matchConditions == null) - { - throw new ArgumentNullException(nameof(matchConditions)); + throw new ArgumentNullException(nameof(setting)); } - if (matchConditions.Count() != 1) + if (string.IsNullOrEmpty(setting.Key)) { - throw new ArgumentException("Requires exactly one MatchConditions value.", nameof(matchConditions)); + throw new ArgumentNullException($"{nameof(setting)}.{nameof(setting.Key)}"); } - MatchConditions condition = matchConditions.First(); - - if (condition.IfNoneMatch.HasValue) - { - throw new ArgumentException("Must have valid IfNoneMatch header.", nameof(matchConditions)); - } - - ConfigurationSetting setting = new ConfigurationSetting(selector.KeyFilter, null, selector.LabelFilter, condition.IfNoneMatch.Value); - try { Response response = await client.GetConfigurationSettingAsync(setting, onlyIfChanged: true, cancellationToken).ConfigureAwait(false); - if (response.GetRawResponse().Status == (int)HttpStatusCode.OK) { return new KeyValueChange @@ -57,7 +42,7 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } } - catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound && condition.IfNoneMatch.Value != default) + catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound && setting.ETag != default) { return new KeyValueChange { @@ -71,7 +56,7 @@ public static async Task GetKeyValueChange(this ConfigurationCli return new KeyValueChange { ChangeType = KeyValueChangeType.None, - Current = null, + Current = setting, Key = setting.Key, Label = setting.Label }; From f4c17ae3e2cfad8e30d0be66ac5a09c98c9d4f40 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Mon, 29 Apr 2024 15:18:32 -0700 Subject: [PATCH 13/72] WIP --- .../AzureAppConfigurationProvider.cs | 49 +++++------ .../ConfigurationClientExtensions.cs | 82 ------------------- .../LogHelper.cs | 9 +- 3 files changed, 23 insertions(+), 117 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 613c133b..50bc9fe5 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -358,8 +358,9 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa { if (!_options.RegisterAllEnabled) { - // TODO reload all flags - changedKeyValuesCollection = await GetRefreshedKeyValueCollections(refreshableMultiKeyWatchers, watchedSettings, client, logDebugBuilder, logInfoBuilder, endpoint, cancellationToken).ConfigureAwait(false); + (data, watchedCollections) = await RefreshKeyValueCollections(data, watchedCollections, client, cancellationToken).ConfigureAwait(false); + + logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); } } else @@ -383,7 +384,7 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa if (!refreshAll) { - watchedSettings = new Dictionary>(_watchedSettings); + watchedSettings = new Dictionary(_watchedSettings); foreach (KeyValueWatcher changeWatcher in refreshableWatchers.Concat(refreshableMultiKeyWatchers)) { @@ -392,6 +393,8 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa foreach (KeyValueChange change in keyValueChanges.Concat(changedKeyValuesCollection)) { + KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); + SettingSelector changeSelector = new SettingSelector() { KeyFilter = change.Key, @@ -401,7 +404,8 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa if (change.ChangeType == KeyValueChangeType.Modified) { ConfigurationSetting setting = change.Current; - watchedSettings[changeSelector] = new List() { new MatchConditions() { IfNoneMatch = setting.ETag } }; + ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + watchedSettings[changeIdentifier] = settingCopy; foreach (Func> func in _options.Mappers) { @@ -420,7 +424,7 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa else if (change.ChangeType == KeyValueChangeType.Deleted) { _mappedData.Remove(change.Key); - watchedSettings.Remove(changeSelector); + watchedSettings.Remove(changeIdentifier); } // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting @@ -739,13 +743,14 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { Dictionary data = null; - Dictionary> watchedSettings = null; + Dictionary> watchedCollections = null; + Dictionary watchedSettings = null; await ExecuteWithFailOverPolicyAsync( clients, async (client) => { - (data, watchedSettings) = await LoadSelectedKeyValues( + (data, watchedCollections) = await LoadSelectedKeyValues( client, cancellationToken) .ConfigureAwait(false); @@ -753,11 +758,10 @@ await ExecuteWithFailOverPolicyAsync( watchedSettings = await LoadKeyValuesRegisteredForRefresh( client, data, - watchedSettings, cancellationToken) .ConfigureAwait(false); - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings); + watchedCollections = UpdateWatchedKeyValueCollections(watchedCollections); }, cancellationToken) .ConfigureAwait(false); @@ -781,7 +785,7 @@ await ExecuteWithFailOverPolicyAsync( if (_options.RegisterAllEnabled) { - foreach (SettingSelector selector in watchedSettings.Keys) + foreach (SettingSelector selector in watchedCollections.Keys) { _options.MultiKeyWatchers.AppendUnique(new KeyValueWatcher() { @@ -905,27 +909,22 @@ await CallWithRequestTracing(async () => return (serverData, watchedSettings); } - private async Task>> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, Dictionary> existingWatchedSettings, CancellationToken cancellationToken) + private async Task> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) { - Dictionary> watchedSettings = new Dictionary>(existingWatchedSettings); + Dictionary watchedSettings = new Dictionary(); foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) { string watchedKey = changeWatcher.Key; string watchedLabel = changeWatcher.Label; - SettingSelector watchedSettingSelector = new SettingSelector() - { - KeyFilter = watchedKey, - LabelFilter = watchedLabel - }; + KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); // Skip the loading for the key-value in case it has already been loaded if (existingSettings.TryGetValue(watchedKey, out ConfigurationSetting loadedKv) - && watchedSettingSelector.KeyFilter == loadedKv.Key - && watchedSettingSelector.LabelFilter == loadedKv.Label) + && watchedKeyLabel.Equals(new KeyValueIdentifier(loadedKv.Key, loadedKv.Label))) { - watchedSettings[watchedSettingSelector] = new List() { new MatchConditions() { IfNoneMatch = loadedKv.ETag } }; + watchedSettings[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); continue; } @@ -943,7 +942,7 @@ private async Task>> Lo // If the key-value was found, store it for updating the settings if (watchedKv != null) { - watchedSettings[watchedSettingSelector] = new List() { new MatchConditions() { IfNoneMatch = loadedKv.ETag } }; + watchedSettings[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); existingSettings[watchedKey] = watchedKv; } } @@ -976,13 +975,10 @@ private Dictionary> UpdateWatchedK return watchedSettingsCopy; } - private async Task> GetRefreshedKeyValueCollections( + private async Task> RefreshKeyValueCollections( IEnumerable multiKeyWatchers, Dictionary> watchedSettings, ConfigurationClient client, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - Uri endpoint, CancellationToken cancellationToken) { var keyValueChanges = new List(); @@ -1004,9 +1000,6 @@ await client.GetKeyValueChangeCollection( RequestTracingEnabled = _requestTracingEnabled, RequestTracingOptions = _requestTracingOptions }, - logDebugBuilder, - logInfoBuilder, - endpoint, cancellationToken) .ConfigureAwait(false)); } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index b21c4725..9734bd6d 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -3,12 +3,10 @@ // using Azure; using Azure.Data.AppConfiguration; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; using System; using System.Collections.Generic; using System.Linq; using System.Net; -using System.Text; using System.Threading; using System.Threading.Tasks; @@ -96,85 +94,5 @@ public static async Task HasAnyKeyValueChanged(this ConfigurationClient cl return false; } - - public static async Task> GetKeyValueChangeCollection( - this ConfigurationClient client, - GetKeyValueChangeCollectionOptions options, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - Uri endpoint, - CancellationToken cancellationToken) - { - if (options == null) - { - throw new ArgumentNullException(nameof(options)); - } - - if (options.Selector == null) - { - throw new ArgumentNullException($"{nameof(options)}.{nameof(options.Selector)}"); - } - - if (string.IsNullOrEmpty(options.Selector.KeyFilter)) - { - throw new ArgumentNullException($"{nameof(options)}.{nameof(options.Selector)}.{nameof(SettingSelector.KeyFilter)}"); - } - - if (options.Selector.LabelFilter != null && options.Selector.LabelFilter.Contains("*")) - { - throw new ArgumentException("The label filter cannot contain '*'", $"{nameof(options)}.{nameof(options.Selector)}.{nameof(options.Selector.LabelFilter)}"); - } - - bool hasKeyValueCollectionChanged = false; - - await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, - async () => hasKeyValueCollectionChanged = await client.HasAnyKeyValueChanged(options.Selector, options.MatchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - - var changes = new List(); - - if (!hasKeyValueCollectionChanged) - { - return changes; - } - - await TracingUtils.CallWithRequestTracing(options.RequestTracingEnabled, RequestType.Watch, options.RequestTracingOptions, - async () => - { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(options.Selector, cancellationToken).ConfigureAwait(false)) - { - if (!eTagMap.TryGetValue(setting.Key, out ETag etag) || !etag.Equals(setting.ETag)) - { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Modified, - Key = setting.Key, - Label = options.Label.NormalizeNull(), - Current = setting - }); - string key = setting.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); - } - - eTagMap.Remove(setting.Key); - } - }).ConfigureAwait(false); - - foreach (var kvp in eTagMap) - { - changes.Add(new KeyValueChange - { - ChangeType = KeyValueChangeType.Deleted, - Key = kvp.Key, - Label = options.Label.NormalizeNull(), - Current = null - }); - string key = kvp.Key.Substring(FeatureManagementConstants.FeatureFlagMarker.Length); - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagReadMessage(key, options.Label.NormalizeNull(), endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildFeatureFlagUpdatedMessage(key)); - } - - return changes; - } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs index 113a1499..0966db94 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs @@ -25,14 +25,9 @@ public static string BuildFeatureFlagsUnchangedMessage(string endpoint) return $"{LoggingConstants.RefreshFeatureFlagsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; } - public static string BuildFeatureFlagReadMessage(string key, string label, string endpoint) + public static string BuildFeatureFlagsUpdatedMessage() { - return $"{LoggingConstants.RefreshFeatureFlagRead} Key:'{key}' Label:'{label}' Endpoint:'{endpoint?.TrimEnd('/')}'"; - } - - public static string BuildFeatureFlagUpdatedMessage(string key) - { - return $"{LoggingConstants.RefreshFeatureFlagUpdated} Key:'{key}'"; + } public static string BuildKeyVaultSecretReadMessage(string key, string label) From 65de1cd16edfe6750cd502843189bb368714eb4e Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 29 May 2024 16:15:56 -0700 Subject: [PATCH 14/72] WIP fixing tests to return response for getconfigurationsettingsasync --- .../AzureAppConfigurationProvider.cs | 102 +++++++++--------- .../Constants/LoggingConstants.cs | 2 +- .../ConfigurationClientExtensions.cs | 14 +-- .../LogHelper.cs | 6 +- .../FeatureManagementTests.cs | 10 +- 5 files changed, 66 insertions(+), 68 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 50bc9fe5..36e48a56 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -33,7 +33,7 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private AzureAppConfigurationOptions _options; private Dictionary _mappedData; private Dictionary _watchedSettings = new Dictionary(); - private Dictionary> _watchedCollections = new Dictionary>(); + private Dictionary> _watchedCollections = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); @@ -249,10 +249,9 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary> watchedCollections = null; + Dictionary> watchedCollections = null; Dictionary watchedSettings = null; List keyValueChanges = null; - List changedKeyValuesCollection = null; Dictionary data = null; bool refreshAll = false; StringBuilder logInfoBuilder = new StringBuilder(); @@ -264,7 +263,6 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => watchedSettings = null; watchedCollections = null; keyValueChanges = new List(); - changedKeyValuesCollection = null; refreshAll = false; Uri endpoint = _configClientManager.GetEndpointForClient(client); logDebugBuilder.Clear(); @@ -336,16 +334,12 @@ await CallWithRequestTracing( { foreach (KeyValueWatcher multiKeyWatcher in refreshableMultiKeyWatchers) { - SettingSelector watchedSettingSelector = new SettingSelector() - { - KeyFilter = multiKeyWatcher.Key, - LabelFilter = multiKeyWatcher.Label - }; + KeyValueIdentifier watchedKeyValueIdentifier = new KeyValueIdentifier(multiKeyWatcher.Key, multiKeyWatcher.Label); - if (_watchedCollections.TryGetValue(watchedSettingSelector, out IEnumerable matchConditions)) + if (_watchedCollections.TryGetValue(watchedKeyValueIdentifier, out IEnumerable matchConditions)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => hasWatchedCollectionsChanged = await client.HasAnyKeyValueChanged(watchedSettingSelector, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + async () => hasWatchedCollectionsChanged = await client.HasAnyKeyValueChanged(watchedKeyValueIdentifier, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } if (hasWatchedCollectionsChanged) @@ -358,7 +352,7 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa { if (!_options.RegisterAllEnabled) { - (data, watchedCollections) = await RefreshKeyValueCollections(data, watchedCollections, client, cancellationToken).ConfigureAwait(false); + keyValueChanges.AddRange(await RefreshKeyValueCollections(_options.MultiKeyWatchers, watchedCollections, client, cancellationToken).ConfigureAwait(false)); logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); } @@ -368,7 +362,8 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); } } - else if (hasWatchedCollectionsChanged && _options.RegisterAllEnabled) + + if (refreshAll || (hasWatchedCollectionsChanged && _options.RegisterAllEnabled)) { // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true // or if RegisterAll was called and any loaded key-value changed @@ -391,7 +386,7 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa UpdateNextRefreshTime(changeWatcher); } - foreach (KeyValueChange change in keyValueChanges.Concat(changedKeyValuesCollection)) + foreach (KeyValueChange change in keyValueChanges) { KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); @@ -451,9 +446,10 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa } } - if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || changedKeyValuesCollection?.Any() == true || keyValueChanges.Any()) + if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any()) { _watchedSettings = watchedSettings; + _watchedCollections = watchedCollections; if (logDebugBuilder.Length > 0) { @@ -743,7 +739,7 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { Dictionary data = null; - Dictionary> watchedCollections = null; + Dictionary> watchedCollections = null; Dictionary watchedSettings = null; await ExecuteWithFailOverPolicyAsync( @@ -785,26 +781,27 @@ await ExecuteWithFailOverPolicyAsync( if (_options.RegisterAllEnabled) { - foreach (SettingSelector selector in watchedCollections.Keys) + foreach (KeyValueIdentifier keyValueIdentifier in watchedCollections.Keys) { _options.MultiKeyWatchers.AppendUnique(new KeyValueWatcher() { - Key = selector.KeyFilter, - Label = selector.LabelFilter, + Key = keyValueIdentifier.Key, + Label = keyValueIdentifier.Label, RefreshInterval = _options.RefreshOptions.RefreshInterval }); } } _watchedSettings = watchedSettings; + _watchedCollections = watchedCollections; _mappedData = mappedData; } } - private async Task<(Dictionary, Dictionary>)> LoadSelectedKeyValues(ConfigurationClient client, CancellationToken cancellationToken) + private async Task<(Dictionary, Dictionary>)> LoadSelectedKeyValues(ConfigurationClient client, CancellationToken cancellationToken) { var serverData = new Dictionary(StringComparer.OrdinalIgnoreCase); - Dictionary> watchedSettings = new Dictionary>(); + Dictionary> watchedCollections = new Dictionary>(); // Use default query if there are no key-values specified for use other than the feature flags bool useDefaultQuery = !_options.KeyValueSelectors.Any(selector => selector.KeyFilter == null || @@ -832,14 +829,11 @@ await CallWithRequestTracing(async () => serverData[setting.Key] = setting; } - if (_options.RegisterAllEnabled) - { - matchConditions.Add(new MatchConditions { IfNoneMatch = page.GetRawResponse().Headers.ETag }); - } + matchConditions.Add(new MatchConditions { IfNoneMatch = page.GetRawResponse().Headers.ETag }); } }).ConfigureAwait(false); - watchedSettings[selector] = matchConditions; + watchedCollections[new KeyValueIdentifier(selector.KeyFilter, selector.LabelFilter)] = matchConditions; } foreach (KeyValueSelector loadOption in _options.KeyValueSelectors) @@ -865,14 +859,11 @@ await CallWithRequestTracing(async () => serverData[setting.Key] = setting; } - if (_options.RegisterAllEnabled) - { - matchConditions.Add(new MatchConditions { IfNoneMatch = page.GetRawResponse().Headers.ETag }); - } + matchConditions.Add(new MatchConditions { IfNoneMatch = page.GetRawResponse().Headers.ETag }); } }).ConfigureAwait(false); - watchedSettings[selector] = matchConditions; + watchedCollections[new KeyValueIdentifier(selector.KeyFilter, selector.LabelFilter)] = matchConditions; } else { @@ -906,7 +897,7 @@ await CallWithRequestTracing(async () => } } - return (serverData, watchedSettings); + return (serverData, watchedCollections); } private async Task> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) @@ -950,34 +941,30 @@ private async Task> LoadKey return watchedSettings; } - private Dictionary> UpdateWatchedKeyValueCollections(Dictionary> watchedSettings) + private Dictionary> UpdateWatchedKeyValueCollections(Dictionary> watchedCollections) { - Dictionary> watchedSettingsCopy = new Dictionary>(); + Dictionary> watchedCollectionsCopy = new Dictionary>(); if (!_options.RegisterAllEnabled) { foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) { - var selector = new SettingSelector() - { - KeyFilter = changeWatcher.Key, - LabelFilter = changeWatcher.Label - }; + KeyValueIdentifier watchedIdentifier = new KeyValueIdentifier(changeWatcher.Key, changeWatcher.Label); - watchedSettingsCopy[selector] = watchedSettings[selector]; + watchedCollectionsCopy[watchedIdentifier] = watchedCollections[watchedIdentifier]; } } else { - return watchedSettings; + return watchedCollections; } - return watchedSettingsCopy; + return watchedCollectionsCopy; } private async Task> RefreshKeyValueCollections( IEnumerable multiKeyWatchers, - Dictionary> watchedSettings, + Dictionary> watchedCollections, ConfigurationClient client, CancellationToken cancellationToken) { @@ -985,23 +972,32 @@ private async Task> RefreshKeyValueCollections( foreach (KeyValueWatcher changeWatcher in multiKeyWatchers) { + KeyValueIdentifier keyValueIdentifier = new KeyValueIdentifier(changeWatcher.Key, changeWatcher.Label); + SettingSelector selector = new SettingSelector() { KeyFilter = changeWatcher.Key, LabelFilter = changeWatcher.Label.NormalizeNull() }; - keyValueChanges.AddRange( - await client.GetKeyValueChangeCollection( - new GetKeyValueChangeCollectionOptions + foreach (MatchConditions condition in watchedCollections[keyValueIdentifier]) + { + selector.MatchConditions.Add(condition); + } + + await CallWithRequestTracing(async () => + { + await foreach (Page page in client.GetConfigurationSettingsAsync(selector, cancellationToken).AsPages().ConfigureAwait(false)) + { + keyValueChanges.AddRange(page.Values.Select(setting => new KeyValueChange() { - Selector = selector, - MatchConditions = watchedSettings[selector], - RequestTracingEnabled = _requestTracingEnabled, - RequestTracingOptions = _requestTracingOptions - }, - cancellationToken) - .ConfigureAwait(false)); + Key = setting.Key, + Label = setting.Label.NormalizeNull(), + Current = setting, + ChangeType = KeyValueChangeType.Modified + })); + } + }).ConfigureAwait(false); } return keyValueChanges; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs index 41c56ca3..7f31b4e1 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs @@ -26,7 +26,7 @@ internal class LoggingConstants public const string RefreshConfigurationUpdatedSuccess = "Configuration reloaded."; public const string RefreshKeyValueSettingUpdated = "Setting updated."; public const string RefreshKeyVaultSettingUpdated = "Setting updated from Key Vault."; - public const string RefreshFeatureFlagUpdated = "Feature flag updated."; + public const string RefreshFeatureFlagsUpdated = "Feature flags updated."; // Other public const string RefreshSkippedNoClientAvailable = "Refresh skipped because no endpoint is accessible."; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 9734bd6d..c7672146 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -3,6 +3,7 @@ // using Azure; using Azure.Data.AppConfiguration; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; using System; using System.Collections.Generic; using System.Linq; @@ -60,13 +61,8 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } - public static async Task HasAnyKeyValueChanged(this ConfigurationClient client, SettingSelector selector, IEnumerable matchConditions, CancellationToken cancellationToken) + public static async Task HasAnyKeyValueChanged(this ConfigurationClient client, KeyValueIdentifier keyValueIdentifier, IEnumerable matchConditions, CancellationToken cancellationToken) { - if (selector == null) - { - throw new ArgumentNullException(nameof(selector)); - } - if (matchConditions == null) { throw new ArgumentNullException(nameof(matchConditions)); @@ -77,6 +73,12 @@ public static async Task HasAnyKeyValueChanged(this ConfigurationClient cl throw new ArgumentException("Requires at least one MatchConditions value.", nameof(matchConditions)); } + SettingSelector selector = new SettingSelector + { + KeyFilter = keyValueIdentifier.Key, + LabelFilter = keyValueIdentifier.Label + }; + foreach (MatchConditions condition in matchConditions) { selector.MatchConditions.Add(condition); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs index 0966db94..b421dc33 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. // +using System.Net; + namespace Microsoft.Extensions.Configuration.AzureAppConfiguration { internal static class LogHelper @@ -26,8 +28,8 @@ public static string BuildFeatureFlagsUnchangedMessage(string endpoint) } public static string BuildFeatureFlagsUpdatedMessage() - { - + { + return LoggingConstants.RefreshFeatureFlagsUnchanged; } public static string BuildKeyVaultSecretReadMessage(string key, string label) diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 46e7bd14..4492e15a 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -704,7 +704,7 @@ public void QueriesFeatureFlags() } [Fact] - public void UsesEtagForFeatureFlagRefresh() + public void DoesNotUseEtagForFeatureFlagRefresh() { var mockClient = new Mock(MockBehavior.Strict); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) @@ -725,7 +725,7 @@ public void UsesEtagForFeatureFlagRefresh() Thread.Sleep(RefreshInterval); refresher.TryRefreshAsync().Wait(); - mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(3)); + mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); } [Fact] @@ -1265,16 +1265,14 @@ public void ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefresh() Thread.Sleep(RefreshInterval); refresher.TryRefreshAsync().Wait(); Assert.Equal("AllUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - Assert.Contains(LogHelper.BuildFeatureFlagReadMessage("myFeature1", null, TestHelpers.PrimaryConfigStoreEndpoint.ToString().TrimEnd('/')), verboseInvocation); - Assert.Contains(LogHelper.BuildFeatureFlagUpdatedMessage("myFeature1"), informationalInvocation); + Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); featureFlags.RemoveAt(0); Thread.Sleep(RefreshInterval); refresher.TryRefreshAsync().Wait(); Assert.Null(config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); - Assert.Contains(LogHelper.BuildFeatureFlagReadMessage("myFeature1", null, TestHelpers.PrimaryConfigStoreEndpoint.ToString().TrimEnd('/')), verboseInvocation); - Assert.Contains(LogHelper.BuildFeatureFlagUpdatedMessage("myFeature1"), informationalInvocation); + Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); } [Fact] From 25c477b7f856e3eeb2de21e79043f1e8de55d92b Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Fri, 31 May 2024 16:11:16 -0700 Subject: [PATCH 15/72] WIP etag for tests --- .../AzureAppConfigurationProvider.cs | 2 +- .../FeatureManagementTests.cs | 21 ++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 36e48a56..5803196d 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -352,7 +352,7 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa { if (!_options.RegisterAllEnabled) { - keyValueChanges.AddRange(await RefreshKeyValueCollections(_options.MultiKeyWatchers, watchedCollections, client, cancellationToken).ConfigureAwait(false)); + keyValueChanges.AddRange(await RefreshKeyValueCollections(_options.MultiKeyWatchers, _watchedCollections, client, cancellationToken).ConfigureAwait(false)); logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); } diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 4492e15a..d3424182 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. // using Azure; +using Azure.Core; using Azure.Core.Diagnostics; using Azure.Core.Testing; using Azure.Data.AppConfiguration; @@ -378,17 +379,25 @@ public void WatchesFeatureFlags() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); - var mockClient = new Mock(MockBehavior.Strict); + string etag = "c3c231fd-39a0-4cb6-3237-4614474b92c1"; - mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + var mockTransport = new MockTransport(req => + { + var response = new MockResponse(200); + response.SetContent(SerializationHelpers.Serialize(featureFlags.ToArray(), TestHelpers.SerializeBatch)); + response.AddHeader(new HttpHeader(HttpHeader.Names.ETag, new ETag(etag).ToString())); + return response; + }); + + var options = new AzureAppConfigurationOptions(); + options.ClientOptions.Transport = mockTransport; + var clientManager = TestHelpers.CreateMockedConfigurationClientManager(options); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { - options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ClientManager = clientManager; options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -430,6 +439,8 @@ public void WatchesFeatureFlags() contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + etag = "c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f"; + featureFlags.Add(_kv2); // Sleep to let the refresh interval elapse From 913587e3ca55f2bc9876a83e3a781b6aabc4e289 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 20 Jun 2024 09:47:00 -0700 Subject: [PATCH 16/72] fix watchedcollections null --- .../AzureAppConfigurationProvider.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 5803196d..da49195b 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -261,7 +261,7 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => { data = null; watchedSettings = null; - watchedCollections = null; + watchedCollections = _watchedCollections; keyValueChanges = new List(); refreshAll = false; Uri endpoint = _configClientManager.GetEndpointForClient(client); @@ -336,7 +336,7 @@ await CallWithRequestTracing( { KeyValueIdentifier watchedKeyValueIdentifier = new KeyValueIdentifier(multiKeyWatcher.Key, multiKeyWatcher.Label); - if (_watchedCollections.TryGetValue(watchedKeyValueIdentifier, out IEnumerable matchConditions)) + if (watchedCollections.TryGetValue(watchedKeyValueIdentifier, out IEnumerable matchConditions)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, async () => hasWatchedCollectionsChanged = await client.HasAnyKeyValueChanged(watchedKeyValueIdentifier, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); @@ -352,7 +352,7 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa { if (!_options.RegisterAllEnabled) { - keyValueChanges.AddRange(await RefreshKeyValueCollections(_options.MultiKeyWatchers, _watchedCollections, client, cancellationToken).ConfigureAwait(false)); + keyValueChanges.AddRange(await RefreshKeyValueCollections(_options.MultiKeyWatchers, watchedCollections, client, cancellationToken).ConfigureAwait(false)); logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); } From 5f842952c5ad3bdfc290b47afc3100962742298d Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 20 Jun 2024 14:35:29 -0700 Subject: [PATCH 17/72] WIP tests, working for examples --- .../Azure.Core.Testing/MockResponse.cs | 4 ++++ .../Tests.AzureAppConfiguration/FeatureManagementTests.cs | 7 ++++++- tests/Tests.AzureAppConfiguration/TestHelper.cs | 7 ++++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs index 886d0a77..d9e217b3 100644 --- a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs +++ b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs @@ -100,6 +100,10 @@ public MockResponse(T value) public override Response GetRawResponse() { + var response = new MockResponse(200); + + response.AddHeader(new HttpHeader("ETag", new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1").ToString())); + throw new NotImplementedException(); } } diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index d3424182..baafaf28 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -16,7 +16,6 @@ using System.Collections.Generic; using System.Diagnostics.Tracing; using System.Linq; -using System.Net; using System.Security.Cryptography; using System.Text; using System.Threading; @@ -1224,6 +1223,12 @@ public void ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefresh() mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) .Returns(new MockAsyncPageable(featureFlags)); + mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((Func>)GetIfChanged); + + mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((Func>)GetTestKey); + string informationalInvocation = ""; string verboseInvocation = ""; using var _ = new AzureEventSourceListener( diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index fe1685d9..216dfad7 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -3,6 +3,7 @@ // using Azure; using Azure.Core; +using Azure.Core.Testing; using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration.AzureAppConfiguration; using Microsoft.Extensions.Logging; @@ -120,8 +121,8 @@ public static List LoadJsonSettingsFromFile(string path) { ConfigurationSetting kv = ConfigurationModelFactory .ConfigurationSetting( - key: setting.GetProperty("key").ToString(), - value: setting.GetProperty("value").GetRawText(), + key: setting.GetProperty("key").ToString(), + value: setting.GetProperty("value").GetRawText(), contentType: setting.GetProperty("contentType").ToString()); _kvCollection.Add(kv); } @@ -157,7 +158,7 @@ public MockAsyncPageable(List collection) public async override IAsyncEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) #pragma warning restore 1998 { - yield return Page.FromValues(_collection, null, new Mock().Object); + yield return Page.FromValues(_collection, null, new MockResponse(200)); } } From e18a8355115bf26d0eeb328d5ef8f97a8f96d054 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 20 Jun 2024 14:38:03 -0700 Subject: [PATCH 18/72] remove unused variables --- .../AzureAppConfigurationOptions.cs | 3 --- .../AzureAppConfigurationRefreshOptions.cs | 3 --- 2 files changed, 6 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index a84a18d9..3eeca707 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -28,7 +28,6 @@ public class AzureAppConfigurationOptions private List _adapters; private List>> _mappers = new List>>(); private List _kvSelectors = new List(); - private List _featureFlagSelectors = new List(); private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); // The following set is sorted in descending order. @@ -242,8 +241,6 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c throw new InvalidOperationException($"Please select feature flags by either the {nameof(options.Select)} method or by setting the {nameof(options.Label)} property, not both."); } - _featureFlagSelectors = options.FeatureFlagSelectors; - if (options.FeatureFlagSelectors.Count() == 0) { // Select clause is not present diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs index c5571511..cdf55b57 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs @@ -1,10 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. // -using Azure.Data.AppConfiguration; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; -using Microsoft.Extensions.Options; using System; using System.Collections.Generic; From d41499e3d4742174bdfd4efa497235040ca0f805 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 20 Jun 2024 14:41:04 -0700 Subject: [PATCH 19/72] update to newest sdk version, remove unused --- .../LogHelper.cs | 1 - ...rosoft.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs index b421dc33..bc8f13e4 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. // -using System.Net; namespace Microsoft.Extensions.Configuration.AzureAppConfiguration { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index ce61574d..132eaac6 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -14,7 +14,7 @@ - + From d6ac25b4ed095513841feaaa4c0c081ca7a5747c Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Mon, 1 Jul 2024 12:40:33 -0700 Subject: [PATCH 20/72] WIP fixing tests --- ...ft.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- .../Azure.Core.Testing/MockResponse.cs | 4 ---- tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs | 6 +++--- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index f6ec8fcf..cd7d8ffa 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -14,7 +14,7 @@ - + diff --git a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs index d9e217b3..886d0a77 100644 --- a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs +++ b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs @@ -100,10 +100,6 @@ public MockResponse(T value) public override Response GetRawResponse() { - var response = new MockResponse(200); - - response.AddHeader(new HttpHeader("ETag", new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1").ToString())); - throw new NotImplementedException(); } } diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 1d110a9e..4261f760 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -665,7 +665,7 @@ public async Task WatchesFeatureFlags() { var featureFlags = new List { _kv }; - string etag = "c3c231fd-39a0-4cb6-3237-4614474b92c1"; + string etag = "\"c3c231fd-39a0-4cb6-3237-4614474b92c1\""; var mockTransport = new MockTransport(req => { @@ -1017,7 +1017,7 @@ public void QueriesFeatureFlags() } [Fact] - public void DoesNotUseEtagForFeatureFlagRefresh() + public async Task DoesNotUseEtagForFeatureFlagRefresh() { var mockClient = new Mock(MockBehavior.Strict); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) @@ -1038,7 +1038,7 @@ public void DoesNotUseEtagForFeatureFlagRefresh() Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); - mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(3)); + mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(4)); } [Fact] From 5410b5d5de97da150e679fa1eb2c714face0bdae Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Mon, 1 Jul 2024 13:23:43 -0700 Subject: [PATCH 21/72] WIP reworking testing to work with new etag approach --- tests/Tests.AzureAppConfiguration/TestHelper.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index ce44ab26..d2ae23c7 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -165,7 +165,6 @@ public async override IAsyncEnumerable> AsPages(strin #pragma warning restore 1998 { yield return Page.FromValues(_collection, null, new MockResponse(200)); - } } @@ -180,7 +179,7 @@ public MockPageable(List collection) public override IEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) { - yield return Page.FromValues(_collection, null, new Mock().Object); + yield return Page.FromValues(_collection, null, new MockResponse(200)); } } } From fc752d3ee18ce1753da0eea7cb2efc104aa25eec Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Tue, 2 Jul 2024 16:58:40 -0700 Subject: [PATCH 22/72] tests passing, fix mockasyncpageable --- .../AzureAppConfigurationProvider.cs | 21 ++++++++ .../FeatureManagementTests.cs | 49 +++++++++++-------- .../KeyVaultReferenceTests.cs | 4 +- .../Tests.AzureAppConfiguration/TestHelper.cs | 46 +++++++++++++++-- 4 files changed, 94 insertions(+), 26 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 30a6e56a..59a76ae4 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -1007,6 +1007,27 @@ await CallWithRequestTracing(async () => }).ConfigureAwait(false); } + HashSet existingSettings = new HashSet(); + + foreach (KeyValueChange change in keyValueChanges) + { + existingSettings.Add(change.Key); + } + + foreach (KeyValuePair kvp in _mappedData) + { + if (!existingSettings.Contains(kvp.Key)) + { + keyValueChanges.Add(new KeyValueChange() + { + Key = kvp.Value.Key, + Label = kvp.Value.Label.NormalizeNull(), + Current = kvp.Value, + ChangeType = KeyValueChangeType.Deleted + }); + } + } + return keyValueChanges; } diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 4261f760..3b0bc0c3 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -750,9 +750,11 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Returns(mockAsyncPageable); var cacheExpirationInterval = TimeSpan.FromSeconds(1); @@ -1038,7 +1040,7 @@ public async Task DoesNotUseEtagForFeatureFlagRefresh() Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); - mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(4)); + mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(3)); } [Fact] @@ -1408,13 +1410,13 @@ public async Task DifferentCacheExpirationsForMultipleFeatureFlagRegistrations() IConfigurationRefresher refresher = null; var featureFlagCollection = new List(_featureFlagCollection); + var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(() => - { - return new MockAsyncPageable(featureFlagCollection.Where(s => + .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlagCollection.Where(s => (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1) || - (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix2) && s.Label == label2 && s.Key != FeatureManagementConstants.FeatureFlagMarker + "App2_Feature3")).ToList()); - }); + (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix2) && s.Label == label2 && s.Key != FeatureManagementConstants.FeatureFlagMarker + "App2_Feature3")).ToList())) + .Returns(mockAsyncPageable); var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => @@ -1595,13 +1597,12 @@ public async Task SelectAndRefreshSingleFeatureFlag() var label1 = "App1_Label"; IConfigurationRefresher refresher = null; var featureFlagCollection = new List(_featureFlagCollection); + var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(() => - { - return new MockAsyncPageable(featureFlagCollection.Where(s => - s.Key.Equals(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1).ToList()); - }); + .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlagCollection.Where(s => + s.Key.Equals(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1).ToList())) + .Returns(mockAsyncPageable); var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => @@ -1660,8 +1661,11 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Func>)GetIfChanged); @@ -1699,10 +1703,10 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre Assert.Equal("MyFeature2", config["feature_management:feature_flags:0:id"]); featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( - key: FeatureManagementConstants.FeatureFlagMarker + "myFeature1", + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature2", value: @" { - ""id"": ""MyFeature"", + ""id"": ""MyFeature2"", ""description"": ""The new beta version of our web site."", ""display_name"": ""Beta Feature"", ""enabled"": true, @@ -1717,12 +1721,11 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); Assert.Equal("AllUsers", config["feature_management:feature_flags:0:conditions:client_filters:0:name"]); - Assert.Equal("MyFeature", config["feature_management:feature_flags:0:id"]); Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); featureFlags.RemoveAt(0); @@ -1731,7 +1734,6 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre Assert.Null(config["feature_management:feature_flags:0:conditions:client_filters:0:name"]); Assert.Contains(LogHelper.BuildFeatureFlagsUpdatedMessage(), informationalInvocation); - } [Fact] @@ -1742,8 +1744,11 @@ public async Task ValidateFeatureFlagsUnchangedLogged() var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Func>)GetIfChanged); @@ -1816,9 +1821,11 @@ public async Task MapTransformFeatureFlagWithRefresh() IConfigurationRefresher refresher = null; var featureFlags = new List { _kv }; var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((Func>)GetIfChanged); @@ -1891,7 +1898,7 @@ public async Task MapTransformFeatureFlagWithRefresh() ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs index 6108519c..ddb0880a 100644 --- a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs +++ b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs @@ -506,9 +506,9 @@ public void DoesNotThrowKeyVaultExceptionWhenProviderIsOptional() .Returns(new MockAsyncPageable(new List { _kv })); var mockKeyValueAdapter = new Mock(MockBehavior.Strict); - mockKeyValueAdapter.Setup(adapter => adapter.CanProcess(_kv)) + mockKeyValueAdapter.Setup(adapter => adapter.CanProcess(It.IsAny())) .Returns(true); - mockKeyValueAdapter.Setup(adapter => adapter.ProcessKeyValue(_kv, It.IsAny(), It.IsAny(), It.IsAny())) + mockKeyValueAdapter.Setup(adapter => adapter.ProcessKeyValue(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Throws(new KeyVaultReferenceException("Key vault error", null)); mockKeyValueAdapter.Setup(adapter => adapter.OnChangeDetected(null)); mockKeyValueAdapter.Setup(adapter => adapter.OnConfigUpdated()); diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index d2ae23c7..e9f96cc7 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -9,8 +9,11 @@ using Microsoft.Extensions.Logging; using Moq; using System; +using System.Collections; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.IO; +using System.Linq; using System.Text; using System.Text.Json; using System.Threading; @@ -153,18 +156,55 @@ public static bool ValidateLog(Mock logger, string expectedMessage, Log class MockAsyncPageable : AsyncPageable { - private readonly List _collection; + private readonly List _collection = new List(); + private int _status; public MockAsyncPageable(List collection) { - _collection = collection; + foreach (ConfigurationSetting setting in collection) + { + var newSetting = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + newSetting.ContentType = setting.ContentType; + + _collection.Add(newSetting); + } + + _status = 304; + } + + public void UpdateFeatureFlags(List newCollection) + { + if (_collection.All(setting => newCollection.Any(newSetting => + setting.Key == newSetting.Key && + setting.Value == newSetting.Value && + setting.Label == newSetting.Label && + setting.ETag == newSetting.ETag))) + { + _status = 304; + } + else + { + _status = 200; + + _collection.Clear(); + + foreach (ConfigurationSetting setting in newCollection) + { + var newSetting = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + newSetting.ContentType = setting.ContentType; + + _collection.Add(newSetting); + } + } } #pragma warning disable 1998 public async override IAsyncEnumerable> AsPages(string continuationToken = null, int? pageSizeHint = null) #pragma warning restore 1998 { - yield return Page.FromValues(_collection, null, new MockResponse(200)); + yield return Page.FromValues(_collection, null, new MockResponse(_status)); } } From 8b02c76e34782d96d0d84c4312c1b594650ddcff Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 3 Jul 2024 10:13:50 -0700 Subject: [PATCH 23/72] update sdk package version --- .../AzureAppConfigurationProvider.cs | 6 ++++-- .../Extensions/ConfigurationClientExtensions.cs | 7 +------ ...t.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 59a76ae4..6125bf13 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -987,14 +987,16 @@ private async Task> RefreshKeyValueCollections( LabelFilter = changeWatcher.Label.NormalizeNull() }; + List matchConditions = new List(); + foreach (MatchConditions condition in watchedCollections[keyValueIdentifier]) { - selector.MatchConditions.Add(condition); + matchConditions.Add(condition); } await CallWithRequestTracing(async () => { - await foreach (Page page in client.GetConfigurationSettingsAsync(selector, cancellationToken).AsPages().ConfigureAwait(false)) + await foreach (Page page in client.GetConfigurationSettingsAsync(selector, cancellationToken).AsPages(matchConditions).ConfigureAwait(false)) { keyValueChanges.AddRange(page.Values.Select(setting => new KeyValueChange() { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index c7672146..0b76c62e 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -79,12 +79,7 @@ public static async Task HasAnyKeyValueChanged(this ConfigurationClient cl LabelFilter = keyValueIdentifier.Label }; - foreach (MatchConditions condition in matchConditions) - { - selector.MatchConditions.Add(condition); - } - - await foreach (Page page in client.GetConfigurationSettingsAsync(selector, cancellationToken).AsPages().ConfigureAwait(false)) + await foreach (Page page in client.GetConfigurationSettingsAsync(selector, cancellationToken).AsPages(matchConditions).ConfigureAwait(false)) { Response response = page.GetRawResponse(); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index cd7d8ffa..f6ec8fcf 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -14,7 +14,7 @@ - + From 3ebf1e4c55866de08525a7f1f7592696441f92db Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 3 Jul 2024 11:47:23 -0700 Subject: [PATCH 24/72] fix loghelper, tests --- .../AzureAppConfigurationProvider.cs | 4 +--- .../LogHelper.cs | 2 +- tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs | 4 +--- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 6125bf13..82faf0b8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -979,8 +979,6 @@ private async Task> RefreshKeyValueCollections( foreach (KeyValueWatcher changeWatcher in multiKeyWatchers) { - KeyValueIdentifier keyValueIdentifier = new KeyValueIdentifier(changeWatcher.Key, changeWatcher.Label); - SettingSelector selector = new SettingSelector() { KeyFilter = changeWatcher.Key, @@ -989,7 +987,7 @@ private async Task> RefreshKeyValueCollections( List matchConditions = new List(); - foreach (MatchConditions condition in watchedCollections[keyValueIdentifier]) + foreach (MatchConditions condition in watchedCollections[new KeyValueIdentifier(changeWatcher.Key, changeWatcher.Label)]) { matchConditions.Add(condition); } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs index 236bf233..dca291e8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs @@ -28,7 +28,7 @@ public static string BuildFeatureFlagsUnchangedMessage(string endpoint) public static string BuildFeatureFlagsUpdatedMessage() { - return LoggingConstants.RefreshFeatureFlagsUnchanged; + return LoggingConstants.RefreshFeatureFlagsUpdated; } public static string BuildKeyVaultSecretReadMessage(string key, string label) diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 3b0bc0c3..a92a4114 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -725,9 +725,7 @@ public async Task WatchesFeatureFlags() ", label: default, contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f")); - - etag = "c3c231fd-39a0-4cb6-3237-4614474b92c1" + "f"; + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); featureFlags.Add(_kv2); From 26d3391779da522da719fab5c3de512fd3809dc1 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Mon, 8 Jul 2024 09:28:37 -0700 Subject: [PATCH 25/72] WIP fixing aspages tests --- .../FeatureManagementTests.cs | 13 ++++++++----- tests/Tests.AzureAppConfiguration/TestHelper.cs | 14 ++++++++++++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index a92a4114..fb0e7fd9 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -746,7 +746,6 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); var mockAsyncPageable = new MockAsyncPageable(featureFlags); @@ -823,11 +822,13 @@ public async Task SkipRefreshIfRefreshIntervalHasNotElapsed() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() @@ -894,11 +895,13 @@ public async Task SkipRefreshIfCacheNotExpired() { var featureFlags = new List { _kv }; - var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index e9f96cc7..b7929872 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -156,8 +156,8 @@ public static bool ValidateLog(Mock logger, string expectedMessage, Log class MockAsyncPageable : AsyncPageable { - private readonly List _collection = new List(); - private int _status; + private static readonly List _collection = new List(); + private static int _status; public MockAsyncPageable(List collection) { @@ -208,6 +208,16 @@ public async override IAsyncEnumerable> AsPages(strin } } + public static class MockConfigurationClientExtensions + { +#pragma warning disable 1998 + public static async IAsyncEnumerable> AsPages(this AsyncPageable asyncPageable, IEnumerable matchConditions, string continuationToken = null, int? pageSizeHint = null) +#pragma warning restore 1998 + { + yield return Page.FromValues(_collection, null, new MockResponse(_status)); + } + } + class MockPageable : Pageable { private readonly List _collection; From aa037b6c157a68f05ee4ee3a68ad5c273cf68474 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 11 Jul 2024 11:55:03 -0700 Subject: [PATCH 26/72] revert watchesfeatureflags test --- .../FeatureManagementTests.cs | 18 +++++------------- .../Tests.AzureAppConfiguration/TestHelper.cs | 10 ---------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index fb0e7fd9..74be1433 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -665,25 +665,17 @@ public async Task WatchesFeatureFlags() { var featureFlags = new List { _kv }; - string etag = "\"c3c231fd-39a0-4cb6-3237-4614474b92c1\""; - - var mockTransport = new MockTransport(req => - { - var response = new MockResponse(200); - response.SetContent(SerializationHelpers.Serialize(featureFlags.ToArray(), TestHelpers.SerializeBatch)); - response.AddHeader(new HttpHeader(HttpHeader.Names.ETag, new ETag(etag).ToString())); - return response; - }); + var mockResponse = new Mock(); + var mockClient = new Mock(MockBehavior.Strict); - var options = new AzureAppConfigurationOptions(); - options.ClientOptions.Transport = mockTransport; - var clientManager = TestHelpers.CreateMockedConfigurationClientManager(options); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns(new MockAsyncPageable(featureFlags)); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { - options.ClientManager = clientManager; + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index b7929872..5ae43058 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -208,16 +208,6 @@ public async override IAsyncEnumerable> AsPages(strin } } - public static class MockConfigurationClientExtensions - { -#pragma warning disable 1998 - public static async IAsyncEnumerable> AsPages(this AsyncPageable asyncPageable, IEnumerable matchConditions, string continuationToken = null, int? pageSizeHint = null) -#pragma warning restore 1998 - { - yield return Page.FromValues(_collection, null, new MockResponse(_status)); - } - } - class MockPageable : Pageable { private readonly List _collection; From 415c57704b5f4dc2ef6e6a09d2ce3e8e7af30635 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 11 Jul 2024 11:56:07 -0700 Subject: [PATCH 27/72] update test again --- tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 74be1433..c852e4f9 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -667,9 +667,11 @@ public async Task WatchesFeatureFlags() var mockResponse = new Mock(); var mockClient = new Mock(MockBehavior.Strict); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); + .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() From 57f5b80f222dc04f83fa115e9936c0cace46a6a7 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 11 Jul 2024 17:49:07 -0700 Subject: [PATCH 28/72] WIP --- .../AzureAppConfigurationOptions.cs | 9 +- .../AzureAppConfigurationProvider.cs | 110 ++++++++++-------- .../AzureAppConfigurationRefreshOptions.cs | 2 +- .../KeyValueChange.cs | 2 + .../Tests.AzureAppConfiguration/TestHelper.cs | 9 +- 5 files changed, 71 insertions(+), 61 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 3eeca707..3898a006 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -39,11 +39,6 @@ public class AzureAppConfigurationOptions /// public bool ReplicaDiscoveryEnabled { get; set; } = true; - /// - /// Flag to indicate whether has been called. - /// - internal bool RegisterAllEnabled { get; private set; } = false; - /// /// Flag to indicate whether load balancing is enabled. /// @@ -84,7 +79,7 @@ public class AzureAppConfigurationOptions /// /// A collection of . /// - internal List MultiKeyWatchers => _multiKeyWatchers; + internal IEnumerable MultiKeyWatchers => _multiKeyWatchers; /// /// A collection of . @@ -400,6 +395,8 @@ public AzureAppConfigurationOptions ConfigureRefresh(Action { data = null; watchedSettings = null; - watchedCollections = _watchedCollections; + watchedCollections = null; keyValueChanges = new List(); refreshAll = false; Uri endpoint = _configClientManager.GetEndpointForClient(client); @@ -332,6 +332,8 @@ await CallWithRequestTracing( if (!refreshAll) { + watchedCollections = _watchedCollections; + foreach (KeyValueWatcher multiKeyWatcher in refreshableMultiKeyWatchers) { KeyValueIdentifier watchedKeyValueIdentifier = new KeyValueIdentifier(multiKeyWatcher.Key, multiKeyWatcher.Label); @@ -348,22 +350,19 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa } } - if (hasWatchedCollectionsChanged) + if (!hasWatchedCollectionsChanged) { - if (!_options.RegisterAllEnabled) - { - keyValueChanges.AddRange(await RefreshKeyValueCollections(_options.MultiKeyWatchers, watchedCollections, client, cancellationToken).ConfigureAwait(false)); - - logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); - } + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); } - else + else if (!_options.RefreshOptions.RegisterAllEnabled) { - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); + keyValueChanges.AddRange(await RefreshKeyValueCollections(_options.MultiKeyWatchers, watchedCollections, _mappedData, client, cancellationToken).ConfigureAwait(false)); + + logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); } } - if (refreshAll || (hasWatchedCollectionsChanged && _options.RegisterAllEnabled)) + if (refreshAll || (hasWatchedCollectionsChanged && _options.RefreshOptions.RegisterAllEnabled)) { // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true // or if RegisterAll was called and any loaded key-value changed @@ -390,17 +389,15 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa { KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); - SettingSelector changeSelector = new SettingSelector() - { - KeyFilter = change.Key, - LabelFilter = change.Label - }; - if (change.ChangeType == KeyValueChangeType.Modified) { ConfigurationSetting setting = change.Current; ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); - watchedSettings[changeIdentifier] = settingCopy; + + if (!change.IsMultiKeyWatcherChange) + { + watchedSettings[changeIdentifier] = settingCopy; + } foreach (Func> func in _options.Mappers) { @@ -419,7 +416,11 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa else if (change.ChangeType == KeyValueChangeType.Deleted) { _mappedData.Remove(change.Key); - watchedSettings.Remove(changeIdentifier); + + if (!change.IsMultiKeyWatcherChange) + { + watchedSettings.Remove(changeIdentifier); + } } // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting @@ -786,11 +787,13 @@ await ExecuteWithFailOverPolicyAsync( Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); - if (_options.RegisterAllEnabled) + if (_options.RefreshOptions.RegisterAllEnabled) { + List updatedMultiKeyWatchers = (List)_options.MultiKeyWatchers; + foreach (KeyValueIdentifier keyValueIdentifier in watchedCollections.Keys) { - _options.MultiKeyWatchers.AppendUnique(new KeyValueWatcher() + updatedMultiKeyWatchers.AppendUnique(new KeyValueWatcher() { Key = keyValueIdentifier.Key, Label = keyValueIdentifier.Label, @@ -799,6 +802,8 @@ await ExecuteWithFailOverPolicyAsync( } } + // TODO set multikeywatchers correctly, figure out why watchedcollections isn't correct here + _watchedSettings = watchedSettings; _watchedCollections = watchedCollections; _mappedData = mappedData; @@ -951,19 +956,12 @@ private async Task> LoadKey private Dictionary> UpdateWatchedKeyValueCollections(Dictionary> watchedCollections) { Dictionary> watchedCollectionsCopy = new Dictionary>(); - - if (!_options.RegisterAllEnabled) + + foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) { - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) - { - KeyValueIdentifier watchedIdentifier = new KeyValueIdentifier(changeWatcher.Key, changeWatcher.Label); + KeyValueIdentifier watchedIdentifier = new KeyValueIdentifier(changeWatcher.Key, changeWatcher.Label); - watchedCollectionsCopy[watchedIdentifier] = watchedCollections[watchedIdentifier]; - } - } - else - { - return watchedCollections; + watchedCollectionsCopy[watchedIdentifier] = watchedCollections[watchedIdentifier]; } return watchedCollectionsCopy; @@ -972,17 +970,20 @@ private Dictionary> UpdateWatch private async Task> RefreshKeyValueCollections( IEnumerable multiKeyWatchers, Dictionary> watchedCollections, + IDictionary existingSettings, ConfigurationClient client, CancellationToken cancellationToken) { - var keyValueChanges = new List(); + List keyValueChanges = new List(); + + HashSet existingKeys = new HashSet(); foreach (KeyValueWatcher changeWatcher in multiKeyWatchers) { SettingSelector selector = new SettingSelector() { KeyFilter = changeWatcher.Key, - LabelFilter = changeWatcher.Label.NormalizeNull() + LabelFilter = changeWatcher.Label }; List matchConditions = new List(); @@ -994,36 +995,45 @@ private async Task> RefreshKeyValueCollections( await CallWithRequestTracing(async () => { - await foreach (Page page in client.GetConfigurationSettingsAsync(selector, cancellationToken).AsPages(matchConditions).ConfigureAwait(false)) + await foreach (Page page in client.GetConfigurationSettingsAsync(selector, cancellationToken).AsPages().ConfigureAwait(false)) { keyValueChanges.AddRange(page.Values.Select(setting => new KeyValueChange() { Key = setting.Key, Label = setting.Label.NormalizeNull(), Current = setting, - ChangeType = KeyValueChangeType.Modified + ChangeType = KeyValueChangeType.Modified, + IsMultiKeyWatcherChange = true })); } }).ConfigureAwait(false); + + IEnumerable existingKeysList = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, existingSettings.Keys); + + foreach (string key in existingKeysList) + { + existingKeys.Add(key); + } } - HashSet existingSettings = new HashSet(); + HashSet loadedSettings = new HashSet(); foreach (KeyValueChange change in keyValueChanges) { - existingSettings.Add(change.Key); + loadedSettings.Add(change.Key); } - foreach (KeyValuePair kvp in _mappedData) + foreach (string key in existingKeys) { - if (!existingSettings.Contains(kvp.Key)) + if (!loadedSettings.Contains(key)) { keyValueChanges.Add(new KeyValueChange() { - Key = kvp.Value.Key, - Label = kvp.Value.Label.NormalizeNull(), - Current = kvp.Value, - ChangeType = KeyValueChangeType.Deleted + Key = key, + Label = null, + Current = null, + ChangeType = KeyValueChangeType.Deleted, + IsMultiKeyWatcherChange = true }); } } @@ -1306,24 +1316,24 @@ private async Task> MapConfigurationSet return mappedData; } - private IEnumerable GetCurrentKeyValueCollection(string key, string label, IEnumerable existingSettings) + private IEnumerable GetCurrentKeyValueCollection(string key, string label, IEnumerable existingKeys) { - IEnumerable currentKeyValues; + IEnumerable currentKeyValues; if (key.EndsWith("*")) { // Get current application settings starting with changeWatcher.Key, excluding the last * character string keyPrefix = key.Substring(0, key.Length - 1); - currentKeyValues = existingSettings.Where(kv => + currentKeyValues = existingKeys.Where(val => { - return kv.Key.StartsWith(keyPrefix) && kv.Label == label.NormalizeNull(); + return val.StartsWith(keyPrefix); }); } else { - currentKeyValues = existingSettings.Where(kv => + currentKeyValues = existingKeys.Where(val => { - return kv.Key.Equals(key) && kv.Label == label.NormalizeNull(); + return val.Equals(key); }); } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs index cdf55b57..a4c9c9ae 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs @@ -14,7 +14,7 @@ public class AzureAppConfigurationRefreshOptions { internal TimeSpan RefreshInterval { get; private set; } = RefreshConstants.DefaultRefreshInterval; internal ISet RefreshRegistrations = new HashSet(); - internal bool RegisterAllEnabled { get; private set; } = false; + internal bool RegisterAllEnabled { get; private set; } /// /// Register the specified individual key-value to be refreshed when the configuration provider's triggers a refresh. diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/KeyValueChange.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/KeyValueChange.cs index 7d41e107..529e67dd 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/KeyValueChange.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/KeyValueChange.cs @@ -21,5 +21,7 @@ internal struct KeyValueChange public string Label { get; set; } public ConfigurationSetting Current { get; set; } + + public bool IsMultiKeyWatcherChange { get; set; } } } diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index 5ae43058..be8f2ffe 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -156,8 +156,8 @@ public static bool ValidateLog(Mock logger, string expectedMessage, Log class MockAsyncPageable : AsyncPageable { - private static readonly List _collection = new List(); - private static int _status; + private readonly List _collection = new List(); + private int _status; public MockAsyncPageable(List collection) { @@ -169,13 +169,14 @@ public MockAsyncPageable(List collection) _collection.Add(newSetting); } + //_collection = collection; - _status = 304; + _status = 200; } public void UpdateFeatureFlags(List newCollection) { - if (_collection.All(setting => newCollection.Any(newSetting => + if (_collection.All(setting => newCollection.Any(newSetting => setting.Key == newSetting.Key && setting.Value == newSetting.Value && setting.Label == newSetting.Label && From 7ac2cd5d72a484d3abb5ba0e7b38551ffd3cf7a7 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Fri, 12 Jul 2024 10:47:04 -0700 Subject: [PATCH 29/72] fixing watchconditions --- .../AzureAppConfigurationProvider.cs | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 430d17bc..9d4e1305 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; +using System.Data; using System.Diagnostics; using System.IO; using System.Linq; @@ -845,7 +846,7 @@ await CallWithRequestTracing(async () => } }).ConfigureAwait(false); - watchedCollections[new KeyValueIdentifier(selector.KeyFilter, selector.LabelFilter)] = matchConditions; + watchedCollections[new KeyValueIdentifier(KeyFilter.Any, LabelFilter.Null)] = matchConditions; } foreach (KeyValueSelector loadOption in _options.KeyValueSelectors) @@ -875,7 +876,7 @@ await CallWithRequestTracing(async () => } }).ConfigureAwait(false); - watchedCollections[new KeyValueIdentifier(selector.KeyFilter, selector.LabelFilter)] = matchConditions; + watchedCollections[new KeyValueIdentifier(loadOption.KeyFilter, loadOption.LabelFilter)] = matchConditions; } else { @@ -957,14 +958,19 @@ private Dictionary> UpdateWatch { Dictionary> watchedCollectionsCopy = new Dictionary>(); - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + if (!_options.RefreshOptions.RegisterAllEnabled) { - KeyValueIdentifier watchedIdentifier = new KeyValueIdentifier(changeWatcher.Key, changeWatcher.Label); + foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + { + KeyValueIdentifier watchedIdentifier = new KeyValueIdentifier(changeWatcher.Key, changeWatcher.Label); - watchedCollectionsCopy[watchedIdentifier] = watchedCollections[watchedIdentifier]; + watchedCollectionsCopy[watchedIdentifier] = watchedCollections[watchedIdentifier]; + } + + return watchedCollectionsCopy; } - return watchedCollectionsCopy; + return watchedCollections; } private async Task> RefreshKeyValueCollections( @@ -986,13 +992,6 @@ private async Task> RefreshKeyValueCollections( LabelFilter = changeWatcher.Label }; - List matchConditions = new List(); - - foreach (MatchConditions condition in watchedCollections[new KeyValueIdentifier(changeWatcher.Key, changeWatcher.Label)]) - { - matchConditions.Add(condition); - } - await CallWithRequestTracing(async () => { await foreach (Page page in client.GetConfigurationSettingsAsync(selector, cancellationToken).AsPages().ConfigureAwait(false)) From ec9a9da550bfbdaf5edfde1303ef13cea4780a1d Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Mon, 15 Jul 2024 14:45:04 -0700 Subject: [PATCH 30/72] separate selected key value collections from feature flag collections, separate selectors, add new methods to support new logic --- .../AzureAppConfigurationOptions.cs | 19 +- .../AzureAppConfigurationProvider.cs | 184 ++++++++++-------- .../Constants/LoggingConstants.cs | 2 + .../ConfigurationClientExtensions.cs | 22 ++- .../Extensions/StringExtensions.cs | 9 +- .../LogHelper.cs | 10 + .../LoadBalancingTests.cs | 4 +- .../LoggingTests.cs | 2 + .../RefreshTests.cs | 4 +- .../Tests.AzureAppConfiguration/TestHelper.cs | 1 - 10 files changed, 151 insertions(+), 106 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 3898a006..e10ba4f6 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -24,10 +24,11 @@ public class AzureAppConfigurationOptions private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(1); private List _changeWatchers = new List(); - private List _multiKeyWatchers = new List(); + private List _featureFlagWatchers = new List(); private List _adapters; private List>> _mappers = new List>>(); private List _kvSelectors = new List(); + private List _featureFlagSelectors = new List(); private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); // The following set is sorted in descending order. @@ -66,6 +67,11 @@ public class AzureAppConfigurationOptions /// internal IEnumerable KeyValueSelectors => _kvSelectors; + /// + /// A collection of . + /// + internal IEnumerable FeatureFlagSelectors => _featureFlagSelectors; + /// /// The configured options for refresh. /// @@ -79,7 +85,7 @@ public class AzureAppConfigurationOptions /// /// A collection of . /// - internal IEnumerable MultiKeyWatchers => _multiKeyWatchers; + internal IEnumerable FeatureFlagWatchers => _featureFlagWatchers; /// /// A collection of . @@ -251,16 +257,19 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c var featureFlagFilter = featureFlagSelector.KeyFilter; var labelFilter = featureFlagSelector.LabelFilter; - Select(featureFlagFilter, labelFilter); + _featureFlagSelectors.AppendUnique(new KeyValueSelector + { + KeyFilter = featureFlagFilter, + LabelFilter = labelFilter + }); - _multiKeyWatchers.AppendUnique(new KeyValueWatcher + _featureFlagWatchers.AppendUnique(new KeyValueWatcher { Key = featureFlagFilter, Label = labelFilter, // If UseFeatureFlags is called multiple times for the same key and label filters, last refresh interval wins RefreshInterval = options.RefreshInterval }); - } return this; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 841cbd5b..580c1823 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -4,12 +4,10 @@ using Azure; using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; -using System.Data; using System.Diagnostics; using System.IO; using System.Linq; @@ -34,9 +32,11 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private AzureAppConfigurationOptions _options; private Dictionary _mappedData; private Dictionary _watchedSettings = new Dictionary(); - private Dictionary> _watchedCollections = new Dictionary>(); + private Dictionary> _watchedSelectedKeyValueCollections = new Dictionary>(); + private Dictionary> _watchedFeatureFlagCollections = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); + private List _selectedKeyValueWatchers = new List(); private readonly TimeSpan MinRefreshInterval; @@ -110,7 +110,7 @@ public AzureAppConfigurationProvider(IConfigurationClientManager configClientMan _options = options ?? throw new ArgumentNullException(nameof(options)); _optional = optional; - IEnumerable watchers = options.ChangeWatchers.Union(options.MultiKeyWatchers); + IEnumerable watchers = options.ChangeWatchers.Union(options.FeatureFlagWatchers); if (watchers.Any()) { @@ -196,12 +196,14 @@ public async Task RefreshAsync(CancellationToken cancellationToken) var utcNow = DateTimeOffset.UtcNow; IEnumerable refreshableWatchers = _options.ChangeWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); - IEnumerable refreshableMultiKeyWatchers = _options.MultiKeyWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); + IEnumerable refreshableFeatureFlagWatchers = _options.FeatureFlagWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); + IEnumerable refreshableSelectedKeyValueWatchers = _selectedKeyValueWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. if (_mappedData != null && !refreshableWatchers.Any() && - !refreshableMultiKeyWatchers.Any() && + !refreshableFeatureFlagWatchers.Any() && + !refreshableSelectedKeyValueWatchers.Any() && !_options.Adapters.Any(adapter => adapter.NeedsRefresh())) { return; @@ -250,7 +252,8 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary> watchedCollections = null; + Dictionary> watchedFeatureFlagCollections = null; + Dictionary> watchedSelectedKeyValueCollections = null; Dictionary watchedSettings = null; List keyValueChanges = null; Dictionary data = null; @@ -262,7 +265,8 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => { data = null; watchedSettings = null; - watchedCollections = null; + watchedSelectedKeyValueCollections = null; + watchedFeatureFlagCollections = null; keyValueChanges = new List(); refreshAll = false; Uri endpoint = _configClientManager.GetEndpointForClient(client); @@ -329,47 +333,50 @@ await CallWithRequestTracing( } } - bool hasWatchedCollectionsChanged = false; - if (!refreshAll) { - watchedCollections = _watchedCollections; + watchedSelectedKeyValueCollections = _watchedSelectedKeyValueCollections; + + watchedFeatureFlagCollections = _watchedFeatureFlagCollections; - foreach (KeyValueWatcher multiKeyWatcher in refreshableMultiKeyWatchers) + if (_options.RefreshOptions.RegisterAllEnabled) { - KeyValueIdentifier watchedKeyValueIdentifier = new KeyValueIdentifier(multiKeyWatcher.Key, multiKeyWatcher.Label); + bool selectedKeyValueCollectionsChanged = await UpdateWatchedCollections(refreshableSelectedKeyValueWatchers, watchedSelectedKeyValueCollections, client, cancellationToken).ConfigureAwait(false); - if (watchedCollections.TryGetValue(watchedKeyValueIdentifier, out IEnumerable matchConditions)) + if (!selectedKeyValueCollectionsChanged) { - await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => hasWatchedCollectionsChanged = await client.HasAnyKeyValueChanged(watchedKeyValueIdentifier, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + logDebugBuilder.AppendLine(LogHelper.BuildSelectedKeyValueCollectionsUnchangedMessage(endpoint.ToString())); } - - if (hasWatchedCollectionsChanged) + else { - break; + keyValueChanges.AddRange(await RefreshKeyValueCollections(_selectedKeyValueWatchers, watchedSelectedKeyValueCollections, _mappedData, client, cancellationToken).ConfigureAwait(false)); + + logInfoBuilder.Append(LogHelper.BuildSelectedKeyValueCollectionsUpdatedMessage()); } } - if (!hasWatchedCollectionsChanged) + bool featureFlagCollectionsChanged = await UpdateWatchedCollections(refreshableFeatureFlagWatchers, watchedFeatureFlagCollections, client, cancellationToken).ConfigureAwait(false); + + if (!featureFlagCollectionsChanged) { logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); } - else if (!_options.RefreshOptions.RegisterAllEnabled) + else { - keyValueChanges.AddRange(await RefreshKeyValueCollections(_options.MultiKeyWatchers, watchedCollections, _mappedData, client, cancellationToken).ConfigureAwait(false)); + keyValueChanges.AddRange(await RefreshKeyValueCollections(_options.FeatureFlagWatchers, watchedFeatureFlagCollections, _mappedData, client, cancellationToken).ConfigureAwait(false)); logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); } } - if (refreshAll || (hasWatchedCollectionsChanged && _options.RefreshOptions.RegisterAllEnabled)) + if (refreshAll) { // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true // or if RegisterAll was called and any loaded key-value changed - (data, watchedCollections) = await LoadSelectedKeyValues(client, cancellationToken).ConfigureAwait(false); + data = new Dictionary(StringComparer.OrdinalIgnoreCase); + watchedSelectedKeyValueCollections = await LoadSelected(data, _options.KeyValueSelectors, client, cancellationToken).ConfigureAwait(false); + watchedFeatureFlagCollections = await LoadSelected(data, _options.FeatureFlagSelectors, client, cancellationToken).ConfigureAwait(false); watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); - watchedCollections = UpdateWatchedKeyValueCollections(watchedCollections); logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); return; } @@ -381,7 +388,7 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa { watchedSettings = new Dictionary(_watchedSettings); - foreach (KeyValueWatcher changeWatcher in refreshableWatchers.Concat(refreshableMultiKeyWatchers)) + foreach (KeyValueWatcher changeWatcher in refreshableWatchers.Concat(refreshableFeatureFlagWatchers).Concat(refreshableSelectedKeyValueWatchers)) { UpdateNextRefreshTime(changeWatcher); } @@ -442,7 +449,7 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa } // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.MultiKeyWatchers)) + foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.FeatureFlagWatchers).Concat(_selectedKeyValueWatchers)) { UpdateNextRefreshTime(changeWatcher); } @@ -451,7 +458,10 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any()) { _watchedSettings = watchedSettings; - _watchedCollections = watchedCollections; + + _watchedFeatureFlagCollections = watchedFeatureFlagCollections; + + _watchedSelectedKeyValueCollections = watchedSelectedKeyValueCollections; if (logDebugBuilder.Length > 0) { @@ -590,7 +600,7 @@ private void SetDirty(TimeSpan? maxDelay) changeWatcher.NextRefreshTime = nextRefreshTime; } - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) + foreach (KeyValueWatcher changeWatcher in _options.FeatureFlagWatchers) { changeWatcher.NextRefreshTime = nextRefreshTime; } @@ -747,15 +757,25 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { - Dictionary data = null; - Dictionary> watchedCollections = null; + Dictionary data = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary> watchedSelectedKeyValueCollections = null; + Dictionary> watchedFeatureFlagCollections = null; Dictionary watchedSettings = null; await ExecuteWithFailOverPolicyAsync( clients, async (client) => { - (data, watchedCollections) = await LoadSelectedKeyValues( + watchedSelectedKeyValueCollections = await LoadSelected( + data, + _options.KeyValueSelectors, + client, + cancellationToken) + .ConfigureAwait(false); + + watchedFeatureFlagCollections = await LoadSelected( + data, + _options.FeatureFlagSelectors, client, cancellationToken) .ConfigureAwait(false); @@ -765,18 +785,10 @@ await ExecuteWithFailOverPolicyAsync( data, cancellationToken) .ConfigureAwait(false); - - watchedCollections = UpdateWatchedKeyValueCollections(watchedCollections); }, cancellationToken) .ConfigureAwait(false); - // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.MultiKeyWatchers)) - { - UpdateNextRefreshTime(changeWatcher); - } - if (data != null) { // Invalidate all the cached KeyVault secrets @@ -788,13 +800,13 @@ await ExecuteWithFailOverPolicyAsync( Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); + List selectedKeyValueWatchers = new List(); + if (_options.RefreshOptions.RegisterAllEnabled) { - List updatedMultiKeyWatchers = (List)_options.MultiKeyWatchers; - - foreach (KeyValueIdentifier keyValueIdentifier in watchedCollections.Keys) + foreach (KeyValueIdentifier keyValueIdentifier in watchedSelectedKeyValueCollections.Keys) { - updatedMultiKeyWatchers.AppendUnique(new KeyValueWatcher() + selectedKeyValueWatchers.AppendUnique(new KeyValueWatcher() { Key = keyValueIdentifier.Key, Label = keyValueIdentifier.Label, @@ -803,22 +815,26 @@ await ExecuteWithFailOverPolicyAsync( } } - // TODO set multikeywatchers correctly, figure out why watchedcollections isn't correct here + // Update the next refresh time for all refresh registered settings and feature flags + foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.FeatureFlagWatchers).Concat(selectedKeyValueWatchers)) + { + UpdateNextRefreshTime(changeWatcher); + } _watchedSettings = watchedSettings; - _watchedCollections = watchedCollections; + _watchedSelectedKeyValueCollections = watchedSelectedKeyValueCollections; + _watchedFeatureFlagCollections = watchedFeatureFlagCollections; _mappedData = mappedData; + _selectedKeyValueWatchers = selectedKeyValueWatchers; } } - private async Task<(Dictionary, Dictionary>)> LoadSelectedKeyValues(ConfigurationClient client, CancellationToken cancellationToken) + private async Task>> LoadSelected(Dictionary existingData, IEnumerable selectors, ConfigurationClient client, CancellationToken cancellationToken) { - var serverData = new Dictionary(StringComparer.OrdinalIgnoreCase); Dictionary> watchedCollections = new Dictionary>(); - // Use default query if there are no key-values specified for use other than the feature flags - bool useDefaultQuery = !_options.KeyValueSelectors.Any(selector => selector.KeyFilter == null || - !selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)); + // Use default query if there are no key-values specified for use other than feature flags + bool useDefaultQuery = !_options.KeyValueSelectors.Any(); if (useDefaultQuery) { @@ -826,7 +842,7 @@ await ExecuteWithFailOverPolicyAsync( var selector = new SettingSelector { KeyFilter = KeyFilter.Any, - LabelFilter = LabelFilter.Null + LabelFilter = LabelFilter.Null ?? LabelFilter.Null }; List matchConditions = new List(); @@ -839,7 +855,7 @@ await CallWithRequestTracing(async () => { foreach (ConfigurationSetting setting in page.Values) { - serverData[setting.Key] = setting; + existingData[setting.Key] = setting; } matchConditions.Add(new MatchConditions { IfNoneMatch = page.GetRawResponse().Headers.ETag }); @@ -849,14 +865,14 @@ await CallWithRequestTracing(async () => watchedCollections[new KeyValueIdentifier(KeyFilter.Any, LabelFilter.Null)] = matchConditions; } - foreach (KeyValueSelector loadOption in _options.KeyValueSelectors) + foreach (KeyValueSelector loadOption in selectors) { if (string.IsNullOrEmpty(loadOption.SnapshotName)) { var selector = new SettingSelector() { KeyFilter = loadOption.KeyFilter, - LabelFilter = loadOption.LabelFilter + LabelFilter = loadOption.LabelFilter ?? LabelFilter.Null }; List matchConditions = new List(); @@ -869,7 +885,7 @@ await CallWithRequestTracing(async () => { foreach (ConfigurationSetting setting in page.Values) { - serverData[setting.Key] = setting; + existingData[setting.Key] = setting; } matchConditions.Add(new MatchConditions { IfNoneMatch = page.GetRawResponse().Headers.ETag }); @@ -904,13 +920,13 @@ await CallWithRequestTracing(async () => { await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) { - serverData[setting.Key] = setting; + existingData[setting.Key] = setting; } }).ConfigureAwait(false); } } - return (serverData, watchedCollections); + return watchedCollections; } private async Task> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) @@ -954,27 +970,8 @@ private async Task> LoadKey return watchedSettings; } - private Dictionary> UpdateWatchedKeyValueCollections(Dictionary> watchedCollections) - { - Dictionary> watchedCollectionsCopy = new Dictionary>(); - - if (!_options.RefreshOptions.RegisterAllEnabled) - { - foreach (KeyValueWatcher changeWatcher in _options.MultiKeyWatchers) - { - KeyValueIdentifier watchedIdentifier = new KeyValueIdentifier(changeWatcher.Key, changeWatcher.Label); - - watchedCollectionsCopy[watchedIdentifier] = watchedCollections[watchedIdentifier]; - } - - return watchedCollectionsCopy; - } - - return watchedCollections; - } - private async Task> RefreshKeyValueCollections( - IEnumerable multiKeyWatchers, + IEnumerable FeatureFlagWatchers, Dictionary> watchedCollections, IDictionary existingSettings, ConfigurationClient client, @@ -984,12 +981,12 @@ private async Task> RefreshKeyValueCollections( HashSet existingKeys = new HashSet(); - foreach (KeyValueWatcher changeWatcher in multiKeyWatchers) + foreach (KeyValueWatcher changeWatcher in FeatureFlagWatchers) { SettingSelector selector = new SettingSelector() { KeyFilter = changeWatcher.Key, - LabelFilter = changeWatcher.Label + LabelFilter = changeWatcher.Label ?? LabelFilter.Null }; await CallWithRequestTracing(async () => @@ -1379,6 +1376,33 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) _configClientBackoffs[endpoint] = clientBackoffStatus; } + private async Task UpdateWatchedCollections(IEnumerable keyValueWatchers, Dictionary> watchedKeyValueCollections, ConfigurationClient client, CancellationToken cancellationToken) + { + bool watchedCollectionsChanged = false; + + foreach (KeyValueWatcher multiKeyWatcher in keyValueWatchers) + { + IEnumerable newMatchConditions = null; + + KeyValueIdentifier watchedKeyValueIdentifier = new KeyValueIdentifier(multiKeyWatcher.Key, multiKeyWatcher.Label); + + if (watchedKeyValueCollections.TryGetValue(watchedKeyValueIdentifier, out IEnumerable matchConditions)) + { + await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, + async () => newMatchConditions = await client.GetNewMatchConditions(watchedKeyValueIdentifier, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + + if (newMatchConditions != null) + { + watchedKeyValueCollections[watchedKeyValueIdentifier] = newMatchConditions; + + watchedCollectionsChanged = true; + } + } + + return watchedCollectionsChanged; + } + public void Dispose() { (_configClientManager as ConfigurationClientManager)?.Dispose(); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs index 24f6ef85..498cd7d6 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs @@ -22,12 +22,14 @@ internal class LoggingConstants public const string RefreshKeyVaultSecretRead = "Secret read from Key Vault for key-value."; public const string RefreshFeatureFlagRead = "Feature flag read from App Configuration."; public const string RefreshFeatureFlagsUnchanged = "Feature flags read from App Configuration. Change:'None'"; + public const string RefreshSelectedKeyValuesCollectionsUnchanged = "Selected key-value collections read from App Configuration. Change:'None'"; // Successful update, information log level public const string RefreshConfigurationUpdatedSuccess = "Configuration reloaded."; public const string RefreshKeyValueSettingUpdated = "Setting updated."; public const string RefreshKeyVaultSettingUpdated = "Setting updated from Key Vault."; public const string RefreshFeatureFlagsUpdated = "Feature flags updated."; + public const string RefreshSelectedKeyValuesCollectionsUpdated = "Selected key-value collections updated."; // Other public const string RefreshSkippedNoClientAvailable = "Refresh skipped because no endpoint is accessible."; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 0b76c62e..e0a46cdb 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -61,35 +61,41 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } - public static async Task HasAnyKeyValueChanged(this ConfigurationClient client, KeyValueIdentifier keyValueIdentifier, IEnumerable matchConditions, CancellationToken cancellationToken) + public static async Task> GetNewMatchConditions(this ConfigurationClient client, KeyValueIdentifier keyValueIdentifier, IEnumerable matchConditions, CancellationToken cancellationToken) { if (matchConditions == null) { throw new ArgumentNullException(nameof(matchConditions)); } - if (!matchConditions.Any()) - { - throw new ArgumentException("Requires at least one MatchConditions value.", nameof(matchConditions)); - } - SettingSelector selector = new SettingSelector { KeyFilter = keyValueIdentifier.Key, LabelFilter = keyValueIdentifier.Label }; + bool hasCollectionChanged = false; + + List newMatchConditions = new List(); + await foreach (Page page in client.GetConfigurationSettingsAsync(selector, cancellationToken).AsPages(matchConditions).ConfigureAwait(false)) { Response response = page.GetRawResponse(); + newMatchConditions.Add(new MatchConditions { IfNoneMatch = response.Headers.ETag }); + if (response.Status == (int)HttpStatusCode.OK) { - return true; + hasCollectionChanged = true; } } - return false; + if (hasCollectionChanged || (!newMatchConditions.Any() && matchConditions.Any())) + { + return newMatchConditions; + } + + return null; } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs index 7bcf7212..e97dfa2f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/StringExtensions.cs @@ -3,18 +3,11 @@ // namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions { - internal static class LabelFilters - { - public static readonly string Null = "\0"; - - public static readonly string Any = "*"; - } - internal static class StringExtensions { public static string NormalizeNull(this string s) { - return s == LabelFilters.Null ? null : s; + return s == LabelFilter.Null ? null : s; } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs index dfbaa01b..3cba6c91 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs @@ -31,6 +31,16 @@ public static string BuildFeatureFlagsUpdatedMessage() return LoggingConstants.RefreshFeatureFlagsUpdated; } + public static string BuildSelectedKeyValueCollectionsUnchangedMessage(string endpoint) + { + return $"{LoggingConstants.RefreshSelectedKeyValuesCollectionsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; + } + + public static string BuildSelectedKeyValueCollectionsUpdatedMessage() + { + return LoggingConstants.RefreshSelectedKeyValuesCollectionsUpdated; + } + public static string BuildKeyVaultSecretReadMessage(string key, string label) { return $"{LoggingConstants.RefreshKeyVaultSecretRead} Key:'{key}' Label:'{label}'"; diff --git a/tests/Tests.AzureAppConfiguration/LoadBalancingTests.cs b/tests/Tests.AzureAppConfiguration/LoadBalancingTests.cs index 4429c7be..e6340ce9 100644 --- a/tests/Tests.AzureAppConfiguration/LoadBalancingTests.cs +++ b/tests/Tests.AzureAppConfiguration/LoadBalancingTests.cs @@ -70,7 +70,7 @@ public async Task LoadBalancingTests_UsesAllEndpoints() }).Build(); // Ensure client 1 was used for startup - mockClient1.Verify(mc => mc.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); + mockClient1.Verify(mc => mc.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); Thread.Sleep(RefreshInterval); await refresher.RefreshAsync(); @@ -134,7 +134,7 @@ public async Task LoadBalancingTests_UsesClientAfterBackoffEnds() }).Build(); // Ensure client 2 was used for startup - mockClient2.Verify(mc => mc.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); + mockClient2.Verify(mc => mc.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); Thread.Sleep(TimeSpan.FromSeconds(2)); await refresher.RefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/LoggingTests.cs b/tests/Tests.AzureAppConfiguration/LoggingTests.cs index 2c614acf..3207021c 100644 --- a/tests/Tests.AzureAppConfiguration/LoggingTests.cs +++ b/tests/Tests.AzureAppConfiguration/LoggingTests.cs @@ -205,6 +205,8 @@ Response GetIfChanged(ConfigurationSetting setting, bool o // No KVR during startup; return KVR during refresh operation to see error because ConfigureKeyVault is missing mockClient.SetupSequence(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns(new MockAsyncPageable(_kvCollection.Select(setting => TestHelpers.CloneSetting(setting)).ToList())) + .Returns(new MockAsyncPageable(_kvCollection.Select(setting => TestHelpers.CloneSetting(setting)).ToList())) .Returns(new MockAsyncPageable(_kvCollection.Select(setting => TestHelpers.CloneSetting(setting)).ToList())) .Returns(new MockAsyncPageable(new List { _kvr })); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index 1faae290..8ad87465 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -497,7 +497,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - Assert.Equal(1, requestCount); + Assert.Equal(2, requestCount); keyValueCollection.First().Value = "newValue"; @@ -509,7 +509,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o await Task.WhenAll(task1, task2, task3); Assert.Equal("newValue", config["TestKey1"]); - Assert.Equal(2, requestCount); + Assert.Equal(3, requestCount); } [Fact] diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index be8f2ffe..c6f942b3 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -169,7 +169,6 @@ public MockAsyncPageable(List collection) _collection.Add(newSetting); } - //_collection = collection; _status = 200; } From e6aadf4d68e2d69b1415f1ad5c5107753da1b16b Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Mon, 15 Jul 2024 14:56:34 -0700 Subject: [PATCH 31/72] comment and naming updates --- .../AzureAppConfigurationOptions.cs | 2 +- .../AzureAppConfigurationProvider.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index e10ba4f6..2740838b 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -223,7 +223,7 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) /// /// Configures options for Azure App Configuration feature flags that will be parsed and transformed into feature management configuration. /// If no filtering is specified via the then all feature flags with no label are loaded. - /// All feature flags loaded by the same key and label filters will be automatically registered for refresh as a collection. + /// All loaded feature flags will be automatically registered for refresh as a collection. /// /// A callback used to configure feature flag options. public AzureAppConfigurationOptions UseFeatureFlags(Action configure = null) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 580c1823..fde85f47 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -349,7 +349,7 @@ await CallWithRequestTracing( } else { - keyValueChanges.AddRange(await RefreshKeyValueCollections(_selectedKeyValueWatchers, watchedSelectedKeyValueCollections, _mappedData, client, cancellationToken).ConfigureAwait(false)); + keyValueChanges.AddRange(await GetRefreshedKeyValueCollections(_selectedKeyValueWatchers, watchedSelectedKeyValueCollections, _mappedData, client, cancellationToken).ConfigureAwait(false)); logInfoBuilder.Append(LogHelper.BuildSelectedKeyValueCollectionsUpdatedMessage()); } @@ -363,7 +363,7 @@ await CallWithRequestTracing( } else { - keyValueChanges.AddRange(await RefreshKeyValueCollections(_options.FeatureFlagWatchers, watchedFeatureFlagCollections, _mappedData, client, cancellationToken).ConfigureAwait(false)); + keyValueChanges.AddRange(await GetRefreshedKeyValueCollections(_options.FeatureFlagWatchers, watchedFeatureFlagCollections, _mappedData, client, cancellationToken).ConfigureAwait(false)); logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); } @@ -970,7 +970,7 @@ private async Task> LoadKey return watchedSettings; } - private async Task> RefreshKeyValueCollections( + private async Task> GetRefreshedKeyValueCollections( IEnumerable FeatureFlagWatchers, Dictionary> watchedCollections, IDictionary existingSettings, From 36f0b80717f1ba64094cca19715ba1a27bf5358d Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Mon, 22 Jul 2024 12:06:25 -0700 Subject: [PATCH 32/72] fixing unit tests, namespace of defining/calling code needs to be same --- .../FeatureManagementTests.cs | 16 ++++++++++++---- tests/Tests.AzureAppConfiguration/TestHelper.cs | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index ada94093..cb58f117 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -16,6 +16,7 @@ using System.Collections.Generic; using System.Diagnostics.Tracing; using System.Linq; +using System.Net; using System.Security.Cryptography; using System.Text; using System.Text.Json; @@ -661,15 +662,22 @@ public void UsesFeatureFlags() [Fact] public async Task WatchesFeatureFlags() { + var mockResponse = new MockResponse(200); + var featureFlags = new List { _kv }; - var mockResponse = new Mock(); + Page page = Page.FromValues( + featureFlags.ToArray(), + "continuationToken", + mockResponse); + + AsyncPageable asyncPageable = AsyncPageable + .FromPages(new[] { page }); + var mockClient = new Mock(MockBehavior.Strict); - var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) - .Returns(mockAsyncPageable); + .Returns(asyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index c6f942b3..eb8dc6a9 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -208,6 +208,21 @@ public async override IAsyncEnumerable> AsPages(strin } } + public static class MockConfigurationClientExtensions + { + public static async IAsyncEnumerable> AsPages( + this AsyncPageable instance, + IEnumerable matches, + string continuationToken = null, + int? pageHint = null) + { + await foreach (var page in instance.AsPages()) + { + yield return page; + } + } + } + class MockPageable : Pageable { private readonly List _collection; From 13db869b23ba8f410449b1ff99b69a835e4aaf0a Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 25 Jul 2024 15:43:49 -0700 Subject: [PATCH 33/72] fixing tests using AsPages --- .../AzureAppConfigurationOptions.cs | 6 +++++ .../AzureAppConfigurationProvider.cs | 2 +- .../ConfigurationSettingPageableManager.cs | 16 ++++++++++++ .../ConfigurationClientExtensions.cs | 6 +++-- .../FeatureManagementTests.cs | 12 +++------ .../MockConfigurationClientExtensions.cs | 25 +++++++++++++++++++ .../Tests.AzureAppConfiguration/TestHelper.cs | 13 +++------- 7 files changed, 59 insertions(+), 21 deletions(-) create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageableManager.cs create mode 100644 tests/Tests.AzureAppConfiguration/MockConfigurationClientExtensions.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 208b49a2..cd480505 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -112,6 +112,12 @@ internal IEnumerable Adapters /// This property is used only for unit testing. internal IConfigurationClientManager ClientManager { get; set; } + /// + /// An optional class used to process pageable results from Azure App Configuration. + /// + /// This property is only set outside of this class if it's used for unit testing. + internal ConfigurationSettingPageableManager PageableManager { get; set; } = new ConfigurationSettingPageableManager(); + /// /// An optional timespan value to set the minimum backoff duration to a value other than the default. /// diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index ddd3942f..447c63cb 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -1389,7 +1389,7 @@ private async Task UpdateWatchedCollections(IEnumerable k if (watchedKeyValueCollections.TryGetValue(watchedKeyValueIdentifier, out IEnumerable matchConditions)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => newMatchConditions = await client.GetNewMatchConditions(watchedKeyValueIdentifier, matchConditions, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + async () => newMatchConditions = await client.GetNewMatchConditions(watchedKeyValueIdentifier, matchConditions, _options.PageableManager, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } if (newMatchConditions != null) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageableManager.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageableManager.cs new file mode 100644 index 00000000..e026eee5 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageableManager.cs @@ -0,0 +1,16 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + internal class ConfigurationSettingPageableManager + { + public virtual IAsyncEnumerable> GetPages(AsyncPageable pageable, IEnumerable matchConditions) + { + return pageable.AsPages(matchConditions); + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index e0a46cdb..1152ec8a 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -61,7 +61,7 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } - public static async Task> GetNewMatchConditions(this ConfigurationClient client, KeyValueIdentifier keyValueIdentifier, IEnumerable matchConditions, CancellationToken cancellationToken) + public static async Task> GetNewMatchConditions(this ConfigurationClient client, KeyValueIdentifier keyValueIdentifier, IEnumerable matchConditions, ConfigurationSettingPageableManager pageableManager, CancellationToken cancellationToken) { if (matchConditions == null) { @@ -78,7 +78,9 @@ public static async Task> GetNewMatchConditions(thi List newMatchConditions = new List(); - await foreach (Page page in client.GetConfigurationSettingsAsync(selector, cancellationToken).AsPages(matchConditions).ConfigureAwait(false)) + AsyncPageable pageable = client.GetConfigurationSettingsAsync(selector, cancellationToken); + + await foreach (Page page in pageableManager.GetPages(pageable, matchConditions).ConfigureAwait(false)) { Response response = page.GetRawResponse(); diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index cb58f117..3d3016fb 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -666,24 +666,20 @@ public async Task WatchesFeatureFlags() var featureFlags = new List { _kv }; - Page page = Page.FromValues( - featureFlags.ToArray(), - "continuationToken", - mockResponse); - - AsyncPageable asyncPageable = AsyncPageable - .FromPages(new[] { page }); + var mockAsyncPageable = new MockAsyncPageable(featureFlags); var mockClient = new Mock(MockBehavior.Strict); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(asyncPageable); + .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.PageableManager = new MockConfigurationSettingPageableManager(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); diff --git a/tests/Tests.AzureAppConfiguration/MockConfigurationClientExtensions.cs b/tests/Tests.AzureAppConfiguration/MockConfigurationClientExtensions.cs new file mode 100644 index 00000000..97745112 --- /dev/null +++ b/tests/Tests.AzureAppConfiguration/MockConfigurationClientExtensions.cs @@ -0,0 +1,25 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions +{ + //public static class MockConfigurationClientExtensions + //{ + // public static async IAsyncEnumerable> AsPages( + // this AsyncPageable instance, + // IEnumerable matches, + // string continuationToken = null, + // int? pageHint = null) + // { + // await foreach (var page in instance.AsPages()) + // { + // yield return page; + // } + // } + //} +} diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index eb8dc6a9..b12ca770 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -208,18 +208,11 @@ public async override IAsyncEnumerable> AsPages(strin } } - public static class MockConfigurationClientExtensions + internal class MockConfigurationSettingPageableManager : ConfigurationSettingPageableManager { - public static async IAsyncEnumerable> AsPages( - this AsyncPageable instance, - IEnumerable matches, - string continuationToken = null, - int? pageHint = null) + public override IAsyncEnumerable> GetPages(AsyncPageable pageable, IEnumerable matchConditions) { - await foreach (var page in instance.AsPages()) - { - yield return page; - } + return pageable.AsPages(); } } From 113143a70232768d47c75b71522b35fef0253da7 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 1 Aug 2024 16:40:43 -0700 Subject: [PATCH 34/72] fix tests with pageablemanager --- .../AzureAppConfigurationProvider.cs | 8 +++++--- .../ConfigurationSettingPageableManager.cs | 7 +++++-- .../Tests.AzureAppConfiguration/ConnectTests.cs | 2 +- .../FeatureManagementTests.cs | 17 +++++++++++++++-- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 14c62eb9..f7983442 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -851,7 +851,7 @@ await CallWithRequestTracing(async () => { AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); - await foreach (Page page in pageableSettings.AsPages().ConfigureAwait(false)) + await foreach (Page page in _options.PageableManager.GetPages(pageableSettings).ConfigureAwait(false)) { foreach (ConfigurationSetting setting in page.Values) { @@ -881,7 +881,7 @@ await CallWithRequestTracing(async () => { AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); - await foreach (Page page in pageableSettings.AsPages().ConfigureAwait(false)) + await foreach (Page page in _options.PageableManager.GetPages(pageableSettings).ConfigureAwait(false)) { foreach (ConfigurationSetting setting in page.Values) { @@ -991,7 +991,9 @@ private async Task> GetRefreshedKeyValueCollections( await CallWithRequestTracing(async () => { - await foreach (Page page in client.GetConfigurationSettingsAsync(selector, cancellationToken).AsPages().ConfigureAwait(false)) + AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); + + await foreach (Page page in _options.PageableManager.GetPages(pageableSettings).ConfigureAwait(false)) { keyValueChanges.AddRange(page.Values.Select(setting => new KeyValueChange() { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageableManager.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageableManager.cs index e026eee5..c71a1c73 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageableManager.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageableManager.cs @@ -1,8 +1,6 @@ using Azure.Data.AppConfiguration; using Azure; -using System; using System.Collections.Generic; -using System.Text; namespace Microsoft.Extensions.Configuration.AzureAppConfiguration { @@ -12,5 +10,10 @@ public virtual IAsyncEnumerable> GetPages(AsyncPageab { return pageable.AsPages(matchConditions); } + + public virtual IAsyncEnumerable> GetPages(AsyncPageable pageable) + { + return pageable.AsPages(); + } } } diff --git a/tests/Tests.AzureAppConfiguration/ConnectTests.cs b/tests/Tests.AzureAppConfiguration/ConnectTests.cs index 22a1507a..73ba5838 100644 --- a/tests/Tests.AzureAppConfiguration/ConnectTests.cs +++ b/tests/Tests.AzureAppConfiguration/ConnectTests.cs @@ -34,7 +34,7 @@ public void ConnectTests_UsesClientInstanceIfSpecified() configBuilder.Build(); // Assert - mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Once); + mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); } [Fact] diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 3d3016fb..1ee8524f 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -752,6 +752,7 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.PageableManager = new MockConfigurationSettingPageableManager(); options.UseFeatureFlags(o => o.CacheExpirationInterval = cacheExpirationInterval); refresher = options.GetRefresher(); @@ -823,6 +824,7 @@ public async Task SkipRefreshIfRefreshIntervalHasNotElapsed() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.PageableManager = new MockConfigurationSettingPageableManager(); options.UseFeatureFlags(o => o.SetRefreshInterval(TimeSpan.FromSeconds(10))); refresher = options.GetRefresher(); @@ -892,6 +894,7 @@ public async Task SkipRefreshIfCacheNotExpired() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.PageableManager = new MockConfigurationSettingPageableManager(); options.UseFeatureFlags(o => o.CacheExpirationInterval = TimeSpan.FromSeconds(10)); refresher = options.GetRefresher(); @@ -1002,15 +1005,20 @@ public void QueriesFeatureFlags() [Fact] public async Task DoesNotUseEtagForFeatureFlagRefresh() { + var mockAsyncPageable = new MockAsyncPageable(new List { _kv }); + var mockClient = new Mock(MockBehavior.Strict); + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(new List { _kv })); + .Callback(() => mockAsyncPageable.UpdateFeatureFlags(new List { _kv })) + .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; var config = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.PageableManager = new MockConfigurationSettingPageableManager(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -1021,7 +1029,7 @@ public async Task DoesNotUseEtagForFeatureFlagRefresh() Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); - mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(3)); + mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(4)); } [Fact] @@ -1041,6 +1049,7 @@ public void SelectFeatureFlags() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(testClient); + options.PageableManager = new MockConfigurationSettingPageableManager(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1402,6 +1411,7 @@ public async Task DifferentCacheExpirationsForMultipleFeatureFlagRegistrations() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.PageableManager = new MockConfigurationSettingPageableManager(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(refreshInterval1); @@ -1571,6 +1581,7 @@ public async Task SelectAndRefreshSingleFeatureFlag() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.PageableManager = new MockConfigurationSettingPageableManager(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1656,6 +1667,7 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; + options.PageableManager = new MockConfigurationSettingPageableManager(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); }) @@ -1733,6 +1745,7 @@ public async Task ValidateFeatureFlagsUnchangedLogged() .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; + options.PageableManager = new MockConfigurationSettingPageableManager(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); options.ConfigureRefresh(refreshOptions => { From 05ede06d613b7a233337fa4d4161c9cb3cb2bb06 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 1 Aug 2024 16:49:40 -0700 Subject: [PATCH 35/72] format --- .../AzureAppConfigurationRefreshOptions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs index a4c9c9ae..cf2847a8 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationRefreshOptions.cs @@ -15,7 +15,7 @@ public class AzureAppConfigurationRefreshOptions internal TimeSpan RefreshInterval { get; private set; } = RefreshConstants.DefaultRefreshInterval; internal ISet RefreshRegistrations = new HashSet(); internal bool RegisterAllEnabled { get; private set; } - + /// /// Register the specified individual key-value to be refreshed when the configuration provider's triggers a refresh. /// The instance can be obtained by calling . From e584d6c7c1a6d39c267deb2e3915592e043d7d69 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 1 Aug 2024 17:12:42 -0700 Subject: [PATCH 36/72] fix tests --- .../Azure.Core.Testing/MockTransport.cs | 2 +- tests/Tests.AzureAppConfiguration/TestHelper.cs | 2 -- tests/Tests.AzureAppConfiguration/Tests.cs | 8 +++++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockTransport.cs b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockTransport.cs index 08a97ba8..16a31387 100644 --- a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockTransport.cs +++ b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockTransport.cs @@ -82,6 +82,6 @@ private async Task ProcessCore(HttpMessage message) } } - public MockRequest SingleRequest => Requests.Single(); + public MockRequest SingleRequest => Requests.First(); } } diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index 261271a2..bddde9b7 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -9,9 +9,7 @@ using Microsoft.Extensions.Logging; using Moq; using System; -using System.Collections; using System.Collections.Generic; -using System.Collections.ObjectModel; using System.IO; using System.Linq; using System.Text; diff --git a/tests/Tests.AzureAppConfiguration/Tests.cs b/tests/Tests.AzureAppConfiguration/Tests.cs index 30010f7d..ab3994c2 100644 --- a/tests/Tests.AzureAppConfiguration/Tests.cs +++ b/tests/Tests.AzureAppConfiguration/Tests.cs @@ -241,10 +241,12 @@ public void TestUserAgentHeader() // 6. Does not contain any additional components string userAgentRegex = @"^Microsoft\.Extensions\.Configuration\.AzureAppConfiguration/\d+\.\d+\.\d+(\+[a-z0-9]+)?(-preview(\.\d+)?)?,azsdk-net-Data.AppConfiguration/[.+\w-]+ \([.;\w\s]+\)$"; - var response = new MockResponse(200); - response.SetContent(SerializationHelpers.Serialize(_kvCollectionPageOne.ToArray(), TestHelpers.SerializeBatch)); + var response1 = new MockResponse(200); + response1.SetContent(SerializationHelpers.Serialize(_kvCollectionPageOne.ToArray(), TestHelpers.SerializeBatch)); + var response2 = new MockResponse(200); + response2.SetContent(SerializationHelpers.Serialize(_kvCollectionPageOne.ToArray(), TestHelpers.SerializeBatch)); - var mockTransport = new MockTransport(response); + var mockTransport = new MockTransport(response1); var options = new AzureAppConfigurationOptions(); options.ClientOptions.Transport = mockTransport; From 76f369bd26afd3d4911bbd8ee2270a1f7189f1da Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 1 Aug 2024 17:17:01 -0700 Subject: [PATCH 37/72] fix tests --- tests/Tests.AzureAppConfiguration/Tests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tests.AzureAppConfiguration/Tests.cs b/tests/Tests.AzureAppConfiguration/Tests.cs index ab3994c2..49680b88 100644 --- a/tests/Tests.AzureAppConfiguration/Tests.cs +++ b/tests/Tests.AzureAppConfiguration/Tests.cs @@ -246,7 +246,7 @@ public void TestUserAgentHeader() var response2 = new MockResponse(200); response2.SetContent(SerializationHelpers.Serialize(_kvCollectionPageOne.ToArray(), TestHelpers.SerializeBatch)); - var mockTransport = new MockTransport(response1); + var mockTransport = new MockTransport(response1, response2); var options = new AzureAppConfigurationOptions(); options.ClientOptions.Transport = mockTransport; From 8391d746da0c2721e94623c881d8a30be4ca0503 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Tue, 8 Oct 2024 10:58:02 -0700 Subject: [PATCH 38/72] remove unused extension test class --- .../MockConfigurationClientExtensions.cs | 25 ------------------- 1 file changed, 25 deletions(-) delete mode 100644 tests/Tests.AzureAppConfiguration/MockConfigurationClientExtensions.cs diff --git a/tests/Tests.AzureAppConfiguration/MockConfigurationClientExtensions.cs b/tests/Tests.AzureAppConfiguration/MockConfigurationClientExtensions.cs deleted file mode 100644 index 97745112..00000000 --- a/tests/Tests.AzureAppConfiguration/MockConfigurationClientExtensions.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Azure.Data.AppConfiguration; -using Azure; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions -{ - //public static class MockConfigurationClientExtensions - //{ - // public static async IAsyncEnumerable> AsPages( - // this AsyncPageable instance, - // IEnumerable matches, - // string continuationToken = null, - // int? pageHint = null) - // { - // await foreach (var page in instance.AsPages()) - // { - // yield return page; - // } - // } - //} -} From e20546d9a1a92c1c815b33007a689526be49af20 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 30 Oct 2024 11:49:43 -0700 Subject: [PATCH 39/72] fix comment, capitalization --- .../AzureAppConfigurationProvider.cs | 4 ++-- .../Extensions/ConfigurationClientExtensions.cs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 5fd28d62..f7310435 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -973,7 +973,7 @@ private async Task> LoadKey } private async Task> GetRefreshedKeyValueCollections( - IEnumerable FeatureFlagWatchers, + IEnumerable featureFlagWatchers, Dictionary> watchedCollections, IDictionary existingSettings, ConfigurationClient client, @@ -983,7 +983,7 @@ private async Task> GetRefreshedKeyValueCollections( HashSet existingKeys = new HashSet(); - foreach (KeyValueWatcher changeWatcher in FeatureFlagWatchers) + foreach (KeyValueWatcher changeWatcher in featureFlagWatchers) { SettingSelector selector = new SettingSelector() { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 1152ec8a..a7f836a0 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -92,6 +92,7 @@ public static async Task> GetNewMatchConditions(thi } } + // Need to check if all pages were deleted since hasCollectionsChanged wouldn't have been set if (hasCollectionChanged || (!newMatchConditions.Any() && matchConditions.Any())) { return newMatchConditions; From 655ec2d63ade729ddb38cfcd7ba360a8077e2f49 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 7 Nov 2024 16:45:10 -0800 Subject: [PATCH 40/72] check etag on 200, fix tests --- .../AzureAppConfigurationProvider.cs | 5 ++--- .../Extensions/ConfigurationClientExtensions.cs | 5 ++++- .../Azure.Core.Testing/MockResponse.cs | 5 +++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index f7310435..8e1902f6 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -370,8 +370,7 @@ await CallWithRequestTracing( logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); } } - - if (refreshAll) + else { // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true // or if RegisterAll was called and any loaded key-value changed @@ -844,7 +843,7 @@ private async Task>> var selector = new SettingSelector { KeyFilter = KeyFilter.Any, - LabelFilter = LabelFilter.Null ?? LabelFilter.Null + LabelFilter = LabelFilter.Null }; List matchConditions = new List(); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index a7f836a0..e70e4511 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -80,13 +80,16 @@ public static async Task> GetNewMatchConditions(thi AsyncPageable pageable = client.GetConfigurationSettingsAsync(selector, cancellationToken); + using IEnumerator existingMatchConditionsEnumerator = matchConditions.GetEnumerator(); + await foreach (Page page in pageableManager.GetPages(pageable, matchConditions).ConfigureAwait(false)) { Response response = page.GetRawResponse(); newMatchConditions.Add(new MatchConditions { IfNoneMatch = response.Headers.ETag }); - if (response.Status == (int)HttpStatusCode.OK) + if (response.Status == (int)HttpStatusCode.OK && + (!existingMatchConditionsEnumerator.MoveNext() || !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(response.Headers.ETag))) { hasCollectionChanged = true; } diff --git a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs index 886d0a77..c60c2a25 100644 --- a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs +++ b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs @@ -17,6 +17,11 @@ public MockResponse(int status, string reasonPhrase = null) { Status = status; ReasonPhrase = reasonPhrase; + + if (status == 200) + { + AddHeader(new HttpHeader(HttpHeader.Names.ETag, "\"" + Guid.NewGuid().ToString() + "\"")); + } } public override int Status { get; } From d9bf761f69b9eb91f30f7105dd80aa7a8827fbd3 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Fri, 8 Nov 2024 10:47:52 -0800 Subject: [PATCH 41/72] add registerall test, fix refresh tests --- .../AzureAppConfigurationOptions.cs | 2 +- .../FeatureManagementTests.cs | 20 ++++----- .../RefreshTests.cs | 42 +++++++++++++++++++ .../Tests.AzureAppConfiguration/TestHelper.cs | 2 +- 4 files changed, 54 insertions(+), 12 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index cd480505..bde4d45a 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -399,7 +399,7 @@ public AzureAppConfigurationOptions ConfigureRefresh(Action(MockBehavior.Strict); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; @@ -850,7 +850,7 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) .Returns(mockAsyncPageable); var cacheExpirationInterval = TimeSpan.FromSeconds(1); @@ -924,7 +924,7 @@ public async Task SkipRefreshIfRefreshIntervalHasNotElapsed() var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; @@ -994,7 +994,7 @@ public async Task SkipRefreshIfCacheNotExpired() var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; @@ -1118,7 +1118,7 @@ public async Task DoesNotUseEtagForFeatureFlagRefresh() var mockClient = new Mock(MockBehavior.Strict); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Callback(() => mockAsyncPageable.UpdateFeatureFlags(new List { _kv })) + .Callback(() => mockAsyncPageable.UpdateCollection(new List { _kv })) .Returns(mockAsyncPageable); IConfigurationRefresher refresher = null; @@ -1510,7 +1510,7 @@ public async Task DifferentCacheExpirationsForMultipleFeatureFlagRegistrations() var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlagCollection.Where(s => + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlagCollection.Where(s => (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1) || (s.Key.StartsWith(FeatureManagementConstants.FeatureFlagMarker + prefix2) && s.Label == label2 && s.Key != FeatureManagementConstants.FeatureFlagMarker + "App2_Feature3")).ToList())) .Returns(mockAsyncPageable); @@ -1681,7 +1681,7 @@ public async Task SelectAndRefreshSingleFeatureFlag() var mockAsyncPageable = new MockAsyncPageable(featureFlagCollection); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlagCollection.Where(s => + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlagCollection.Where(s => s.Key.Equals(FeatureManagementConstants.FeatureFlagMarker + prefix1) && s.Label == label1).ToList())) .Returns(mockAsyncPageable); @@ -1745,7 +1745,7 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) @@ -1829,7 +1829,7 @@ public async Task ValidateFeatureFlagsUnchangedLogged() var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) @@ -1907,7 +1907,7 @@ public async Task MapTransformFeatureFlagWithRefresh() var mockAsyncPageable = new MockAsyncPageable(featureFlags); mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Callback(() => mockAsyncPageable.UpdateFeatureFlags(featureFlags)) + .Callback(() => mockAsyncPageable.UpdateCollection(featureFlags)) .Returns(mockAsyncPageable); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index ddd8ae0b..fa672412 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -1056,6 +1056,48 @@ public void RefreshTests_RefreshIsCancelled() Assert.Equal("TestValue1", config["TestKey1"]); } + [Fact] + public async Task RefreshTests_SelectedKeysRefreshWithRegisterAll() + { + IConfigurationRefresher refresher = null; + var mockClient = GetMockConfigurationClient(); + + var mockAsyncPageable = new MockAsyncPageable(_kvCollection); + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Callback(() => mockAsyncPageable.UpdateCollection(_kvCollection)) + .Returns(mockAsyncPageable); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.Select("TestKey*"); + options.PageableManager = new MockConfigurationSettingPageableManager(); + options.ConfigureRefresh(refreshOptions => + { + refreshOptions.RegisterAll() + .SetRefreshInterval(TimeSpan.FromSeconds(1)); + }); + + refresher = options.GetRefresher(); + }) + .Build(); + + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("TestValue3", config["TestKey3"]); + FirstKeyValue.Value = "newValue1"; + _kvCollection[2].Value = "newValue3"; + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Equal("newValue3", config["TestKey3"]); + } + #if NET8_0 [Fact] public void RefreshTests_ChainedConfigurationProviderUsedAsRootForRefresherProvider() diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index bddde9b7..cfa0335b 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -174,7 +174,7 @@ public MockAsyncPageable(List collection) _status = 200; } - public void UpdateFeatureFlags(List newCollection) + public void UpdateCollection(List newCollection) { if (_collection.All(setting => newCollection.Any(newSetting => setting.Key == newSetting.Key && From b92ae2cb34a26e44d2acf76f02c0f54227e98ad7 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 13 Nov 2024 10:32:37 -0800 Subject: [PATCH 42/72] fix condition for pages and old match conditions --- .../Extensions/ConfigurationClientExtensions.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 0912c2ef..65e7ce07 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -98,8 +98,8 @@ public static async Task> GetNewMatchConditions(thi } } - // Need to check if all pages were deleted since hasCollectionsChanged wouldn't have been set - if (hasCollectionChanged || (!newMatchConditions.Any() && matchConditions.Any())) + // Need to check if pages were deleted since hasCollectionsChanged wouldn't have been set + if (hasCollectionChanged || existingMatchConditionsEnumerator.MoveNext()) { return newMatchConditions; } From 81f844ccacab48165284781a2012c542e8499ec8 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Tue, 19 Nov 2024 13:28:47 -0800 Subject: [PATCH 43/72] WIP fixing PR comments, tests --- .../AzureAppConfigurationOptions.cs | 6 +- .../AzureAppConfigurationProvider.cs | 165 +++++++++--------- .../ConfigurationClientExtensions.cs | 4 +- .../KeyValueChange.cs | 2 +- .../LogHelper.cs | 1 - .../Azure.Core.Testing/MockResponse.cs | 5 +- 6 files changed, 89 insertions(+), 94 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index bde4d45a..3fbbdba6 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -263,11 +263,7 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c var featureFlagFilter = featureFlagSelector.KeyFilter; var labelFilter = featureFlagSelector.LabelFilter; - _featureFlagSelectors.AppendUnique(new KeyValueSelector - { - KeyFilter = featureFlagFilter, - LabelFilter = labelFilter - }); + _featureFlagSelectors.AppendUnique(featureFlagSelector); _featureFlagWatchers.AppendUnique(new KeyValueWatcher { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index edc7aad5..9303ef03 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -32,11 +32,11 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private AzureAppConfigurationOptions _options; private Dictionary _mappedData; private Dictionary _watchedSettings = new Dictionary(); - private Dictionary> _watchedSelectedKeyValueCollections = new Dictionary>(); + private Dictionary> _watchedSelectedKvCollections = new Dictionary>(); private Dictionary> _watchedFeatureFlagCollections = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); - private List _selectedKeyValueWatchers = new List(); + private List _selectedKvWatchers = new List(); private readonly TimeSpan MinRefreshInterval; @@ -198,7 +198,7 @@ public async Task RefreshAsync(CancellationToken cancellationToken) var utcNow = DateTimeOffset.UtcNow; IEnumerable refreshableWatchers = _options.ChangeWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); IEnumerable refreshableFeatureFlagWatchers = _options.FeatureFlagWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); - IEnumerable refreshableSelectedKeyValueWatchers = _selectedKeyValueWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); + IEnumerable refreshableSelectedKeyValueWatchers = _selectedKvWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. if (_mappedData != null && @@ -254,7 +254,7 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification Dictionary> watchedFeatureFlagCollections = null; - Dictionary> watchedSelectedKeyValueCollections = null; + Dictionary> watchedSelectedKvCollections = null; Dictionary watchedSettings = null; List keyValueChanges = null; Dictionary data = null; @@ -266,7 +266,7 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => { data = null; watchedSettings = null; - watchedSelectedKeyValueCollections = null; + watchedSelectedKvCollections = null; watchedFeatureFlagCollections = null; keyValueChanges = new List(); refreshAll = false; @@ -289,6 +289,8 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + + change.IsWatchedSetting = true; } else { @@ -311,7 +313,8 @@ await CallWithRequestTracing( Key = watchedKv.Key, Label = watchedKv.Label.NormalizeNull(), Current = watchedKv, - ChangeType = KeyValueChangeType.Modified + ChangeType = KeyValueChangeType.Modified, + IsWatchedSetting = true }; } } @@ -337,21 +340,21 @@ await CallWithRequestTracing( if (!refreshAll) { - watchedSelectedKeyValueCollections = _watchedSelectedKeyValueCollections; + watchedSelectedKvCollections = _watchedSelectedKvCollections; watchedFeatureFlagCollections = _watchedFeatureFlagCollections; if (_options.RefreshOptions.RegisterAllEnabled) { - bool selectedKeyValueCollectionsChanged = await UpdateWatchedCollections(refreshableSelectedKeyValueWatchers, watchedSelectedKeyValueCollections, client, cancellationToken).ConfigureAwait(false); + bool selectedKvCollectionsChanged = await UpdateWatchedCollections(refreshableSelectedKeyValueWatchers, watchedSelectedKvCollections, client, cancellationToken).ConfigureAwait(false); - if (!selectedKeyValueCollectionsChanged) + if (!selectedKvCollectionsChanged) { logDebugBuilder.AppendLine(LogHelper.BuildSelectedKeyValueCollectionsUnchangedMessage(endpoint.ToString())); } else { - keyValueChanges.AddRange(await GetRefreshedKeyValueCollections(_selectedKeyValueWatchers, watchedSelectedKeyValueCollections, _mappedData, client, cancellationToken).ConfigureAwait(false)); + keyValueChanges.AddRange(await GetRefreshedCollections(_options.KeyValueSelectors, _mappedData, client, cancellationToken).ConfigureAwait(false)); logInfoBuilder.Append(LogHelper.BuildSelectedKeyValueCollectionsUpdatedMessage()); } @@ -365,7 +368,7 @@ await CallWithRequestTracing( } else { - keyValueChanges.AddRange(await GetRefreshedKeyValueCollections(_options.FeatureFlagWatchers, watchedFeatureFlagCollections, _mappedData, client, cancellationToken).ConfigureAwait(false)); + keyValueChanges.AddRange(await GetRefreshedCollections(_options.FeatureFlagSelectors, _mappedData, client, cancellationToken).ConfigureAwait(false)); logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); } @@ -373,9 +376,8 @@ await CallWithRequestTracing( else { // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true - // or if RegisterAll was called and any loaded key-value changed data = new Dictionary(StringComparer.OrdinalIgnoreCase); - watchedSelectedKeyValueCollections = await LoadSelected(data, _options.KeyValueSelectors, client, cancellationToken).ConfigureAwait(false); + watchedSelectedKvCollections = await LoadSelected(data, _options.KeyValueSelectors, client, cancellationToken).ConfigureAwait(false); watchedFeatureFlagCollections = await LoadSelected(data, _options.FeatureFlagSelectors, client, cancellationToken).ConfigureAwait(false); watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); @@ -403,7 +405,7 @@ await CallWithRequestTracing( ConfigurationSetting setting = change.Current; ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); - if (!change.IsMultiKeyWatcherChange) + if (change.IsWatchedSetting) { watchedSettings[changeIdentifier] = settingCopy; } @@ -426,7 +428,7 @@ await CallWithRequestTracing( { _mappedData.Remove(change.Key); - if (!change.IsMultiKeyWatcherChange) + if (change.IsWatchedSetting) { watchedSettings.Remove(changeIdentifier); } @@ -458,7 +460,7 @@ await CallWithRequestTracing( } // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.FeatureFlagWatchers).Concat(_selectedKeyValueWatchers)) + foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.FeatureFlagWatchers).Concat(_selectedKvWatchers)) { UpdateNextRefreshTime(changeWatcher); } @@ -470,7 +472,7 @@ await CallWithRequestTracing( _watchedFeatureFlagCollections = watchedFeatureFlagCollections; - _watchedSelectedKeyValueCollections = watchedSelectedKeyValueCollections; + _watchedSelectedKvCollections = watchedSelectedKvCollections; if (logDebugBuilder.Length > 0) { @@ -798,13 +800,13 @@ await ExecuteWithFailOverPolicyAsync( Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); - List selectedKeyValueWatchers = new List(); + var selectedKeyValueWatchers = new List(); if (_options.RefreshOptions.RegisterAllEnabled) { foreach (KeyValueIdentifier keyValueIdentifier in watchedSelectedKeyValueCollections.Keys) { - selectedKeyValueWatchers.AppendUnique(new KeyValueWatcher() + selectedKeyValueWatchers.Add(new KeyValueWatcher() { Key = keyValueIdentifier.Key, Label = keyValueIdentifier.Label, @@ -820,21 +822,19 @@ await ExecuteWithFailOverPolicyAsync( } _watchedSettings = watchedSettings; - _watchedSelectedKeyValueCollections = watchedSelectedKeyValueCollections; + _watchedSelectedKvCollections = watchedSelectedKeyValueCollections; _watchedFeatureFlagCollections = watchedFeatureFlagCollections; _mappedData = mappedData; - _selectedKeyValueWatchers = selectedKeyValueWatchers; + _selectedKvWatchers = selectedKeyValueWatchers; } } private async Task>> LoadSelected(Dictionary existingData, IEnumerable selectors, ConfigurationClient client, CancellationToken cancellationToken) { - Dictionary> watchedCollections = new Dictionary>(); - - // Use default query if there are no key-values specified for use other than feature flags - bool useDefaultQuery = !_options.KeyValueSelectors.Any(); + var watchedCollections = new Dictionary>(); - if (useDefaultQuery) + // Use default query if there are no key-values specified for use + if (!_options.KeyValueSelectors.Any()) { // Load all key-values with the null label. var selector = new SettingSelector @@ -843,7 +843,7 @@ private async Task>> LabelFilter = LabelFilter.Null }; - List matchConditions = new List(); + var matchConditions = new List(); await CallWithRequestTracing(async () => { @@ -873,7 +873,7 @@ await CallWithRequestTracing(async () => LabelFilter = loadOption.LabelFilter ?? LabelFilter.Null }; - List matchConditions = new List(); + var matchConditions = new List(); await CallWithRequestTracing(async () => { @@ -929,7 +929,7 @@ await CallWithRequestTracing(async () => private async Task> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) { - Dictionary watchedSettings = new Dictionary(); + var watchedSettings = new Dictionary(); foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) { @@ -968,51 +968,78 @@ private async Task> LoadKey return watchedSettings; } - private async Task> GetRefreshedKeyValueCollections( - IEnumerable featureFlagWatchers, - Dictionary> watchedCollections, + private async Task> GetRefreshedCollections( + IEnumerable selectors, IDictionary existingSettings, ConfigurationClient client, CancellationToken cancellationToken) { - List keyValueChanges = new List(); + var keyValueChanges = new List(); - HashSet existingKeys = new HashSet(); + var existingKeys = new HashSet(); - foreach (KeyValueWatcher changeWatcher in featureFlagWatchers) + AsyncPageable pageableSettings; + + foreach (KeyValueSelector loadOption in selectors) { - SettingSelector selector = new SettingSelector() + if (string.IsNullOrEmpty(loadOption.SnapshotName)) { - KeyFilter = changeWatcher.Key, - LabelFilter = changeWatcher.Label ?? LabelFilter.Null - }; + SettingSelector selector = new SettingSelector() + { + KeyFilter = loadOption.KeyFilter, + LabelFilter = loadOption.LabelFilter ?? LabelFilter.Null + }; - await CallWithRequestTracing(async () => + pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); + } + else { - AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); + ConfigurationSnapshot snapshot; - await foreach (Page page in _options.PageableManager.GetPages(pageableSettings).ConfigureAwait(false)) + try { - keyValueChanges.AddRange(page.Values.Select(setting => new KeyValueChange() - { - Key = setting.Key, - Label = setting.Label.NormalizeNull(), - Current = setting, - ChangeType = KeyValueChangeType.Modified, - IsMultiKeyWatcherChange = true - })); + snapshot = await client.GetSnapshotAsync(loadOption.SnapshotName).ConfigureAwait(false); + } + catch (RequestFailedException rfe) when (rfe.Status == (int)HttpStatusCode.NotFound) + { + throw new InvalidOperationException($"Could not find snapshot with name '{loadOption.SnapshotName}'.", rfe); + } + + if (snapshot.SnapshotComposition != SnapshotComposition.Key) + { + throw new InvalidOperationException($"{nameof(snapshot.SnapshotComposition)} for the selected snapshot with name '{snapshot.Name}' must be 'key', found '{snapshot.SnapshotComposition}'."); } - }).ConfigureAwait(false); - IEnumerable existingKeysList = GetCurrentKeyValueCollection(changeWatcher.Key, changeWatcher.Label, existingSettings.Keys); + pageableSettings = client.GetConfigurationSettingsForSnapshotAsync( + loadOption.SnapshotName, + cancellationToken); + } - foreach (string key in existingKeysList) + await CallWithRequestTracing(async () => { - existingKeys.Add(key); - } + await foreach (Page page in _options.PageableManager.GetPages(pageableSettings).ConfigureAwait(false)) + { + foreach (ConfigurationSetting setting in page.Values) + { + keyValueChanges.Add(new KeyValueChange() + { + Key = setting.Key, + Label = setting.Label, + Current = setting, + ChangeType = KeyValueChangeType.Modified, + IsWatchedSetting = false + }); + + if (existingSettings.ContainsKey(setting.Key)) + { + existingKeys.Add(setting.Key); + } + } + } + }).ConfigureAwait(false); } - HashSet loadedSettings = new HashSet(); + var loadedSettings = new HashSet(); foreach (KeyValueChange change in keyValueChanges) { @@ -1029,7 +1056,7 @@ await CallWithRequestTracing(async () => Label = null, Current = null, ChangeType = KeyValueChangeType.Deleted, - IsMultiKeyWatcherChange = true + IsWatchedSetting = false }); } } @@ -1302,30 +1329,6 @@ private async Task> MapConfigurationSet return mappedData; } - private IEnumerable GetCurrentKeyValueCollection(string key, string label, IEnumerable existingKeys) - { - IEnumerable currentKeyValues; - - if (key.EndsWith("*")) - { - // Get current application settings starting with changeWatcher.Key, excluding the last * character - string keyPrefix = key.Substring(0, key.Length - 1); - currentKeyValues = existingKeys.Where(val => - { - return val.StartsWith(keyPrefix); - }); - } - else - { - currentKeyValues = existingKeys.Where(val => - { - return val.Equals(key); - }); - } - - return currentKeyValues; - } - private void EnsureAssemblyInspected() { if (!_isAssemblyInspected) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 65e7ce07..99dd18a5 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -6,7 +6,6 @@ using Microsoft.Extensions.Configuration.AzureAppConfiguration.Models; using System; using System.Collections.Generic; -using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -79,7 +78,7 @@ public static async Task> GetNewMatchConditions(thi bool hasCollectionChanged = false; - List newMatchConditions = new List(); + var newMatchConditions = new List(); AsyncPageable pageable = client.GetConfigurationSettingsAsync(selector, cancellationToken); @@ -91,6 +90,7 @@ public static async Task> GetNewMatchConditions(thi newMatchConditions.Add(new MatchConditions { IfNoneMatch = response.Headers.ETag }); + // Set hasCollectionChanged to true if the lists of etags are different, and continue iterating to get all of newMatchConditions if (response.Status == (int)HttpStatusCode.OK && (!existingMatchConditionsEnumerator.MoveNext() || !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(response.Headers.ETag))) { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/KeyValueChange.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/KeyValueChange.cs index 81a23475..5f319602 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/KeyValueChange.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/KeyValueChange.cs @@ -24,6 +24,6 @@ internal struct KeyValueChange public ConfigurationSetting Previous { get; set; } - public bool IsMultiKeyWatcherChange { get; set; } + public bool IsWatchedSetting { get; set; } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs index 3cba6c91..d22a772d 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. // - namespace Microsoft.Extensions.Configuration.AzureAppConfiguration { internal static class LogHelper diff --git a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs index c60c2a25..aaee0b9c 100644 --- a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs +++ b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockResponse.cs @@ -18,10 +18,7 @@ public MockResponse(int status, string reasonPhrase = null) Status = status; ReasonPhrase = reasonPhrase; - if (status == 200) - { - AddHeader(new HttpHeader(HttpHeader.Names.ETag, "\"" + Guid.NewGuid().ToString() + "\"")); - } + AddHeader(new HttpHeader(HttpHeader.Names.ETag, "\"" + Guid.NewGuid().ToString() + "\"")); } public override int Status { get; } From 6212daa3eddebe223e72f1477ddb25baaecb9914 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Tue, 19 Nov 2024 13:31:23 -0800 Subject: [PATCH 44/72] check status after advancing existing etag enumerator --- .../Extensions/ConfigurationClientExtensions.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 99dd18a5..a9333b82 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -91,8 +91,9 @@ public static async Task> GetNewMatchConditions(thi newMatchConditions.Add(new MatchConditions { IfNoneMatch = response.Headers.ETag }); // Set hasCollectionChanged to true if the lists of etags are different, and continue iterating to get all of newMatchConditions - if (response.Status == (int)HttpStatusCode.OK && - (!existingMatchConditionsEnumerator.MoveNext() || !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(response.Headers.ETag))) + if ((!existingMatchConditionsEnumerator.MoveNext() || + !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(response.Headers.ETag)) && + response.Status == (int)HttpStatusCode.OK) { hasCollectionChanged = true; } From e27dea1a816ec649c6123835a339068b9d568b26 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Tue, 19 Nov 2024 14:40:23 -0800 Subject: [PATCH 45/72] move around refresh logic --- .../AzureAppConfigurationProvider.cs | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 9303ef03..758ae06e 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -338,12 +338,21 @@ await CallWithRequestTracing( } } - if (!refreshAll) + if (refreshAll) { + // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true + data = new Dictionary(StringComparer.OrdinalIgnoreCase); + watchedSelectedKvCollections = await LoadSelected(data, _options.KeyValueSelectors, client, cancellationToken).ConfigureAwait(false); + watchedFeatureFlagCollections = await LoadSelected(data, _options.FeatureFlagSelectors, client, cancellationToken).ConfigureAwait(false); + watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); + logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); + return; + } + else + { + // Get key value collection changes if RegisterAll was called watchedSelectedKvCollections = _watchedSelectedKvCollections; - watchedFeatureFlagCollections = _watchedFeatureFlagCollections; - if (_options.RefreshOptions.RegisterAllEnabled) { bool selectedKvCollectionsChanged = await UpdateWatchedCollections(refreshableSelectedKeyValueWatchers, watchedSelectedKvCollections, client, cancellationToken).ConfigureAwait(false); @@ -360,6 +369,9 @@ await CallWithRequestTracing( } } + // Get feature flag changes + watchedFeatureFlagCollections = _watchedFeatureFlagCollections; + bool featureFlagCollectionsChanged = await UpdateWatchedCollections(refreshableFeatureFlagWatchers, watchedFeatureFlagCollections, client, cancellationToken).ConfigureAwait(false); if (!featureFlagCollectionsChanged) @@ -373,16 +385,6 @@ await CallWithRequestTracing( logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); } } - else - { - // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true - data = new Dictionary(StringComparer.OrdinalIgnoreCase); - watchedSelectedKvCollections = await LoadSelected(data, _options.KeyValueSelectors, client, cancellationToken).ConfigureAwait(false); - watchedFeatureFlagCollections = await LoadSelected(data, _options.FeatureFlagSelectors, client, cancellationToken).ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); - logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); - return; - } }, cancellationToken) .ConfigureAwait(false); From 78b83d3ad230aa429d2190d1ce6d0f02372acf0d Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 20 Nov 2024 12:38:03 -0800 Subject: [PATCH 46/72] null check page etag, revert break to existing keys check in getrefreshedcollections --- .../AzureAppConfigurationProvider.cs | 54 ++++++++++++++++--- .../Constants/ErrorMessages.cs | 1 + .../ConfigurationClientExtensions.cs | 11 +++- .../RefreshTests.cs | 10 ++++ .../Tests.AzureAppConfiguration/TestHelper.cs | 3 +- 5 files changed, 69 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 758ae06e..6d8b7fba 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -853,12 +853,19 @@ await CallWithRequestTracing(async () => await foreach (Page page in _options.PageableManager.GetPages(pageableSettings).ConfigureAwait(false)) { + ETag serverEtag = (ETag)page?.GetRawResponse()?.Headers.ETag; + + if (serverEtag == null || page.Values == null) + { + throw new RequestFailedException(ErrorMessages.InvalidConfigurationSettingPage); + } + foreach (ConfigurationSetting setting in page.Values) { existingData[setting.Key] = setting; } - matchConditions.Add(new MatchConditions { IfNoneMatch = page.GetRawResponse().Headers.ETag }); + matchConditions.Add(new MatchConditions { IfNoneMatch = serverEtag }); } }).ConfigureAwait(false); @@ -883,12 +890,19 @@ await CallWithRequestTracing(async () => await foreach (Page page in _options.PageableManager.GetPages(pageableSettings).ConfigureAwait(false)) { + ETag serverEtag = (ETag)page?.GetRawResponse()?.Headers.ETag; + + if (serverEtag == null || page.Values == null) + { + throw new RequestFailedException(ErrorMessages.InvalidConfigurationSettingPage); + } + foreach (ConfigurationSetting setting in page.Values) { existingData[setting.Key] = setting; } - matchConditions.Add(new MatchConditions { IfNoneMatch = page.GetRawResponse().Headers.ETag }); + matchConditions.Add(new MatchConditions { IfNoneMatch = serverEtag }); } }).ConfigureAwait(false); @@ -1031,14 +1045,16 @@ await CallWithRequestTracing(async () => ChangeType = KeyValueChangeType.Modified, IsWatchedSetting = false }); - - if (existingSettings.ContainsKey(setting.Key)) - { - existingKeys.Add(setting.Key); - } } } }).ConfigureAwait(false); + + IEnumerable existingKeysList = GetCurrentKeyValueCollection(loadOption.KeyFilter, existingSettings.Keys); + + foreach (string key in existingKeysList) + { + existingKeys.Add(key); + } } var loadedSettings = new HashSet(); @@ -1309,6 +1325,30 @@ innerException is SocketException || innerException is IOException; } + private IEnumerable GetCurrentKeyValueCollection(string key, IEnumerable existingKeys) + { + IEnumerable currentKeyValues; + + if (key.EndsWith("*")) + { + // Get current application settings starting with changeWatcher.Key, excluding the last * character + string keyPrefix = key.Substring(0, key.Length - 1); + currentKeyValues = existingKeys.Where(val => + { + return val.StartsWith(keyPrefix); + }); + } + else + { + currentKeyValues = existingKeys.Where(val => + { + return val.Equals(key); + }); + } + + return currentKeyValues; + } + private async Task> MapConfigurationSettings(Dictionary data) { Dictionary mappedData = new Dictionary(StringComparer.OrdinalIgnoreCase); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/ErrorMessages.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/ErrorMessages.cs index c7974736..527062fc 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/ErrorMessages.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/ErrorMessages.cs @@ -10,5 +10,6 @@ internal class ErrorMessages public const string FeatureFlagInvalidJsonProperty = "Invalid property '{0}' for feature flag. Key: '{1}'. Found type: '{2}'. Expected type: '{3}'."; public const string FeatureFlagInvalidFormat = "Invalid json format for feature flag. Key: '{0}'."; public const string InvalidKeyVaultReference = "Invalid Key Vault reference."; + public const string InvalidConfigurationSettingPage = "Invalid page while loading configuration settings."; } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index a9333b82..3805ad48 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -86,13 +86,20 @@ public static async Task> GetNewMatchConditions(thi await foreach (Page page in pageableManager.GetPages(pageable, matchConditions).ConfigureAwait(false)) { + ETag serverEtag = (ETag)page?.GetRawResponse()?.Headers.ETag; + + if (serverEtag == null || page.Values == null) + { + throw new RequestFailedException(ErrorMessages.InvalidConfigurationSettingPage); + } + Response response = page.GetRawResponse(); - newMatchConditions.Add(new MatchConditions { IfNoneMatch = response.Headers.ETag }); + newMatchConditions.Add(new MatchConditions { IfNoneMatch = serverEtag }); // Set hasCollectionChanged to true if the lists of etags are different, and continue iterating to get all of newMatchConditions if ((!existingMatchConditionsEnumerator.MoveNext() || - !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(response.Headers.ETag)) && + !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(serverEtag)) && response.Status == (int)HttpStatusCode.OK) { hasCollectionChanged = true; diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index fa672412..d58859bc 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -1096,6 +1096,16 @@ public async Task RefreshTests_SelectedKeysRefreshWithRegisterAll() Assert.Equal("newValue1", config["TestKey1"]); Assert.Equal("newValue3", config["TestKey3"]); + + _kvCollection.RemoveAt(2); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Null(config["TestKey3"]); } #if NET8_0 diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index cfa0335b..21f03607 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -176,7 +176,8 @@ public MockAsyncPageable(List collection) public void UpdateCollection(List newCollection) { - if (_collection.All(setting => newCollection.Any(newSetting => + if (_collection.Count() == newCollection.Count() && + _collection.All(setting => newCollection.Any(newSetting => setting.Key == newSetting.Key && setting.Value == newSetting.Value && setting.Label == newSetting.Label && From ee368911c48591571c88961f931232a48a0c7b59 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 20 Nov 2024 16:23:04 -0800 Subject: [PATCH 47/72] fix loadselected, replace selectedkvwatchers with registerall refresh time --- .../AzureAppConfigurationOptions.cs | 47 ++++++-- .../AzureAppConfigurationProvider.cs | 112 ++++++------------ .../ConnectTests.cs | 2 +- .../FeatureManagementTests.cs | 2 +- .../LoadBalancingTests.cs | 4 +- .../LoggingTests.cs | 2 - .../RefreshTests.cs | 4 +- 7 files changed, 84 insertions(+), 89 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 3fbbdba6..92203d76 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -27,9 +27,10 @@ public class AzureAppConfigurationOptions private List _featureFlagWatchers = new List(); private List _adapters; private List>> _mappers = new List>>(); - private List _kvSelectors = new List(); + private List _kvSelectors; private List _featureFlagSelectors = new List(); private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); + private bool _selectCalled = false; // The following set is sorted in descending order. // Since multiple prefixes could start with the same characters, we need to trim the longest prefix first. @@ -159,6 +160,9 @@ public AzureAppConfigurationOptions() new JsonKeyValueAdapter(), new FeatureManagementKeyValueAdapter(FeatureFlagTracing) }; + + // Adds the default query to App Configuration if no select API is called + _kvSelectors = new List { new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null } }; } /// @@ -182,20 +186,18 @@ public AzureAppConfigurationOptions() /// public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter = LabelFilter.Null) { - if (string.IsNullOrEmpty(keyFilter)) - { - throw new ArgumentNullException(nameof(keyFilter)); - } + ValidateSelectFilters(keyFilter, labelFilter); if (string.IsNullOrWhiteSpace(labelFilter)) { labelFilter = LabelFilter.Null; } - // Do not support * and , for label filter for now. - if (labelFilter.Contains('*') || labelFilter.Contains(',')) + if (!_selectCalled) { - throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + _kvSelectors.Clear(); + + _selectCalled = true; } _kvSelectors.AppendUnique(new KeyValueSelector @@ -203,6 +205,7 @@ public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter KeyFilter = keyFilter, LabelFilter = labelFilter }); + return this; } @@ -218,6 +221,13 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) throw new ArgumentNullException(nameof(name)); } + if (!_selectCalled) + { + _kvSelectors.Clear(); + + _selectCalled = true; + } + _kvSelectors.AppendUnique(new KeyValueSelector { SnapshotName = name @@ -263,6 +273,13 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c var featureFlagFilter = featureFlagSelector.KeyFilter; var labelFilter = featureFlagSelector.LabelFilter; + ValidateSelectFilters(featureFlagFilter, labelFilter); + + if (string.IsNullOrWhiteSpace(labelFilter)) + { + labelFilter = LabelFilter.Null; + } + _featureFlagSelectors.AppendUnique(featureFlagSelector); _featureFlagWatchers.AppendUnique(new KeyValueWatcher @@ -477,5 +494,19 @@ private static ConfigurationClientOptions GetDefaultClientOptions() return clientOptions; } + + private static void ValidateSelectFilters(string keyFilter, string labelFilter) + { + if (string.IsNullOrEmpty(keyFilter)) + { + throw new ArgumentNullException(nameof(keyFilter)); + } + + // Do not support * and , for label filter for now. + if (labelFilter != null && (labelFilter.Contains('*') || labelFilter.Contains(','))) + { + throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + } + } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 6d8b7fba..c6001b6f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -36,7 +36,7 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private Dictionary> _watchedFeatureFlagCollections = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); - private List _selectedKvWatchers = new List(); + private DateTimeOffset _registerAllNextRefreshTime; private readonly TimeSpan MinRefreshInterval; @@ -198,13 +198,13 @@ public async Task RefreshAsync(CancellationToken cancellationToken) var utcNow = DateTimeOffset.UtcNow; IEnumerable refreshableWatchers = _options.ChangeWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); IEnumerable refreshableFeatureFlagWatchers = _options.FeatureFlagWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); - IEnumerable refreshableSelectedKeyValueWatchers = _selectedKvWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); + bool registerAllIsRefreshable = utcNow >= _registerAllNextRefreshTime; // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. if (_mappedData != null && !refreshableWatchers.Any() && !refreshableFeatureFlagWatchers.Any() && - !refreshableSelectedKeyValueWatchers.Any() && + !registerAllIsRefreshable && !_options.Adapters.Any(adapter => adapter.NeedsRefresh())) { return; @@ -353,9 +353,9 @@ await CallWithRequestTracing( // Get key value collection changes if RegisterAll was called watchedSelectedKvCollections = _watchedSelectedKvCollections; - if (_options.RefreshOptions.RegisterAllEnabled) + if (_options.RefreshOptions.RegisterAllEnabled && registerAllIsRefreshable) { - bool selectedKvCollectionsChanged = await UpdateWatchedCollections(refreshableSelectedKeyValueWatchers, watchedSelectedKvCollections, client, cancellationToken).ConfigureAwait(false); + bool selectedKvCollectionsChanged = await UpdateWatchedCollections(_options.KeyValueSelectors, watchedSelectedKvCollections, client, cancellationToken).ConfigureAwait(false); if (!selectedKvCollectionsChanged) { @@ -372,7 +372,11 @@ await CallWithRequestTracing( // Get feature flag changes watchedFeatureFlagCollections = _watchedFeatureFlagCollections; - bool featureFlagCollectionsChanged = await UpdateWatchedCollections(refreshableFeatureFlagWatchers, watchedFeatureFlagCollections, client, cancellationToken).ConfigureAwait(false); + bool featureFlagCollectionsChanged = await UpdateWatchedCollections( + refreshableFeatureFlagWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), + watchedFeatureFlagCollections, + client, + cancellationToken).ConfigureAwait(false); if (!featureFlagCollectionsChanged) { @@ -393,11 +397,16 @@ await CallWithRequestTracing( { watchedSettings = new Dictionary(_watchedSettings); - foreach (KeyValueWatcher changeWatcher in refreshableWatchers.Concat(refreshableFeatureFlagWatchers).Concat(refreshableSelectedKeyValueWatchers)) + foreach (KeyValueWatcher changeWatcher in refreshableWatchers.Concat(refreshableFeatureFlagWatchers)) { UpdateNextRefreshTime(changeWatcher); } + if (_options.RefreshOptions.RegisterAllEnabled && registerAllIsRefreshable) + { + _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.RefreshOptions.RefreshInterval); + } + foreach (KeyValueChange change in keyValueChanges) { KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); @@ -462,10 +471,15 @@ await CallWithRequestTracing( } // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.FeatureFlagWatchers).Concat(_selectedKvWatchers)) + foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.FeatureFlagWatchers)) { UpdateNextRefreshTime(changeWatcher); } + + if (_options.RefreshOptions.RegisterAllEnabled) + { + _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.RefreshOptions.RefreshInterval); + } } if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any()) @@ -760,7 +774,7 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { Dictionary data = new Dictionary(StringComparer.OrdinalIgnoreCase); - Dictionary> watchedSelectedKeyValueCollections = null; + Dictionary> watchedSelectedKvCollections = null; Dictionary> watchedFeatureFlagCollections = null; Dictionary watchedSettings = null; @@ -768,7 +782,7 @@ await ExecuteWithFailOverPolicyAsync( clients, async (client) => { - watchedSelectedKeyValueCollections = await LoadSelected( + watchedSelectedKvCollections = await LoadSelected( data, _options.KeyValueSelectors, client, @@ -791,6 +805,17 @@ await ExecuteWithFailOverPolicyAsync( cancellationToken) .ConfigureAwait(false); + // Update the next refresh time for all refresh registered settings and feature flags + foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.FeatureFlagWatchers)) + { + UpdateNextRefreshTime(changeWatcher); + } + + if (_options.RefreshOptions.RegisterAllEnabled) + { + _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.RefreshOptions.RefreshInterval); + } + if (data != null) { // Invalidate all the cached KeyVault secrets @@ -802,32 +827,10 @@ await ExecuteWithFailOverPolicyAsync( Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); - var selectedKeyValueWatchers = new List(); - - if (_options.RefreshOptions.RegisterAllEnabled) - { - foreach (KeyValueIdentifier keyValueIdentifier in watchedSelectedKeyValueCollections.Keys) - { - selectedKeyValueWatchers.Add(new KeyValueWatcher() - { - Key = keyValueIdentifier.Key, - Label = keyValueIdentifier.Label, - RefreshInterval = _options.RefreshOptions.RefreshInterval - }); - } - } - - // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.FeatureFlagWatchers).Concat(selectedKeyValueWatchers)) - { - UpdateNextRefreshTime(changeWatcher); - } - _watchedSettings = watchedSettings; - _watchedSelectedKvCollections = watchedSelectedKeyValueCollections; + _watchedSelectedKvCollections = watchedSelectedKvCollections; _watchedFeatureFlagCollections = watchedFeatureFlagCollections; _mappedData = mappedData; - _selectedKvWatchers = selectedKeyValueWatchers; } } @@ -835,43 +838,6 @@ private async Task>> { var watchedCollections = new Dictionary>(); - // Use default query if there are no key-values specified for use - if (!_options.KeyValueSelectors.Any()) - { - // Load all key-values with the null label. - var selector = new SettingSelector - { - KeyFilter = KeyFilter.Any, - LabelFilter = LabelFilter.Null - }; - - var matchConditions = new List(); - - await CallWithRequestTracing(async () => - { - AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); - - await foreach (Page page in _options.PageableManager.GetPages(pageableSettings).ConfigureAwait(false)) - { - ETag serverEtag = (ETag)page?.GetRawResponse()?.Headers.ETag; - - if (serverEtag == null || page.Values == null) - { - throw new RequestFailedException(ErrorMessages.InvalidConfigurationSettingPage); - } - - foreach (ConfigurationSetting setting in page.Values) - { - existingData[setting.Key] = setting; - } - - matchConditions.Add(new MatchConditions { IfNoneMatch = serverEtag }); - } - }).ConfigureAwait(false); - - watchedCollections[new KeyValueIdentifier(KeyFilter.Any, LabelFilter.Null)] = matchConditions; - } - foreach (KeyValueSelector loadOption in selectors) { if (string.IsNullOrEmpty(loadOption.SnapshotName)) @@ -1416,15 +1382,15 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) _configClientBackoffs[endpoint] = clientBackoffStatus; } - private async Task UpdateWatchedCollections(IEnumerable keyValueWatchers, Dictionary> watchedKeyValueCollections, ConfigurationClient client, CancellationToken cancellationToken) + private async Task UpdateWatchedCollections(IEnumerable selectors, Dictionary> watchedKeyValueCollections, ConfigurationClient client, CancellationToken cancellationToken) { bool watchedCollectionsChanged = false; - foreach (KeyValueWatcher multiKeyWatcher in keyValueWatchers) + foreach (KeyValueSelector selector in selectors) { IEnumerable newMatchConditions = null; - KeyValueIdentifier watchedKeyValueIdentifier = new KeyValueIdentifier(multiKeyWatcher.Key, multiKeyWatcher.Label); + KeyValueIdentifier watchedKeyValueIdentifier = new KeyValueIdentifier(selector.KeyFilter, selector.LabelFilter); if (watchedKeyValueCollections.TryGetValue(watchedKeyValueIdentifier, out IEnumerable matchConditions)) { diff --git a/tests/Tests.AzureAppConfiguration/ConnectTests.cs b/tests/Tests.AzureAppConfiguration/ConnectTests.cs index 9c478742..9f256175 100644 --- a/tests/Tests.AzureAppConfiguration/ConnectTests.cs +++ b/tests/Tests.AzureAppConfiguration/ConnectTests.cs @@ -33,7 +33,7 @@ public void ConnectTests_UsesClientInstanceIfSpecified() configBuilder.Build(); // Assert - mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); + mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); } [Fact] diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index bbcdfc52..23c2afd2 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -1137,7 +1137,7 @@ public async Task DoesNotUseEtagForFeatureFlagRefresh() Thread.Sleep(RefreshInterval); await refresher.TryRefreshAsync(); - mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(4)); + mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(3)); } [Fact] diff --git a/tests/Tests.AzureAppConfiguration/LoadBalancingTests.cs b/tests/Tests.AzureAppConfiguration/LoadBalancingTests.cs index e6340ce9..4429c7be 100644 --- a/tests/Tests.AzureAppConfiguration/LoadBalancingTests.cs +++ b/tests/Tests.AzureAppConfiguration/LoadBalancingTests.cs @@ -70,7 +70,7 @@ public async Task LoadBalancingTests_UsesAllEndpoints() }).Build(); // Ensure client 1 was used for startup - mockClient1.Verify(mc => mc.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); + mockClient1.Verify(mc => mc.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); Thread.Sleep(RefreshInterval); await refresher.RefreshAsync(); @@ -134,7 +134,7 @@ public async Task LoadBalancingTests_UsesClientAfterBackoffEnds() }).Build(); // Ensure client 2 was used for startup - mockClient2.Verify(mc => mc.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); + mockClient2.Verify(mc => mc.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); Thread.Sleep(TimeSpan.FromSeconds(2)); await refresher.RefreshAsync(); diff --git a/tests/Tests.AzureAppConfiguration/LoggingTests.cs b/tests/Tests.AzureAppConfiguration/LoggingTests.cs index 5fa1a6a9..547c65bd 100644 --- a/tests/Tests.AzureAppConfiguration/LoggingTests.cs +++ b/tests/Tests.AzureAppConfiguration/LoggingTests.cs @@ -204,8 +204,6 @@ Response GetIfChanged(ConfigurationSetting setting, bool o // No KVR during startup; return KVR during refresh operation to see error because ConfigureKeyVault is missing mockClient.SetupSequence(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(_kvCollection.Select(setting => TestHelpers.CloneSetting(setting)).ToList())) - .Returns(new MockAsyncPageable(_kvCollection.Select(setting => TestHelpers.CloneSetting(setting)).ToList())) .Returns(new MockAsyncPageable(_kvCollection.Select(setting => TestHelpers.CloneSetting(setting)).ToList())) .Returns(new MockAsyncPageable(new List { _kvr })); mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index d58859bc..8844e7b0 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -497,7 +497,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o .Build(); Assert.Equal("TestValue1", config["TestKey1"]); - Assert.Equal(2, requestCount); + Assert.Equal(1, requestCount); keyValueCollection.First().Value = "newValue"; @@ -509,7 +509,7 @@ Response GetIfChanged(ConfigurationSetting setting, bool o await Task.WhenAll(task1, task2, task3); Assert.Equal("newValue", config["TestKey1"]); - Assert.Equal(3, requestCount); + Assert.Equal(2, requestCount); } [Fact] From 569656f79f23108492635ac26c38056defa6d104 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 20 Nov 2024 16:25:06 -0800 Subject: [PATCH 48/72] fix comment in options --- .../AzureAppConfigurationOptions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 92203d76..46065f55 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -161,7 +161,7 @@ public AzureAppConfigurationOptions() new FeatureManagementKeyValueAdapter(FeatureFlagTracing) }; - // Adds the default query to App Configuration if no select API is called + // Adds the default query to App Configuration if and are never called. _kvSelectors = new List { new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null } }; } From d049c2d9ab1ae3b6a3af674030109b7f5b780efd Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 20 Nov 2024 16:44:54 -0800 Subject: [PATCH 49/72] clean up tests --- .../Azure.Core.Testing/MockTransport.cs | 2 +- tests/Tests.AzureAppConfiguration/ConnectTests.cs | 2 +- .../Tests.AzureAppConfiguration/FeatureManagementTests.cs | 2 -- tests/Tests.AzureAppConfiguration/Tests.cs | 8 +++----- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockTransport.cs b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockTransport.cs index 16a31387..08a97ba8 100644 --- a/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockTransport.cs +++ b/tests/Tests.AzureAppConfiguration/Azure.Core.Testing/MockTransport.cs @@ -82,6 +82,6 @@ private async Task ProcessCore(HttpMessage message) } } - public MockRequest SingleRequest => Requests.First(); + public MockRequest SingleRequest => Requests.Single(); } } diff --git a/tests/Tests.AzureAppConfiguration/ConnectTests.cs b/tests/Tests.AzureAppConfiguration/ConnectTests.cs index 9f256175..6a8ddc8d 100644 --- a/tests/Tests.AzureAppConfiguration/ConnectTests.cs +++ b/tests/Tests.AzureAppConfiguration/ConnectTests.cs @@ -33,7 +33,7 @@ public void ConnectTests_UsesClientInstanceIfSpecified() configBuilder.Build(); // Assert - mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Exactly(1)); + mockClient.Verify(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny()), Times.Once); } [Fact] diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 23c2afd2..079ad25a 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. // using Azure; -using Azure.Core; using Azure.Core.Diagnostics; using Azure.Core.Testing; using Azure.Data.AppConfiguration; @@ -16,7 +15,6 @@ using System.Collections.Generic; using System.Diagnostics.Tracing; using System.Linq; -using System.Net; using System.Security.Cryptography; using System.Text; using System.Text.Json; diff --git a/tests/Tests.AzureAppConfiguration/Tests.cs b/tests/Tests.AzureAppConfiguration/Tests.cs index 49680b88..30010f7d 100644 --- a/tests/Tests.AzureAppConfiguration/Tests.cs +++ b/tests/Tests.AzureAppConfiguration/Tests.cs @@ -241,12 +241,10 @@ public void TestUserAgentHeader() // 6. Does not contain any additional components string userAgentRegex = @"^Microsoft\.Extensions\.Configuration\.AzureAppConfiguration/\d+\.\d+\.\d+(\+[a-z0-9]+)?(-preview(\.\d+)?)?,azsdk-net-Data.AppConfiguration/[.+\w-]+ \([.;\w\s]+\)$"; - var response1 = new MockResponse(200); - response1.SetContent(SerializationHelpers.Serialize(_kvCollectionPageOne.ToArray(), TestHelpers.SerializeBatch)); - var response2 = new MockResponse(200); - response2.SetContent(SerializationHelpers.Serialize(_kvCollectionPageOne.ToArray(), TestHelpers.SerializeBatch)); + var response = new MockResponse(200); + response.SetContent(SerializationHelpers.Serialize(_kvCollectionPageOne.ToArray(), TestHelpers.SerializeBatch)); - var mockTransport = new MockTransport(response1, response2); + var mockTransport = new MockTransport(response); var options = new AzureAppConfigurationOptions(); options.ClientOptions.Transport = mockTransport; From 1375b285342a8dfd267de0d6be0a275669b1f3ef Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 21 Nov 2024 13:53:46 -0800 Subject: [PATCH 50/72] PR comments --- .../AzureAppConfigurationOptions.cs | 13 +++++-------- .../AzureAppConfigurationProvider.cs | 15 ++++++--------- .../Extensions/ConfigurationClientExtensions.cs | 2 +- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 46065f55..7c8a8faa 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -270,22 +270,19 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c foreach (var featureFlagSelector in options.FeatureFlagSelectors) { - var featureFlagFilter = featureFlagSelector.KeyFilter; - var labelFilter = featureFlagSelector.LabelFilter; + ValidateSelectFilters(featureFlagSelector.KeyFilter, featureFlagSelector.LabelFilter); - ValidateSelectFilters(featureFlagFilter, labelFilter); - - if (string.IsNullOrWhiteSpace(labelFilter)) + if (string.IsNullOrWhiteSpace(featureFlagSelector.LabelFilter)) { - labelFilter = LabelFilter.Null; + featureFlagSelector.LabelFilter = LabelFilter.Null; } _featureFlagSelectors.AppendUnique(featureFlagSelector); _featureFlagWatchers.AppendUnique(new KeyValueWatcher { - Key = featureFlagFilter, - Label = labelFilter, + Key = featureFlagSelector.KeyFilter, + Label = featureFlagSelector.LabelFilter, // If UseFeatureFlags is called multiple times for the same key and label filters, last refresh interval wins RefreshInterval = options.RefreshInterval }); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index c6001b6f..41898d4f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -845,7 +845,7 @@ private async Task>> var selector = new SettingSelector() { KeyFilter = loadOption.KeyFilter, - LabelFilter = loadOption.LabelFilter ?? LabelFilter.Null + LabelFilter = loadOption.LabelFilter }; var matchConditions = new List(); @@ -960,6 +960,8 @@ private async Task> GetRefreshedCollections( var existingKeys = new HashSet(); + var loadedSettings = new HashSet(); + AsyncPageable pageableSettings; foreach (KeyValueSelector loadOption in selectors) @@ -969,7 +971,7 @@ private async Task> GetRefreshedCollections( SettingSelector selector = new SettingSelector() { KeyFilter = loadOption.KeyFilter, - LabelFilter = loadOption.LabelFilter ?? LabelFilter.Null + LabelFilter = loadOption.LabelFilter }; pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); @@ -1011,6 +1013,8 @@ await CallWithRequestTracing(async () => ChangeType = KeyValueChangeType.Modified, IsWatchedSetting = false }); + + loadedSettings.Add(setting.Key); } } }).ConfigureAwait(false); @@ -1023,13 +1027,6 @@ await CallWithRequestTracing(async () => } } - var loadedSettings = new HashSet(); - - foreach (KeyValueChange change in keyValueChanges) - { - loadedSettings.Add(change.Key); - } - foreach (string key in existingKeys) { if (!loadedSettings.Contains(key)) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 3805ad48..839884d9 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -88,7 +88,7 @@ public static async Task> GetNewMatchConditions(thi { ETag serverEtag = (ETag)page?.GetRawResponse()?.Headers.ETag; - if (serverEtag == null || page.Values == null) + if (page?.Values == null) { throw new RequestFailedException(ErrorMessages.InvalidConfigurationSettingPage); } From 47cd0ba6469163a6ea20d1c532b3d788c299bc42 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Mon, 2 Dec 2024 11:06:15 -0800 Subject: [PATCH 51/72] PR comments --- .../AzureAppConfigurationOptions.cs | 53 ++++++++++--------- .../AzureAppConfigurationProvider.cs | 42 +++++++-------- .../ConfigurationClientExtensions.cs | 6 +-- 3 files changed, 50 insertions(+), 51 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 7c8a8faa..4a80a191 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -74,9 +74,14 @@ public class AzureAppConfigurationOptions internal IEnumerable FeatureFlagSelectors => _featureFlagSelectors; /// - /// The configured options for refresh. + /// Indicates if was called. /// - internal AzureAppConfigurationRefreshOptions RefreshOptions { get; private set; } = new AzureAppConfigurationRefreshOptions(); + internal bool RegisterAllEnabled { get; private set; } + + /// + /// Refresh interval for selected key-value collections when is called. + /// + internal TimeSpan KvCollectionRefreshInterval { get; private set; } = RefreshConstants.DefaultRefreshInterval; /// /// A collection of . @@ -186,7 +191,16 @@ public AzureAppConfigurationOptions() /// public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter = LabelFilter.Null) { - ValidateSelectFilters(keyFilter, labelFilter); + if (string.IsNullOrEmpty(keyFilter)) + { + throw new ArgumentNullException(nameof(keyFilter)); + } + + // Do not support * and , for label filter for now. + if (labelFilter != null && (labelFilter.Contains('*') || labelFilter.Contains(','))) + { + throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); + } if (string.IsNullOrWhiteSpace(labelFilter)) { @@ -268,15 +282,8 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c }); } - foreach (var featureFlagSelector in options.FeatureFlagSelectors) + foreach (KeyValueSelector featureFlagSelector in options.FeatureFlagSelectors) { - ValidateSelectFilters(featureFlagSelector.KeyFilter, featureFlagSelector.LabelFilter); - - if (string.IsNullOrWhiteSpace(featureFlagSelector.LabelFilter)) - { - featureFlagSelector.LabelFilter = LabelFilter.Null; - } - _featureFlagSelectors.AppendUnique(featureFlagSelector); _featureFlagWatchers.AppendUnique(new KeyValueWatcher @@ -420,7 +427,15 @@ public AzureAppConfigurationOptions ConfigureRefresh(Action _mappedData; private Dictionary _watchedSettings = new Dictionary(); - private Dictionary> _watchedSelectedKvCollections = new Dictionary>(); - private Dictionary> _watchedFeatureFlagCollections = new Dictionary>(); + private Dictionary> _watchedSelectedKvCollections = new Dictionary>(); + private Dictionary> _watchedFeatureFlagCollections = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); private DateTimeOffset _registerAllNextRefreshTime; @@ -253,8 +253,8 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary> watchedFeatureFlagCollections = null; - Dictionary> watchedSelectedKvCollections = null; + Dictionary> watchedFeatureFlagCollections = null; + Dictionary> watchedSelectedKvCollections = null; Dictionary watchedSettings = null; List keyValueChanges = null; Dictionary data = null; @@ -353,7 +353,7 @@ await CallWithRequestTracing( // Get key value collection changes if RegisterAll was called watchedSelectedKvCollections = _watchedSelectedKvCollections; - if (_options.RefreshOptions.RegisterAllEnabled && registerAllIsRefreshable) + if (_options.RegisterAllEnabled && registerAllIsRefreshable) { bool selectedKvCollectionsChanged = await UpdateWatchedCollections(_options.KeyValueSelectors, watchedSelectedKvCollections, client, cancellationToken).ConfigureAwait(false); @@ -402,9 +402,9 @@ await CallWithRequestTracing( UpdateNextRefreshTime(changeWatcher); } - if (_options.RefreshOptions.RegisterAllEnabled && registerAllIsRefreshable) + if (_options.RegisterAllEnabled && registerAllIsRefreshable) { - _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.RefreshOptions.RefreshInterval); + _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); } foreach (KeyValueChange change in keyValueChanges) @@ -476,9 +476,9 @@ await CallWithRequestTracing( UpdateNextRefreshTime(changeWatcher); } - if (_options.RefreshOptions.RegisterAllEnabled) + if (_options.RegisterAllEnabled) { - _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.RefreshOptions.RefreshInterval); + _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); } } @@ -774,8 +774,8 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { Dictionary data = new Dictionary(StringComparer.OrdinalIgnoreCase); - Dictionary> watchedSelectedKvCollections = null; - Dictionary> watchedFeatureFlagCollections = null; + Dictionary> watchedSelectedKvCollections = null; + Dictionary> watchedFeatureFlagCollections = null; Dictionary watchedSettings = null; await ExecuteWithFailOverPolicyAsync( @@ -811,9 +811,9 @@ await ExecuteWithFailOverPolicyAsync( UpdateNextRefreshTime(changeWatcher); } - if (_options.RefreshOptions.RegisterAllEnabled) + if (_options.RegisterAllEnabled) { - _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.RefreshOptions.RefreshInterval); + _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); } if (data != null) @@ -834,9 +834,9 @@ await ExecuteWithFailOverPolicyAsync( } } - private async Task>> LoadSelected(Dictionary existingData, IEnumerable selectors, ConfigurationClient client, CancellationToken cancellationToken) + private async Task>> LoadSelected(Dictionary existingData, IEnumerable selectors, ConfigurationClient client, CancellationToken cancellationToken) { - var watchedCollections = new Dictionary>(); + var watchedCollections = new Dictionary>(); foreach (KeyValueSelector loadOption in selectors) { @@ -872,7 +872,7 @@ await CallWithRequestTracing(async () => } }).ConfigureAwait(false); - watchedCollections[new KeyValueIdentifier(loadOption.KeyFilter, loadOption.LabelFilter)] = matchConditions; + watchedCollections[loadOption] = matchConditions; } else { @@ -1379,7 +1379,7 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) _configClientBackoffs[endpoint] = clientBackoffStatus; } - private async Task UpdateWatchedCollections(IEnumerable selectors, Dictionary> watchedKeyValueCollections, ConfigurationClient client, CancellationToken cancellationToken) + private async Task UpdateWatchedCollections(IEnumerable selectors, Dictionary> watchedKeyValueCollections, ConfigurationClient client, CancellationToken cancellationToken) { bool watchedCollectionsChanged = false; @@ -1387,17 +1387,15 @@ private async Task UpdateWatchedCollections(IEnumerable { IEnumerable newMatchConditions = null; - KeyValueIdentifier watchedKeyValueIdentifier = new KeyValueIdentifier(selector.KeyFilter, selector.LabelFilter); - - if (watchedKeyValueCollections.TryGetValue(watchedKeyValueIdentifier, out IEnumerable matchConditions)) + if (watchedKeyValueCollections.TryGetValue(selector, out IEnumerable matchConditions)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => newMatchConditions = await client.GetNewMatchConditions(watchedKeyValueIdentifier, matchConditions, _options.PageableManager, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + async () => newMatchConditions = await client.GetNewMatchConditions(selector, matchConditions, _options.PageableManager, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } if (newMatchConditions != null) { - watchedKeyValueCollections[watchedKeyValueIdentifier] = newMatchConditions; + watchedKeyValueCollections[selector] = newMatchConditions; watchedCollectionsChanged = true; } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 839884d9..c23b8398 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -63,7 +63,7 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } - public static async Task> GetNewMatchConditions(this ConfigurationClient client, KeyValueIdentifier keyValueIdentifier, IEnumerable matchConditions, ConfigurationSettingPageableManager pageableManager, CancellationToken cancellationToken) + public static async Task> GetNewMatchConditions(this ConfigurationClient client, KeyValueSelector keyValueSelector, IEnumerable matchConditions, ConfigurationSettingPageableManager pageableManager, CancellationToken cancellationToken) { if (matchConditions == null) { @@ -72,8 +72,8 @@ public static async Task> GetNewMatchConditions(thi SettingSelector selector = new SettingSelector { - KeyFilter = keyValueIdentifier.Key, - LabelFilter = keyValueIdentifier.Label + KeyFilter = keyValueSelector.KeyFilter, + LabelFilter = keyValueSelector.LabelFilter }; bool hasCollectionChanged = false; From fff31b32cec28f43581bcb825e45fcbd1d16ce1c Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Mon, 2 Dec 2024 11:42:35 -0800 Subject: [PATCH 52/72] don't allow both registerall and register --- .../AzureAppConfigurationOptions.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 4a80a191..b6652d6b 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -421,6 +421,12 @@ public AzureAppConfigurationOptions ConfigureRefresh(Action Date: Thu, 5 Dec 2024 16:46:40 -0800 Subject: [PATCH 53/72] fix check for calls to both register methods --- .../AzureAppConfigurationOptions.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index b6652d6b..9a5141d6 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -421,7 +421,10 @@ public AzureAppConfigurationOptions ConfigureRefresh(Action Date: Mon, 16 Dec 2024 13:27:05 -0800 Subject: [PATCH 54/72] PR comments for rename/small changes --- .../AzureAppConfigurationOptions.cs | 53 +++---- .../AzureAppConfigurationProvider.cs | 129 ++++++++++-------- .../Constants/LoggingConstants.cs | 4 +- 3 files changed, 102 insertions(+), 84 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 9a5141d6..ba994eaf 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -23,8 +23,8 @@ public class AzureAppConfigurationOptions private const int MaxRetries = 2; private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(1); - private List _changeWatchers = new List(); - private List _featureFlagWatchers = new List(); + private List _sentinelKvWatchers = new List(); + private List _ffWatchers = new List(); private List _adapters; private List>> _mappers = new List>>(); private List _kvSelectors; @@ -64,12 +64,12 @@ public class AzureAppConfigurationOptions internal TokenCredential Credential { get; private set; } /// - /// A collection of . + /// Key Value selectors specified by user. /// internal IEnumerable KeyValueSelectors => _kvSelectors; /// - /// A collection of . + /// Feature Flag selectors specified by user. /// internal IEnumerable FeatureFlagSelectors => _featureFlagSelectors; @@ -81,17 +81,17 @@ public class AzureAppConfigurationOptions /// /// Refresh interval for selected key-value collections when is called. /// - internal TimeSpan KvCollectionRefreshInterval { get; private set; } = RefreshConstants.DefaultRefreshInterval; + internal TimeSpan KvCollectionRefreshInterval { get; private set; } /// /// A collection of . /// - internal IEnumerable ChangeWatchers => _changeWatchers; + internal IEnumerable KvWatchers => _sentinelKvWatchers; /// /// A collection of . /// - internal IEnumerable FeatureFlagWatchers => _featureFlagWatchers; + internal IEnumerable FeatureFlagWatchers => _ffWatchers; /// /// A collection of . @@ -278,7 +278,7 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c options.FeatureFlagSelectors.Add(new KeyValueSelector { KeyFilter = FeatureManagementConstants.FeatureFlagMarker + "*", - LabelFilter = options.Label == null ? LabelFilter.Null : options.Label + LabelFilter = string.IsNullOrWhiteSpace(options.Label) ? LabelFilter.Null : options.Label }); } @@ -286,7 +286,7 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c { _featureFlagSelectors.AppendUnique(featureFlagSelector); - _featureFlagWatchers.AppendUnique(new KeyValueWatcher + _ffWatchers.AppendUnique(new KeyValueWatcher { Key = featureFlagSelector.KeyFilter, Label = featureFlagSelector.LabelFilter, @@ -413,38 +413,41 @@ public AzureAppConfigurationOptions ConfigureClientOptions(ActionA callback used to configure Azure App Configuration refresh options. public AzureAppConfigurationOptions ConfigureRefresh(Action configure) { + if (RegisterAllEnabled) + { + throw new ArgumentException($"{nameof(ConfigureRefresh)}() cannot be invoked multiple times when {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} has been invoked."); + } + var refreshOptions = new AzureAppConfigurationRefreshOptions(); configure?.Invoke(refreshOptions); - if (!refreshOptions.RefreshRegistrations.Any() && !refreshOptions.RegisterAllEnabled) + bool isRegisterCalled = refreshOptions.RefreshRegistrations.Any(); + RegisterAllEnabled = refreshOptions.RegisterAllEnabled; + + if (!isRegisterCalled && !RegisterAllEnabled) { - throw new ArgumentException($"{nameof(ConfigureRefresh)}() must have at least one key-value registered for refresh."); + throw new ArgumentException($"{nameof(ConfigureRefresh)}() must register at least one key-value for refresh or enable refresh of all selected key-values."); } // Check if both register methods are called at any point - if ((RegisterAllEnabled && refreshOptions.RefreshRegistrations.Any()) || - (refreshOptions.RegisterAllEnabled && _changeWatchers.Any()) || - (refreshOptions.RefreshRegistrations.Any() && refreshOptions.RegisterAllEnabled)) + if (RegisterAllEnabled && (_sentinelKvWatchers.Any() || isRegisterCalled)) { throw new ArgumentException($"Cannot call both {nameof(AzureAppConfigurationRefreshOptions.RegisterAll)} and " + $"{nameof(AzureAppConfigurationRefreshOptions.Register)}."); } - foreach (var item in refreshOptions.RefreshRegistrations) - { - item.RefreshInterval = refreshOptions.RefreshInterval; - _changeWatchers.Add(item); - } - - if (refreshOptions.RegisterAllEnabled) - { - RegisterAllEnabled = refreshOptions.RegisterAllEnabled; - } - if (RegisterAllEnabled) { KvCollectionRefreshInterval = refreshOptions.RefreshInterval; } + else + { + foreach (KeyValueWatcher item in refreshOptions.RefreshRegistrations) + { + item.RefreshInterval = refreshOptions.RefreshInterval; + _sentinelKvWatchers.Add(item); + } + } return this; } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 2c6c1734..3391b30e 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -31,9 +31,9 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private Uri _lastSuccessfulEndpoint; private AzureAppConfigurationOptions _options; private Dictionary _mappedData; - private Dictionary _watchedSettings = new Dictionary(); - private Dictionary> _watchedSelectedKvCollections = new Dictionary>(); - private Dictionary> _watchedFeatureFlagCollections = new Dictionary>(); + private Dictionary _watchedSentinelKvs = new Dictionary(); + private Dictionary> _watchedKvCollections = new Dictionary>(); + private Dictionary> _watchedFfCollections = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); private DateTimeOffset _registerAllNextRefreshTime; @@ -110,12 +110,16 @@ public AzureAppConfigurationProvider(IConfigurationClientManager configClientMan _options = options ?? throw new ArgumentNullException(nameof(options)); _optional = optional; - IEnumerable watchers = options.ChangeWatchers.Union(options.FeatureFlagWatchers); + IEnumerable watchers = options.KvWatchers.Union(options.FeatureFlagWatchers); if (watchers.Any()) { MinRefreshInterval = watchers.Min(w => w.RefreshInterval); } + else if (options.RegisterAllEnabled) + { + MinRefreshInterval = options.KvCollectionRefreshInterval; + } else { MinRefreshInterval = RefreshConstants.DefaultRefreshInterval; @@ -196,13 +200,13 @@ public async Task RefreshAsync(CancellationToken cancellationToken) EnsureAssemblyInspected(); var utcNow = DateTimeOffset.UtcNow; - IEnumerable refreshableWatchers = _options.ChangeWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); - IEnumerable refreshableFeatureFlagWatchers = _options.FeatureFlagWatchers.Where(changeWatcher => utcNow >= changeWatcher.NextRefreshTime); + IEnumerable refreshableKvWatchers = _options.KvWatchers.Where(kvWatcher => utcNow >= kvWatcher.NextRefreshTime); + IEnumerable refreshableFeatureFlagWatchers = _options.FeatureFlagWatchers.Where(ffWatcher => utcNow >= ffWatcher.NextRefreshTime); bool registerAllIsRefreshable = utcNow >= _registerAllNextRefreshTime; // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. if (_mappedData != null && - !refreshableWatchers.Any() && + !refreshableKvWatchers.Any() && !refreshableFeatureFlagWatchers.Any() && !registerAllIsRefreshable && !_options.Adapters.Any(adapter => adapter.NeedsRefresh())) @@ -253,9 +257,9 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary> watchedFeatureFlagCollections = null; - Dictionary> watchedSelectedKvCollections = null; - Dictionary watchedSettings = null; + Dictionary> watchedFfCollections = null; + Dictionary> watchedKvCollections = null; + Dictionary watchedSentinelKvs = null; List keyValueChanges = null; Dictionary data = null; bool refreshAll = false; @@ -265,19 +269,19 @@ public async Task RefreshAsync(CancellationToken cancellationToken) await ExecuteWithFailOverPolicyAsync(clients, async (client) => { data = null; - watchedSettings = null; - watchedSelectedKvCollections = null; - watchedFeatureFlagCollections = null; + watchedSentinelKvs = null; + watchedKvCollections = null; + watchedFfCollections = null; keyValueChanges = new List(); refreshAll = false; Uri endpoint = _configClientManager.GetEndpointForClient(client); logDebugBuilder.Clear(); logInfoBuilder.Clear(); - foreach (KeyValueWatcher changeWatcher in refreshableWatchers) + foreach (KeyValueWatcher kvWatcher in refreshableKvWatchers) { - string watchedKey = changeWatcher.Key; - string watchedLabel = changeWatcher.Label; + string watchedKey = kvWatcher.Key; + string watchedLabel = kvWatcher.Label; KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); @@ -285,7 +289,7 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => // // Find if there is a change associated with watcher - if (_watchedSettings.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) + if (_watchedSentinelKvs.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); @@ -326,7 +330,7 @@ await CallWithRequestTracing( logInfoBuilder.AppendLine(LogHelper.BuildKeyValueSettingUpdatedMessage(change.Key)); keyValueChanges.Add(change); - if (changeWatcher.RefreshAll) + if (kvWatcher.RefreshAll) { refreshAll = true; break; @@ -342,20 +346,20 @@ await CallWithRequestTracing( { // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true data = new Dictionary(StringComparer.OrdinalIgnoreCase); - watchedSelectedKvCollections = await LoadSelected(data, _options.KeyValueSelectors, client, cancellationToken).ConfigureAwait(false); - watchedFeatureFlagCollections = await LoadSelected(data, _options.FeatureFlagSelectors, client, cancellationToken).ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); + watchedKvCollections = await LoadSelected(client, data, _options.KeyValueSelectors, cancellationToken).ConfigureAwait(false); + watchedFfCollections = await LoadSelected(client, data, _options.FeatureFlagSelectors, cancellationToken).ConfigureAwait(false); + watchedSentinelKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); return; } else { // Get key value collection changes if RegisterAll was called - watchedSelectedKvCollections = _watchedSelectedKvCollections; + watchedKvCollections = _watchedKvCollections; if (_options.RegisterAllEnabled && registerAllIsRefreshable) { - bool selectedKvCollectionsChanged = await UpdateWatchedCollections(_options.KeyValueSelectors, watchedSelectedKvCollections, client, cancellationToken).ConfigureAwait(false); + bool selectedKvCollectionsChanged = await UpdateWatchedCollections(_options.KeyValueSelectors, watchedKvCollections, client, cancellationToken).ConfigureAwait(false); if (!selectedKvCollectionsChanged) { @@ -370,11 +374,11 @@ await CallWithRequestTracing( } // Get feature flag changes - watchedFeatureFlagCollections = _watchedFeatureFlagCollections; + watchedFfCollections = _watchedFfCollections; bool featureFlagCollectionsChanged = await UpdateWatchedCollections( refreshableFeatureFlagWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), - watchedFeatureFlagCollections, + watchedFfCollections, client, cancellationToken).ConfigureAwait(false); @@ -395,9 +399,9 @@ await CallWithRequestTracing( if (!refreshAll) { - watchedSettings = new Dictionary(_watchedSettings); + watchedSentinelKvs = new Dictionary(_watchedSentinelKvs); - foreach (KeyValueWatcher changeWatcher in refreshableWatchers.Concat(refreshableFeatureFlagWatchers)) + foreach (KeyValueWatcher changeWatcher in refreshableKvWatchers.Concat(refreshableFeatureFlagWatchers)) { UpdateNextRefreshTime(changeWatcher); } @@ -418,7 +422,7 @@ await CallWithRequestTracing( if (change.IsWatchedSetting) { - watchedSettings[changeIdentifier] = settingCopy; + watchedSentinelKvs[changeIdentifier] = settingCopy; } foreach (Func> func in _options.Mappers) @@ -441,7 +445,7 @@ await CallWithRequestTracing( if (change.IsWatchedSetting) { - watchedSettings.Remove(changeIdentifier); + watchedSentinelKvs.Remove(changeIdentifier); } } @@ -471,7 +475,7 @@ await CallWithRequestTracing( } // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.FeatureFlagWatchers)) + foreach (KeyValueWatcher changeWatcher in _options.KvWatchers.Concat(_options.FeatureFlagWatchers)) { UpdateNextRefreshTime(changeWatcher); } @@ -484,11 +488,11 @@ await CallWithRequestTracing( if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any()) { - _watchedSettings = watchedSettings; + _watchedSentinelKvs = watchedSentinelKvs; - _watchedFeatureFlagCollections = watchedFeatureFlagCollections; + _watchedFfCollections = watchedFfCollections; - _watchedSelectedKvCollections = watchedSelectedKvCollections; + _watchedKvCollections = watchedKvCollections; if (logDebugBuilder.Length > 0) { @@ -622,14 +626,21 @@ private void SetDirty(TimeSpan? maxDelay) { DateTimeOffset nextRefreshTime = AddRandomDelay(DateTimeOffset.UtcNow, maxDelay ?? DefaultMaxSetDirtyDelay); - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) + if (_options.RegisterAllEnabled) + { + _registerAllNextRefreshTime = nextRefreshTime; + } + else { - changeWatcher.NextRefreshTime = nextRefreshTime; + foreach (KeyValueWatcher kvWatcher in _options.KvWatchers) + { + kvWatcher.NextRefreshTime = nextRefreshTime; + } } - foreach (KeyValueWatcher changeWatcher in _options.FeatureFlagWatchers) + foreach (KeyValueWatcher featureFlagWatcher in _options.FeatureFlagWatchers) { - changeWatcher.NextRefreshTime = nextRefreshTime; + featureFlagWatcher.NextRefreshTime = nextRefreshTime; } } @@ -774,29 +785,29 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { Dictionary data = new Dictionary(StringComparer.OrdinalIgnoreCase); - Dictionary> watchedSelectedKvCollections = null; - Dictionary> watchedFeatureFlagCollections = null; - Dictionary watchedSettings = null; + Dictionary> watchedKvCollections = null; + Dictionary> watchedFfCollections = null; + Dictionary watchedSentinelKvs = null; await ExecuteWithFailOverPolicyAsync( clients, async (client) => { - watchedSelectedKvCollections = await LoadSelected( + watchedKvCollections = await LoadSelected( + client, data, _options.KeyValueSelectors, - client, cancellationToken) .ConfigureAwait(false); - watchedFeatureFlagCollections = await LoadSelected( + watchedFfCollections = await LoadSelected( + client, data, _options.FeatureFlagSelectors, - client, cancellationToken) .ConfigureAwait(false); - watchedSettings = await LoadKeyValuesRegisteredForRefresh( + watchedSentinelKvs = await LoadKeyValuesRegisteredForRefresh( client, data, cancellationToken) @@ -806,7 +817,7 @@ await ExecuteWithFailOverPolicyAsync( .ConfigureAwait(false); // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.FeatureFlagWatchers)) + foreach (KeyValueWatcher changeWatcher in _options.KvWatchers.Concat(_options.FeatureFlagWatchers)) { UpdateNextRefreshTime(changeWatcher); } @@ -827,14 +838,18 @@ await ExecuteWithFailOverPolicyAsync( Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); - _watchedSettings = watchedSettings; - _watchedSelectedKvCollections = watchedSelectedKvCollections; - _watchedFeatureFlagCollections = watchedFeatureFlagCollections; + _watchedSentinelKvs = watchedSentinelKvs; + _watchedKvCollections = watchedKvCollections; + _watchedFfCollections = watchedFfCollections; _mappedData = mappedData; } } - private async Task>> LoadSelected(Dictionary existingData, IEnumerable selectors, ConfigurationClient client, CancellationToken cancellationToken) + private async Task>> LoadSelected( + ConfigurationClient client, + Dictionary existingData, + IEnumerable selectors, + CancellationToken cancellationToken) { var watchedCollections = new Dictionary>(); @@ -911,12 +926,12 @@ await CallWithRequestTracing(async () => private async Task> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) { - var watchedSettings = new Dictionary(); + var watchedSentinelKvs = new Dictionary(); - foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers) + foreach (KeyValueWatcher kvWatcher in _options.KvWatchers) { - string watchedKey = changeWatcher.Key; - string watchedLabel = changeWatcher.Label; + string watchedKey = kvWatcher.Key; + string watchedLabel = kvWatcher.Label; KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); @@ -924,7 +939,7 @@ private async Task> LoadKey if (existingSettings.TryGetValue(watchedKey, out ConfigurationSetting loadedKv) && watchedKeyLabel.Equals(new KeyValueIdentifier(loadedKv.Key, loadedKv.Label))) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); + watchedSentinelKvs[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); continue; } @@ -942,12 +957,12 @@ private async Task> LoadKey // If the key-value was found, store it for updating the settings if (watchedKv != null) { - watchedSettings[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); + watchedSentinelKvs[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); existingSettings[watchedKey] = watchedKv; } } - return watchedSettings; + return watchedSentinelKvs; } private async Task> GetRefreshedCollections( diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs index 498cd7d6..69200ad4 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs @@ -22,14 +22,14 @@ internal class LoggingConstants public const string RefreshKeyVaultSecretRead = "Secret read from Key Vault for key-value."; public const string RefreshFeatureFlagRead = "Feature flag read from App Configuration."; public const string RefreshFeatureFlagsUnchanged = "Feature flags read from App Configuration. Change:'None'"; - public const string RefreshSelectedKeyValuesCollectionsUnchanged = "Selected key-value collections read from App Configuration. Change:'None'"; + public const string RefreshSelectedKeyValueCollectionsUnchanged = "Selected key-value collections read from App Configuration. Change:'None'"; // Successful update, information log level public const string RefreshConfigurationUpdatedSuccess = "Configuration reloaded."; public const string RefreshKeyValueSettingUpdated = "Setting updated."; public const string RefreshKeyVaultSettingUpdated = "Setting updated from Key Vault."; public const string RefreshFeatureFlagsUpdated = "Feature flags updated."; - public const string RefreshSelectedKeyValuesCollectionsUpdated = "Selected key-value collections updated."; + public const string RefreshSelectedKeyValueCollectionsUpdated = "Selected key-value collections updated."; // Other public const string RefreshSkippedNoClientAvailable = "Refresh skipped because no endpoint is accessible."; From 3ab86720e22d285df08b53bff3b7c2a083ebe2c4 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Mon, 16 Dec 2024 13:42:38 -0800 Subject: [PATCH 55/72] fix compile error --- .../LogHelper.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs index d22a772d..24cbaaf1 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs @@ -32,12 +32,12 @@ public static string BuildFeatureFlagsUpdatedMessage() public static string BuildSelectedKeyValueCollectionsUnchangedMessage(string endpoint) { - return $"{LoggingConstants.RefreshSelectedKeyValuesCollectionsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; + return $"{LoggingConstants.RefreshSelectedKeyValueCollectionsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; } public static string BuildSelectedKeyValueCollectionsUpdatedMessage() { - return LoggingConstants.RefreshSelectedKeyValuesCollectionsUpdated; + return LoggingConstants.RefreshSelectedKeyValueCollectionsUpdated; } public static string BuildKeyVaultSecretReadMessage(string key, string label) From c2bca820b19cbccbd59344632438557ada97cad8 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Tue, 17 Dec 2024 12:34:17 -0800 Subject: [PATCH 56/72] simplify refreshasync path, fix naming from comments --- .../AzureAppConfigurationOptions.cs | 8 +- .../AzureAppConfigurationProvider.cs | 308 +++++++++++------- 2 files changed, 201 insertions(+), 115 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index ba994eaf..89850e93 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -23,7 +23,7 @@ public class AzureAppConfigurationOptions private const int MaxRetries = 2; private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromMinutes(1); - private List _sentinelKvWatchers = new List(); + private List _individualKvWatchers = new List(); private List _ffWatchers = new List(); private List _adapters; private List>> _mappers = new List>>(); @@ -86,7 +86,7 @@ public class AzureAppConfigurationOptions /// /// A collection of . /// - internal IEnumerable KvWatchers => _sentinelKvWatchers; + internal IEnumerable IndividualKvWatchers => _individualKvWatchers; /// /// A collection of . @@ -430,7 +430,7 @@ public AzureAppConfigurationOptions ConfigureRefresh(Action watchers = options.KvWatchers.Union(options.FeatureFlagWatchers); + IEnumerable watchers = options.IndividualKvWatchers.Union(options.FeatureFlagWatchers); - if (watchers.Any()) + if (options.RegisterAllEnabled) { - MinRefreshInterval = watchers.Min(w => w.RefreshInterval); + if (options.FeatureFlagWatchers.Any()) + { + TimeSpan minFfWatcherRefreshInterval = options.FeatureFlagWatchers.Min(w => w.RefreshInterval); + + MinRefreshInterval = minFfWatcherRefreshInterval < options.KvCollectionRefreshInterval ? minFfWatcherRefreshInterval : options.KvCollectionRefreshInterval; + } + else + { + MinRefreshInterval = options.KvCollectionRefreshInterval; + } } - else if (options.RegisterAllEnabled) + else if (watchers.Any()) { - MinRefreshInterval = options.KvCollectionRefreshInterval; + MinRefreshInterval = watchers.Min(w => w.RefreshInterval); } else { @@ -200,14 +209,14 @@ public async Task RefreshAsync(CancellationToken cancellationToken) EnsureAssemblyInspected(); var utcNow = DateTimeOffset.UtcNow; - IEnumerable refreshableKvWatchers = _options.KvWatchers.Where(kvWatcher => utcNow >= kvWatcher.NextRefreshTime); - IEnumerable refreshableFeatureFlagWatchers = _options.FeatureFlagWatchers.Where(ffWatcher => utcNow >= ffWatcher.NextRefreshTime); + IEnumerable refreshableIndividualKvWatchers = _options.IndividualKvWatchers.Where(kvWatcher => utcNow >= kvWatcher.NextRefreshTime); + IEnumerable refreshableFfWatchers = _options.FeatureFlagWatchers.Where(ffWatcher => utcNow >= ffWatcher.NextRefreshTime); bool registerAllIsRefreshable = utcNow >= _registerAllNextRefreshTime; // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. if (_mappedData != null && - !refreshableKvWatchers.Any() && - !refreshableFeatureFlagWatchers.Any() && + !refreshableIndividualKvWatchers.Any() && + !refreshableFfWatchers.Any() && !registerAllIsRefreshable && !_options.Adapters.Any(adapter => adapter.NeedsRefresh())) { @@ -278,120 +287,71 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => logDebugBuilder.Clear(); logInfoBuilder.Clear(); - foreach (KeyValueWatcher kvWatcher in refreshableKvWatchers) + if (!_options.RegisterAllEnabled) { - string watchedKey = kvWatcher.Key; - string watchedLabel = kvWatcher.Label; - - KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); - - KeyValueChange change = default; - - // - // Find if there is a change associated with watcher - if (_watchedSentinelKvs.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) - { - await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - - change.IsWatchedSetting = true; - } - else - { - // Load the key-value in case the previous load attempts had failed - - try - { - await CallWithRequestTracing( - async () => watchedKv = await client.GetConfigurationSettingAsync(watchedKey, watchedLabel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - } - catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound) - { - watchedKv = null; - } - - if (watchedKv != null) - { - change = new KeyValueChange() - { - Key = watchedKv.Key, - Label = watchedKv.Label.NormalizeNull(), - Current = watchedKv, - ChangeType = KeyValueChangeType.Modified, - IsWatchedSetting = true - }; - } - } + refreshAll = await RefreshIndividualKvWatchers( + client, + keyValueChanges, + refreshableIndividualKvWatchers, + endpoint, + logDebugBuilder, + logInfoBuilder, + cancellationToken).ConfigureAwait(false); - // Check if a change has been detected in the key-value registered for refresh - if (change.ChangeType != KeyValueChangeType.None) + if (!refreshAll) { - logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); - logInfoBuilder.AppendLine(LogHelper.BuildKeyValueSettingUpdatedMessage(change.Key)); - keyValueChanges.Add(change); - - if (kvWatcher.RefreshAll) - { - refreshAll = true; - break; - } + // Get feature flag changes + watchedFfCollections = new Dictionary>(_watchedFfCollections); + + await RefreshFfCollections(client, + keyValueChanges, + refreshableFfWatchers, + watchedFfCollections, + endpoint, + logDebugBuilder, + logInfoBuilder, + cancellationToken).ConfigureAwait(false); } else { - logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); + // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true + data = new Dictionary(StringComparer.OrdinalIgnoreCase); + watchedKvCollections = await LoadSelected(client, data, _options.KeyValueSelectors, cancellationToken).ConfigureAwait(false); + watchedFfCollections = await LoadSelected(client, data, _options.FeatureFlagSelectors, cancellationToken).ConfigureAwait(false); + watchedSentinelKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); + logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); + return; } } - - if (refreshAll) - { - // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true - data = new Dictionary(StringComparer.OrdinalIgnoreCase); - watchedKvCollections = await LoadSelected(client, data, _options.KeyValueSelectors, cancellationToken).ConfigureAwait(false); - watchedFfCollections = await LoadSelected(client, data, _options.FeatureFlagSelectors, cancellationToken).ConfigureAwait(false); - watchedSentinelKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); - logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); - return; - } else { // Get key value collection changes if RegisterAll was called - watchedKvCollections = _watchedKvCollections; + watchedKvCollections = new Dictionary>(_watchedKvCollections); if (_options.RegisterAllEnabled && registerAllIsRefreshable) { - bool selectedKvCollectionsChanged = await UpdateWatchedCollections(_options.KeyValueSelectors, watchedKvCollections, client, cancellationToken).ConfigureAwait(false); - - if (!selectedKvCollectionsChanged) - { - logDebugBuilder.AppendLine(LogHelper.BuildSelectedKeyValueCollectionsUnchangedMessage(endpoint.ToString())); - } - else - { - keyValueChanges.AddRange(await GetRefreshedCollections(_options.KeyValueSelectors, _mappedData, client, cancellationToken).ConfigureAwait(false)); - - logInfoBuilder.Append(LogHelper.BuildSelectedKeyValueCollectionsUpdatedMessage()); - } + await RefreshKvCollections( + client, + keyValueChanges, + watchedKvCollections, + endpoint, + logDebugBuilder, + logInfoBuilder, + cancellationToken).ConfigureAwait(false); } // Get feature flag changes - watchedFfCollections = _watchedFfCollections; + watchedFfCollections = new Dictionary>(_watchedFfCollections); - bool featureFlagCollectionsChanged = await UpdateWatchedCollections( - refreshableFeatureFlagWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), - watchedFfCollections, + await RefreshFfCollections( client, + keyValueChanges, + refreshableFfWatchers, + watchedFfCollections, + endpoint, + logDebugBuilder, + logInfoBuilder, cancellationToken).ConfigureAwait(false); - - if (!featureFlagCollectionsChanged) - { - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); - } - else - { - keyValueChanges.AddRange(await GetRefreshedCollections(_options.FeatureFlagSelectors, _mappedData, client, cancellationToken).ConfigureAwait(false)); - - logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); - } } }, cancellationToken) @@ -401,7 +361,7 @@ await CallWithRequestTracing( { watchedSentinelKvs = new Dictionary(_watchedSentinelKvs); - foreach (KeyValueWatcher changeWatcher in refreshableKvWatchers.Concat(refreshableFeatureFlagWatchers)) + foreach (KeyValueWatcher changeWatcher in refreshableIndividualKvWatchers.Concat(refreshableFfWatchers)) { UpdateNextRefreshTime(changeWatcher); } @@ -475,7 +435,7 @@ await CallWithRequestTracing( } // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.KvWatchers.Concat(_options.FeatureFlagWatchers)) + foreach (KeyValueWatcher changeWatcher in _options.IndividualKvWatchers.Concat(_options.FeatureFlagWatchers)) { UpdateNextRefreshTime(changeWatcher); } @@ -632,7 +592,7 @@ private void SetDirty(TimeSpan? maxDelay) } else { - foreach (KeyValueWatcher kvWatcher in _options.KvWatchers) + foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) { kvWatcher.NextRefreshTime = nextRefreshTime; } @@ -817,7 +777,7 @@ await ExecuteWithFailOverPolicyAsync( .ConfigureAwait(false); // Update the next refresh time for all refresh registered settings and feature flags - foreach (KeyValueWatcher changeWatcher in _options.KvWatchers.Concat(_options.FeatureFlagWatchers)) + foreach (KeyValueWatcher changeWatcher in _options.IndividualKvWatchers.Concat(_options.FeatureFlagWatchers)) { UpdateNextRefreshTime(changeWatcher); } @@ -928,7 +888,7 @@ private async Task> LoadKey { var watchedSentinelKvs = new Dictionary(); - foreach (KeyValueWatcher kvWatcher in _options.KvWatchers) + foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) { string watchedKey = kvWatcher.Key; string watchedLabel = kvWatcher.Label; @@ -1060,6 +1020,132 @@ await CallWithRequestTracing(async () => return keyValueChanges; } + private async Task RefreshIndividualKvWatchers( + ConfigurationClient client, + List keyValueChanges, + IEnumerable refreshableIndividualKvWatchers, + Uri endpoint, + StringBuilder logDebugBuilder, + StringBuilder logInfoBuilder, + CancellationToken cancellationToken) + { + foreach (KeyValueWatcher kvWatcher in refreshableIndividualKvWatchers) + { + string watchedKey = kvWatcher.Key; + string watchedLabel = kvWatcher.Label; + + KeyValueIdentifier watchedKeyLabel = new KeyValueIdentifier(watchedKey, watchedLabel); + + KeyValueChange change = default; + + // + // Find if there is a change associated with watcher + if (_watchedSentinelKvs.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) + { + await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, + async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + + change.IsWatchedSetting = true; + } + else + { + // Load the key-value in case the previous load attempts had failed + + try + { + await CallWithRequestTracing( + async () => watchedKv = await client.GetConfigurationSettingAsync(watchedKey, watchedLabel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + } + catch (RequestFailedException e) when (e.Status == (int)HttpStatusCode.NotFound) + { + watchedKv = null; + } + + if (watchedKv != null) + { + change = new KeyValueChange() + { + Key = watchedKv.Key, + Label = watchedKv.Label.NormalizeNull(), + Current = watchedKv, + ChangeType = KeyValueChangeType.Modified, + IsWatchedSetting = true + }; + } + } + + // Check if a change has been detected in the key-value registered for refresh + if (change.ChangeType != KeyValueChangeType.None) + { + logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); + logInfoBuilder.AppendLine(LogHelper.BuildKeyValueSettingUpdatedMessage(change.Key)); + keyValueChanges.Add(change); + + if (kvWatcher.RefreshAll) + { + return true; + } + } + else + { + logDebugBuilder.AppendLine(LogHelper.BuildKeyValueReadMessage(change.ChangeType, change.Key, change.Label, endpoint.ToString())); + } + } + + return false; + } + + private async Task RefreshFfCollections( + ConfigurationClient client, + List keyValueChanges, + IEnumerable refreshableFfWatchers, + Dictionary> watchedFfCollections, + Uri endpoint, + StringBuilder logDebugBuilder, + StringBuilder logInfoBuilder, + CancellationToken cancellationToken) + { + bool featureFlagCollectionsChanged = await UpdateWatchedCollections( + refreshableFfWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), + watchedFfCollections, + client, + cancellationToken).ConfigureAwait(false); + + if (!featureFlagCollectionsChanged) + { + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); + } + else + { + keyValueChanges.AddRange(await GetRefreshedCollections(_options.FeatureFlagSelectors, _mappedData, client, cancellationToken).ConfigureAwait(false)); + + logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); + } + } + + private async Task RefreshKvCollections( + ConfigurationClient client, + List keyValueChanges, + Dictionary> watchedKvCollections, + Uri endpoint, + StringBuilder logDebugBuilder, + StringBuilder logInfoBuilder, + CancellationToken cancellationToken) + { + bool selectedKvCollectionsChanged = await UpdateWatchedCollections(_options.KeyValueSelectors, watchedKvCollections, client, cancellationToken).ConfigureAwait(false); + + if (!selectedKvCollectionsChanged) + { + logDebugBuilder.AppendLine(LogHelper.BuildSelectedKeyValueCollectionsUnchangedMessage(endpoint.ToString())); + } + else + { + keyValueChanges.AddRange(await GetRefreshedCollections(_options.KeyValueSelectors, _mappedData, client, cancellationToken).ConfigureAwait(false)); + + logInfoBuilder.Append(LogHelper.BuildSelectedKeyValueCollectionsUpdatedMessage()); + } + } + private void SetData(IDictionary data) { // Set the application data for the configuration provider @@ -1394,7 +1480,7 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) _configClientBackoffs[endpoint] = clientBackoffStatus; } - private async Task UpdateWatchedCollections(IEnumerable selectors, Dictionary> watchedKeyValueCollections, ConfigurationClient client, CancellationToken cancellationToken) + private async Task UpdateWatchedCollections(IEnumerable selectors, Dictionary> watchedCollections, ConfigurationClient client, CancellationToken cancellationToken) { bool watchedCollectionsChanged = false; @@ -1402,7 +1488,7 @@ private async Task UpdateWatchedCollections(IEnumerable { IEnumerable newMatchConditions = null; - if (watchedKeyValueCollections.TryGetValue(selector, out IEnumerable matchConditions)) + if (watchedCollections.TryGetValue(selector, out IEnumerable matchConditions)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, async () => newMatchConditions = await client.GetNewMatchConditions(selector, matchConditions, _options.PageableManager, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); @@ -1410,7 +1496,7 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa if (newMatchConditions != null) { - watchedKeyValueCollections[selector] = newMatchConditions; + watchedCollections[selector] = newMatchConditions; watchedCollectionsChanged = true; } From 6d1cc1acc4b591975d798e13eead3ecbdc06913e Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 19 Dec 2024 10:35:33 -0800 Subject: [PATCH 57/72] remove redundant if check --- .../AzureAppConfigurationProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 0bbe0730..e47b65b2 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -328,7 +328,7 @@ await RefreshFfCollections(client, // Get key value collection changes if RegisterAll was called watchedKvCollections = new Dictionary>(_watchedKvCollections); - if (_options.RegisterAllEnabled && registerAllIsRefreshable) + if (registerAllIsRefreshable) { await RefreshKvCollections( client, From aa4829bbdc426da1b9c12be281306bedf8945f5c Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Mon, 30 Dec 2024 11:34:34 -0800 Subject: [PATCH 58/72] simplify logic for minrefreshinterval --- .../AzureAppConfigurationProvider.cs | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index e47b65b2..cf544350 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -112,26 +112,16 @@ public AzureAppConfigurationProvider(IConfigurationClientManager configClientMan IEnumerable watchers = options.IndividualKvWatchers.Union(options.FeatureFlagWatchers); - if (options.RegisterAllEnabled) - { - if (options.FeatureFlagWatchers.Any()) - { - TimeSpan minFfWatcherRefreshInterval = options.FeatureFlagWatchers.Min(w => w.RefreshInterval); + bool hasWatchers = watchers.Any(); + TimeSpan minWatcherRefreshInterval = hasWatchers ? watchers.Min(w => w.RefreshInterval) : TimeSpan.MaxValue; - MinRefreshInterval = minFfWatcherRefreshInterval < options.KvCollectionRefreshInterval ? minFfWatcherRefreshInterval : options.KvCollectionRefreshInterval; - } - else - { - MinRefreshInterval = options.KvCollectionRefreshInterval; - } - } - else if (watchers.Any()) + if (options.RegisterAllEnabled) { - MinRefreshInterval = watchers.Min(w => w.RefreshInterval); + MinRefreshInterval = TimeSpan.FromTicks(Math.Min(minWatcherRefreshInterval.Ticks, options.KvCollectionRefreshInterval.Ticks)); } else { - MinRefreshInterval = RefreshConstants.DefaultRefreshInterval; + MinRefreshInterval = hasWatchers ? minWatcherRefreshInterval : RefreshConstants.DefaultRefreshInterval; } // Enable request tracing if not opt-out From 9afe17a5f25b9af7a1d07f6e8f563c968ba0fbcb Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 2 Jan 2025 12:19:42 -0800 Subject: [PATCH 59/72] fix smaller comments --- .../AzureAppConfigurationProvider.cs | 103 +++++++++--------- .../ConfigurationClientExtensions.cs | 10 ++ 2 files changed, 60 insertions(+), 53 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index cf544350..0e5ecab4 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -31,9 +31,9 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private Uri _lastSuccessfulEndpoint; private AzureAppConfigurationOptions _options; private Dictionary _mappedData; - private Dictionary _watchedSentinelKvs = new Dictionary(); - private Dictionary> _watchedKvCollections = new Dictionary>(); - private Dictionary> _watchedFfCollections = new Dictionary>(); + private Dictionary _watchedIndividualKvs = new Dictionary(); + private Dictionary> _watchedKvEtags = new Dictionary>(); + private Dictionary> _watchedFfEtags = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); private DateTimeOffset _registerAllNextRefreshTime; @@ -256,9 +256,9 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary> watchedFfCollections = null; - Dictionary> watchedKvCollections = null; - Dictionary watchedSentinelKvs = null; + Dictionary> watchedFfEtags = null; + Dictionary> watchedKvEtags = null; + Dictionary watchedIndividualKvs = null; List keyValueChanges = null; Dictionary data = null; bool refreshAll = false; @@ -268,9 +268,9 @@ public async Task RefreshAsync(CancellationToken cancellationToken) await ExecuteWithFailOverPolicyAsync(clients, async (client) => { data = null; - watchedSentinelKvs = null; - watchedKvCollections = null; - watchedFfCollections = null; + watchedIndividualKvs = null; + watchedKvEtags = null; + watchedFfEtags = null; keyValueChanges = new List(); refreshAll = false; Uri endpoint = _configClientManager.GetEndpointForClient(client); @@ -291,12 +291,13 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => if (!refreshAll) { // Get feature flag changes - watchedFfCollections = new Dictionary>(_watchedFfCollections); + watchedFfEtags = new Dictionary>(_watchedFfEtags); - await RefreshFfCollections(client, + await RefreshFfCollections( + client, keyValueChanges, refreshableFfWatchers, - watchedFfCollections, + watchedFfEtags, endpoint, logDebugBuilder, logInfoBuilder, @@ -306,9 +307,9 @@ await RefreshFfCollections(client, { // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true data = new Dictionary(StringComparer.OrdinalIgnoreCase); - watchedKvCollections = await LoadSelected(client, data, _options.KeyValueSelectors, cancellationToken).ConfigureAwait(false); - watchedFfCollections = await LoadSelected(client, data, _options.FeatureFlagSelectors, cancellationToken).ConfigureAwait(false); - watchedSentinelKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); + watchedKvEtags = await LoadSelected(client, data, _options.KeyValueSelectors, cancellationToken).ConfigureAwait(false); + watchedFfEtags = await LoadSelected(client, data, _options.FeatureFlagSelectors, cancellationToken).ConfigureAwait(false); + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); return; } @@ -316,14 +317,14 @@ await RefreshFfCollections(client, else { // Get key value collection changes if RegisterAll was called - watchedKvCollections = new Dictionary>(_watchedKvCollections); + watchedKvEtags = new Dictionary>(_watchedKvEtags); if (registerAllIsRefreshable) { await RefreshKvCollections( client, keyValueChanges, - watchedKvCollections, + watchedKvEtags, endpoint, logDebugBuilder, logInfoBuilder, @@ -331,13 +332,13 @@ await RefreshKvCollections( } // Get feature flag changes - watchedFfCollections = new Dictionary>(_watchedFfCollections); + watchedFfEtags = new Dictionary>(_watchedFfEtags); await RefreshFfCollections( client, keyValueChanges, refreshableFfWatchers, - watchedFfCollections, + watchedFfEtags, endpoint, logDebugBuilder, logInfoBuilder, @@ -349,7 +350,7 @@ await RefreshFfCollections( if (!refreshAll) { - watchedSentinelKvs = new Dictionary(_watchedSentinelKvs); + watchedIndividualKvs = new Dictionary(_watchedIndividualKvs); foreach (KeyValueWatcher changeWatcher in refreshableIndividualKvWatchers.Concat(refreshableFfWatchers)) { @@ -372,7 +373,7 @@ await RefreshFfCollections( if (change.IsWatchedSetting) { - watchedSentinelKvs[changeIdentifier] = settingCopy; + watchedIndividualKvs[changeIdentifier] = settingCopy; } foreach (Func> func in _options.Mappers) @@ -395,7 +396,7 @@ await RefreshFfCollections( if (change.IsWatchedSetting) { - watchedSentinelKvs.Remove(changeIdentifier); + watchedIndividualKvs.Remove(changeIdentifier); } } @@ -429,20 +430,15 @@ await RefreshFfCollections( { UpdateNextRefreshTime(changeWatcher); } - - if (_options.RegisterAllEnabled) - { - _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); - } } if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any()) { - _watchedSentinelKvs = watchedSentinelKvs; + _watchedIndividualKvs = watchedIndividualKvs; - _watchedFfCollections = watchedFfCollections; + _watchedFfEtags = watchedFfEtags; - _watchedKvCollections = watchedKvCollections; + _watchedKvEtags = watchedKvEtags; if (logDebugBuilder.Length > 0) { @@ -735,29 +731,29 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { Dictionary data = new Dictionary(StringComparer.OrdinalIgnoreCase); - Dictionary> watchedKvCollections = null; - Dictionary> watchedFfCollections = null; - Dictionary watchedSentinelKvs = null; + Dictionary> watchedKvEtags = null; + Dictionary> watchedFfEtags = null; + Dictionary watchedIndividualKvs = null; await ExecuteWithFailOverPolicyAsync( clients, async (client) => { - watchedKvCollections = await LoadSelected( + watchedKvEtags = await LoadSelected( client, data, _options.KeyValueSelectors, cancellationToken) .ConfigureAwait(false); - watchedFfCollections = await LoadSelected( + watchedFfEtags = await LoadSelected( client, data, _options.FeatureFlagSelectors, cancellationToken) .ConfigureAwait(false); - watchedSentinelKvs = await LoadKeyValuesRegisteredForRefresh( + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh( client, data, cancellationToken) @@ -788,9 +784,9 @@ await ExecuteWithFailOverPolicyAsync( Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); - _watchedSentinelKvs = watchedSentinelKvs; - _watchedKvCollections = watchedKvCollections; - _watchedFfCollections = watchedFfCollections; + _watchedIndividualKvs = watchedIndividualKvs; + _watchedKvEtags = watchedKvEtags; + _watchedFfEtags = watchedFfEtags; _mappedData = mappedData; } } @@ -821,6 +817,7 @@ await CallWithRequestTracing(async () => await foreach (Page page in _options.PageableManager.GetPages(pageableSettings).ConfigureAwait(false)) { + // Server etag will never be null or empty because non-conditional List key-values API always returns 200 ETag serverEtag = (ETag)page?.GetRawResponse()?.Headers.ETag; if (serverEtag == null || page.Values == null) @@ -876,7 +873,7 @@ await CallWithRequestTracing(async () => private async Task> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) { - var watchedSentinelKvs = new Dictionary(); + var watchedIndividualKvs = new Dictionary(); foreach (KeyValueWatcher kvWatcher in _options.IndividualKvWatchers) { @@ -889,7 +886,7 @@ private async Task> LoadKey if (existingSettings.TryGetValue(watchedKey, out ConfigurationSetting loadedKv) && watchedKeyLabel.Equals(new KeyValueIdentifier(loadedKv.Key, loadedKv.Label))) { - watchedSentinelKvs[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); + watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); continue; } @@ -907,17 +904,17 @@ private async Task> LoadKey // If the key-value was found, store it for updating the settings if (watchedKv != null) { - watchedSentinelKvs[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); + watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); existingSettings[watchedKey] = watchedKv; } } - return watchedSentinelKvs; + return watchedIndividualKvs; } private async Task> GetRefreshedCollections( IEnumerable selectors, - IDictionary existingSettings, + IEnumerable existingDataKeys, ConfigurationClient client, CancellationToken cancellationToken) { @@ -984,7 +981,7 @@ await CallWithRequestTracing(async () => } }).ConfigureAwait(false); - IEnumerable existingKeysList = GetCurrentKeyValueCollection(loadOption.KeyFilter, existingSettings.Keys); + IEnumerable existingKeysList = GetExistingKeys(loadOption.KeyFilter, existingDataKeys); foreach (string key in existingKeysList) { @@ -1030,7 +1027,7 @@ private async Task RefreshIndividualKvWatchers( // // Find if there is a change associated with watcher - if (_watchedSentinelKvs.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) + if (_watchedIndividualKvs.TryGetValue(watchedKeyLabel, out ConfigurationSetting watchedKv)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); @@ -1089,7 +1086,7 @@ private async Task RefreshFfCollections( ConfigurationClient client, List keyValueChanges, IEnumerable refreshableFfWatchers, - Dictionary> watchedFfCollections, + Dictionary> watchedFfEtags, Uri endpoint, StringBuilder logDebugBuilder, StringBuilder logInfoBuilder, @@ -1097,7 +1094,7 @@ private async Task RefreshFfCollections( { bool featureFlagCollectionsChanged = await UpdateWatchedCollections( refreshableFfWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), - watchedFfCollections, + watchedFfEtags, client, cancellationToken).ConfigureAwait(false); @@ -1107,7 +1104,7 @@ private async Task RefreshFfCollections( } else { - keyValueChanges.AddRange(await GetRefreshedCollections(_options.FeatureFlagSelectors, _mappedData, client, cancellationToken).ConfigureAwait(false)); + keyValueChanges.AddRange(await GetRefreshedCollections(_options.FeatureFlagSelectors, _mappedData.Keys, client, cancellationToken).ConfigureAwait(false)); logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); } @@ -1116,13 +1113,13 @@ private async Task RefreshFfCollections( private async Task RefreshKvCollections( ConfigurationClient client, List keyValueChanges, - Dictionary> watchedKvCollections, + Dictionary> watchedKvEtags, Uri endpoint, StringBuilder logDebugBuilder, StringBuilder logInfoBuilder, CancellationToken cancellationToken) { - bool selectedKvCollectionsChanged = await UpdateWatchedCollections(_options.KeyValueSelectors, watchedKvCollections, client, cancellationToken).ConfigureAwait(false); + bool selectedKvCollectionsChanged = await UpdateWatchedCollections(_options.KeyValueSelectors, watchedKvEtags, client, cancellationToken).ConfigureAwait(false); if (!selectedKvCollectionsChanged) { @@ -1130,7 +1127,7 @@ private async Task RefreshKvCollections( } else { - keyValueChanges.AddRange(await GetRefreshedCollections(_options.KeyValueSelectors, _mappedData, client, cancellationToken).ConfigureAwait(false)); + keyValueChanges.AddRange(await GetRefreshedCollections(_options.KeyValueSelectors, _mappedData.Keys, client, cancellationToken).ConfigureAwait(false)); logInfoBuilder.Append(LogHelper.BuildSelectedKeyValueCollectionsUpdatedMessage()); } @@ -1379,7 +1376,7 @@ innerException is SocketException || innerException is IOException; } - private IEnumerable GetCurrentKeyValueCollection(string key, IEnumerable existingKeys) + private IEnumerable GetExistingKeys(string key, IEnumerable existingKeys) { IEnumerable currentKeyValues; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index c23b8398..5227f814 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -70,6 +70,16 @@ public static async Task> GetNewMatchConditions(thi throw new ArgumentNullException(nameof(matchConditions)); } + if (keyValueSelector == null) + { + throw new ArgumentNullException(nameof(keyValueSelector)); + } + + if (pageableManager == null) + { + throw new ArgumentNullException(nameof(pageableManager)); + } + SettingSelector selector = new SettingSelector { KeyFilter = keyValueSelector.KeyFilter, From 69fb9ccb55f63d854fb7b45f7ac45c8eafc877c4 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 8 Jan 2025 13:19:08 -0800 Subject: [PATCH 60/72] call loadselected when refreshing collection, separate data for individual refresh --- .../AzureAppConfigurationProvider.cs | 313 ++++++++---------- .../ConfigurationClientExtensions.cs | 25 +- .../GetKeyValueChangeCollectionOptions.cs | 17 - .../KeyValueChange.cs | 2 - 4 files changed, 159 insertions(+), 198 deletions(-) delete mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 0e5ecab4..91d44f97 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -30,7 +30,9 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private readonly IConfigurationClientManager _configClientManager; private Uri _lastSuccessfulEndpoint; private AzureAppConfigurationOptions _options; - private Dictionary _mappedData; + private Dictionary _individualKvData; + private Dictionary _kvCollectionData; + private Dictionary _ffCollectionData; private Dictionary _watchedIndividualKvs = new Dictionary(); private Dictionary> _watchedKvEtags = new Dictionary>(); private Dictionary> _watchedFfEtags = new Dictionary>(); @@ -204,7 +206,9 @@ public async Task RefreshAsync(CancellationToken cancellationToken) bool registerAllIsRefreshable = utcNow >= _registerAllNextRefreshTime; // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. - if (_mappedData != null && + if (_individualKvData == null && + _kvCollectionData == null && + _ffCollectionData == null && !refreshableIndividualKvWatchers.Any() && !refreshableFfWatchers.Any() && !registerAllIsRefreshable && @@ -242,7 +246,7 @@ public async Task RefreshAsync(CancellationToken cancellationToken) } // Check if initial configuration load had failed - if (_mappedData == null) + if (_individualKvData == null && _kvCollectionData == null && _ffCollectionData == null) { if (InitializationCacheExpires < utcNow) { @@ -260,14 +264,20 @@ public async Task RefreshAsync(CancellationToken cancellationToken) Dictionary> watchedKvEtags = null; Dictionary watchedIndividualKvs = null; List keyValueChanges = null; - Dictionary data = null; + Dictionary individualKvData = null; + Dictionary kvCollectionData = null; + Dictionary ffCollectionData = null; + bool kvCollectionUpdated = false; + bool ffCollectionUpdated = false; bool refreshAll = false; StringBuilder logInfoBuilder = new StringBuilder(); StringBuilder logDebugBuilder = new StringBuilder(); await ExecuteWithFailOverPolicyAsync(clients, async (client) => { - data = null; + kvCollectionData = null; + ffCollectionData = null; + individualKvData = null; watchedIndividualKvs = null; watchedKvEtags = null; watchedFfEtags = null; @@ -293,9 +303,11 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => // Get feature flag changes watchedFfEtags = new Dictionary>(_watchedFfEtags); - await RefreshFfCollections( + ffCollectionData = new Dictionary(); + + ffCollectionUpdated = await RefreshFfCollections( client, - keyValueChanges, + ffCollectionData, refreshableFfWatchers, watchedFfEtags, endpoint, @@ -306,10 +318,13 @@ await RefreshFfCollections( else { // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true - data = new Dictionary(StringComparer.OrdinalIgnoreCase); - watchedKvEtags = await LoadSelected(client, data, _options.KeyValueSelectors, cancellationToken).ConfigureAwait(false); - watchedFfEtags = await LoadSelected(client, data, _options.FeatureFlagSelectors, cancellationToken).ConfigureAwait(false); - watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); + kvCollectionData = new Dictionary(); + ffCollectionData = new Dictionary(); + individualKvData = new Dictionary(); + + watchedKvEtags = await LoadSelected(client, kvCollectionData, _options.KeyValueSelectors, cancellationToken).ConfigureAwait(false); + watchedFfEtags = await LoadSelected(client, ffCollectionData, _options.FeatureFlagSelectors, cancellationToken).ConfigureAwait(false); + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, kvCollectionData, individualKvData, cancellationToken).ConfigureAwait(false); logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); return; } @@ -321,9 +336,11 @@ await RefreshFfCollections( if (registerAllIsRefreshable) { - await RefreshKvCollections( + kvCollectionData = new Dictionary(); + + kvCollectionUpdated = await RefreshKvCollections( client, - keyValueChanges, + kvCollectionData, watchedKvEtags, endpoint, logDebugBuilder, @@ -334,9 +351,11 @@ await RefreshKvCollections( // Get feature flag changes watchedFfEtags = new Dictionary>(_watchedFfEtags); - await RefreshFfCollections( + ffCollectionData = new Dictionary(); + + ffCollectionUpdated = await RefreshFfCollections( client, - keyValueChanges, + ffCollectionData, refreshableFfWatchers, watchedFfEtags, endpoint, @@ -357,11 +376,6 @@ await RefreshFfCollections( UpdateNextRefreshTime(changeWatcher); } - if (_options.RegisterAllEnabled && registerAllIsRefreshable) - { - _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); - } - foreach (KeyValueChange change in keyValueChanges) { KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); @@ -371,10 +385,7 @@ await RefreshFfCollections( ConfigurationSetting setting = change.Current; ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); - if (change.IsWatchedSetting) - { - watchedIndividualKvs[changeIdentifier] = settingCopy; - } + watchedIndividualKvs[changeIdentifier] = settingCopy; foreach (Func> func in _options.Mappers) { @@ -383,21 +394,18 @@ await RefreshFfCollections( if (setting == null) { - _mappedData.Remove(change.Key); + _individualKvData.Remove(change.Key); } else { - _mappedData[change.Key] = setting; + _individualKvData[change.Key] = setting; } } else if (change.ChangeType == KeyValueChangeType.Deleted) { - _mappedData.Remove(change.Key); + _individualKvData.Remove(change.Key); - if (change.IsWatchedSetting) - { - watchedIndividualKvs.Remove(changeIdentifier); - } + watchedIndividualKvs.Remove(changeIdentifier); } // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting @@ -415,9 +423,14 @@ await RefreshFfCollections( } } } - else + + if (refreshAll) { - _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); + _kvCollectionData = await MapConfigurationSettings(kvCollectionData).ConfigureAwait(false); + + _individualKvData = await MapConfigurationSettings(individualKvData).ConfigureAwait(false); + + _ffCollectionData = await MapConfigurationSettings(ffCollectionData).ConfigureAwait(false); // Invalidate all the cached KeyVault secrets foreach (IKeyValueAdapter adapter in _options.Adapters) @@ -431,8 +444,28 @@ await RefreshFfCollections( UpdateNextRefreshTime(changeWatcher); } } + else + { + if (_options.RegisterAllEnabled) + { + if (registerAllIsRefreshable) + { + _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); + } - if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any()) + if (kvCollectionUpdated) + { + _kvCollectionData = await MapConfigurationSettings(kvCollectionData).ConfigureAwait(false); + } + } + + if (ffCollectionUpdated) + { + _ffCollectionData = await MapConfigurationSettings(ffCollectionData).ConfigureAwait(false); + } + } + + if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any() || kvCollectionUpdated || ffCollectionUpdated) { _watchedIndividualKvs = watchedIndividualKvs; @@ -449,10 +482,13 @@ await RefreshFfCollections( { _logger.LogInformation(logInfoBuilder.ToString().Trim()); } + + Dictionary combinedData = GetCombinedData(_kvCollectionData, _individualKvData, _ffCollectionData); + // PrepareData makes calls to KeyVault and may throw exceptions. But, we still update watchers before // SetData because repeating appconfig calls (by not updating watchers) won't help anything for keyvault calls. // As long as adapter.NeedsRefresh is true, we will attempt to update keyvault again the next time RefreshAsync is called. - SetData(await PrepareData(_mappedData, cancellationToken).ConfigureAwait(false)); + SetData(await PrepareData(combinedData, cancellationToken).ConfigureAwait(false)); } } finally @@ -730,7 +766,9 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { - Dictionary data = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary kvCollectionData = new Dictionary(); + Dictionary ffCollectionData = new Dictionary(); + Dictionary individualKvData = new Dictionary(); Dictionary> watchedKvEtags = null; Dictionary> watchedFfEtags = null; Dictionary watchedIndividualKvs = null; @@ -741,21 +779,22 @@ await ExecuteWithFailOverPolicyAsync( { watchedKvEtags = await LoadSelected( client, - data, + kvCollectionData, _options.KeyValueSelectors, cancellationToken) .ConfigureAwait(false); watchedFfEtags = await LoadSelected( client, - data, + ffCollectionData, _options.FeatureFlagSelectors, cancellationToken) .ConfigureAwait(false); watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh( client, - data, + kvCollectionData, + individualKvData, cancellationToken) .ConfigureAwait(false); }, @@ -773,7 +812,7 @@ await ExecuteWithFailOverPolicyAsync( _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); } - if (data != null) + if (watchedKvEtags != null && watchedFfEtags != null && watchedIndividualKvs != null) { // Invalidate all the cached KeyVault secrets foreach (IKeyValueAdapter adapter in _options.Adapters) @@ -781,13 +820,22 @@ await ExecuteWithFailOverPolicyAsync( adapter.OnChangeDetected(); } - Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); + kvCollectionData = await MapConfigurationSettings(kvCollectionData).ConfigureAwait(false); + + individualKvData = await MapConfigurationSettings(individualKvData).ConfigureAwait(false); + + ffCollectionData = await MapConfigurationSettings(ffCollectionData).ConfigureAwait(false); + + Dictionary combinedData = GetCombinedData(kvCollectionData, individualKvData, ffCollectionData); + + SetData(await PrepareData(combinedData, cancellationToken).ConfigureAwait(false)); _watchedIndividualKvs = watchedIndividualKvs; _watchedKvEtags = watchedKvEtags; _watchedFfEtags = watchedFfEtags; - _mappedData = mappedData; + _kvCollectionData = kvCollectionData; + _individualKvData = individualKvData; + _ffCollectionData = ffCollectionData; } } @@ -797,7 +845,7 @@ private async Task>> L IEnumerable selectors, CancellationToken cancellationToken) { - var watchedCollections = new Dictionary>(); + var watchedEtags = new Dictionary>(); foreach (KeyValueSelector loadOption in selectors) { @@ -834,7 +882,7 @@ await CallWithRequestTracing(async () => } }).ConfigureAwait(false); - watchedCollections[loadOption] = matchConditions; + watchedEtags[loadOption] = matchConditions; } else { @@ -868,10 +916,14 @@ await CallWithRequestTracing(async () => } } - return watchedCollections; + return watchedEtags; } - private async Task> LoadKeyValuesRegisteredForRefresh(ConfigurationClient client, IDictionary existingSettings, CancellationToken cancellationToken) + private async Task> LoadKeyValuesRegisteredForRefresh( + ConfigurationClient client, + IDictionary existingSettings, + IDictionary individualKvData, + CancellationToken cancellationToken) { var watchedIndividualKvs = new Dictionary(); @@ -887,6 +939,7 @@ private async Task> LoadKey && watchedKeyLabel.Equals(new KeyValueIdentifier(loadedKv.Key, loadedKv.Label))) { watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); + individualKvData[watchedKey] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); continue; } @@ -905,108 +958,13 @@ private async Task> LoadKey if (watchedKv != null) { watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); - existingSettings[watchedKey] = watchedKv; + individualKvData[watchedKey] = watchedKv; } } return watchedIndividualKvs; } - private async Task> GetRefreshedCollections( - IEnumerable selectors, - IEnumerable existingDataKeys, - ConfigurationClient client, - CancellationToken cancellationToken) - { - var keyValueChanges = new List(); - - var existingKeys = new HashSet(); - - var loadedSettings = new HashSet(); - - AsyncPageable pageableSettings; - - foreach (KeyValueSelector loadOption in selectors) - { - if (string.IsNullOrEmpty(loadOption.SnapshotName)) - { - SettingSelector selector = new SettingSelector() - { - KeyFilter = loadOption.KeyFilter, - LabelFilter = loadOption.LabelFilter - }; - - pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); - } - else - { - ConfigurationSnapshot snapshot; - - try - { - snapshot = await client.GetSnapshotAsync(loadOption.SnapshotName).ConfigureAwait(false); - } - catch (RequestFailedException rfe) when (rfe.Status == (int)HttpStatusCode.NotFound) - { - throw new InvalidOperationException($"Could not find snapshot with name '{loadOption.SnapshotName}'.", rfe); - } - - if (snapshot.SnapshotComposition != SnapshotComposition.Key) - { - throw new InvalidOperationException($"{nameof(snapshot.SnapshotComposition)} for the selected snapshot with name '{snapshot.Name}' must be 'key', found '{snapshot.SnapshotComposition}'."); - } - - pageableSettings = client.GetConfigurationSettingsForSnapshotAsync( - loadOption.SnapshotName, - cancellationToken); - } - - await CallWithRequestTracing(async () => - { - await foreach (Page page in _options.PageableManager.GetPages(pageableSettings).ConfigureAwait(false)) - { - foreach (ConfigurationSetting setting in page.Values) - { - keyValueChanges.Add(new KeyValueChange() - { - Key = setting.Key, - Label = setting.Label, - Current = setting, - ChangeType = KeyValueChangeType.Modified, - IsWatchedSetting = false - }); - - loadedSettings.Add(setting.Key); - } - } - }).ConfigureAwait(false); - - IEnumerable existingKeysList = GetExistingKeys(loadOption.KeyFilter, existingDataKeys); - - foreach (string key in existingKeysList) - { - existingKeys.Add(key); - } - } - - foreach (string key in existingKeys) - { - if (!loadedSettings.Contains(key)) - { - keyValueChanges.Add(new KeyValueChange() - { - Key = key, - Label = null, - Current = null, - ChangeType = KeyValueChangeType.Deleted, - IsWatchedSetting = false - }); - } - } - - return keyValueChanges; - } - private async Task RefreshIndividualKvWatchers( ConfigurationClient client, List keyValueChanges, @@ -1031,8 +989,6 @@ private async Task RefreshIndividualKvWatchers( { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, async () => change = await client.GetKeyValueChange(watchedKv, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - - change.IsWatchedSetting = true; } else { @@ -1055,8 +1011,7 @@ await CallWithRequestTracing( Key = watchedKv.Key, Label = watchedKv.Label.NormalizeNull(), Current = watchedKv, - ChangeType = KeyValueChangeType.Modified, - IsWatchedSetting = true + ChangeType = KeyValueChangeType.Modified }; } } @@ -1082,9 +1037,9 @@ await CallWithRequestTracing( return false; } - private async Task RefreshFfCollections( + private async Task RefreshFfCollections( ConfigurationClient client, - List keyValueChanges, + Dictionary data, IEnumerable refreshableFfWatchers, Dictionary> watchedFfEtags, Uri endpoint, @@ -1092,45 +1047,49 @@ private async Task RefreshFfCollections( StringBuilder logInfoBuilder, CancellationToken cancellationToken) { - bool featureFlagCollectionsChanged = await UpdateWatchedCollections( + bool ffCollectionsChanged = await UpdateWatchedCollections( refreshableFfWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), watchedFfEtags, client, cancellationToken).ConfigureAwait(false); - if (!featureFlagCollectionsChanged) + if (!ffCollectionsChanged) { logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); } else { - keyValueChanges.AddRange(await GetRefreshedCollections(_options.FeatureFlagSelectors, _mappedData.Keys, client, cancellationToken).ConfigureAwait(false)); + watchedFfEtags = await LoadSelected(client, data, _options.FeatureFlagSelectors, cancellationToken).ConfigureAwait(false); logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); } + + return ffCollectionsChanged; } - private async Task RefreshKvCollections( + private async Task RefreshKvCollections( ConfigurationClient client, - List keyValueChanges, + Dictionary data, Dictionary> watchedKvEtags, Uri endpoint, StringBuilder logDebugBuilder, StringBuilder logInfoBuilder, CancellationToken cancellationToken) { - bool selectedKvCollectionsChanged = await UpdateWatchedCollections(_options.KeyValueSelectors, watchedKvEtags, client, cancellationToken).ConfigureAwait(false); + bool kvCollectionsChanged = await UpdateWatchedCollections(_options.KeyValueSelectors, watchedKvEtags, client, cancellationToken).ConfigureAwait(false); - if (!selectedKvCollectionsChanged) + if (!kvCollectionsChanged) { logDebugBuilder.AppendLine(LogHelper.BuildSelectedKeyValueCollectionsUnchangedMessage(endpoint.ToString())); } else { - keyValueChanges.AddRange(await GetRefreshedCollections(_options.KeyValueSelectors, _mappedData.Keys, client, cancellationToken).ConfigureAwait(false)); + watchedKvEtags = await LoadSelected(client, data, _options.KeyValueSelectors, cancellationToken).ConfigureAwait(false); logInfoBuilder.Append(LogHelper.BuildSelectedKeyValueCollectionsUpdatedMessage()); } + + return kvCollectionsChanged; } private void SetData(IDictionary data) @@ -1467,31 +1426,53 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) _configClientBackoffs[endpoint] = clientBackoffStatus; } - private async Task UpdateWatchedCollections(IEnumerable selectors, Dictionary> watchedCollections, ConfigurationClient client, CancellationToken cancellationToken) + private async Task UpdateWatchedCollections(IEnumerable selectors, Dictionary> watchedEtags, ConfigurationClient client, CancellationToken cancellationToken) { bool watchedCollectionsChanged = false; foreach (KeyValueSelector selector in selectors) { - IEnumerable newMatchConditions = null; - - if (watchedCollections.TryGetValue(selector, out IEnumerable matchConditions)) + if (watchedEtags.TryGetValue(selector, out IEnumerable matchConditions)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => newMatchConditions = await client.GetNewMatchConditions(selector, matchConditions, _options.PageableManager, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + async () => watchedCollectionsChanged = await client.HasWatchedCollectionsChanged(selector, matchConditions, _options.PageableManager, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } - if (newMatchConditions != null) + if (watchedCollectionsChanged) { - watchedCollections[selector] = newMatchConditions; - - watchedCollectionsChanged = true; + return watchedCollectionsChanged; } } return watchedCollectionsChanged; } + private Dictionary GetCombinedData( + Dictionary kvCollectionData, + Dictionary individualKvData, + Dictionary ffCollectionData) + { + // Merge all key-value and feature flag data + Dictionary combinedData = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (KeyValuePair kvp in kvCollectionData) + { + combinedData[kvp.Key] = kvp.Value; + } + + foreach (KeyValuePair kvp in individualKvData) + { + combinedData[kvp.Key] = kvp.Value; + } + + foreach (KeyValuePair kvp in ffCollectionData) + { + combinedData[kvp.Key] = kvp.Value; + } + + return combinedData; + } + public void Dispose() { (_configClientManager as ConfigurationClientManager)?.Dispose(); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index 5227f814..ab7aa029 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -63,7 +63,7 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } - public static async Task> GetNewMatchConditions(this ConfigurationClient client, KeyValueSelector keyValueSelector, IEnumerable matchConditions, ConfigurationSettingPageableManager pageableManager, CancellationToken cancellationToken) + public static async Task HasWatchedCollectionsChanged(this ConfigurationClient client, KeyValueSelector keyValueSelector, IEnumerable matchConditions, ConfigurationSettingPageableManager pageableManager, CancellationToken cancellationToken) { if (matchConditions == null) { @@ -75,6 +75,11 @@ public static async Task> GetNewMatchConditions(thi throw new ArgumentNullException(nameof(keyValueSelector)); } + if (keyValueSelector.SnapshotName != null) + { + throw new ArgumentException("Cannot check snapshot for changes.", $"{nameof(keyValueSelector)}.{nameof(keyValueSelector.SnapshotName)}"); + } + if (pageableManager == null) { throw new ArgumentNullException(nameof(pageableManager)); @@ -86,10 +91,6 @@ public static async Task> GetNewMatchConditions(thi LabelFilter = keyValueSelector.LabelFilter }; - bool hasCollectionChanged = false; - - var newMatchConditions = new List(); - AsyncPageable pageable = client.GetConfigurationSettingsAsync(selector, cancellationToken); using IEnumerator existingMatchConditionsEnumerator = matchConditions.GetEnumerator(); @@ -105,24 +106,22 @@ public static async Task> GetNewMatchConditions(thi Response response = page.GetRawResponse(); - newMatchConditions.Add(new MatchConditions { IfNoneMatch = serverEtag }); - - // Set hasCollectionChanged to true if the lists of etags are different, and continue iterating to get all of newMatchConditions + // Return true if the lists of etags are different if ((!existingMatchConditionsEnumerator.MoveNext() || !existingMatchConditionsEnumerator.Current.IfNoneMatch.Equals(serverEtag)) && response.Status == (int)HttpStatusCode.OK) { - hasCollectionChanged = true; + return true; } } - // Need to check if pages were deleted since hasCollectionsChanged wouldn't have been set - if (hasCollectionChanged || existingMatchConditionsEnumerator.MoveNext()) + // Need to check if pages were deleted and no change was found within the new shorter list of match conditions + if (existingMatchConditionsEnumerator.MoveNext()) { - return newMatchConditions; + return true; } - return null; + return false; } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs deleted file mode 100644 index 89d5e3f8..00000000 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/GetKeyValueChangeCollectionOptions.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. -// -using Azure; -using Azure.Data.AppConfiguration; -using System.Collections.Generic; - -namespace Microsoft.Extensions.Configuration.AzureAppConfiguration -{ - internal class GetKeyValueChangeCollectionOptions - { - public SettingSelector Selector { get; set; } - public IEnumerable MatchConditions { get; set; } - public bool RequestTracingEnabled { get; set; } - public RequestTracingOptions RequestTracingOptions { get; set; } - } -} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/KeyValueChange.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/KeyValueChange.cs index 5f319602..2286016d 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/KeyValueChange.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/KeyValueChange.cs @@ -23,7 +23,5 @@ internal struct KeyValueChange public ConfigurationSetting Current { get; set; } public ConfigurationSetting Previous { get; set; } - - public bool IsWatchedSetting { get; set; } } } From 1155b788eda1dcbf2c5025d29f7df63979597be1 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Fri, 10 Jan 2025 12:50:37 -0800 Subject: [PATCH 61/72] in progress change to registerall include ff --- .../AzureAppConfigurationProvider.cs | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 91d44f97..7f035ba9 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -30,9 +30,7 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private readonly IConfigurationClientManager _configClientManager; private Uri _lastSuccessfulEndpoint; private AzureAppConfigurationOptions _options; - private Dictionary _individualKvData; - private Dictionary _kvCollectionData; - private Dictionary _ffCollectionData; + private Dictionary _mappedData; private Dictionary _watchedIndividualKvs = new Dictionary(); private Dictionary> _watchedKvEtags = new Dictionary>(); private Dictionary> _watchedFfEtags = new Dictionary>(); @@ -206,9 +204,7 @@ public async Task RefreshAsync(CancellationToken cancellationToken) bool registerAllIsRefreshable = utcNow >= _registerAllNextRefreshTime; // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. - if (_individualKvData == null && - _kvCollectionData == null && - _ffCollectionData == null && + if (_mappedData != null && !refreshableIndividualKvWatchers.Any() && !refreshableFfWatchers.Any() && !registerAllIsRefreshable && @@ -246,7 +242,7 @@ public async Task RefreshAsync(CancellationToken cancellationToken) } // Check if initial configuration load had failed - if (_individualKvData == null && _kvCollectionData == null && _ffCollectionData == null) + if (_mappedData == null) { if (InitializationCacheExpires < utcNow) { @@ -264,8 +260,7 @@ public async Task RefreshAsync(CancellationToken cancellationToken) Dictionary> watchedKvEtags = null; Dictionary watchedIndividualKvs = null; List keyValueChanges = null; - Dictionary individualKvData = null; - Dictionary kvCollectionData = null; + Dictionary data = null; Dictionary ffCollectionData = null; bool kvCollectionUpdated = false; bool ffCollectionUpdated = false; @@ -275,9 +270,8 @@ public async Task RefreshAsync(CancellationToken cancellationToken) await ExecuteWithFailOverPolicyAsync(clients, async (client) => { - kvCollectionData = null; + data = null; ffCollectionData = null; - individualKvData = null; watchedIndividualKvs = null; watchedKvEtags = null; watchedFfEtags = null; @@ -318,13 +312,11 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => else { // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true - kvCollectionData = new Dictionary(); ffCollectionData = new Dictionary(); - individualKvData = new Dictionary(); + data = new Dictionary(); - watchedKvEtags = await LoadSelected(client, kvCollectionData, _options.KeyValueSelectors, cancellationToken).ConfigureAwait(false); - watchedFfEtags = await LoadSelected(client, ffCollectionData, _options.FeatureFlagSelectors, cancellationToken).ConfigureAwait(false); - watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, kvCollectionData, individualKvData, cancellationToken).ConfigureAwait(false); + watchedKvEtags = await LoadSelected(client, data, _options.Selectors, cancellationToken).ConfigureAwait(false); + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); return; } @@ -922,7 +914,6 @@ await CallWithRequestTracing(async () => private async Task> LoadKeyValuesRegisteredForRefresh( ConfigurationClient client, IDictionary existingSettings, - IDictionary individualKvData, CancellationToken cancellationToken) { var watchedIndividualKvs = new Dictionary(); @@ -939,7 +930,6 @@ private async Task> LoadKey && watchedKeyLabel.Equals(new KeyValueIdentifier(loadedKv.Key, loadedKv.Label))) { watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); - individualKvData[watchedKey] = new ConfigurationSetting(loadedKv.Key, loadedKv.Value, loadedKv.Label, loadedKv.ETag); continue; } @@ -958,7 +948,7 @@ private async Task> LoadKey if (watchedKv != null) { watchedIndividualKvs[watchedKeyLabel] = new ConfigurationSetting(watchedKv.Key, watchedKv.Value, watchedKv.Label, watchedKv.ETag); - individualKvData[watchedKey] = watchedKv; + existingSettings[watchedKey] = watchedKv; } } From 9ed77afef26812ce962fd72a732059e5ffd04edd Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Fri, 10 Jan 2025 16:12:53 -0800 Subject: [PATCH 62/72] fix load order --- .../AzureAppConfigurationOptions.cs | 27 +- .../AzureAppConfigurationProvider.cs | 278 +++++++----------- .../FeatureManagement/FeatureFlagOptions.cs | 3 +- .../Models/KeyValueSelector.cs | 5 + 4 files changed, 131 insertions(+), 182 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 89850e93..36c80e8a 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -27,8 +27,7 @@ public class AzureAppConfigurationOptions private List _ffWatchers = new List(); private List _adapters; private List>> _mappers = new List>>(); - private List _kvSelectors; - private List _featureFlagSelectors = new List(); + private List _selectors; private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); private bool _selectCalled = false; @@ -64,14 +63,9 @@ public class AzureAppConfigurationOptions internal TokenCredential Credential { get; private set; } /// - /// Key Value selectors specified by user. + /// Key Value and Feature Flag selectors specified by user. /// - internal IEnumerable KeyValueSelectors => _kvSelectors; - - /// - /// Feature Flag selectors specified by user. - /// - internal IEnumerable FeatureFlagSelectors => _featureFlagSelectors; + internal IEnumerable Selectors => _selectors; /// /// Indicates if was called. @@ -167,7 +161,7 @@ public AzureAppConfigurationOptions() }; // Adds the default query to App Configuration if and are never called. - _kvSelectors = new List { new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null } }; + _selectors = new List { new KeyValueSelector { KeyFilter = KeyFilter.Any, LabelFilter = LabelFilter.Null } }; } /// @@ -209,12 +203,12 @@ public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter if (!_selectCalled) { - _kvSelectors.Clear(); + _selectors.Clear(); _selectCalled = true; } - _kvSelectors.AppendUnique(new KeyValueSelector + _selectors.AppendUnique(new KeyValueSelector { KeyFilter = keyFilter, LabelFilter = labelFilter @@ -237,12 +231,12 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) if (!_selectCalled) { - _kvSelectors.Clear(); + _selectors.Clear(); _selectCalled = true; } - _kvSelectors.AppendUnique(new KeyValueSelector + _selectors.AppendUnique(new KeyValueSelector { SnapshotName = name }); @@ -278,13 +272,14 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c options.FeatureFlagSelectors.Add(new KeyValueSelector { KeyFilter = FeatureManagementConstants.FeatureFlagMarker + "*", - LabelFilter = string.IsNullOrWhiteSpace(options.Label) ? LabelFilter.Null : options.Label + LabelFilter = string.IsNullOrWhiteSpace(options.Label) ? LabelFilter.Null : options.Label, + IsFeatureFlagSelector = true }); } foreach (KeyValueSelector featureFlagSelector in options.FeatureFlagSelectors) { - _featureFlagSelectors.AppendUnique(featureFlagSelector); + _selectors.AppendUnique(featureFlagSelector); _ffWatchers.AppendUnique(new KeyValueWatcher { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 7f035ba9..38775c0d 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -262,7 +262,7 @@ public async Task RefreshAsync(CancellationToken cancellationToken) List keyValueChanges = null; Dictionary data = null; Dictionary ffCollectionData = null; - bool kvCollectionUpdated = false; + bool collectionsUpdated = false; bool ffCollectionUpdated = false; bool refreshAll = false; StringBuilder logInfoBuilder = new StringBuilder(); @@ -276,6 +276,8 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => watchedKvEtags = null; watchedFfEtags = null; keyValueChanges = new List(); + collectionsUpdated = false; + ffCollectionUpdated = false; refreshAll = false; Uri endpoint = _configClientManager.GetEndpointForClient(client); logDebugBuilder.Clear(); @@ -297,25 +299,37 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => // Get feature flag changes watchedFfEtags = new Dictionary>(_watchedFfEtags); - ffCollectionData = new Dictionary(); - - ffCollectionUpdated = await RefreshFfCollections( - client, - ffCollectionData, - refreshableFfWatchers, + ffCollectionUpdated = await UpdateWatchedCollections( + refreshableFfWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), watchedFfEtags, - endpoint, - logDebugBuilder, - logInfoBuilder, + client, cancellationToken).ConfigureAwait(false); + + if (!ffCollectionUpdated) + { + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); + } + else + { + watchedFfEtags = new Dictionary>(); + + ffCollectionData = await LoadSelected( + client, + new Dictionary>(), + watchedFfEtags, + _options.Selectors.Where(selector => selector.IsFeatureFlagSelector), + cancellationToken).ConfigureAwait(false); + + logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); + } } else { // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true - ffCollectionData = new Dictionary(); - data = new Dictionary(); + watchedKvEtags = new Dictionary>(); + watchedFfEtags = new Dictionary>(); - watchedKvEtags = await LoadSelected(client, data, _options.Selectors, cancellationToken).ConfigureAwait(false); + data = await LoadSelected(client, watchedKvEtags, watchedFfEtags, _options.Selectors, cancellationToken).ConfigureAwait(false); watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); return; @@ -328,32 +342,57 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => if (registerAllIsRefreshable) { - kvCollectionData = new Dictionary(); - - kvCollectionUpdated = await RefreshKvCollections( - client, - kvCollectionData, + collectionsUpdated = await UpdateWatchedCollections( + _options.Selectors.Where(selector => !selector.IsFeatureFlagSelector).ToList(), watchedKvEtags, - endpoint, - logDebugBuilder, - logInfoBuilder, + client, cancellationToken).ConfigureAwait(false); + + if (!collectionsUpdated) + { + logDebugBuilder.AppendLine(LogHelper.BuildSelectedKeyValueCollectionsUnchangedMessage(endpoint.ToString())); + } + else + { + watchedKvEtags = new Dictionary>(); + + watchedFfEtags = new Dictionary>(); + + data = await LoadSelected(client, watchedKvEtags, watchedFfEtags, _options.Selectors, cancellationToken).ConfigureAwait(false); + + logInfoBuilder.Append(LogHelper.BuildSelectedKeyValueCollectionsUpdatedMessage()); + } } - // Get feature flag changes - watchedFfEtags = new Dictionary>(_watchedFfEtags); + if (!collectionsUpdated) + { + // Get feature flag changes + watchedFfEtags = new Dictionary>(_watchedFfEtags); - ffCollectionData = new Dictionary(); + ffCollectionUpdated = await UpdateWatchedCollections( + refreshableFfWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), + watchedFfEtags, + client, + cancellationToken).ConfigureAwait(false); - ffCollectionUpdated = await RefreshFfCollections( - client, - ffCollectionData, - refreshableFfWatchers, - watchedFfEtags, - endpoint, - logDebugBuilder, - logInfoBuilder, - cancellationToken).ConfigureAwait(false); + if (!ffCollectionUpdated) + { + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); + } + else + { + watchedFfEtags = new Dictionary>(); + + ffCollectionData = await LoadSelected( + client, + new Dictionary>(), + watchedFfEtags, + _options.Selectors.Where(selector => selector.IsFeatureFlagSelector), + cancellationToken).ConfigureAwait(false); + + logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); + } + } } }, cancellationToken) @@ -386,16 +425,16 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => if (setting == null) { - _individualKvData.Remove(change.Key); + _mappedData.Remove(change.Key); } else { - _individualKvData[change.Key] = setting; + _mappedData[change.Key] = setting; } } else if (change.ChangeType == KeyValueChangeType.Deleted) { - _individualKvData.Remove(change.Key); + _mappedData.Remove(change.Key); watchedIndividualKvs.Remove(changeIdentifier); } @@ -418,11 +457,7 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => if (refreshAll) { - _kvCollectionData = await MapConfigurationSettings(kvCollectionData).ConfigureAwait(false); - - _individualKvData = await MapConfigurationSettings(individualKvData).ConfigureAwait(false); - - _ffCollectionData = await MapConfigurationSettings(ffCollectionData).ConfigureAwait(false); + _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); // Invalidate all the cached KeyVault secrets foreach (IKeyValueAdapter adapter in _options.Adapters) @@ -445,19 +480,24 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); } - if (kvCollectionUpdated) + if (collectionsUpdated) { - _kvCollectionData = await MapConfigurationSettings(kvCollectionData).ConfigureAwait(false); + _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); } } if (ffCollectionUpdated) { - _ffCollectionData = await MapConfigurationSettings(ffCollectionData).ConfigureAwait(false); + Dictionary mappedFfData = await MapConfigurationSettings(ffCollectionData).ConfigureAwait(false); + + foreach (KeyValuePair kvp in mappedFfData) + { + _mappedData[kvp.Key] = kvp.Value; + } } } - if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any() || kvCollectionUpdated || ffCollectionUpdated) + if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any() || collectionsUpdated || ffCollectionUpdated) { _watchedIndividualKvs = watchedIndividualKvs; @@ -475,12 +515,10 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => _logger.LogInformation(logInfoBuilder.ToString().Trim()); } - Dictionary combinedData = GetCombinedData(_kvCollectionData, _individualKvData, _ffCollectionData); - // PrepareData makes calls to KeyVault and may throw exceptions. But, we still update watchers before // SetData because repeating appconfig calls (by not updating watchers) won't help anything for keyvault calls. // As long as adapter.NeedsRefresh is true, we will attempt to update keyvault again the next time RefreshAsync is called. - SetData(await PrepareData(combinedData, cancellationToken).ConfigureAwait(false)); + SetData(await PrepareData(_mappedData, cancellationToken).ConfigureAwait(false)); } } finally @@ -758,35 +796,26 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { - Dictionary kvCollectionData = new Dictionary(); - Dictionary ffCollectionData = new Dictionary(); - Dictionary individualKvData = new Dictionary(); - Dictionary> watchedKvEtags = null; - Dictionary> watchedFfEtags = null; + Dictionary data = null; + Dictionary> watchedKvEtags = new Dictionary>(); + Dictionary> watchedFfEtags = new Dictionary>(); Dictionary watchedIndividualKvs = null; await ExecuteWithFailOverPolicyAsync( clients, async (client) => { - watchedKvEtags = await LoadSelected( + data = await LoadSelected( client, - kvCollectionData, - _options.KeyValueSelectors, - cancellationToken) - .ConfigureAwait(false); - - watchedFfEtags = await LoadSelected( - client, - ffCollectionData, - _options.FeatureFlagSelectors, + watchedKvEtags, + watchedFfEtags, + _options.Selectors, cancellationToken) .ConfigureAwait(false); watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh( client, - kvCollectionData, - individualKvData, + data, cancellationToken) .ConfigureAwait(false); }, @@ -804,7 +833,7 @@ await ExecuteWithFailOverPolicyAsync( _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); } - if (watchedKvEtags != null && watchedFfEtags != null && watchedIndividualKvs != null) + if (data != null) { // Invalidate all the cached KeyVault secrets foreach (IKeyValueAdapter adapter in _options.Adapters) @@ -812,32 +841,25 @@ await ExecuteWithFailOverPolicyAsync( adapter.OnChangeDetected(); } - kvCollectionData = await MapConfigurationSettings(kvCollectionData).ConfigureAwait(false); - - individualKvData = await MapConfigurationSettings(individualKvData).ConfigureAwait(false); + Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - ffCollectionData = await MapConfigurationSettings(ffCollectionData).ConfigureAwait(false); - - Dictionary combinedData = GetCombinedData(kvCollectionData, individualKvData, ffCollectionData); - - SetData(await PrepareData(combinedData, cancellationToken).ConfigureAwait(false)); + SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); _watchedIndividualKvs = watchedIndividualKvs; _watchedKvEtags = watchedKvEtags; _watchedFfEtags = watchedFfEtags; - _kvCollectionData = kvCollectionData; - _individualKvData = individualKvData; - _ffCollectionData = ffCollectionData; + _mappedData = mappedData; } } - private async Task>> LoadSelected( + private async Task> LoadSelected( ConfigurationClient client, - Dictionary existingData, + Dictionary> watchedKvEtags, + Dictionary> watchedFfEtags, IEnumerable selectors, CancellationToken cancellationToken) { - var watchedEtags = new Dictionary>(); + Dictionary data = new Dictionary(); foreach (KeyValueSelector loadOption in selectors) { @@ -867,14 +889,21 @@ await CallWithRequestTracing(async () => foreach (ConfigurationSetting setting in page.Values) { - existingData[setting.Key] = setting; + data[setting.Key] = setting; } matchConditions.Add(new MatchConditions { IfNoneMatch = serverEtag }); } }).ConfigureAwait(false); - watchedEtags[loadOption] = matchConditions; + if (loadOption.IsFeatureFlagSelector) + { + watchedFfEtags[loadOption] = matchConditions; + } + else + { + watchedKvEtags[loadOption] = matchConditions; + } } else { @@ -902,13 +931,13 @@ await CallWithRequestTracing(async () => { await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) { - existingData[setting.Key] = setting; + data[setting.Key] = setting; } }).ConfigureAwait(false); } } - return watchedEtags; + return data; } private async Task> LoadKeyValuesRegisteredForRefresh( @@ -1027,61 +1056,6 @@ await CallWithRequestTracing( return false; } - private async Task RefreshFfCollections( - ConfigurationClient client, - Dictionary data, - IEnumerable refreshableFfWatchers, - Dictionary> watchedFfEtags, - Uri endpoint, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - CancellationToken cancellationToken) - { - bool ffCollectionsChanged = await UpdateWatchedCollections( - refreshableFfWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), - watchedFfEtags, - client, - cancellationToken).ConfigureAwait(false); - - if (!ffCollectionsChanged) - { - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); - } - else - { - watchedFfEtags = await LoadSelected(client, data, _options.FeatureFlagSelectors, cancellationToken).ConfigureAwait(false); - - logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); - } - - return ffCollectionsChanged; - } - - private async Task RefreshKvCollections( - ConfigurationClient client, - Dictionary data, - Dictionary> watchedKvEtags, - Uri endpoint, - StringBuilder logDebugBuilder, - StringBuilder logInfoBuilder, - CancellationToken cancellationToken) - { - bool kvCollectionsChanged = await UpdateWatchedCollections(_options.KeyValueSelectors, watchedKvEtags, client, cancellationToken).ConfigureAwait(false); - - if (!kvCollectionsChanged) - { - logDebugBuilder.AppendLine(LogHelper.BuildSelectedKeyValueCollectionsUnchangedMessage(endpoint.ToString())); - } - else - { - watchedKvEtags = await LoadSelected(client, data, _options.KeyValueSelectors, cancellationToken).ConfigureAwait(false); - - logInfoBuilder.Append(LogHelper.BuildSelectedKeyValueCollectionsUpdatedMessage()); - } - - return kvCollectionsChanged; - } - private void SetData(IDictionary data) { // Set the application data for the configuration provider @@ -1437,32 +1411,6 @@ await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Wa return watchedCollectionsChanged; } - private Dictionary GetCombinedData( - Dictionary kvCollectionData, - Dictionary individualKvData, - Dictionary ffCollectionData) - { - // Merge all key-value and feature flag data - Dictionary combinedData = new Dictionary(StringComparer.OrdinalIgnoreCase); - - foreach (KeyValuePair kvp in kvCollectionData) - { - combinedData[kvp.Key] = kvp.Value; - } - - foreach (KeyValuePair kvp in individualKvData) - { - combinedData[kvp.Key] = kvp.Value; - } - - foreach (KeyValuePair kvp in ffCollectionData) - { - combinedData[kvp.Key] = kvp.Value; - } - - return combinedData; - } - public void Dispose() { (_configClientManager as ConfigurationClientManager)?.Dispose(); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs index 1e8beae6..26390762 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureFlagOptions.cs @@ -102,7 +102,8 @@ public FeatureFlagOptions Select(string featureFlagFilter, string labelFilter = FeatureFlagSelectors.AppendUnique(new KeyValueSelector { KeyFilter = featureFlagPrefix, - LabelFilter = labelFilter + LabelFilter = labelFilter, + IsFeatureFlagSelector = true }); return this; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs index 5491d04d..54bda1a4 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs @@ -24,6 +24,11 @@ public class KeyValueSelector /// public string SnapshotName { get; set; } + /// + /// A boolean that signifies whether this selector is intended to select feature flags. + /// + public bool IsFeatureFlagSelector { get; set; } + /// /// Determines whether the specified object is equal to the current object. /// From cd62a68949aae584f68f95cb9270dc8971fd2398 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Mon, 13 Jan 2025 14:15:06 -0800 Subject: [PATCH 63/72] fix comments, rename logging constants to match new behavior --- .../AzureAppConfigurationOptions.cs | 2 +- .../AzureAppConfigurationProvider.cs | 2 +- .../Constants/LoggingConstants.cs | 2 +- .../LogHelper.cs | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 36c80e8a..de8d533f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -63,7 +63,7 @@ public class AzureAppConfigurationOptions internal TokenCredential Credential { get; private set; } /// - /// Key Value and Feature Flag selectors specified by user. + /// A collection of specified by user. /// internal IEnumerable Selectors => _selectors; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 38775c0d..ef46f227 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -360,7 +360,7 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => data = await LoadSelected(client, watchedKvEtags, watchedFfEtags, _options.Selectors, cancellationToken).ConfigureAwait(false); - logInfoBuilder.Append(LogHelper.BuildSelectedKeyValueCollectionsUpdatedMessage()); + logInfoBuilder.Append(LogHelper.BuildSelectedKeyValuesAndFeatureFlagsUpdatedMessage()); } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs index 69200ad4..20560be4 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs @@ -29,7 +29,7 @@ internal class LoggingConstants public const string RefreshKeyValueSettingUpdated = "Setting updated."; public const string RefreshKeyVaultSettingUpdated = "Setting updated from Key Vault."; public const string RefreshFeatureFlagsUpdated = "Feature flags updated."; - public const string RefreshSelectedKeyValueCollectionsUpdated = "Selected key-value collections updated."; + public const string RefreshSelectedKeyValuesAndFeatureFlagsUpdated = "Selected key-value collections and feature flags updated."; // Other public const string RefreshSkippedNoClientAvailable = "Refresh skipped because no endpoint is accessible."; diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs index 24cbaaf1..4f999406 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/LogHelper.cs @@ -35,9 +35,9 @@ public static string BuildSelectedKeyValueCollectionsUnchangedMessage(string end return $"{LoggingConstants.RefreshSelectedKeyValueCollectionsUnchanged} Endpoint:'{endpoint?.TrimEnd('/')}'"; } - public static string BuildSelectedKeyValueCollectionsUpdatedMessage() + public static string BuildSelectedKeyValuesAndFeatureFlagsUpdatedMessage() { - return LoggingConstants.RefreshSelectedKeyValueCollectionsUpdated; + return LoggingConstants.RefreshSelectedKeyValuesAndFeatureFlagsUpdated; } public static string BuildKeyVaultSecretReadMessage(string key, string label) From beab2374176b8219766006e0c7675f3f6540b417 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Tue, 14 Jan 2025 16:25:48 -0800 Subject: [PATCH 64/72] pr comments, refactor refreshasync --- .../AzureAppConfigurationOptions.cs | 8 +- .../AzureAppConfigurationProvider.cs | 374 ++++++++---------- .../ConfigurationSettingPageableManager.cs | 19 - .../ConfigurationClientExtensions.cs | 25 +- .../IPageableConfigurationSettings.cs | 40 ++ .../FeatureManagementTests.cs | 20 +- .../RefreshTests.cs | 2 +- .../Tests.AzureAppConfiguration/TestHelper.cs | 9 +- 8 files changed, 227 insertions(+), 270 deletions(-) delete mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageableManager.cs create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IPageableConfigurationSettings.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index de8d533f..005dc94e 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -107,16 +107,14 @@ internal IEnumerable Adapters internal IEnumerable KeyPrefixes => _keyPrefixes; /// - /// An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. + /// For use in tests only. An optional configuration client manager that can be used to provide clients to communicate with Azure App Configuration. /// - /// This property is used only for unit testing. internal IConfigurationClientManager ClientManager { get; set; } /// - /// An optional class used to process pageable results from Azure App Configuration. + /// For use in tests only. An optional class used to process pageable results from Azure App Configuration. /// - /// This property is only set outside of this class if it's used for unit testing. - internal ConfigurationSettingPageableManager PageableManager { get; set; } = new ConfigurationSettingPageableManager(); + internal IPageableConfigurationSettings PageableConfigurationSettings { get; set; } /// /// An optional timespan value to set the minimum backoff duration to a value other than the default. diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index ef46f227..3f08713f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -32,8 +32,8 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private AzureAppConfigurationOptions _options; private Dictionary _mappedData; private Dictionary _watchedIndividualKvs = new Dictionary(); - private Dictionary> _watchedKvEtags = new Dictionary>(); - private Dictionary> _watchedFfEtags = new Dictionary>(); + private Dictionary> _kvEtags = new Dictionary>(); + private Dictionary> _ffEtags = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); private DateTimeOffset _registerAllNextRefreshTime; @@ -201,13 +201,13 @@ public async Task RefreshAsync(CancellationToken cancellationToken) var utcNow = DateTimeOffset.UtcNow; IEnumerable refreshableIndividualKvWatchers = _options.IndividualKvWatchers.Where(kvWatcher => utcNow >= kvWatcher.NextRefreshTime); IEnumerable refreshableFfWatchers = _options.FeatureFlagWatchers.Where(ffWatcher => utcNow >= ffWatcher.NextRefreshTime); - bool registerAllIsRefreshable = utcNow >= _registerAllNextRefreshTime; + bool isRefreshDue = utcNow >= _registerAllNextRefreshTime; // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. if (_mappedData != null && !refreshableIndividualKvWatchers.Any() && !refreshableFfWatchers.Any() && - !registerAllIsRefreshable && + !isRefreshDue && !_options.Adapters.Any(adapter => adapter.NeedsRefresh())) { return; @@ -256,204 +256,100 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary> watchedFfEtags = null; - Dictionary> watchedKvEtags = null; + Dictionary> ffEtags = null; + Dictionary> kvEtags = null; Dictionary watchedIndividualKvs = null; List keyValueChanges = null; Dictionary data = null; Dictionary ffCollectionData = null; - bool collectionsUpdated = false; bool ffCollectionUpdated = false; bool refreshAll = false; StringBuilder logInfoBuilder = new StringBuilder(); StringBuilder logDebugBuilder = new StringBuilder(); await ExecuteWithFailOverPolicyAsync(clients, async (client) => + { + data = null; + ffCollectionData = null; + watchedIndividualKvs = null; + kvEtags = null; + ffEtags = null; + keyValueChanges = new List(); + ffCollectionUpdated = false; + refreshAll = false; + Uri endpoint = _configClientManager.GetEndpointForClient(client); + logDebugBuilder.Clear(); + logInfoBuilder.Clear(); + + if (_options.RegisterAllEnabled) { - data = null; - ffCollectionData = null; - watchedIndividualKvs = null; - watchedKvEtags = null; - watchedFfEtags = null; - keyValueChanges = new List(); - collectionsUpdated = false; - ffCollectionUpdated = false; - refreshAll = false; - Uri endpoint = _configClientManager.GetEndpointForClient(client); - logDebugBuilder.Clear(); - logInfoBuilder.Clear(); - - if (!_options.RegisterAllEnabled) + // Get key value collection changes if RegisterAll was called + kvEtags = new Dictionary>(_kvEtags); + + if (isRefreshDue) { - refreshAll = await RefreshIndividualKvWatchers( + refreshAll = await HaveCollectionsChanged( + _options.Selectors.Where(selector => !selector.IsFeatureFlagSelector).ToList(), + kvEtags, client, - keyValueChanges, - refreshableIndividualKvWatchers, - endpoint, - logDebugBuilder, - logInfoBuilder, cancellationToken).ConfigureAwait(false); - - if (!refreshAll) - { - // Get feature flag changes - watchedFfEtags = new Dictionary>(_watchedFfEtags); - - ffCollectionUpdated = await UpdateWatchedCollections( - refreshableFfWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), - watchedFfEtags, - client, - cancellationToken).ConfigureAwait(false); - - if (!ffCollectionUpdated) - { - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); - } - else - { - watchedFfEtags = new Dictionary>(); - - ffCollectionData = await LoadSelected( - client, - new Dictionary>(), - watchedFfEtags, - _options.Selectors.Where(selector => selector.IsFeatureFlagSelector), - cancellationToken).ConfigureAwait(false); - - logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); - } - } - else - { - // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true - watchedKvEtags = new Dictionary>(); - watchedFfEtags = new Dictionary>(); - - data = await LoadSelected(client, watchedKvEtags, watchedFfEtags, _options.Selectors, cancellationToken).ConfigureAwait(false); - watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); - logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); - return; - } } - else - { - // Get key value collection changes if RegisterAll was called - watchedKvEtags = new Dictionary>(_watchedKvEtags); - - if (registerAllIsRefreshable) - { - collectionsUpdated = await UpdateWatchedCollections( - _options.Selectors.Where(selector => !selector.IsFeatureFlagSelector).ToList(), - watchedKvEtags, - client, - cancellationToken).ConfigureAwait(false); - - if (!collectionsUpdated) - { - logDebugBuilder.AppendLine(LogHelper.BuildSelectedKeyValueCollectionsUnchangedMessage(endpoint.ToString())); - } - else - { - watchedKvEtags = new Dictionary>(); - - watchedFfEtags = new Dictionary>(); - - data = await LoadSelected(client, watchedKvEtags, watchedFfEtags, _options.Selectors, cancellationToken).ConfigureAwait(false); - - logInfoBuilder.Append(LogHelper.BuildSelectedKeyValuesAndFeatureFlagsUpdatedMessage()); - } - } - - if (!collectionsUpdated) - { - // Get feature flag changes - watchedFfEtags = new Dictionary>(_watchedFfEtags); - - ffCollectionUpdated = await UpdateWatchedCollections( - refreshableFfWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), - watchedFfEtags, - client, - cancellationToken).ConfigureAwait(false); - - if (!ffCollectionUpdated) - { - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); - } - else - { - watchedFfEtags = new Dictionary>(); - - ffCollectionData = await LoadSelected( - client, - new Dictionary>(), - watchedFfEtags, - _options.Selectors.Where(selector => selector.IsFeatureFlagSelector), - cancellationToken).ConfigureAwait(false); - - logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); - } - } - } - }, - cancellationToken) - .ConfigureAwait(false); - - if (!refreshAll) - { - watchedIndividualKvs = new Dictionary(_watchedIndividualKvs); - - foreach (KeyValueWatcher changeWatcher in refreshableIndividualKvWatchers.Concat(refreshableFfWatchers)) + } + else { - UpdateNextRefreshTime(changeWatcher); + refreshAll = await RefreshIndividualKvWatchers( + client, + keyValueChanges, + refreshableIndividualKvWatchers, + endpoint, + logDebugBuilder, + logInfoBuilder, + cancellationToken).ConfigureAwait(false); } - foreach (KeyValueChange change in keyValueChanges) + if (refreshAll) + { + // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true + kvEtags = new Dictionary>(); + ffEtags = new Dictionary>(); + + data = await LoadSelected(client, kvEtags, ffEtags, _options.Selectors, cancellationToken).ConfigureAwait(false); + watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); + logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); + return; + } + else { - KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); + // Get feature flag changes + ffEtags = new Dictionary>(_ffEtags); - if (change.ChangeType == KeyValueChangeType.Modified) - { - ConfigurationSetting setting = change.Current; - ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); - - watchedIndividualKvs[changeIdentifier] = settingCopy; - - foreach (Func> func in _options.Mappers) - { - setting = await func(setting).ConfigureAwait(false); - } - - if (setting == null) - { - _mappedData.Remove(change.Key); - } - else - { - _mappedData[change.Key] = setting; - } - } - else if (change.ChangeType == KeyValueChangeType.Deleted) + ffCollectionUpdated = await HaveCollectionsChanged( + refreshableFfWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), + ffEtags, + client, + cancellationToken).ConfigureAwait(false); + + if (ffCollectionUpdated) { - _mappedData.Remove(change.Key); + ffEtags = new Dictionary>(); - watchedIndividualKvs.Remove(changeIdentifier); - } + ffCollectionData = await LoadSelected( + client, + new Dictionary>(), + ffEtags, + _options.Selectors.Where(selector => selector.IsFeatureFlagSelector), + cancellationToken).ConfigureAwait(false); - // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting - foreach (IKeyValueAdapter adapter in _options.Adapters) + logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); + } + else { - // If the current setting is null, try to pass the previous setting instead - if (change.Current != null) - { - adapter.OnChangeDetected(change.Current); - } - else if (change.Previous != null) - { - adapter.OnChangeDetected(change.Previous); - } + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); } } - } + }, + cancellationToken) + .ConfigureAwait(false); if (refreshAll) { @@ -473,18 +369,9 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => } else { - if (_options.RegisterAllEnabled) - { - if (registerAllIsRefreshable) - { - _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); - } + watchedIndividualKvs = new Dictionary(_watchedIndividualKvs); - if (collectionsUpdated) - { - _mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - } - } + await ProcessKeyValueChangesAsync(keyValueChanges, _mappedData, watchedIndividualKvs).ConfigureAwait(false); if (ffCollectionUpdated) { @@ -495,15 +382,27 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => _mappedData[kvp.Key] = kvp.Value; } } + + // + // update the next refresh time for all refresh registered settings and feature flags + foreach (KeyValueWatcher changeWatcher in refreshableIndividualKvWatchers.Concat(refreshableFfWatchers)) + { + UpdateNextRefreshTime(changeWatcher); + } + } + + if (isRefreshDue) + { + _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); } - if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any() || collectionsUpdated || ffCollectionUpdated) + if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any() || refreshAll || ffCollectionUpdated) { _watchedIndividualKvs = watchedIndividualKvs; - _watchedFfEtags = watchedFfEtags; + _ffEtags = ffEtags; - _watchedKvEtags = watchedKvEtags; + _kvEtags = kvEtags; if (logDebugBuilder.Length > 0) { @@ -797,8 +696,8 @@ private async Task TryInitializeAsync(IEnumerable cli private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) { Dictionary data = null; - Dictionary> watchedKvEtags = new Dictionary>(); - Dictionary> watchedFfEtags = new Dictionary>(); + Dictionary> kvEtags = new Dictionary>(); + Dictionary> ffEtags = new Dictionary>(); Dictionary watchedIndividualKvs = null; await ExecuteWithFailOverPolicyAsync( @@ -807,8 +706,8 @@ await ExecuteWithFailOverPolicyAsync( { data = await LoadSelected( client, - watchedKvEtags, - watchedFfEtags, + kvEtags, + ffEtags, _options.Selectors, cancellationToken) .ConfigureAwait(false); @@ -846,16 +745,16 @@ await ExecuteWithFailOverPolicyAsync( SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); _watchedIndividualKvs = watchedIndividualKvs; - _watchedKvEtags = watchedKvEtags; - _watchedFfEtags = watchedFfEtags; + _kvEtags = kvEtags; + _ffEtags = ffEtags; _mappedData = mappedData; } } private async Task> LoadSelected( ConfigurationClient client, - Dictionary> watchedKvEtags, - Dictionary> watchedFfEtags, + Dictionary> kvEtags, + Dictionary> ffEtags, IEnumerable selectors, CancellationToken cancellationToken) { @@ -877,15 +776,11 @@ await CallWithRequestTracing(async () => { AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); - await foreach (Page page in _options.PageableManager.GetPages(pageableSettings).ConfigureAwait(false)) + await foreach (Page page in pageableSettings.AsPages().ConfigureAwait(false)) { - // Server etag will never be null or empty because non-conditional List key-values API always returns 200 - ETag serverEtag = (ETag)page?.GetRawResponse()?.Headers.ETag; + using Response response = page.GetRawResponse(); - if (serverEtag == null || page.Values == null) - { - throw new RequestFailedException(ErrorMessages.InvalidConfigurationSettingPage); - } + ETag serverEtag = (ETag)response.Headers.ETag; foreach (ConfigurationSetting setting in page.Values) { @@ -898,11 +793,11 @@ await CallWithRequestTracing(async () => if (loadOption.IsFeatureFlagSelector) { - watchedFfEtags[loadOption] = matchConditions; + ffEtags[loadOption] = matchConditions; } else { - watchedKvEtags[loadOption] = matchConditions; + kvEtags[loadOption] = matchConditions; } } else @@ -1390,25 +1285,78 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) _configClientBackoffs[endpoint] = clientBackoffStatus; } - private async Task UpdateWatchedCollections(IEnumerable selectors, Dictionary> watchedEtags, ConfigurationClient client, CancellationToken cancellationToken) + private async Task HaveCollectionsChanged(IEnumerable selectors, Dictionary> watchedEtags, ConfigurationClient client, CancellationToken cancellationToken) { - bool watchedCollectionsChanged = false; + bool haveCollectionsChanged = false; foreach (KeyValueSelector selector in selectors) { if (watchedEtags.TryGetValue(selector, out IEnumerable matchConditions)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => watchedCollectionsChanged = await client.HasWatchedCollectionsChanged(selector, matchConditions, _options.PageableManager, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + async () => haveCollectionsChanged = await client.HaveCollectionsChanged(selector, matchConditions, _options.PageableConfigurationSettings, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } - if (watchedCollectionsChanged) + if (haveCollectionsChanged) { - return watchedCollectionsChanged; + return haveCollectionsChanged; } } - return watchedCollectionsChanged; + return haveCollectionsChanged; + } + + private async Task ProcessKeyValueChangesAsync( + IEnumerable keyValueChanges, + Dictionary mappedData, + Dictionary watchedIndividualKvs) + { + foreach (KeyValueChange change in keyValueChanges) + { + KeyValueIdentifier changeIdentifier = new KeyValueIdentifier(change.Key, change.Label); + + if (change.ChangeType == KeyValueChangeType.Modified) + { + ConfigurationSetting setting = change.Current; + ConfigurationSetting settingCopy = new ConfigurationSetting(setting.Key, setting.Value, setting.Label, setting.ETag); + + watchedIndividualKvs[changeIdentifier] = settingCopy; + + foreach (Func> func in _options.Mappers) + { + setting = await func(setting).ConfigureAwait(false); + } + + if (setting == null) + { + mappedData.Remove(change.Key); + } + else + { + mappedData[change.Key] = setting; + } + } + else if (change.ChangeType == KeyValueChangeType.Deleted) + { + mappedData.Remove(change.Key); + + watchedIndividualKvs.Remove(changeIdentifier); + } + + // Invalidate the cached Key Vault secret (if any) for this ConfigurationSetting + foreach (IKeyValueAdapter adapter in _options.Adapters) + { + // If the current setting is null, try to pass the previous setting instead + if (change.Current != null) + { + adapter.OnChangeDetected(change.Current); + } + else if (change.Previous != null) + { + adapter.OnChangeDetected(change.Previous); + } + } + } } public void Dispose() diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageableManager.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageableManager.cs deleted file mode 100644 index c71a1c73..00000000 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageableManager.cs +++ /dev/null @@ -1,19 +0,0 @@ -using Azure.Data.AppConfiguration; -using Azure; -using System.Collections.Generic; - -namespace Microsoft.Extensions.Configuration.AzureAppConfiguration -{ - internal class ConfigurationSettingPageableManager - { - public virtual IAsyncEnumerable> GetPages(AsyncPageable pageable, IEnumerable matchConditions) - { - return pageable.AsPages(matchConditions); - } - - public virtual IAsyncEnumerable> GetPages(AsyncPageable pageable) - { - return pageable.AsPages(); - } - } -} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index ab7aa029..d1e9b4e5 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -63,7 +63,7 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } - public static async Task HasWatchedCollectionsChanged(this ConfigurationClient client, KeyValueSelector keyValueSelector, IEnumerable matchConditions, ConfigurationSettingPageableManager pageableManager, CancellationToken cancellationToken) + public static async Task HaveCollectionsChanged(this ConfigurationClient client, KeyValueSelector keyValueSelector, IEnumerable matchConditions, IPageableConfigurationSettings pageableConfigurationSettings, CancellationToken cancellationToken) { if (matchConditions == null) { @@ -80,11 +80,6 @@ public static async Task HasWatchedCollectionsChanged(this ConfigurationCl throw new ArgumentException("Cannot check snapshot for changes.", $"{nameof(keyValueSelector)}.{nameof(keyValueSelector.SnapshotName)}"); } - if (pageableManager == null) - { - throw new ArgumentNullException(nameof(pageableManager)); - } - SettingSelector selector = new SettingSelector { KeyFilter = keyValueSelector.KeyFilter, @@ -95,16 +90,11 @@ public static async Task HasWatchedCollectionsChanged(this ConfigurationCl using IEnumerator existingMatchConditionsEnumerator = matchConditions.GetEnumerator(); - await foreach (Page page in pageableManager.GetPages(pageable, matchConditions).ConfigureAwait(false)) + await foreach (Page page in pageable.AsPages(pageableConfigurationSettings, matchConditions).ConfigureAwait(false)) { - ETag serverEtag = (ETag)page?.GetRawResponse()?.Headers.ETag; + using Response response = page.GetRawResponse(); - if (page?.Values == null) - { - throw new RequestFailedException(ErrorMessages.InvalidConfigurationSettingPage); - } - - Response response = page.GetRawResponse(); + ETag serverEtag = (ETag)response.Headers.ETag; // Return true if the lists of etags are different if ((!existingMatchConditionsEnumerator.MoveNext() || @@ -116,12 +106,7 @@ public static async Task HasWatchedCollectionsChanged(this ConfigurationCl } // Need to check if pages were deleted and no change was found within the new shorter list of match conditions - if (existingMatchConditionsEnumerator.MoveNext()) - { - return true; - } - - return false; + return existingMatchConditionsEnumerator.MoveNext(); } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IPageableConfigurationSettings.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IPageableConfigurationSettings.cs new file mode 100644 index 00000000..73aca597 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IPageableConfigurationSettings.cs @@ -0,0 +1,40 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System.Collections.Generic; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + internal interface IPageableConfigurationSettings + { + IAsyncEnumerable> IteratePages(AsyncPageable pageable); + + IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions); + } + + static class ConfigurationSettingPageExtensions + { + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IPageableConfigurationSettings pageableConfigurationSettings) + { + // + // Allow custom iteration + if (pageableConfigurationSettings != null) + { + return pageableConfigurationSettings.IteratePages(pageable); + } + + return pageable.AsPages(); + } + + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IPageableConfigurationSettings pageableConfigurationSettings, IEnumerable matchConditions) + { + // + // Allow custom iteration + if (pageableConfigurationSettings != null) + { + return pageableConfigurationSettings.IteratePages(pageable, matchConditions); + } + + return pageable.AsPages(matchConditions); + } + } +} diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 079ad25a..f8d21ae2 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -785,7 +785,7 @@ public async Task WatchesFeatureFlags() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableManager = new MockConfigurationSettingPageableManager(); + options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -858,7 +858,7 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableManager = new MockConfigurationSettingPageableManager(); + options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); options.UseFeatureFlags(o => o.CacheExpirationInterval = cacheExpirationInterval); refresher = options.GetRefresher(); @@ -930,7 +930,7 @@ public async Task SkipRefreshIfRefreshIntervalHasNotElapsed() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableManager = new MockConfigurationSettingPageableManager(); + options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); options.UseFeatureFlags(o => o.SetRefreshInterval(TimeSpan.FromSeconds(10))); refresher = options.GetRefresher(); @@ -1000,7 +1000,7 @@ public async Task SkipRefreshIfCacheNotExpired() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableManager = new MockConfigurationSettingPageableManager(); + options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); options.UseFeatureFlags(o => o.CacheExpirationInterval = TimeSpan.FromSeconds(10)); refresher = options.GetRefresher(); @@ -1124,7 +1124,7 @@ public async Task DoesNotUseEtagForFeatureFlagRefresh() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableManager = new MockConfigurationSettingPageableManager(); + options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -1155,7 +1155,7 @@ public void SelectFeatureFlags() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(testClient); - options.PageableManager = new MockConfigurationSettingPageableManager(); + options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1517,7 +1517,7 @@ public async Task DifferentCacheExpirationsForMultipleFeatureFlagRegistrations() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableManager = new MockConfigurationSettingPageableManager(); + options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(refreshInterval1); @@ -1687,7 +1687,7 @@ public async Task SelectAndRefreshSingleFeatureFlag() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableManager = new MockConfigurationSettingPageableManager(); + options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1774,7 +1774,7 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; - options.PageableManager = new MockConfigurationSettingPageableManager(); + options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); }) @@ -1852,7 +1852,7 @@ public async Task ValidateFeatureFlagsUnchangedLogged() .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; - options.PageableManager = new MockConfigurationSettingPageableManager(); + options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); options.ConfigureRefresh(refreshOptions => { diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index 8844e7b0..fa1bce7b 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -1073,7 +1073,7 @@ public async Task RefreshTests_SelectedKeysRefreshWithRegisterAll() { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); options.Select("TestKey*"); - options.PageableManager = new MockConfigurationSettingPageableManager(); + options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); options.ConfigureRefresh(refreshOptions => { refreshOptions.RegisterAll() diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index 21f03607..1db20e55 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -210,9 +210,14 @@ public async override IAsyncEnumerable> AsPages(strin } } - internal class MockConfigurationSettingPageableManager : ConfigurationSettingPageableManager + internal class MockPageableConfigurationSettings : IPageableConfigurationSettings { - public override IAsyncEnumerable> GetPages(AsyncPageable pageable, IEnumerable matchConditions) + public IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions) + { + return pageable.AsPages(); + } + + public IAsyncEnumerable> IteratePages(AsyncPageable pageable) { return pageable.AsPages(); } From 19283570a5c22e7127745d2ad8564bcb49f599fb Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Tue, 14 Jan 2025 17:02:25 -0800 Subject: [PATCH 65/72] clean up etags dictionary creation --- .../AzureAppConfigurationProvider.cs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 3f08713f..de31e3a4 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -284,13 +284,11 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => if (_options.RegisterAllEnabled) { // Get key value collection changes if RegisterAll was called - kvEtags = new Dictionary>(_kvEtags); - if (isRefreshDue) { refreshAll = await HaveCollectionsChanged( _options.Selectors.Where(selector => !selector.IsFeatureFlagSelector).ToList(), - kvEtags, + _kvEtags, client, cancellationToken).ConfigureAwait(false); } @@ -321,11 +319,9 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => else { // Get feature flag changes - ffEtags = new Dictionary>(_ffEtags); - ffCollectionUpdated = await HaveCollectionsChanged( refreshableFfWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), - ffEtags, + _ffEtags, client, cancellationToken).ConfigureAwait(false); @@ -1285,13 +1281,13 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) _configClientBackoffs[endpoint] = clientBackoffStatus; } - private async Task HaveCollectionsChanged(IEnumerable selectors, Dictionary> watchedEtags, ConfigurationClient client, CancellationToken cancellationToken) + private async Task HaveCollectionsChanged(IEnumerable selectors, Dictionary> pageEtags, ConfigurationClient client, CancellationToken cancellationToken) { bool haveCollectionsChanged = false; foreach (KeyValueSelector selector in selectors) { - if (watchedEtags.TryGetValue(selector, out IEnumerable matchConditions)) + if (pageEtags.TryGetValue(selector, out IEnumerable matchConditions)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, async () => haveCollectionsChanged = await client.HaveCollectionsChanged(selector, matchConditions, _options.PageableConfigurationSettings, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); From 9135d42603cb3bf5d8784fd067dc3fb29b175b5f Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 15 Jan 2025 12:15:29 -0800 Subject: [PATCH 66/72] PR comments --- .../AzureAppConfigurationOptions.cs | 2 +- .../AzureAppConfigurationProvider.cs | 102 ++++++++---------- .../ConfigurationSettingPageExtensions.cs | 33 ++++++ .../ConfigurationClientExtensions.cs | 4 +- .../IConfigurationSettingPageIterator.cs | 13 +++ .../IPageableConfigurationSettings.cs | 40 ------- .../FeatureManagementTests.cs | 20 ++-- .../RefreshTests.cs | 2 +- .../Tests.AzureAppConfiguration/TestHelper.cs | 2 +- 9 files changed, 106 insertions(+), 112 deletions(-) create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs delete mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IPageableConfigurationSettings.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 005dc94e..5c1d7382 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -114,7 +114,7 @@ internal IEnumerable Adapters /// /// For use in tests only. An optional class used to process pageable results from Azure App Configuration. /// - internal IPageableConfigurationSettings PageableConfigurationSettings { get; set; } + internal IConfigurationSettingPageIterator ConfigurationSettingPageIterator { get; set; } /// /// An optional timespan value to set the minimum backoff duration to a value other than the default. diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index de31e3a4..44c00b4c 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -36,7 +36,7 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private Dictionary> _ffEtags = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; private Dictionary _configClientBackoffs = new Dictionary(); - private DateTimeOffset _registerAllNextRefreshTime; + private DateTimeOffset _nextCollectionRefreshTime; private readonly TimeSpan MinRefreshInterval; @@ -119,9 +119,13 @@ public AzureAppConfigurationProvider(IConfigurationClientManager configClientMan { MinRefreshInterval = TimeSpan.FromTicks(Math.Min(minWatcherRefreshInterval.Ticks, options.KvCollectionRefreshInterval.Ticks)); } + else if (hasWatchers) + { + MinRefreshInterval = minWatcherRefreshInterval; + } else { - MinRefreshInterval = hasWatchers ? minWatcherRefreshInterval : RefreshConstants.DefaultRefreshInterval; + MinRefreshInterval = RefreshConstants.DefaultRefreshInterval; } // Enable request tracing if not opt-out @@ -201,7 +205,7 @@ public async Task RefreshAsync(CancellationToken cancellationToken) var utcNow = DateTimeOffset.UtcNow; IEnumerable refreshableIndividualKvWatchers = _options.IndividualKvWatchers.Where(kvWatcher => utcNow >= kvWatcher.NextRefreshTime); IEnumerable refreshableFfWatchers = _options.FeatureFlagWatchers.Where(ffWatcher => utcNow >= ffWatcher.NextRefreshTime); - bool isRefreshDue = utcNow >= _registerAllNextRefreshTime; + bool isRefreshDue = utcNow >= _nextCollectionRefreshTime; // Skip refresh if mappedData is loaded, but none of the watchers or adapters are refreshable. if (_mappedData != null && @@ -287,7 +291,7 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => if (isRefreshDue) { refreshAll = await HaveCollectionsChanged( - _options.Selectors.Where(selector => !selector.IsFeatureFlagSelector).ToList(), + _options.Selectors.Where(selector => !selector.IsFeatureFlagSelector), _kvEtags, client, cancellationToken).ConfigureAwait(false); @@ -307,7 +311,8 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => if (refreshAll) { - // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true + // Trigger a single load-all operation if a change was detected in one or more key-values with refreshAll: true, + // or if any key-value collection change was detected. kvEtags = new Dictionary>(); ffEtags = new Dictionary>(); @@ -316,32 +321,30 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); return; } - else + + // Get feature flag changes + ffCollectionUpdated = await HaveCollectionsChanged( + refreshableFfWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), + _ffEtags, + client, + cancellationToken).ConfigureAwait(false); + + if (ffCollectionUpdated) { - // Get feature flag changes - ffCollectionUpdated = await HaveCollectionsChanged( - refreshableFfWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), - _ffEtags, + ffEtags = new Dictionary>(); + + ffCollectionData = await LoadSelected( client, + new Dictionary>(), + ffEtags, + _options.Selectors.Where(selector => selector.IsFeatureFlagSelector), cancellationToken).ConfigureAwait(false); - if (ffCollectionUpdated) - { - ffEtags = new Dictionary>(); - - ffCollectionData = await LoadSelected( - client, - new Dictionary>(), - ffEtags, - _options.Selectors.Where(selector => selector.IsFeatureFlagSelector), - cancellationToken).ConfigureAwait(false); - - logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); - } - else - { - logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); - } + logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); + } + else + { + logDebugBuilder.AppendLine(LogHelper.BuildFeatureFlagsUnchangedMessage(endpoint.ToString())); } }, cancellationToken) @@ -389,7 +392,7 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => if (isRefreshDue) { - _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); + _nextCollectionRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); } if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any() || refreshAll || ffCollectionUpdated) @@ -535,7 +538,7 @@ private void SetDirty(TimeSpan? maxDelay) if (_options.RegisterAllEnabled) { - _registerAllNextRefreshTime = nextRefreshTime; + _nextCollectionRefreshTime = nextRefreshTime; } else { @@ -725,7 +728,7 @@ await ExecuteWithFailOverPolicyAsync( if (_options.RegisterAllEnabled) { - _registerAllNextRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); + _nextCollectionRefreshTime = DateTimeOffset.UtcNow.Add(_options.KvCollectionRefreshInterval); } if (data != null) @@ -772,7 +775,7 @@ await CallWithRequestTracing(async () => { AsyncPageable pageableSettings = client.GetConfigurationSettingsAsync(selector, cancellationToken); - await foreach (Page page in pageableSettings.AsPages().ConfigureAwait(false)) + await foreach (Page page in pageableSettings.AsPages(_options.ConfigurationSettingPageIterator).ConfigureAwait(false)) { using Response response = page.GetRawResponse(); @@ -1190,30 +1193,6 @@ innerException is SocketException || innerException is IOException; } - private IEnumerable GetExistingKeys(string key, IEnumerable existingKeys) - { - IEnumerable currentKeyValues; - - if (key.EndsWith("*")) - { - // Get current application settings starting with changeWatcher.Key, excluding the last * character - string keyPrefix = key.Substring(0, key.Length - 1); - currentKeyValues = existingKeys.Where(val => - { - return val.StartsWith(keyPrefix); - }); - } - else - { - currentKeyValues = existingKeys.Where(val => - { - return val.Equals(key); - }); - } - - return currentKeyValues; - } - private async Task> MapConfigurationSettings(Dictionary data) { Dictionary mappedData = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -1281,7 +1260,12 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) _configClientBackoffs[endpoint] = clientBackoffStatus; } - private async Task HaveCollectionsChanged(IEnumerable selectors, Dictionary> pageEtags, ConfigurationClient client, CancellationToken cancellationToken) + private async Task HaveCollectionsChanged( + IEnumerable selectors, + Dictionary> pageEtags, + ConfigurationClient client, + CancellationToken cancellationToken) { bool haveCollectionsChanged = false; @@ -1290,7 +1274,11 @@ private async Task HaveCollectionsChanged(IEnumerable se if (pageEtags.TryGetValue(selector, out IEnumerable matchConditions)) { await TracingUtils.CallWithRequestTracing(_requestTracingEnabled, RequestType.Watch, _requestTracingOptions, - async () => haveCollectionsChanged = await client.HaveCollectionsChanged(selector, matchConditions, _options.PageableConfigurationSettings, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + async () => haveCollectionsChanged = await client.HaveCollectionsChanged( + selector, + matchConditions, + _options.ConfigurationSettingPageIterator, + cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); } if (haveCollectionsChanged) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs new file mode 100644 index 00000000..aba7684b --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationSettingPageExtensions.cs @@ -0,0 +1,33 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System.Collections.Generic; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + static class ConfigurationSettingPageExtensions + { + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IConfigurationSettingPageIterator pageIterator) + { + // + // Allow custom iteration + if (pageIterator != null) + { + return pageIterator.IteratePages(pageable); + } + + return pageable.AsPages(); + } + + public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IConfigurationSettingPageIterator pageIterator, IEnumerable matchConditions) + { + // + // Allow custom iteration + if (pageIterator != null) + { + return pageIterator.IteratePages(pageable, matchConditions); + } + + return pageable.AsPages(matchConditions); + } + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs index d1e9b4e5..67712231 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/ConfigurationClientExtensions.cs @@ -63,7 +63,7 @@ public static async Task GetKeyValueChange(this ConfigurationCli }; } - public static async Task HaveCollectionsChanged(this ConfigurationClient client, KeyValueSelector keyValueSelector, IEnumerable matchConditions, IPageableConfigurationSettings pageableConfigurationSettings, CancellationToken cancellationToken) + public static async Task HaveCollectionsChanged(this ConfigurationClient client, KeyValueSelector keyValueSelector, IEnumerable matchConditions, IConfigurationSettingPageIterator pageIterator, CancellationToken cancellationToken) { if (matchConditions == null) { @@ -90,7 +90,7 @@ public static async Task HaveCollectionsChanged(this ConfigurationClient c using IEnumerator existingMatchConditionsEnumerator = matchConditions.GetEnumerator(); - await foreach (Page page in pageable.AsPages(pageableConfigurationSettings, matchConditions).ConfigureAwait(false)) + await foreach (Page page in pageable.AsPages(pageIterator, matchConditions).ConfigureAwait(false)) { using Response response = page.GetRawResponse(); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs new file mode 100644 index 00000000..08c95751 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationSettingPageIterator.cs @@ -0,0 +1,13 @@ +using Azure.Data.AppConfiguration; +using Azure; +using System.Collections.Generic; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + internal interface IConfigurationSettingPageIterator + { + IAsyncEnumerable> IteratePages(AsyncPageable pageable); + + IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions); + } +} diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IPageableConfigurationSettings.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IPageableConfigurationSettings.cs deleted file mode 100644 index 73aca597..00000000 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IPageableConfigurationSettings.cs +++ /dev/null @@ -1,40 +0,0 @@ -using Azure.Data.AppConfiguration; -using Azure; -using System.Collections.Generic; - -namespace Microsoft.Extensions.Configuration.AzureAppConfiguration -{ - internal interface IPageableConfigurationSettings - { - IAsyncEnumerable> IteratePages(AsyncPageable pageable); - - IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions); - } - - static class ConfigurationSettingPageExtensions - { - public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IPageableConfigurationSettings pageableConfigurationSettings) - { - // - // Allow custom iteration - if (pageableConfigurationSettings != null) - { - return pageableConfigurationSettings.IteratePages(pageable); - } - - return pageable.AsPages(); - } - - public static IAsyncEnumerable> AsPages(this AsyncPageable pageable, IPageableConfigurationSettings pageableConfigurationSettings, IEnumerable matchConditions) - { - // - // Allow custom iteration - if (pageableConfigurationSettings != null) - { - return pageableConfigurationSettings.IteratePages(pageable, matchConditions); - } - - return pageable.AsPages(matchConditions); - } - } -} diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index f8d21ae2..f7ebf127 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -785,7 +785,7 @@ public async Task WatchesFeatureFlags() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -858,7 +858,7 @@ public async Task WatchesFeatureFlagsUsingCacheExpirationInterval() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.CacheExpirationInterval = cacheExpirationInterval); refresher = options.GetRefresher(); @@ -930,7 +930,7 @@ public async Task SkipRefreshIfRefreshIntervalHasNotElapsed() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(TimeSpan.FromSeconds(10))); refresher = options.GetRefresher(); @@ -1000,7 +1000,7 @@ public async Task SkipRefreshIfCacheNotExpired() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.CacheExpirationInterval = TimeSpan.FromSeconds(10)); refresher = options.GetRefresher(); @@ -1124,7 +1124,7 @@ public async Task DoesNotUseEtagForFeatureFlagRefresh() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); @@ -1155,7 +1155,7 @@ public void SelectFeatureFlags() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(testClient); - options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1517,7 +1517,7 @@ public async Task DifferentCacheExpirationsForMultipleFeatureFlagRegistrations() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(refreshInterval1); @@ -1687,7 +1687,7 @@ public async Task SelectAndRefreshSingleFeatureFlag() .AddAzureAppConfiguration(options => { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(ff => { ff.SetRefreshInterval(RefreshInterval); @@ -1774,7 +1774,7 @@ public async Task ValidateCorrectFeatureFlagLoggedIfModifiedOrRemovedDuringRefre .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; - options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); refresher = options.GetRefresher(); }) @@ -1852,7 +1852,7 @@ public async Task ValidateFeatureFlagsUnchangedLogged() .AddAzureAppConfiguration(options => { options.ClientManager = mockClientManager; - options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.UseFeatureFlags(o => o.SetRefreshInterval(RefreshInterval)); options.ConfigureRefresh(refreshOptions => { diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index fa1bce7b..cac256d9 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -1073,7 +1073,7 @@ public async Task RefreshTests_SelectedKeysRefreshWithRegisterAll() { options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); options.Select("TestKey*"); - options.PageableConfigurationSettings = new MockPageableConfigurationSettings(); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); options.ConfigureRefresh(refreshOptions => { refreshOptions.RegisterAll() diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index 1db20e55..0445f137 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -210,7 +210,7 @@ public async override IAsyncEnumerable> AsPages(strin } } - internal class MockPageableConfigurationSettings : IPageableConfigurationSettings + internal class MockPageableConfigurationSettings : IConfigurationSettingPageIterator { public IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions) { From c55aa604750378d29c889942d372e25bf5fe94d8 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Wed, 15 Jan 2025 12:31:29 -0800 Subject: [PATCH 67/72] add uncommitted changes to testhelper --- tests/Tests.AzureAppConfiguration/TestHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tests.AzureAppConfiguration/TestHelper.cs b/tests/Tests.AzureAppConfiguration/TestHelper.cs index 0445f137..e3bf5500 100644 --- a/tests/Tests.AzureAppConfiguration/TestHelper.cs +++ b/tests/Tests.AzureAppConfiguration/TestHelper.cs @@ -210,7 +210,7 @@ public async override IAsyncEnumerable> AsPages(strin } } - internal class MockPageableConfigurationSettings : IConfigurationSettingPageIterator + internal class MockConfigurationSettingPageIterator : IConfigurationSettingPageIterator { public IAsyncEnumerable> IteratePages(AsyncPageable pageable, IEnumerable matchConditions) { From 0b6aecb1faee9a38975d5968ee94391e6593d458 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 16 Jan 2025 11:58:44 -0800 Subject: [PATCH 68/72] update tests for registerall with feature flags, check ff keys to remove flags on refresh --- .../AzureAppConfigurationProvider.cs | 38 ++++-- .../RefreshTests.cs | 113 ++++++++++++++++++ 2 files changed, 143 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 44c00b4c..b17a9457 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -32,6 +32,7 @@ internal class AzureAppConfigurationProvider : ConfigurationProvider, IConfigura private AzureAppConfigurationOptions _options; private Dictionary _mappedData; private Dictionary _watchedIndividualKvs = new Dictionary(); + private HashSet _ffKeys = new HashSet(); private Dictionary> _kvEtags = new Dictionary>(); private Dictionary> _ffEtags = new Dictionary>(); private RequestTracingOptions _requestTracingOptions; @@ -262,6 +263,7 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // Avoid instance state modification Dictionary> ffEtags = null; Dictionary> kvEtags = null; + HashSet ffKeys = null; Dictionary watchedIndividualKvs = null; List keyValueChanges = null; Dictionary data = null; @@ -273,17 +275,18 @@ public async Task RefreshAsync(CancellationToken cancellationToken) await ExecuteWithFailOverPolicyAsync(clients, async (client) => { - data = null; - ffCollectionData = null; - watchedIndividualKvs = null; kvEtags = null; ffEtags = null; + ffKeys = null; + watchedIndividualKvs = null; keyValueChanges = new List(); + data = null; + ffCollectionData = null; ffCollectionUpdated = false; refreshAll = false; - Uri endpoint = _configClientManager.GetEndpointForClient(client); logDebugBuilder.Clear(); logInfoBuilder.Clear(); + Uri endpoint = _configClientManager.GetEndpointForClient(client); if (_options.RegisterAllEnabled) { @@ -315,8 +318,9 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => // or if any key-value collection change was detected. kvEtags = new Dictionary>(); ffEtags = new Dictionary>(); + ffKeys = new HashSet(); - data = await LoadSelected(client, kvEtags, ffEtags, _options.Selectors, cancellationToken).ConfigureAwait(false); + data = await LoadSelected(client, kvEtags, ffEtags, _options.Selectors, ffKeys, cancellationToken).ConfigureAwait(false); watchedIndividualKvs = await LoadKeyValuesRegisteredForRefresh(client, data, cancellationToken).ConfigureAwait(false); logInfoBuilder.AppendLine(LogHelper.BuildConfigurationUpdatedMessage()); return; @@ -332,12 +336,14 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => if (ffCollectionUpdated) { ffEtags = new Dictionary>(); + ffKeys = new HashSet(); ffCollectionData = await LoadSelected( client, new Dictionary>(), ffEtags, _options.Selectors.Where(selector => selector.IsFeatureFlagSelector), + ffKeys, cancellationToken).ConfigureAwait(false); logInfoBuilder.Append(LogHelper.BuildFeatureFlagsUpdatedMessage()); @@ -374,6 +380,12 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => if (ffCollectionUpdated) { + // Remove all feature flag keys that are not present in the latest loading of feature flags, but were loaded previously + foreach (string key in _ffKeys.Except(ffKeys)) + { + _mappedData.Remove(key); + } + Dictionary mappedFfData = await MapConfigurationSettings(ffCollectionData).ConfigureAwait(false); foreach (KeyValuePair kvp in mappedFfData) @@ -397,11 +409,13 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => if (_options.Adapters.Any(adapter => adapter.NeedsRefresh()) || keyValueChanges.Any() || refreshAll || ffCollectionUpdated) { - _watchedIndividualKvs = watchedIndividualKvs; + _watchedIndividualKvs = watchedIndividualKvs ?? _watchedIndividualKvs; + + _ffEtags = ffEtags ?? _ffEtags; - _ffEtags = ffEtags; + _kvEtags = kvEtags ?? _kvEtags; - _kvEtags = kvEtags; + _ffKeys = ffKeys ?? _ffKeys; if (logDebugBuilder.Length > 0) { @@ -698,6 +712,7 @@ private async Task InitializeAsync(IEnumerable clients, Can Dictionary> kvEtags = new Dictionary>(); Dictionary> ffEtags = new Dictionary>(); Dictionary watchedIndividualKvs = null; + HashSet ffKeys = new HashSet(); await ExecuteWithFailOverPolicyAsync( clients, @@ -708,6 +723,7 @@ await ExecuteWithFailOverPolicyAsync( kvEtags, ffEtags, _options.Selectors, + ffKeys, cancellationToken) .ConfigureAwait(false); @@ -755,6 +771,7 @@ private async Task> LoadSelected( Dictionary> kvEtags, Dictionary> ffEtags, IEnumerable selectors, + HashSet ffKeys, CancellationToken cancellationToken) { Dictionary data = new Dictionary(); @@ -784,6 +801,11 @@ await CallWithRequestTracing(async () => foreach (ConfigurationSetting setting in page.Values) { data[setting.Key] = setting; + + if (loadOption.IsFeatureFlagSelector) + { + ffKeys.Add(setting.Key); + } } matchConditions.Add(new MatchConditions { IfNoneMatch = serverEtag }); diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index cac256d9..a7fdbd18 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -7,6 +7,7 @@ using Azure.Identity; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration.AzureAppConfiguration; +using Microsoft.Extensions.Configuration.AzureAppConfiguration.FeatureManagement; using Microsoft.Extensions.Logging.Abstractions; using Moq; using System; @@ -1108,6 +1109,118 @@ public async Task RefreshTests_SelectedKeysRefreshWithRegisterAll() Assert.Null(config["TestKey3"]); } + [Fact] + public async Task RefreshTests_RegisterAllRefreshesFeatureFlags() + { + IConfigurationRefresher refresher = null; + var mockClient = GetMockConfigurationClient(); + + var featureFlags = new List { + ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", + value: @" + { + ""id"": ""MyFeature"", + ""description"": ""The new beta version of our web site."", + ""display_name"": ""Beta Feature"", + ""enabled"": true, + ""conditions"": { + ""client_filters"": [ + { + ""name"": ""SuperUsers"" + } + ] + } + } + ", + label: default, + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")) + }; + + var mockAsyncPageableKv = new MockAsyncPageable(_kvCollection); + + var mockAsyncPageableFf = new MockAsyncPageable(featureFlags); + + MockAsyncPageable GetTestKeys(SettingSelector selector, CancellationToken ct) + { + if (selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)) + { + mockAsyncPageableFf.UpdateCollection(featureFlags); + + return mockAsyncPageableFf; + } + + mockAsyncPageableKv.UpdateCollection(_kvCollection); + + return mockAsyncPageableKv; + } + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns((Func)GetTestKeys); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.Select("TestKey*"); + options.ConfigurationSettingPageIterator = new MockConfigurationSettingPageIterator(); + options.ConfigureRefresh(refreshOptions => + { + refreshOptions.RegisterAll() + .SetRefreshInterval(TimeSpan.FromSeconds(1)); + }); + options.UseFeatureFlags(); + + refresher = options.GetRefresher(); + }) + .Build(); + + Assert.Equal("TestValue1", config["TestKey1"]); + Assert.Equal("SuperUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); + + FirstKeyValue.Value = "newValue1"; + featureFlags[0] = ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "myFeature", + value: @" + { + ""id"": ""MyFeature"", + ""description"": ""The new beta version of our web site."", + ""display_name"": ""Beta Feature"", + ""enabled"": true, + ""conditions"": { + ""client_filters"": [ + { + ""name"": ""AllUsers"" + } + ] + } + } + ", + label: default, + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newValue1", config["TestKey1"]); + Assert.Equal("AllUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); + + FirstKeyValue.Value = "newerValue1"; + featureFlags.RemoveAt(0); + + // Wait for the cache to expire + Thread.Sleep(1500); + + await refresher.RefreshAsync(); + + Assert.Equal("newerValue1", config["TestKey1"]); + Assert.Null(config["FeatureManagement:MyFeature"]); + } + #if NET8_0 [Fact] public void RefreshTests_ChainedConfigurationProviderUsedAsRootForRefresherProvider() From 8b3f49a1e1d701d325500c516833ecfe9e842b1e Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Fri, 17 Jan 2025 10:19:15 -0800 Subject: [PATCH 69/72] PR comments --- .../AzureAppConfigurationProvider.cs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index b17a9457..32bb8105 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -261,8 +261,8 @@ public async Task RefreshAsync(CancellationToken cancellationToken) // // Avoid instance state modification - Dictionary> ffEtags = null; Dictionary> kvEtags = null; + Dictionary> ffEtags = null; HashSet ffKeys = null; Dictionary watchedIndividualKvs = null; List keyValueChanges = null; @@ -328,7 +328,12 @@ await ExecuteWithFailOverPolicyAsync(clients, async (client) => // Get feature flag changes ffCollectionUpdated = await HaveCollectionsChanged( - refreshableFfWatchers.Select(watcher => new KeyValueSelector { KeyFilter = watcher.Key, LabelFilter = watcher.Label }), + refreshableFfWatchers.Select(watcher => new KeyValueSelector + { + KeyFilter = watcher.Key, + LabelFilter = watcher.Label, + IsFeatureFlagSelector = true + }), _ffEtags, client, cancellationToken).ConfigureAwait(false); @@ -759,10 +764,11 @@ await ExecuteWithFailOverPolicyAsync( SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); - _watchedIndividualKvs = watchedIndividualKvs; + _mappedData = mappedData; _kvEtags = kvEtags; _ffEtags = ffEtags; - _mappedData = mappedData; + _watchedIndividualKvs = watchedIndividualKvs; + _ffKeys = ffKeys; } } @@ -1284,8 +1290,7 @@ private void UpdateClientBackoffStatus(Uri endpoint, bool successful) private async Task HaveCollectionsChanged( IEnumerable selectors, - Dictionary> pageEtags, + Dictionary> pageEtags, ConfigurationClient client, CancellationToken cancellationToken) { From 00ac014eeaa687f5734334bc289e2f811b276282 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Tue, 21 Jan 2025 09:41:54 -0800 Subject: [PATCH 70/72] PR comments --- .../AzureAppConfigurationOptions.cs | 5 +++-- .../AzureAppConfigurationProvider.cs | 2 +- .../Constants/LoggingConstants.cs | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 5c1d7382..ebece3ba 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -408,7 +408,7 @@ public AzureAppConfigurationOptions ConfigureRefresh(Action Date: Tue, 21 Jan 2025 14:33:25 -0800 Subject: [PATCH 71/72] use invalidoperationexception in configurerefresh, update loggingconstants to match behavior --- .../AzureAppConfigurationOptions.cs | 4 ++-- .../Constants/LoggingConstants.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index ebece3ba..9391f21b 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -419,14 +419,14 @@ public AzureAppConfigurationOptions ConfigureRefresh(Action Date: Wed, 22 Jan 2025 10:36:53 -0800 Subject: [PATCH 72/72] remove unused changes --- .../Constants/ErrorMessages.cs | 1 - .../Constants/LoggingConstants.cs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/ErrorMessages.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/ErrorMessages.cs index 527062fc..c7974736 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/ErrorMessages.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/ErrorMessages.cs @@ -10,6 +10,5 @@ internal class ErrorMessages public const string FeatureFlagInvalidJsonProperty = "Invalid property '{0}' for feature flag. Key: '{1}'. Found type: '{2}'. Expected type: '{3}'."; public const string FeatureFlagInvalidFormat = "Invalid json format for feature flag. Key: '{0}'."; public const string InvalidKeyVaultReference = "Invalid Key Vault reference."; - public const string InvalidConfigurationSettingPage = "Invalid page while loading configuration settings."; } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs index 376540a6..3bdcaecd 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/LoggingConstants.cs @@ -28,7 +28,7 @@ internal class LoggingConstants public const string RefreshKeyValueSettingUpdated = "Setting updated."; public const string RefreshKeyVaultSettingUpdated = "Setting updated from Key Vault."; public const string RefreshFeatureFlagsUpdated = "Feature flags reloaded."; - public const string RefreshSelectedKeyValuesAndFeatureFlagsUpdated = "Selected key-value collections and feature flags updated."; + public const string RefreshSelectedKeyValuesAndFeatureFlagsUpdated = "Selected key-value collections and feature flags reloaded."; // Other public const string RefreshSkippedNoClientAvailable = "Refresh skipped because no endpoint is accessible.";