Fix handling of non-mergeable char classes in regex prefix analyzer

Author: stephentoub

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs

@@ -329,6 +329,18 @@ public void AddCharClass(RegexCharClass cc)
             }
         }
 
+        /// <summary>Adds a regex char class if the classes are mergeable.</summary>
+        public bool TryAddCharClass(RegexCharClass cc)
+        {
+            if (cc.CanMerge && CanMerge)
+            {
+                AddCharClass(cc);
+                return true;
+            }
+
+            return false;
+        }
+
         private StringBuilder EnsureCategories() =>
             _categories ??= new StringBuilder();
 

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexPrefixAnalyzer.cs

@@ -223,7 +223,11 @@ public static (string CharClass, bool CaseInsensitive)[]? ComputeMultipleCharCla
                                         (classes[classPos++] ??= new RegexCharClass()).AddChar(concatChild.Ch);
                                         break;
                                     case RegexNode.Set:
-                                        (classes[classPos++] ??= new RegexCharClass()).AddCharClass(RegexCharClass.Parse(concatChild.Str!));
+                                        if (!(classes[classPos++] ??= new RegexCharClass()).TryAddCharClass(RegexCharClass.Parse(concatChild.Str!)))
+                                        {
+                                            // If the classes can't be merged, give up.
+                                            return null;
+                                        }
                                         break;
                                     case RegexNode.Multi:
                                         for (int c = 0; c < concatChild.Str!.Length && classPos < classes.Length; c++)

src/libraries/System.Text.RegularExpressions/tests/Regex.Match.Tests.cs

@@ -284,6 +284,7 @@ public static IEnumerable<object[]> Match_Basic_TestData()
             yield return new object[] { "(?(cat)|dog)", "oof", RegexOptions.None, 0, 3, false, string.Empty };
             yield return new object[] { "(?(a:b))", "a", RegexOptions.None, 0, 1, true, string.Empty };
             yield return new object[] { "(?(a:))", "a", RegexOptions.None, 0, 1, true, string.Empty };
+            yield return new object[] { "[^a-z0-9]etag|[^a-z0-9]digest", "this string has .digest as a substring", RegexOptions.None, 16, 7, true, ".digest" };
 
             // No Negation
             yield return new object[] { "[abcd-[abcd]]+", "abcxyzABCXYZ`!@#$%^&*()_-+= \t\n", RegexOptions.None, 0, 30, false, string.Empty };