[release/6.0] Fix issue with incorrect handling of optimized method layouts

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunFileLayoutOptimizer.cs

@@ -26,6 +26,7 @@ public enum ReadyToRunMethodLayoutAlgorithm
         HotWarmCold,
         CallFrequency,
         PettisHansen,
+        Random,
     }
 
     public enum ReadyToRunFileLayoutAlgorithm
@@ -166,6 +167,17 @@ int ComputeHotWarmColdRegion(MethodWithGCInfo method)
                     methods = PettisHansenSort(methods);
                     break;
 
+                case ReadyToRunMethodLayoutAlgorithm.Random:
+                    Random rand = new Random(0);
+                    for (int i = 0; i < methods.Count - 1; i++)
+                    {
+                        int j = rand.Next(i, methods.Count);
+                        MethodWithGCInfo temp = methods[i];
+                        methods[i] = methods[j];
+                        methods[j] = temp;
+                    }
+                    break;
+
                 default:
                     throw new NotImplementedException(_methodLayoutAlgorithm.ToString());
             }

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunTableManager.cs

@@ -101,16 +101,30 @@ public IEnumerable<IMethodNode> GetCompiledMethods(EcmaModule moduleToEnumerate,
             {
                 if (!_sortedMethods)
                 {
-                    TypeSystemComparer comparer = new TypeSystemComparer();
-                    Comparison<IMethodNode> sortHelper = (x, y) => comparer.Compare(x.Method, y.Method);
+                    CompilerComparer comparer = new CompilerComparer();
+                    SortableDependencyNode.ObjectNodeComparer objectNodeComparer = new SortableDependencyNode.ObjectNodeComparer(comparer);
+                    Comparison<IMethodNode> sortHelper = (x, y) =>
+                    {
+                        int nodeComparerResult = objectNodeComparer.Compare((SortableDependencyNode)x, (SortableDependencyNode)y);
+#if DEBUG
+                        int methodOnlyResult = comparer.Compare(x.Method, y.Method);
+
+                        // Assert the two sorting techniques produce the same result unless there is a CustomSort applied
+                        Debug.Assert((nodeComparerResult == methodOnlyResult) || 
+                            ((x is SortableDependencyNode sortableX && sortableX.CustomSort != Int32.MaxValue) ||
+                             (y is SortableDependencyNode sortableY && sortableY.CustomSort != Int32.MaxValue)));
+#endif
+                        return nodeComparerResult;
+                    };
+                    Comparison<IMethodNode> sortHelperNoCustomSort = (x, y) => comparer.Compare(x, y);
 
                     List<PerModuleMethodsGenerated> perModuleDatas = new List<PerModuleMethodsGenerated>(_methodsGenerated.Values);
                     perModuleDatas.Sort((x, y) => x.Module.CompareTo(y.Module));
 
                     foreach (var perModuleData in perModuleDatas)
                     {
-                        perModuleData.MethodsGenerated.MergeSort(sortHelper);
-                        perModuleData.GenericMethodsGenerated.MergeSort(sortHelper);
+                        perModuleData.MethodsGenerated.MergeSort(sortHelperNoCustomSort);
+                        perModuleData.GenericMethodsGenerated.MergeSort(sortHelperNoCustomSort);
                         _completeSortedMethods.AddRange(perModuleData.MethodsGenerated);
                         _completeSortedMethods.AddRange(perModuleData.GenericMethodsGenerated);
                         _completeSortedGenericMethods.AddRange(perModuleData.GenericMethodsGenerated);

src/coreclr/tools/aot/crossgen2/Program.cs

@@ -156,6 +156,7 @@ private void ProcessCommandLine(string[] args)
                     "hotwarmcold" => ReadyToRunMethodLayoutAlgorithm.HotWarmCold,
                     "callfrequency" => ReadyToRunMethodLayoutAlgorithm.CallFrequency,
                     "pettishansen" => ReadyToRunMethodLayoutAlgorithm.PettisHansen,
+                    "random" => ReadyToRunMethodLayoutAlgorithm.Random,
                     _ => throw new CommandLineException(SR.InvalidMethodLayout)
                 };
             }

src/coreclr/tools/aot/crossgen2/Properties/Resources.resx

@@ -148,7 +148,7 @@
     <value>Method layout must be either DefaultSort or MethodOrder.</value>
   </data>
   <data name="InvalidMethodLayout" xml:space="preserve">
-    <value>Method layout must be either DefaultSort, ExclusiveWeight, HotCold, HotWarmCold, CallFrequency or PettisHansen.</value>
+    <value>Method layout must be either DefaultSort, ExclusiveWeight, HotCold, HotWarmCold, CallFrequency, PettisHansen, or Random.</value>
   </data>
   <data name="CompileNoMethodsOption" xml:space="preserve">
     <value>True to skip compiling methods into the R2R image (default = false)</value>

src/tests/Common/CLRTest.CrossGen.targets

@@ -225,6 +225,7 @@ if defined RunCrossGen2 (
     echo -o:!__OutputFile!>>!__ResponseFile!
     echo --targetarch:$(TargetArchitecture)>>!__ResponseFile!
     echo --verify-type-and-field-layout>>!__ResponseFile!
+    echo --method-layout:random>>!__ResponseFile!
     echo -r:!CORE_ROOT!\System.*.dll>>!__ResponseFile!
     echo -r:!CORE_ROOT!\Microsoft.*.dll>>!__ResponseFile!
     echo -r:!CORE_ROOT!\mscorlib.dll>>!__ResponseFile!