Skip to content

Commit c255ac7

Browse files
authored
maxModels instead of time for AutoML unit test (#5471)
Uses the internal `maxModels` parameter instead of `MaxExperimentTimeInSeconds` for the exit criteria of AutoML. This is to increase the test stability in case the test is run on a slower machine.
1 parent 4a30bf5 commit c255ac7

File tree

1 file changed

+7
-24
lines changed

1 file changed

+7
-24
lines changed

test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs

Lines changed: 7 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -121,11 +121,11 @@ public void AutoFitRegressionTest(string culture)
121121
// the sweeper encounters problems when parsing some strings.
122122
// So testing in another culture is necessary.
123123
// Furthermore, these issues might only occur after ~70
124-
// iterations, so more experiment time is needed for this to
125-
// occur.
126-
uint experimentTime = (uint) (culture == "en-US" ? 0 : 180);
124+
// iterations, so setting the internal maxModels parameter.
125+
int maxModels = culture == "en-US" ? 1 : 75;
126+
127+
var experimentSettings = new RegressionExperimentSettings { MaxModels = maxModels };
127128

128-
var experimentSettings = new RegressionExperimentSettings { MaxExperimentTimeInSeconds = experimentTime};
129129
if (!Environment.Is64BitProcess)
130130
{
131131
// LightGBM isn't available on x86 machines
@@ -144,27 +144,10 @@ public void AutoFitRegressionTest(string culture)
144144
.Execute(trainData, validationData,
145145
new ColumnInformation() { LabelColumnName = DatasetUtil.MlNetGeneratedRegressionLabel });
146146

147-
Assert.True(result.RunDetails.Max(i => i?.ValidationMetrics?.RSquared) > 0.9);
147+
Assert.True(result.RunDetails.Max(i => i?.ValidationMetrics?.RSquared) > 0.99);
148148

149-
// Ensure experimentTime allows enough iterations to fully test the internationalization code
150-
// If the below assertion fails, increase the experiment time so the number of iterations is met
151-
Assert.True(culture == "en-US" || result.RunDetails.Count() >= 75, $"RunDetails.Count() = {result.RunDetails.Count()}, below 75");
152-
}
153-
catch (AggregateException ae)
154-
{
155-
// During CI unit testing, the host machines can run slower than normal, which
156-
// can increase the run time of unit tests and throw OperationCanceledExceptions
157-
// from multiple threads in the form of a single AggregateException.
158-
foreach (var ex in ae.Flatten().InnerExceptions)
159-
{
160-
var ignoredExceptions = new List<Exception>();
161-
if (ex is OperationCanceledException)
162-
continue;
163-
else
164-
ignoredExceptions.Add(ex);
165-
if (ignoredExceptions.Count > 0)
166-
throw new AggregateException(ignoredExceptions);
167-
}
149+
// Test the internal maxModels parameter
150+
Assert.True(culture == "en-US" || result.RunDetails.Count() == 75, $"RunDetails.Count() = {result.RunDetails.Count()}, is not 75");
168151
}
169152
finally
170153
{

0 commit comments

Comments
 (0)