From f9f328d52cd5b4281ad38b7a6af20c219dd0fd44 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Thu, 17 Dec 2020 10:33:02 -0800 Subject: [PATCH 1/5] runSpecific in YAML --- .vsts-dotnet-ci.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index d3315c00c5..388c4ee921 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -27,7 +27,8 @@ jobs: _config_short: RI _includeBenchmarkData: true _targetFramework: netcoreapp3.1 - innerLoop: true + innerLoop: false + runSpecific: true pool: name: Hosted Ubuntu 1604 @@ -36,7 +37,8 @@ jobs: name: Ubuntu_x64_NetCoreApp21 buildScript: ./build.sh container: UbuntuContainer - innerLoop: true + innerLoop: false + runSpecific: true pool: name: Hosted Ubuntu 1604 @@ -44,7 +46,8 @@ jobs: parameters: name: MacOS_x64_NetCoreApp21 buildScript: ./build.sh - innerLoop: true + innerLoop: false + runSpecific: true pool: name: Hosted macOS @@ -63,7 +66,8 @@ jobs: _config_short: RI _includeBenchmarkData: true _targetFramework: netcoreapp3.1 - innerLoop: true + innerLoop: false + runSpecific: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v3.0" pool: name: Hosted VS2017 @@ -72,7 +76,8 @@ jobs: parameters: name: Windows_x64_NetCoreApp21 buildScript: build.cmd - innerLoop: true + innerLoop: false + runSpecific: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017 @@ -92,7 +97,8 @@ jobs: _config_short: RFX _includeBenchmarkData: false _targetFramework: win-x64 - innerLoop: true + innerLoop: false + runSpecific: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v4.0" pool: name: Hosted VS2017 @@ -102,7 +108,8 @@ jobs: name: Windows_x86_NetCoreApp21 architecture: x86 buildScript: build.cmd - innerLoop: true + innerLoop: false + runSpecific: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017 From fed12b26ae71e7a95d2dd1f4703541138a780d75 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Thu, 17 Dec 2020 10:46:43 -0800 Subject: [PATCH 2/5] RunSpecific in test --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index d5af1f1be5..c85e939a32 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -442,9 +442,11 @@ public void AutoFitWithPresplittedData() } - [LightGBMFact] - public void AutoFitMaxExperimentTimeTest() + [Theory, IterationData(10)] + [TestCategory("RunSpecificTest")] + public void AutoFitMaxExperimentTimeTest(int iteration) { + Console.WriteLine(iteration); // A single binary classification experiment takes less than 5 seconds. // System.OperationCanceledException is thrown when ongoing experiment // is canceled and at least one model has been generated. @@ -455,8 +457,12 @@ public void AutoFitMaxExperimentTimeTest() var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.UciAdultLabel); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(dataPath); + + var experSettings = new BinaryExperimentSettings { MaxExperimentTimeInSeconds = 60}; + experSettings.Trainers.Remove(BinaryClassificationTrainer.LightGbm); + var experiment = context.Auto() - .CreateBinaryClassificationExperiment(15) + .CreateBinaryClassificationExperiment(experSettings) .Execute(trainData, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }); // Ensure the (last) model that was training when maximum experiment time was reached has been stopped, From 79407db4187fe723ec6917702a202cc78bf5aa33 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Thu, 17 Dec 2020 11:25:08 -0800 Subject: [PATCH 3/5] Add SymSgdNative reference --- test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj b/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj index af86334edf..2800f55994 100644 --- a/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj +++ b/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj @@ -41,6 +41,7 @@ + From c2685eaffabb16be5709d7fed169bdff67d4d779 Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Thu, 17 Dec 2020 11:26:18 -0800 Subject: [PATCH 4/5] Revert "RunSpecific in test" This reverts commit fed12b26ae71e7a95d2dd1f4703541138a780d75. --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index c85e939a32..d5af1f1be5 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -442,11 +442,9 @@ public void AutoFitWithPresplittedData() } - [Theory, IterationData(10)] - [TestCategory("RunSpecificTest")] - public void AutoFitMaxExperimentTimeTest(int iteration) + [LightGBMFact] + public void AutoFitMaxExperimentTimeTest() { - Console.WriteLine(iteration); // A single binary classification experiment takes less than 5 seconds. // System.OperationCanceledException is thrown when ongoing experiment // is canceled and at least one model has been generated. @@ -457,12 +455,8 @@ public void AutoFitMaxExperimentTimeTest(int iteration) var columnInference = context.Auto().InferColumns(dataPath, DatasetUtil.UciAdultLabel); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(dataPath); - - var experSettings = new BinaryExperimentSettings { MaxExperimentTimeInSeconds = 60}; - experSettings.Trainers.Remove(BinaryClassificationTrainer.LightGbm); - var experiment = context.Auto() - .CreateBinaryClassificationExperiment(experSettings) + .CreateBinaryClassificationExperiment(15) .Execute(trainData, new ColumnInformation() { LabelColumnName = DatasetUtil.UciAdultLabel }); // Ensure the (last) model that was training when maximum experiment time was reached has been stopped, From 0afe59f8d85d3fcdf8d3a8b3764970b42cf0b35f Mon Sep 17 00:00:00 2001 From: Antonio Velazquez Date: Thu, 17 Dec 2020 11:26:27 -0800 Subject: [PATCH 5/5] Revert "runSpecific in YAML" This reverts commit f9f328d52cd5b4281ad38b7a6af20c219dd0fd44. --- .vsts-dotnet-ci.yml | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 388c4ee921..d3315c00c5 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -27,8 +27,7 @@ jobs: _config_short: RI _includeBenchmarkData: true _targetFramework: netcoreapp3.1 - innerLoop: false - runSpecific: true + innerLoop: true pool: name: Hosted Ubuntu 1604 @@ -37,8 +36,7 @@ jobs: name: Ubuntu_x64_NetCoreApp21 buildScript: ./build.sh container: UbuntuContainer - innerLoop: false - runSpecific: true + innerLoop: true pool: name: Hosted Ubuntu 1604 @@ -46,8 +44,7 @@ jobs: parameters: name: MacOS_x64_NetCoreApp21 buildScript: ./build.sh - innerLoop: false - runSpecific: true + innerLoop: true pool: name: Hosted macOS @@ -66,8 +63,7 @@ jobs: _config_short: RI _includeBenchmarkData: true _targetFramework: netcoreapp3.1 - innerLoop: false - runSpecific: true + innerLoop: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v3.0" pool: name: Hosted VS2017 @@ -76,8 +72,7 @@ jobs: parameters: name: Windows_x64_NetCoreApp21 buildScript: build.cmd - innerLoop: false - runSpecific: true + innerLoop: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017 @@ -97,8 +92,7 @@ jobs: _config_short: RFX _includeBenchmarkData: false _targetFramework: win-x64 - innerLoop: false - runSpecific: true + innerLoop: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v4.0" pool: name: Hosted VS2017 @@ -108,8 +102,7 @@ jobs: name: Windows_x86_NetCoreApp21 architecture: x86 buildScript: build.cmd - innerLoop: false - runSpecific: true + innerLoop: true vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1" pool: name: Hosted VS2017