From 3dd2e57be9fd39087b82845c4c77caed689663f1 Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Wed, 1 May 2019 15:48:54 -0700 Subject: [PATCH 01/15] early stopping in CLI --- .../Utils/BestResultUtil.cs | 27 ++++ src/mlnet/AutoML/AutoMLEngine.cs | 38 +++-- src/mlnet/AutoML/IAutoMLEngine.cs | 7 +- .../CodeGenerator/CodeGenerationHelper.cs | 131 ++++++++++++------ src/mlnet/Strings.resx | 3 + src/mlnet/Utilities/ProgressHandlers.cs | 103 ++++++++++---- src/mlnet/strings.Designer.cs | 9 ++ 7 files changed, 235 insertions(+), 83 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs index a16d5617ff..4ab3b89da0 100644 --- a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs +++ b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs @@ -4,11 +4,36 @@ using System.Collections.Generic; using System.Linq; +using Microsoft.ML.Data; namespace Microsoft.ML.AutoML { internal class BestResultUtil { + public static RunDetail GetBestRun(IEnumerable> results, + BinaryClassificationMetric metric) + { + var metricsAgent = new BinaryMetricsAgent(null, metric); + var metricInfo = new OptimizingMetricInfo(metric); + return GetBestRun(results, metricsAgent, metricInfo.IsMaximizing); + } + + public static RunDetail GetBestRun(IEnumerable> results, + RegressionMetric metric) + { + var metricsAgent = new RegressionMetricsAgent(null, metric); + var metricInfo = new OptimizingMetricInfo(metric); + return GetBestRun(results, metricsAgent, metricInfo.IsMaximizing); + } + + public static RunDetail GetBestRun(IEnumerable> results, + MulticlassClassificationMetric metric) + { + var metricsAgent = new MultiMetricsAgent(null, metric); + var metricInfo = new OptimizingMetricInfo(metric); + return GetBestRun(results, metricsAgent, metricInfo.IsMaximizing); + } + public static RunDetail GetBestRun(IEnumerable> results, IMetricsAgent metricsAgent, bool isMetricMaximizing) { @@ -29,6 +54,8 @@ public static CrossValidationRunDetail GetBestRun(IEnumerabl return results.ElementAt(indexOfBestScore); } + + public static IEnumerable<(RunDetail, int)> GetTopNRunResults(IEnumerable> results, IMetricsAgent metricsAgent, int n, bool isMetricMaximizing) { diff --git a/src/mlnet/AutoML/AutoMLEngine.cs b/src/mlnet/AutoML/AutoMLEngine.cs index 7acfe5f84a..8340e4984f 100644 --- a/src/mlnet/AutoML/AutoMLEngine.cs +++ b/src/mlnet/AutoML/AutoMLEngine.cs @@ -2,7 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; +using System.Threading.Tasks; using Microsoft.ML.AutoML; using Microsoft.ML.CLI.Data; using Microsoft.ML.CLI.ShellProgressBar; @@ -44,45 +46,57 @@ public ColumnInferenceResults InferColumns(MLContext context, ColumnInformation return columnInference; } - ExperimentResult IAutoMLEngine.ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, ProgressBar progressBar) + ExperimentResult IAutoMLEngine.ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, out List> completedIterations, ProgressBar progressBar) { - var progressReporter = new ProgressHandlers.BinaryClassificationHandler(optimizationMetric, progressBar); - var result = context.Auto() + completedIterations = new List>(); + var progressReporter = new ProgressHandlers.BinaryClassificationHandler(optimizationMetric, completedIterations, progressBar); + ExperimentResult result = null; + Task.Run(() => result = context.Auto() .CreateBinaryClassificationExperiment(new BinaryExperimentSettings() { MaxExperimentTimeInSeconds = settings.MaxExplorationTime, CacheBeforeTrainer = this.cacheBeforeTrainer, OptimizingMetric = optimizationMetric }) - .Execute(trainData, validationData, columnInformation, progressHandler: progressReporter); + .Execute(trainData, validationData, columnInformation, progressHandler: progressReporter)).Wait(experimentTimeout); + + progressReporter.Stop(); logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline); return result; } - ExperimentResult IAutoMLEngine.ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, ProgressBar progressBar) + ExperimentResult IAutoMLEngine.ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, out List> completedIterations, ProgressBar progressBar) { - var progressReporter = new ProgressHandlers.RegressionHandler(optimizationMetric, progressBar); - var result = context.Auto() + completedIterations = new List>(); + var progressReporter = new ProgressHandlers.RegressionHandler(optimizationMetric, completedIterations, progressBar); + ExperimentResult result = null; + Task.Run(() => result = context.Auto() .CreateRegressionExperiment(new RegressionExperimentSettings() { MaxExperimentTimeInSeconds = settings.MaxExplorationTime, OptimizingMetric = optimizationMetric, CacheBeforeTrainer = this.cacheBeforeTrainer - }).Execute(trainData, validationData, columnInformation, progressHandler: progressReporter); + }).Execute(trainData, validationData, columnInformation, progressHandler: progressReporter)).Wait(experimentTimeout); + + progressReporter.Stop(); logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline); return result; } - ExperimentResult IAutoMLEngine.ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, ProgressBar progressBar) + ExperimentResult IAutoMLEngine.ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, out List> completedIterations, ProgressBar progressBar) { - var progressReporter = new ProgressHandlers.MulticlassClassificationHandler(optimizationMetric, progressBar); - var result = context.Auto() + completedIterations = new List>(); + var progressReporter = new ProgressHandlers.MulticlassClassificationHandler(optimizationMetric, completedIterations, progressBar); + ExperimentResult result = null; + Task.Run(() => result = context.Auto() .CreateMulticlassClassificationExperiment(new MulticlassExperimentSettings() { MaxExperimentTimeInSeconds = settings.MaxExplorationTime, CacheBeforeTrainer = this.cacheBeforeTrainer, OptimizingMetric = optimizationMetric - }).Execute(trainData, validationData, columnInformation, progressHandler: progressReporter); + }).Execute(trainData, validationData, columnInformation, progressHandler: progressReporter)).Wait(experimentTimeout); + // wait for experiment time + progressReporter.Stop(); logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline); return result; } diff --git a/src/mlnet/AutoML/IAutoMLEngine.cs b/src/mlnet/AutoML/IAutoMLEngine.cs index 882ff654fe..87a3adf38d 100644 --- a/src/mlnet/AutoML/IAutoMLEngine.cs +++ b/src/mlnet/AutoML/IAutoMLEngine.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; using Microsoft.ML.AutoML; using Microsoft.ML.CLI.ShellProgressBar; @@ -13,11 +14,11 @@ internal interface IAutoMLEngine { ColumnInferenceResults InferColumns(MLContext context, ColumnInformation columnInformation); - ExperimentResult ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, ProgressBar progressBar = null); + ExperimentResult ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, out List> completedIterations, ProgressBar progressBar = null); - ExperimentResult ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, ProgressBar progressBar = null); + ExperimentResult ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, out List> completedIterations, ProgressBar progressBar = null); - ExperimentResult ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, ProgressBar progressBar = null); + ExperimentResult ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, out List> completedIterations, ProgressBar progressBar = null); } } diff --git a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs index f2153fd785..5cd9e726be 100644 --- a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs +++ b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; @@ -36,7 +37,7 @@ public void GenerateCode() { Stopwatch watch = Stopwatch.StartNew(); var context = new MLContext(); - ConsumeAutoMLSDKLogs(context); + context.Log += ConsumeAutoMLSDKLog; var verboseLevel = Utils.GetVerbosity(settings.Verbosity); @@ -76,6 +77,11 @@ public void GenerateCode() ExperimentResult binaryExperimentResult = default; ExperimentResult multiclassExperimentResult = default; ExperimentResult regressionExperimentResult = default; + + List> completedBinaryRuns = default; + List> completedMulticlassRuns = default; + List> completedRegressionRuns = default; + if (verboseLevel > LogLevel.Trace) { Console.Write($"{Strings.ExplorePipeline}: "); @@ -113,13 +119,13 @@ public void GenerateCode() { // TODO: It may be a good idea to convert the below Threads to Tasks or get rid of this progress bar all together and use an existing one in opensource. case TaskKind.BinaryClassification: - t = new Thread(() => SafeExecute(() => automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, pbar), out ex, out binaryExperimentResult, pbar)); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, wait, out completedBinaryRuns, pbar), out ex, out binaryExperimentResult, pbar)); break; case TaskKind.Regression: - t = new Thread(() => SafeExecute(() => automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, pbar), out ex, out regressionExperimentResult, pbar)); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, wait, out completedRegressionRuns, pbar), out ex, out regressionExperimentResult, pbar)); break; case TaskKind.MulticlassClassification: - t = new Thread(() => SafeExecute(() => automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, pbar), out ex, out multiclassExperimentResult, pbar)); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, wait, out completedMulticlassRuns, pbar), out ex, out multiclassExperimentResult, pbar)); break; default: logger.Log(LogLevel.Error, Strings.UnsupportedMlTask); @@ -130,18 +136,8 @@ public void GenerateCode() if (!pbar.CompletedHandle.WaitOne(wait)) pbar.Message = $"{nameof(FixedDurationBar)} did not signal {nameof(FixedDurationBar.CompletedHandle)} after {wait}"; - if (t.IsAlive == true) - { - string waitingMessage = Strings.WaitingForLastIteration; - string originalMessage = pbar.Message; - pbar.Message = waitingMessage; - t.Join(); - if (waitingMessage.Equals(pbar.Message)) - { - // Corner cases where thread was alive but has completed all iterations. - pbar.Message = originalMessage; - } - } + t.Join(); + if (ex != null) { throw ex; @@ -153,27 +149,29 @@ public void GenerateCode() switch (taskKind) { case TaskKind.BinaryClassification: - binaryExperimentResult = automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric); + binaryExperimentResult = automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, wait, out completedBinaryRuns); break; case TaskKind.Regression: - regressionExperimentResult = automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric); + regressionExperimentResult = automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, wait, out completedRegressionRuns); break; case TaskKind.MulticlassClassification: - multiclassExperimentResult = automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric); + multiclassExperimentResult = automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, wait, out completedMulticlassRuns); break; default: logger.Log(LogLevel.Error, Strings.UnsupportedMlTask); break; } } - - } catch (Exception) { logger.Log(LogLevel.Error, $"{Strings.ExplorePipelineException}:"); throw; } + finally + { + context.Log -= ConsumeAutoMLSDKLog; + } var elapsedTime = watch.Elapsed.TotalSeconds; @@ -185,25 +183,55 @@ public void GenerateCode() switch (taskKind) { case TaskKind.BinaryClassification: - var bestBinaryIteration = binaryExperimentResult.BestRun; - bestPipeline = bestBinaryIteration.Pipeline; - bestModel = bestBinaryIteration.Model; - ConsolePrinter.ExperimentResultsHeader(LogLevel.Info, settings.MlTask, settings.Dataset.Name, columnInformation.LabelColumnName, elapsedTime.ToString("F2"), binaryExperimentResult.RunDetails.Count()); - ConsolePrinter.PrintIterationSummary(binaryExperimentResult.RunDetails, new BinaryExperimentSettings().OptimizingMetric, 5); + if (completedBinaryRuns.Count > 0) + { + var binaryMetric = new BinaryExperimentSettings().OptimizingMetric; + var bestBinaryIteration = BestResultUtil.GetBestRun(completedBinaryRuns, binaryMetric); + bestPipeline = bestBinaryIteration.Pipeline; + bestModel = bestBinaryIteration.Model; + ConsolePrinter.ExperimentResultsHeader(LogLevel.Info, settings.MlTask, settings.Dataset.Name, columnInformation.LabelColumnName, elapsedTime.ToString("F2"), completedBinaryRuns.Count()); + ConsolePrinter.PrintIterationSummary(completedBinaryRuns, binaryMetric, 5); + } + else + { + logger.Log(LogLevel.Error, Strings.CouldNotFinshOnTime); + logger.Log(LogLevel.Info, Strings.Exiting); + return; + } break; case TaskKind.Regression: - var bestRegressionIteration = regressionExperimentResult.BestRun; - bestPipeline = bestRegressionIteration.Pipeline; - bestModel = bestRegressionIteration.Model; - ConsolePrinter.ExperimentResultsHeader(LogLevel.Info, settings.MlTask, settings.Dataset.Name, columnInformation.LabelColumnName, elapsedTime.ToString("F2"), regressionExperimentResult.RunDetails.Count()); - ConsolePrinter.PrintIterationSummary(regressionExperimentResult.RunDetails, new RegressionExperimentSettings().OptimizingMetric, 5); + if (completedRegressionRuns.Count > 0) + { + var regressionMetric = new RegressionExperimentSettings().OptimizingMetric; + var bestRegressionIteration = BestResultUtil.GetBestRun(completedRegressionRuns, regressionMetric); + bestPipeline = bestRegressionIteration.Pipeline; + bestModel = bestRegressionIteration.Model; + ConsolePrinter.ExperimentResultsHeader(LogLevel.Info, settings.MlTask, settings.Dataset.Name, columnInformation.LabelColumnName, elapsedTime.ToString("F2"), completedRegressionRuns.Count()); + ConsolePrinter.PrintIterationSummary(completedRegressionRuns, regressionMetric, 5); + } + else + { + logger.Log(LogLevel.Error, Strings.CouldNotFinshOnTime); + logger.Log(LogLevel.Info, Strings.Exiting); + return; + } break; case TaskKind.MulticlassClassification: - var bestMulticlassIteration = multiclassExperimentResult.BestRun; - bestPipeline = bestMulticlassIteration.Pipeline; - bestModel = bestMulticlassIteration.Model; - ConsolePrinter.ExperimentResultsHeader(LogLevel.Info, settings.MlTask, settings.Dataset.Name, columnInformation.LabelColumnName, elapsedTime.ToString("F2"), multiclassExperimentResult.RunDetails.Count()); - ConsolePrinter.PrintIterationSummary(multiclassExperimentResult.RunDetails, new MulticlassExperimentSettings().OptimizingMetric, 5); + if (completedMulticlassRuns.Count > 0) + { + var muliclassMetric = new MulticlassExperimentSettings().OptimizingMetric; + var bestMulticlassIteration = BestResultUtil.GetBestRun(completedMulticlassRuns, muliclassMetric); + bestPipeline = bestMulticlassIteration.Pipeline; + bestModel = bestMulticlassIteration.Model; + ConsolePrinter.ExperimentResultsHeader(LogLevel.Info, settings.MlTask, settings.Dataset.Name, columnInformation.LabelColumnName, elapsedTime.ToString("F2"), completedMulticlassRuns.Count()); + ConsolePrinter.PrintIterationSummary(completedMulticlassRuns, muliclassMetric, 5); + } + else + { + logger.Log(LogLevel.Error, Strings.CouldNotFinshOnTime); + logger.Log(LogLevel.Info, Strings.Exiting); + return; + } break; } } @@ -284,16 +312,13 @@ internal void GenerateProject(ColumnInferenceResults columnInference, Pipeline p return (trainData, validationData); } - private void ConsumeAutoMLSDKLogs(MLContext context) + private static void ConsumeAutoMLSDKLog(object sender, LoggingEventArgs args) { - context.Log += (object sender, LoggingEventArgs loggingEventArgs) => + var logMessage = args.Message; + if (logMessage.Contains(AutoMLLogger.ChannelName)) { - var logMessage = loggingEventArgs.Message; - if (logMessage.Contains(AutoMLLogger.ChannelName)) - { - logger.Trace(loggingEventArgs.Message); - } - }; + logger.Trace(args.Message); + } } private void SafeExecute(Func> p, out Exception ex, out ExperimentResult binaryExperimentResult, FixedDurationBar pbar) @@ -303,6 +328,12 @@ private void SafeExecute(Func> p, binaryExperimentResult = p.Invoke(); ex = null; } + catch (ThreadAbortException) + { + binaryExperimentResult = null; + pbar.Dispose(); // or ((ManualResetEvent)pbar.CompletedHandle).Set(); + throw; + } catch (Exception e) { ex = e; @@ -318,6 +349,12 @@ private void SafeExecute(Func> p, out Except regressionExperimentResult = p.Invoke(); ex = null; } + catch (ThreadAbortException) + { + regressionExperimentResult = null; + pbar.Dispose(); // or ((ManualResetEvent)pbar.CompletedHandle).Set(); + throw; + } catch (Exception e) { ex = e; @@ -333,6 +370,12 @@ private void SafeExecute(Func> multiClassExperimentResult = p.Invoke(); ex = null; } + catch (ThreadAbortException) + { + multiClassExperimentResult = null; + pbar.Dispose(); // or ((ManualResetEvent)pbar.CompletedHandle).Set(); + throw; + } catch (Exception e) { ex = e; diff --git a/src/mlnet/Strings.resx b/src/mlnet/Strings.resx index 81b4ad8e13..47381f8f38 100644 --- a/src/mlnet/Strings.resx +++ b/src/mlnet/Strings.resx @@ -198,4 +198,7 @@ Exception occured while saving the model + + Training could not finish in the specified exploration time. Please set the exploration time to a higher value. + \ No newline at end of file diff --git a/src/mlnet/Utilities/ProgressHandlers.cs b/src/mlnet/Utilities/ProgressHandlers.cs index 39123432ba..1fc5996816 100644 --- a/src/mlnet/Utilities/ProgressHandlers.cs +++ b/src/mlnet/Utilities/ProgressHandlers.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using Microsoft.ML.AutoML; using Microsoft.ML.CLI.ShellProgressBar; using Microsoft.ML.Data; @@ -23,13 +24,16 @@ internal class RegressionHandler : IProgress> private readonly Func, double> GetScore; private RunDetail bestResult; private int iterationIndex; + private List> completedIterations; private ProgressBar progressBar; private string optimizationMetric = string.Empty; + private bool isStopped; - public RegressionHandler(RegressionMetric optimizationMetric, ShellProgressBar.ProgressBar progressBar) + public RegressionHandler(RegressionMetric optimizationMetric, List> completedIterations, ShellProgressBar.ProgressBar progressBar) { this.isMaximizing = new OptimizingMetricInfo(optimizationMetric).IsMaximizing; this.optimizationMetric = optimizationMetric.ToString(); + this.completedIterations = completedIterations; this.progressBar = progressBar; GetScore = (RunDetail result) => new RegressionMetricsAgent(null, optimizationMetric).GetScore(result?.ValidationMetrics); ConsolePrinter.PrintRegressionMetricsHeader(LogLevel.Trace); @@ -37,14 +41,28 @@ public RegressionHandler(RegressionMetric optimizationMetric, ShellProgressBar.P public void Report(RunDetail iterationResult) { - iterationIndex++; - UpdateBestResult(iterationResult); - if (progressBar != null) - progressBar.Message = $"Best quality({this.optimizationMetric}): {GetScore(bestResult):F4}, Best Algorithm: {bestResult?.TrainerName}, Last Algorithm: {iterationResult?.TrainerName}"; - ConsolePrinter.PrintMetrics(iterationIndex, iterationResult?.TrainerName, iterationResult?.ValidationMetrics, GetScore(bestResult), iterationResult?.RuntimeInSeconds, LogLevel.Trace); - if (iterationResult.Exception != null) + lock (this) { - ConsolePrinter.PrintException(iterationResult.Exception, LogLevel.Trace); + if (this.isStopped) + return; + + iterationIndex++; + completedIterations.Add(iterationResult); + UpdateBestResult(iterationResult); + if (progressBar != null) + progressBar.Message = $"Best quality({this.optimizationMetric}): {GetScore(bestResult):F4}, Best Algorithm: {bestResult?.TrainerName}, Last Algorithm: {iterationResult?.TrainerName}"; + ConsolePrinter.PrintMetrics(iterationIndex, iterationResult?.TrainerName, iterationResult?.ValidationMetrics, GetScore(bestResult), iterationResult?.RuntimeInSeconds, LogLevel.Trace); + if (iterationResult.Exception != null) + { + ConsolePrinter.PrintException(iterationResult.Exception, LogLevel.Trace); + } + } + } + public void Stop() + { + lock (this) + { + this.isStopped = true; } } @@ -65,11 +83,14 @@ internal class BinaryClassificationHandler : IProgress> completedIterations; + private bool isStopped; - public BinaryClassificationHandler(BinaryClassificationMetric optimizationMetric, ProgressBar progressBar) + public BinaryClassificationHandler(BinaryClassificationMetric optimizationMetric, List> completedIterations, ProgressBar progressBar) { this.isMaximizing = new OptimizingMetricInfo(optimizationMetric).IsMaximizing; this.optimizationMetric = optimizationMetric; + this.completedIterations = completedIterations; this.progressBar = progressBar; GetScore = (RunDetail result) => new BinaryMetricsAgent(null, optimizationMetric).GetScore(result?.ValidationMetrics); ConsolePrinter.PrintBinaryClassificationMetricsHeader(LogLevel.Trace); @@ -77,14 +98,20 @@ public BinaryClassificationHandler(BinaryClassificationMetric optimizationMetric public void Report(RunDetail iterationResult) { - iterationIndex++; - UpdateBestResult(iterationResult); - if (progressBar != null) - progressBar.Message = GetProgressBarMessage(iterationResult); - ConsolePrinter.PrintMetrics(iterationIndex, iterationResult?.TrainerName, iterationResult?.ValidationMetrics, GetScore(bestResult), iterationResult?.RuntimeInSeconds, LogLevel.Trace); - if (iterationResult.Exception != null) + lock (this) { - ConsolePrinter.PrintException(iterationResult.Exception, LogLevel.Trace); + if (this.isStopped) + return; + iterationIndex++; + completedIterations.Add(iterationResult); + UpdateBestResult(iterationResult); + if (progressBar != null) + progressBar.Message = GetProgressBarMessage(iterationResult); + ConsolePrinter.PrintMetrics(iterationIndex, iterationResult?.TrainerName, iterationResult?.ValidationMetrics, GetScore(bestResult), iterationResult?.RuntimeInSeconds, LogLevel.Trace); + if (iterationResult.Exception != null) + { + ConsolePrinter.PrintException(iterationResult.Exception, LogLevel.Trace); + } } } @@ -98,6 +125,14 @@ private string GetProgressBarMessage(RunDetail iter return $"Best {this.optimizationMetric}: {GetScore(bestResult):F4}, Best Algorithm: {bestResult?.TrainerName}, Last Algorithm: {iterationResult?.TrainerName}"; } + public void Stop() + { + lock (this) + { + this.isStopped = true; + } + } + private void UpdateBestResult(RunDetail iterationResult) { if (MetricComparator(GetScore(iterationResult), GetScore(bestResult), isMaximizing) > 0) @@ -115,11 +150,14 @@ internal class MulticlassClassificationHandler : IProgress> completedIterations; + private bool isStopped; - public MulticlassClassificationHandler(MulticlassClassificationMetric optimizationMetric, ProgressBar progressBar) + public MulticlassClassificationHandler(MulticlassClassificationMetric optimizationMetric, List> completedIterations, ProgressBar progressBar) { this.isMaximizing = new OptimizingMetricInfo(optimizationMetric).IsMaximizing; this.optimizationMetric = optimizationMetric; + this.completedIterations = completedIterations; this.progressBar = progressBar; GetScore = (RunDetail result) => new MultiMetricsAgent(null, optimizationMetric).GetScore(result?.ValidationMetrics); ConsolePrinter.PrintMulticlassClassificationMetricsHeader(LogLevel.Trace); @@ -127,14 +165,31 @@ public MulticlassClassificationHandler(MulticlassClassificationMetric optimizati public void Report(RunDetail iterationResult) { - iterationIndex++; - UpdateBestResult(iterationResult); - if (progressBar != null) - progressBar.Message = GetProgressBarMessage(iterationResult); - ConsolePrinter.PrintMetrics(iterationIndex, iterationResult?.TrainerName, iterationResult?.ValidationMetrics, GetScore(bestResult), iterationResult?.RuntimeInSeconds, LogLevel.Trace); - if (iterationResult.Exception != null) + lock (this) + { + if (this.isStopped) + { + return; + } + + iterationIndex++; + completedIterations.Add(iterationResult); + UpdateBestResult(iterationResult); + if (progressBar != null) + progressBar.Message = GetProgressBarMessage(iterationResult); + ConsolePrinter.PrintMetrics(iterationIndex, iterationResult?.TrainerName, iterationResult?.ValidationMetrics, GetScore(bestResult), iterationResult?.RuntimeInSeconds, LogLevel.Trace); + if (iterationResult.Exception != null) + { + ConsolePrinter.PrintException(iterationResult.Exception, LogLevel.Trace); + } + } + } + + public void Stop() + { + lock (this) { - ConsolePrinter.PrintException(iterationResult.Exception, LogLevel.Trace); + this.isStopped = true; } } diff --git a/src/mlnet/strings.Designer.cs b/src/mlnet/strings.Designer.cs index 7ae8e6bebc..f3a4bbdf8c 100644 --- a/src/mlnet/strings.Designer.cs +++ b/src/mlnet/strings.Designer.cs @@ -69,6 +69,15 @@ internal static string BestPipeline { } } + /// + /// Looks up a localized string similar to Training could not finish in the specified exploration time. Please set the exploration time to a higher value.. + /// + internal static string CouldNotFinshOnTime { + get { + return ResourceManager.GetString("CouldNotFinshOnTime", resourceCulture); + } + } + /// /// Looks up a localized string similar to Creating Data loader .... /// From e53d60e17c12453e53d05db18c3cc08bc16d923e Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Wed, 1 May 2019 15:57:35 -0700 Subject: [PATCH 02/15] remove unused variables --- .../CodeGenerator/CodeGenerationHelper.cs | 51 +++++-------------- 1 file changed, 14 insertions(+), 37 deletions(-) diff --git a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs index 5cd9e726be..2d55d3ecb6 100644 --- a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs +++ b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs @@ -74,10 +74,6 @@ public void GenerateCode() // The reason why we are doing this way of defining 3 different results is because of the AutoML API // i.e there is no common class/interface to handle all three tasks together. - ExperimentResult binaryExperimentResult = default; - ExperimentResult multiclassExperimentResult = default; - ExperimentResult regressionExperimentResult = default; - List> completedBinaryRuns = default; List> completedMulticlassRuns = default; List> completedRegressionRuns = default; @@ -119,13 +115,13 @@ public void GenerateCode() { // TODO: It may be a good idea to convert the below Threads to Tasks or get rid of this progress bar all together and use an existing one in opensource. case TaskKind.BinaryClassification: - t = new Thread(() => SafeExecute(() => automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, wait, out completedBinaryRuns, pbar), out ex, out binaryExperimentResult, pbar)); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, wait, out completedBinaryRuns, pbar), out ex, pbar)); break; case TaskKind.Regression: - t = new Thread(() => SafeExecute(() => automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, wait, out completedRegressionRuns, pbar), out ex, out regressionExperimentResult, pbar)); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, wait, out completedRegressionRuns, pbar), out ex, pbar)); break; case TaskKind.MulticlassClassification: - t = new Thread(() => SafeExecute(() => automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, wait, out completedMulticlassRuns, pbar), out ex, out multiclassExperimentResult, pbar)); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, wait, out completedMulticlassRuns, pbar), out ex, pbar)); break; default: logger.Log(LogLevel.Error, Strings.UnsupportedMlTask); @@ -149,13 +145,13 @@ public void GenerateCode() switch (taskKind) { case TaskKind.BinaryClassification: - binaryExperimentResult = automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, wait, out completedBinaryRuns); + automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, wait, out completedBinaryRuns); break; case TaskKind.Regression: - regressionExperimentResult = automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, wait, out completedRegressionRuns); + automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, wait, out completedRegressionRuns); break; case TaskKind.MulticlassClassification: - multiclassExperimentResult = automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, wait, out completedMulticlassRuns); + automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, wait, out completedMulticlassRuns); break; default: logger.Log(LogLevel.Error, Strings.UnsupportedMlTask); @@ -321,65 +317,46 @@ private static void ConsumeAutoMLSDKLog(object sender, LoggingEventArgs args) } } - private void SafeExecute(Func> p, out Exception ex, out ExperimentResult binaryExperimentResult, FixedDurationBar pbar) + private void SafeExecute(Func> p, out Exception ex, FixedDurationBar pbar) { try { - binaryExperimentResult = p.Invoke(); + p.Invoke(); ex = null; } - catch (ThreadAbortException) - { - binaryExperimentResult = null; - pbar.Dispose(); // or ((ManualResetEvent)pbar.CompletedHandle).Set(); - throw; - } catch (Exception e) { ex = e; - binaryExperimentResult = null; + pbar.Dispose(); return; } } - private void SafeExecute(Func> p, out Exception ex, out ExperimentResult regressionExperimentResult, FixedDurationBar pbar) + private void SafeExecute(Func> p, out Exception ex, FixedDurationBar pbar) { try { - regressionExperimentResult = p.Invoke(); + p.Invoke(); ex = null; } - catch (ThreadAbortException) - { - regressionExperimentResult = null; - pbar.Dispose(); // or ((ManualResetEvent)pbar.CompletedHandle).Set(); - throw; - } catch (Exception e) { ex = e; - regressionExperimentResult = null; + pbar.Dispose(); return; } } - private void SafeExecute(Func> p, out Exception ex, out ExperimentResult multiClassExperimentResult, FixedDurationBar pbar) + private void SafeExecute(Func> p, out Exception ex, FixedDurationBar pbar) { try { - multiClassExperimentResult = p.Invoke(); + p.Invoke(); ex = null; } - catch (ThreadAbortException) - { - multiClassExperimentResult = null; - pbar.Dispose(); // or ((ManualResetEvent)pbar.CompletedHandle).Set(); - throw; - } catch (Exception e) { ex = e; - multiClassExperimentResult = null; pbar.Dispose(); // or ((ManualResetEvent)pbar.CompletedHandle).Set(); return; } From 82eebf546b129ba6a4db0638bf24e5c2fc87e5d1 Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Wed, 1 May 2019 18:34:51 -0700 Subject: [PATCH 03/15] change back to thread --- src/mlnet/AutoML/AutoMLEngine.cs | 32 ++++----- src/mlnet/AutoML/IAutoMLEngine.cs | 7 +- .../CodeGenerator/CodeGenerationHelper.cs | 72 ++++++++++++++++--- src/mlnet/Program.cs | 3 + 4 files changed, 82 insertions(+), 32 deletions(-) diff --git a/src/mlnet/AutoML/AutoMLEngine.cs b/src/mlnet/AutoML/AutoMLEngine.cs index 8340e4984f..3c5d5b7573 100644 --- a/src/mlnet/AutoML/AutoMLEngine.cs +++ b/src/mlnet/AutoML/AutoMLEngine.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using Microsoft.ML.AutoML; using Microsoft.ML.CLI.Data; @@ -46,57 +47,52 @@ public ColumnInferenceResults InferColumns(MLContext context, ColumnInformation return columnInference; } - ExperimentResult IAutoMLEngine.ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, out List> completedIterations, ProgressBar progressBar) + ExperimentResult IAutoMLEngine.ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.BinaryClassificationHandler handler, ProgressBar progressBar) { - completedIterations = new List>(); - var progressReporter = new ProgressHandlers.BinaryClassificationHandler(optimizationMetric, completedIterations, progressBar); ExperimentResult result = null; - Task.Run(() => result = context.Auto() + result = context.Auto() .CreateBinaryClassificationExperiment(new BinaryExperimentSettings() { MaxExperimentTimeInSeconds = settings.MaxExplorationTime, CacheBeforeTrainer = this.cacheBeforeTrainer, OptimizingMetric = optimizationMetric }) - .Execute(trainData, validationData, columnInformation, progressHandler: progressReporter)).Wait(experimentTimeout); + .Execute(trainData, validationData, columnInformation, progressHandler: handler); - progressReporter.Stop(); + //progressReporter.Stop(); logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline); return result; } - ExperimentResult IAutoMLEngine.ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, out List> completedIterations, ProgressBar progressBar) + ExperimentResult IAutoMLEngine.ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.RegressionHandler handler, ProgressBar progressBar) { - completedIterations = new List>(); - var progressReporter = new ProgressHandlers.RegressionHandler(optimizationMetric, completedIterations, progressBar); ExperimentResult result = null; - Task.Run(() => result = context.Auto() + result = context.Auto() .CreateRegressionExperiment(new RegressionExperimentSettings() { MaxExperimentTimeInSeconds = settings.MaxExplorationTime, OptimizingMetric = optimizationMetric, CacheBeforeTrainer = this.cacheBeforeTrainer - }).Execute(trainData, validationData, columnInformation, progressHandler: progressReporter)).Wait(experimentTimeout); + }).Execute(trainData, validationData, columnInformation, progressHandler: handler); - progressReporter.Stop(); + //progressReporter.Stop(); logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline); return result; } - ExperimentResult IAutoMLEngine.ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, out List> completedIterations, ProgressBar progressBar) + ExperimentResult IAutoMLEngine.ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.MulticlassClassificationHandler handler, ProgressBar progressBar) { - completedIterations = new List>(); - var progressReporter = new ProgressHandlers.MulticlassClassificationHandler(optimizationMetric, completedIterations, progressBar); ExperimentResult result = null; - Task.Run(() => result = context.Auto() + result = context.Auto() .CreateMulticlassClassificationExperiment(new MulticlassExperimentSettings() { MaxExperimentTimeInSeconds = settings.MaxExplorationTime, CacheBeforeTrainer = this.cacheBeforeTrainer, OptimizingMetric = optimizationMetric - }).Execute(trainData, validationData, columnInformation, progressHandler: progressReporter)).Wait(experimentTimeout); + }).Execute(trainData, validationData, columnInformation, progressHandler: handler); + // wait for experiment time - progressReporter.Stop(); + //progressReporter.Stop(); logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline); return result; } diff --git a/src/mlnet/AutoML/IAutoMLEngine.cs b/src/mlnet/AutoML/IAutoMLEngine.cs index 87a3adf38d..ee2f53c26a 100644 --- a/src/mlnet/AutoML/IAutoMLEngine.cs +++ b/src/mlnet/AutoML/IAutoMLEngine.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using Microsoft.ML.AutoML; using Microsoft.ML.CLI.ShellProgressBar; +using Microsoft.ML.CLI.Utilities; using Microsoft.ML.Data; namespace Microsoft.ML.CLI.CodeGenerator @@ -14,11 +15,11 @@ internal interface IAutoMLEngine { ColumnInferenceResults InferColumns(MLContext context, ColumnInformation columnInformation); - ExperimentResult ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, out List> completedIterations, ProgressBar progressBar = null); + ExperimentResult ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.BinaryClassificationHandler handler, ProgressBar progressBar = null); - ExperimentResult ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, out List> completedIterations, ProgressBar progressBar = null); + ExperimentResult ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.MulticlassClassificationHandler handler, ProgressBar progressBar = null); - ExperimentResult ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, out List> completedIterations, ProgressBar progressBar = null); + ExperimentResult ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.RegressionHandler handler, ProgressBar progressBar = null); } } diff --git a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs index 2d55d3ecb6..01db3c07a0 100644 --- a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs +++ b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs @@ -74,9 +74,13 @@ public void GenerateCode() // The reason why we are doing this way of defining 3 different results is because of the AutoML API // i.e there is no common class/interface to handle all three tasks together. - List> completedBinaryRuns = default; - List> completedMulticlassRuns = default; - List> completedRegressionRuns = default; + List> completedBinaryRuns = new List>(); + List> completedMulticlassRuns = new List>(); + List> completedRegressionRuns = new List>(); + + ProgressHandlers.BinaryClassificationHandler binaryHandler = default; + ProgressHandlers.RegressionHandler regressionHandler = default; + ProgressHandlers.MulticlassClassificationHandler multiClassHandler = default; if (verboseLevel > LogLevel.Trace) { @@ -115,13 +119,16 @@ public void GenerateCode() { // TODO: It may be a good idea to convert the below Threads to Tasks or get rid of this progress bar all together and use an existing one in opensource. case TaskKind.BinaryClassification: - t = new Thread(() => SafeExecute(() => automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, wait, out completedBinaryRuns, pbar), out ex, pbar)); + binaryHandler = new ProgressHandlers.BinaryClassificationHandler(new BinaryExperimentSettings().OptimizingMetric, completedBinaryRuns, pbar); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, wait, binaryHandler, pbar), out ex, pbar)); break; case TaskKind.Regression: - t = new Thread(() => SafeExecute(() => automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, wait, out completedRegressionRuns, pbar), out ex, pbar)); + regressionHandler = new ProgressHandlers.RegressionHandler(new RegressionExperimentSettings().OptimizingMetric, completedRegressionRuns, pbar); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, wait, regressionHandler, pbar), out ex, pbar)); break; case TaskKind.MulticlassClassification: - t = new Thread(() => SafeExecute(() => automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, wait, out completedMulticlassRuns, pbar), out ex, pbar)); + multiClassHandler = new ProgressHandlers.MulticlassClassificationHandler(new MulticlassExperimentSettings().OptimizingMetric, completedMulticlassRuns, pbar); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, wait, multiClassHandler, pbar), out ex, pbar)); break; default: logger.Log(LogLevel.Error, Strings.UnsupportedMlTask); @@ -131,8 +138,22 @@ public void GenerateCode() if (!pbar.CompletedHandle.WaitOne(wait)) pbar.Message = $"{nameof(FixedDurationBar)} did not signal {nameof(FixedDurationBar.CompletedHandle)} after {wait}"; - - t.Join(); + context.Log -= ConsumeAutoMLSDKLog; + switch (taskKind) + { + case TaskKind.BinaryClassification: + binaryHandler.Stop(); + break; + case TaskKind.Regression: + regressionHandler.Stop(); + break; + case TaskKind.MulticlassClassification: + multiClassHandler.Stop(); + break; + default: + logger.Log(LogLevel.Error, Strings.UnsupportedMlTask); + break; + } if (ex != null) { @@ -142,21 +163,50 @@ public void GenerateCode() } else { + Exception ex = null; + Thread t = default; switch (taskKind) { + // TODO: It may be a good idea to convert the below Threads to Tasks or get rid of this progress bar all together and use an existing one in opensource. case TaskKind.BinaryClassification: - automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, wait, out completedBinaryRuns); + binaryHandler = new ProgressHandlers.BinaryClassificationHandler(new BinaryExperimentSettings().OptimizingMetric, completedBinaryRuns, null); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, wait, binaryHandler, null), out ex, null)); break; case TaskKind.Regression: - automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, wait, out completedRegressionRuns); + regressionHandler = new ProgressHandlers.RegressionHandler(new RegressionExperimentSettings().OptimizingMetric, completedRegressionRuns, null); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, wait, regressionHandler, null), out ex, null)); break; case TaskKind.MulticlassClassification: - automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, wait, out completedMulticlassRuns); + multiClassHandler = new ProgressHandlers.MulticlassClassificationHandler(new MulticlassExperimentSettings().OptimizingMetric, completedMulticlassRuns, null); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, wait, multiClassHandler, null), out ex, null)); break; default: logger.Log(LogLevel.Error, Strings.UnsupportedMlTask); break; } + t.Start(); + Thread.Sleep(wait); + context.Log -= ConsumeAutoMLSDKLog; + switch (taskKind) + { + case TaskKind.BinaryClassification: + binaryHandler.Stop(); + break; + case TaskKind.Regression: + regressionHandler.Stop(); + break; + case TaskKind.MulticlassClassification: + multiClassHandler.Stop(); + break; + default: + logger.Log(LogLevel.Error, Strings.UnsupportedMlTask); + break; + } + + if (ex != null) + { + throw ex; + } } } catch (Exception) diff --git a/src/mlnet/Program.cs b/src/mlnet/Program.cs index eb049ba827..dcadf0b1b2 100644 --- a/src/mlnet/Program.cs +++ b/src/mlnet/Program.cs @@ -7,6 +7,7 @@ using System.CommandLine.Invocation; using System.IO; using System.Linq; +using System.Threading; using Microsoft.DotNet.Cli.Telemetry; using Microsoft.ML.CLI.Commands; using Microsoft.ML.CLI.Commands.New; @@ -101,6 +102,8 @@ public static void Main(string[] args) } parser.InvokeAsync(parseResult).Wait(); + + Thread.Sleep(25000); } } } From b5ae9cc4f175240ba87fffedb5ae583ca32353f9 Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Wed, 1 May 2019 18:35:10 -0700 Subject: [PATCH 04/15] remove sleep --- src/mlnet/Program.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mlnet/Program.cs b/src/mlnet/Program.cs index dcadf0b1b2..6854142590 100644 --- a/src/mlnet/Program.cs +++ b/src/mlnet/Program.cs @@ -102,8 +102,6 @@ public static void Main(string[] args) } parser.InvokeAsync(parseResult).Wait(); - - Thread.Sleep(25000); } } } From 877be159a323455d7c95e115755dff7a2a5bc205 Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Wed, 1 May 2019 18:52:47 -0700 Subject: [PATCH 05/15] fix review comments --- src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs | 2 -- src/mlnet/AutoML/AutoMLEngine.cs | 16 +++------------- src/mlnet/AutoML/IAutoMLEngine.cs | 6 +++--- src/mlnet/CodeGenerator/CodeGenerationHelper.cs | 11 ++++++++--- 4 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs index 4ab3b89da0..05cba5e8a7 100644 --- a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs +++ b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs @@ -54,8 +54,6 @@ public static CrossValidationRunDetail GetBestRun(IEnumerabl return results.ElementAt(indexOfBestScore); } - - public static IEnumerable<(RunDetail, int)> GetTopNRunResults(IEnumerable> results, IMetricsAgent metricsAgent, int n, bool isMetricMaximizing) { diff --git a/src/mlnet/AutoML/AutoMLEngine.cs b/src/mlnet/AutoML/AutoMLEngine.cs index 3c5d5b7573..19520bfbaf 100644 --- a/src/mlnet/AutoML/AutoMLEngine.cs +++ b/src/mlnet/AutoML/AutoMLEngine.cs @@ -3,9 +3,6 @@ // See the LICENSE file in the project root for more information. using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; using Microsoft.ML.AutoML; using Microsoft.ML.CLI.Data; using Microsoft.ML.CLI.ShellProgressBar; @@ -47,7 +44,7 @@ public ColumnInferenceResults InferColumns(MLContext context, ColumnInformation return columnInference; } - ExperimentResult IAutoMLEngine.ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.BinaryClassificationHandler handler, ProgressBar progressBar) + void IAutoMLEngine.ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.BinaryClassificationHandler handler, ProgressBar progressBar) { ExperimentResult result = null; result = context.Auto() @@ -59,12 +56,10 @@ ExperimentResult IAutoMLEngine.ExploreBinaryClassif }) .Execute(trainData, validationData, columnInformation, progressHandler: handler); - //progressReporter.Stop(); logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline); - return result; } - ExperimentResult IAutoMLEngine.ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.RegressionHandler handler, ProgressBar progressBar) + void IAutoMLEngine.ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.RegressionHandler handler, ProgressBar progressBar) { ExperimentResult result = null; result = context.Auto() @@ -75,12 +70,10 @@ ExperimentResult IAutoMLEngine.ExploreRegressionModels(MLCont CacheBeforeTrainer = this.cacheBeforeTrainer }).Execute(trainData, validationData, columnInformation, progressHandler: handler); - //progressReporter.Stop(); logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline); - return result; } - ExperimentResult IAutoMLEngine.ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.MulticlassClassificationHandler handler, ProgressBar progressBar) + void IAutoMLEngine.ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.MulticlassClassificationHandler handler, ProgressBar progressBar) { ExperimentResult result = null; result = context.Auto() @@ -91,10 +84,7 @@ ExperimentResult IAutoMLEngine.ExploreMultiClas OptimizingMetric = optimizationMetric }).Execute(trainData, validationData, columnInformation, progressHandler: handler); - // wait for experiment time - //progressReporter.Stop(); logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline); - return result; } } diff --git a/src/mlnet/AutoML/IAutoMLEngine.cs b/src/mlnet/AutoML/IAutoMLEngine.cs index ee2f53c26a..fc2a32de3a 100644 --- a/src/mlnet/AutoML/IAutoMLEngine.cs +++ b/src/mlnet/AutoML/IAutoMLEngine.cs @@ -15,11 +15,11 @@ internal interface IAutoMLEngine { ColumnInferenceResults InferColumns(MLContext context, ColumnInformation columnInformation); - ExperimentResult ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.BinaryClassificationHandler handler, ProgressBar progressBar = null); + void ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.BinaryClassificationHandler handler, ProgressBar progressBar = null); - ExperimentResult ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.MulticlassClassificationHandler handler, ProgressBar progressBar = null); + void ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.MulticlassClassificationHandler handler, ProgressBar progressBar = null); - ExperimentResult ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.RegressionHandler handler, ProgressBar progressBar = null); + void ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.RegressionHandler handler, ProgressBar progressBar = null); } } diff --git a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs index 01db3c07a0..ff3c1e8764 100644 --- a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs +++ b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs @@ -96,6 +96,10 @@ public void GenerateCode() logger.Log(LogLevel.Trace, $"{Strings.ExplorePipeline}: {settings.MlTask}"); logger.Log(LogLevel.Trace, $"{Strings.FurtherLearning}: {Strings.LearningHttpLink}"); + + // TODO the below region needs more refactoring to be done especially with so many switch cases. + + #region RunAutoMLEngine try { var options = new ProgressBarOptions @@ -286,6 +290,7 @@ public void GenerateCode() logger.Log(LogLevel.Info, Strings.ErrorBestPipeline); throw; } + #endregion // Save the model var modelprojectDir = Path.Combine(settings.OutputPath.FullName, $"{settings.Name}.Model"); @@ -377,7 +382,7 @@ private void SafeExecute(Func> p, catch (Exception e) { ex = e; - pbar.Dispose(); + pbar?.Dispose(); return; } } @@ -392,7 +397,7 @@ private void SafeExecute(Func> p, out Except catch (Exception e) { ex = e; - pbar.Dispose(); + pbar?.Dispose(); return; } } @@ -407,7 +412,7 @@ private void SafeExecute(Func> catch (Exception e) { ex = e; - pbar.Dispose(); // or ((ManualResetEvent)pbar.CompletedHandle).Set(); + pbar?.Dispose(); // or ((ManualResetEvent)pbar.CompletedHandle).Set(); return; } } From d62e9f61bf1df2a235308c80715e3795e6fa6a59 Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Wed, 1 May 2019 19:01:26 -0700 Subject: [PATCH 06/15] remove ununsed usings --- src/mlnet/Program.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlnet/Program.cs b/src/mlnet/Program.cs index 6854142590..eb049ba827 100644 --- a/src/mlnet/Program.cs +++ b/src/mlnet/Program.cs @@ -7,7 +7,6 @@ using System.CommandLine.Invocation; using System.IO; using System.Linq; -using System.Threading; using Microsoft.DotNet.Cli.Telemetry; using Microsoft.ML.CLI.Commands; using Microsoft.ML.CLI.Commands.New; From c3cd5a504907d07958760477cb422be59693fc93 Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Wed, 1 May 2019 19:26:53 -0700 Subject: [PATCH 07/15] format message --- .../CodeGenerator/CodeGenerationHelper.cs | 38 ++----------------- src/mlnet/Strings.resx | 2 +- src/mlnet/strings.Designer.cs | 2 +- 3 files changed, 6 insertions(+), 36 deletions(-) diff --git a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs index ff3c1e8764..b95cc76404 100644 --- a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs +++ b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs @@ -244,7 +244,7 @@ public void GenerateCode() } else { - logger.Log(LogLevel.Error, Strings.CouldNotFinshOnTime); + logger.Log(LogLevel.Error, string.Format(Strings.CouldNotFinshOnTime, settings.MaxExplorationTime)); logger.Log(LogLevel.Info, Strings.Exiting); return; } @@ -261,7 +261,7 @@ public void GenerateCode() } else { - logger.Log(LogLevel.Error, Strings.CouldNotFinshOnTime); + logger.Log(LogLevel.Error, string.Format(Strings.CouldNotFinshOnTime, settings.MaxExplorationTime)); logger.Log(LogLevel.Info, Strings.Exiting); return; } @@ -278,7 +278,7 @@ public void GenerateCode() } else { - logger.Log(LogLevel.Error, Strings.CouldNotFinshOnTime); + logger.Log(LogLevel.Error, string.Format(Strings.CouldNotFinshOnTime, settings.MaxExplorationTime)); logger.Log(LogLevel.Info, Strings.Exiting); return; } @@ -372,7 +372,7 @@ private static void ConsumeAutoMLSDKLog(object sender, LoggingEventArgs args) } } - private void SafeExecute(Func> p, out Exception ex, FixedDurationBar pbar) + private void SafeExecute(Action p, out Exception ex, FixedDurationBar pbar) { try { @@ -386,35 +386,5 @@ private void SafeExecute(Func> p, return; } } - - private void SafeExecute(Func> p, out Exception ex, FixedDurationBar pbar) - { - try - { - p.Invoke(); - ex = null; - } - catch (Exception e) - { - ex = e; - pbar?.Dispose(); - return; - } - } - - private void SafeExecute(Func> p, out Exception ex, FixedDurationBar pbar) - { - try - { - p.Invoke(); - ex = null; - } - catch (Exception e) - { - ex = e; - pbar?.Dispose(); // or ((ManualResetEvent)pbar.CompletedHandle).Set(); - return; - } - } } } diff --git a/src/mlnet/Strings.resx b/src/mlnet/Strings.resx index 47381f8f38..8a326883a4 100644 --- a/src/mlnet/Strings.resx +++ b/src/mlnet/Strings.resx @@ -199,6 +199,6 @@ Exception occured while saving the model - Training could not finish in the specified exploration time. Please set the exploration time to a higher value. + {0} sec was not enough to train at least one model for your dataset. Try with a longer time. Learn about recommended training time at https://aka.ms/cli-trainingtime \ No newline at end of file diff --git a/src/mlnet/strings.Designer.cs b/src/mlnet/strings.Designer.cs index f3a4bbdf8c..3fc5d7ab22 100644 --- a/src/mlnet/strings.Designer.cs +++ b/src/mlnet/strings.Designer.cs @@ -70,7 +70,7 @@ internal static string BestPipeline { } /// - /// Looks up a localized string similar to Training could not finish in the specified exploration time. Please set the exploration time to a higher value.. + /// Looks up a localized string similar to {0} sec was not enough to train at least one model for your dataset. Try with a longer time. Learn about recommended training time at https://aka.ms/cli-trainingtime. /// internal static string CouldNotFinshOnTime { get { From f61335c7f582ebb2dd4ec8743076d3efc7aff99a Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Wed, 1 May 2019 21:41:31 -0700 Subject: [PATCH 08/15] collapse declaration --- src/mlnet/AutoML/AutoMLEngine.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/mlnet/AutoML/AutoMLEngine.cs b/src/mlnet/AutoML/AutoMLEngine.cs index 19520bfbaf..0224099074 100644 --- a/src/mlnet/AutoML/AutoMLEngine.cs +++ b/src/mlnet/AutoML/AutoMLEngine.cs @@ -46,8 +46,7 @@ public ColumnInferenceResults InferColumns(MLContext context, ColumnInformation void IAutoMLEngine.ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.BinaryClassificationHandler handler, ProgressBar progressBar) { - ExperimentResult result = null; - result = context.Auto() + ExperimentResult result = context.Auto() .CreateBinaryClassificationExperiment(new BinaryExperimentSettings() { MaxExperimentTimeInSeconds = settings.MaxExplorationTime, @@ -61,8 +60,7 @@ void IAutoMLEngine.ExploreBinaryClassificationModels(MLContext context, IDataVie void IAutoMLEngine.ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.RegressionHandler handler, ProgressBar progressBar) { - ExperimentResult result = null; - result = context.Auto() + ExperimentResult result = context.Auto() .CreateRegressionExperiment(new RegressionExperimentSettings() { MaxExperimentTimeInSeconds = settings.MaxExplorationTime, @@ -75,8 +73,7 @@ void IAutoMLEngine.ExploreRegressionModels(MLContext context, IDataView trainDat void IAutoMLEngine.ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.MulticlassClassificationHandler handler, ProgressBar progressBar) { - ExperimentResult result = null; - result = context.Auto() + ExperimentResult result = context.Auto() .CreateMulticlassClassificationExperiment(new MulticlassExperimentSettings() { MaxExperimentTimeInSeconds = settings.MaxExplorationTime, From 80989b27d9fef75a7b0e2df5808498ed4216b4d8 Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Wed, 1 May 2019 21:57:17 -0700 Subject: [PATCH 09/15] remove unused param --- src/mlnet/AutoML/AutoMLEngine.cs | 6 +++--- src/mlnet/AutoML/IAutoMLEngine.cs | 6 +++--- src/mlnet/CodeGenerator/CodeGenerationHelper.cs | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/mlnet/AutoML/AutoMLEngine.cs b/src/mlnet/AutoML/AutoMLEngine.cs index 0224099074..7ad28b3cd4 100644 --- a/src/mlnet/AutoML/AutoMLEngine.cs +++ b/src/mlnet/AutoML/AutoMLEngine.cs @@ -44,7 +44,7 @@ public ColumnInferenceResults InferColumns(MLContext context, ColumnInformation return columnInference; } - void IAutoMLEngine.ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.BinaryClassificationHandler handler, ProgressBar progressBar) + void IAutoMLEngine.ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, ProgressHandlers.BinaryClassificationHandler handler, ProgressBar progressBar) { ExperimentResult result = context.Auto() .CreateBinaryClassificationExperiment(new BinaryExperimentSettings() @@ -58,7 +58,7 @@ void IAutoMLEngine.ExploreBinaryClassificationModels(MLContext context, IDataVie logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline); } - void IAutoMLEngine.ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.RegressionHandler handler, ProgressBar progressBar) + void IAutoMLEngine.ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, ProgressHandlers.RegressionHandler handler, ProgressBar progressBar) { ExperimentResult result = context.Auto() .CreateRegressionExperiment(new RegressionExperimentSettings() @@ -71,7 +71,7 @@ void IAutoMLEngine.ExploreRegressionModels(MLContext context, IDataView trainDat logger.Log(LogLevel.Trace, Strings.RetrieveBestPipeline); } - void IAutoMLEngine.ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.MulticlassClassificationHandler handler, ProgressBar progressBar) + void IAutoMLEngine.ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, ProgressHandlers.MulticlassClassificationHandler handler, ProgressBar progressBar) { ExperimentResult result = context.Auto() .CreateMulticlassClassificationExperiment(new MulticlassExperimentSettings() diff --git a/src/mlnet/AutoML/IAutoMLEngine.cs b/src/mlnet/AutoML/IAutoMLEngine.cs index fc2a32de3a..4af8fcf2df 100644 --- a/src/mlnet/AutoML/IAutoMLEngine.cs +++ b/src/mlnet/AutoML/IAutoMLEngine.cs @@ -15,11 +15,11 @@ internal interface IAutoMLEngine { ColumnInferenceResults InferColumns(MLContext context, ColumnInformation columnInformation); - void ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.BinaryClassificationHandler handler, ProgressBar progressBar = null); + void ExploreBinaryClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, BinaryClassificationMetric optimizationMetric, ProgressHandlers.BinaryClassificationHandler handler, ProgressBar progressBar = null); - void ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.MulticlassClassificationHandler handler, ProgressBar progressBar = null); + void ExploreMultiClassificationModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, MulticlassClassificationMetric optimizationMetric, ProgressHandlers.MulticlassClassificationHandler handler, ProgressBar progressBar = null); - void ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, TimeSpan experimentTimeout, ProgressHandlers.RegressionHandler handler, ProgressBar progressBar = null); + void ExploreRegressionModels(MLContext context, IDataView trainData, IDataView validationData, ColumnInformation columnInformation, RegressionMetric optimizationMetric, ProgressHandlers.RegressionHandler handler, ProgressBar progressBar = null); } } diff --git a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs index b95cc76404..022ddf5952 100644 --- a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs +++ b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs @@ -124,15 +124,15 @@ public void GenerateCode() // TODO: It may be a good idea to convert the below Threads to Tasks or get rid of this progress bar all together and use an existing one in opensource. case TaskKind.BinaryClassification: binaryHandler = new ProgressHandlers.BinaryClassificationHandler(new BinaryExperimentSettings().OptimizingMetric, completedBinaryRuns, pbar); - t = new Thread(() => SafeExecute(() => automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, wait, binaryHandler, pbar), out ex, pbar)); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, binaryHandler, pbar), out ex, pbar)); break; case TaskKind.Regression: regressionHandler = new ProgressHandlers.RegressionHandler(new RegressionExperimentSettings().OptimizingMetric, completedRegressionRuns, pbar); - t = new Thread(() => SafeExecute(() => automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, wait, regressionHandler, pbar), out ex, pbar)); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, regressionHandler, pbar), out ex, pbar)); break; case TaskKind.MulticlassClassification: multiClassHandler = new ProgressHandlers.MulticlassClassificationHandler(new MulticlassExperimentSettings().OptimizingMetric, completedMulticlassRuns, pbar); - t = new Thread(() => SafeExecute(() => automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, wait, multiClassHandler, pbar), out ex, pbar)); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, multiClassHandler, pbar), out ex, pbar)); break; default: logger.Log(LogLevel.Error, Strings.UnsupportedMlTask); @@ -174,15 +174,15 @@ public void GenerateCode() // TODO: It may be a good idea to convert the below Threads to Tasks or get rid of this progress bar all together and use an existing one in opensource. case TaskKind.BinaryClassification: binaryHandler = new ProgressHandlers.BinaryClassificationHandler(new BinaryExperimentSettings().OptimizingMetric, completedBinaryRuns, null); - t = new Thread(() => SafeExecute(() => automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, wait, binaryHandler, null), out ex, null)); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, binaryHandler, null), out ex, null)); break; case TaskKind.Regression: regressionHandler = new ProgressHandlers.RegressionHandler(new RegressionExperimentSettings().OptimizingMetric, completedRegressionRuns, null); - t = new Thread(() => SafeExecute(() => automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, wait, regressionHandler, null), out ex, null)); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, regressionHandler, null), out ex, null)); break; case TaskKind.MulticlassClassification: multiClassHandler = new ProgressHandlers.MulticlassClassificationHandler(new MulticlassExperimentSettings().OptimizingMetric, completedMulticlassRuns, null); - t = new Thread(() => SafeExecute(() => automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, wait, multiClassHandler, null), out ex, null)); + t = new Thread(() => SafeExecute(() => automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, multiClassHandler, null), out ex, null)); break; default: logger.Log(LogLevel.Error, Strings.UnsupportedMlTask); From 12af51f0c4621aa663f46038852d12050d9cf6f1 Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Thu, 2 May 2019 07:37:50 -0700 Subject: [PATCH 10/15] added environment.exit and removal of error message --- src/mlnet/CodeGenerator/CodeGenerationHelper.cs | 3 +-- src/mlnet/Program.cs | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs index 022ddf5952..a58088ad78 100644 --- a/src/mlnet/CodeGenerator/CodeGenerationHelper.cs +++ b/src/mlnet/CodeGenerator/CodeGenerationHelper.cs @@ -140,8 +140,7 @@ public void GenerateCode() } t.Start(); - if (!pbar.CompletedHandle.WaitOne(wait)) - pbar.Message = $"{nameof(FixedDurationBar)} did not signal {nameof(FixedDurationBar.CompletedHandle)} after {wait}"; + pbar.CompletedHandle.WaitOne(wait); context.Log -= ConsumeAutoMLSDKLog; switch (taskKind) { diff --git a/src/mlnet/Program.cs b/src/mlnet/Program.cs index eb049ba827..135b17648e 100644 --- a/src/mlnet/Program.cs +++ b/src/mlnet/Program.cs @@ -101,6 +101,7 @@ public static void Main(string[] args) } parser.InvokeAsync(parseResult).Wait(); + Environment.Exit(0); } } } From 6a9694e64c97b1a3033d96ef3b45429f4131b29a Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Thu, 2 May 2019 07:46:06 -0700 Subject: [PATCH 11/15] correction in message --- src/mlnet/Strings.resx | 2 +- src/mlnet/strings.Designer.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlnet/Strings.resx b/src/mlnet/Strings.resx index 8a326883a4..56726f9589 100644 --- a/src/mlnet/Strings.resx +++ b/src/mlnet/Strings.resx @@ -199,6 +199,6 @@ Exception occured while saving the model - {0} sec was not enough to train at least one model for your dataset. Try with a longer time. Learn about recommended training time at https://aka.ms/cli-trainingtime + {0} secs was not enough to train at least one model for your dataset. Try with a longer time. Learn about recommended training time at https://aka.ms/cli-trainingtime \ No newline at end of file diff --git a/src/mlnet/strings.Designer.cs b/src/mlnet/strings.Designer.cs index 3fc5d7ab22..03811eb998 100644 --- a/src/mlnet/strings.Designer.cs +++ b/src/mlnet/strings.Designer.cs @@ -70,7 +70,7 @@ internal static string BestPipeline { } /// - /// Looks up a localized string similar to {0} sec was not enough to train at least one model for your dataset. Try with a longer time. Learn about recommended training time at https://aka.ms/cli-trainingtime. + /// Looks up a localized string similar to {0} secs was not enough to train at least one model for your dataset. Try with a longer time. Learn about recommended training time at https://aka.ms/cli-trainingtime. /// internal static string CouldNotFinshOnTime { get { From 4a1e019fc3b56c41aec543a713ab631ec90c50f3 Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Thu, 2 May 2019 07:47:45 -0700 Subject: [PATCH 12/15] secs-> seconds --- src/mlnet/Strings.resx | 2 +- src/mlnet/strings.Designer.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlnet/Strings.resx b/src/mlnet/Strings.resx index 56726f9589..f220f9ca2c 100644 --- a/src/mlnet/Strings.resx +++ b/src/mlnet/Strings.resx @@ -199,6 +199,6 @@ Exception occured while saving the model - {0} secs was not enough to train at least one model for your dataset. Try with a longer time. Learn about recommended training time at https://aka.ms/cli-trainingtime + {0} seconds was not enough to train at least one model for your dataset. Try with a longer time. Learn about recommended training time at https://aka.ms/cli-trainingtime \ No newline at end of file diff --git a/src/mlnet/strings.Designer.cs b/src/mlnet/strings.Designer.cs index 03811eb998..e9791c1318 100644 --- a/src/mlnet/strings.Designer.cs +++ b/src/mlnet/strings.Designer.cs @@ -70,7 +70,7 @@ internal static string BestPipeline { } /// - /// Looks up a localized string similar to {0} secs was not enough to train at least one model for your dataset. Try with a longer time. Learn about recommended training time at https://aka.ms/cli-trainingtime. + /// Looks up a localized string similar to {0} seconds was not enough to train at least one model for your dataset. Try with a longer time. Learn about recommended training time at https://aka.ms/cli-trainingtime. /// internal static string CouldNotFinshOnTime { get { From ef73cd8c5fe2bff6c35d6eb96636d4848d13db8f Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Thu, 2 May 2019 10:25:39 -0700 Subject: [PATCH 13/15] exit code --- src/mlnet/Program.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlnet/Program.cs b/src/mlnet/Program.cs index 135b17648e..957437b1b0 100644 --- a/src/mlnet/Program.cs +++ b/src/mlnet/Program.cs @@ -23,7 +23,7 @@ class Program public static void Main(string[] args) { var telemetry = new MlTelemetry(); - + int exitCode = 0; // Create handler outside so that commandline and the handler is decoupled and testable. var handler = CommandHandler.Create( (options) => @@ -70,7 +70,7 @@ public static void Main(string[] args) logger.Log(LogLevel.Debug, e.ToString()); logger.Log(LogLevel.Info, Strings.LookIntoLogFile); logger.Log(LogLevel.Error, Strings.Exiting); - return; + exitCode = -1; } }); @@ -101,7 +101,7 @@ public static void Main(string[] args) } parser.InvokeAsync(parseResult).Wait(); - Environment.Exit(0); + Environment.Exit(exitCode); } } } From 0f409e3c47a914ad616fdc175d6582ec45f92b6a Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Thu, 2 May 2019 10:32:13 -0700 Subject: [PATCH 14/15] change value to 1 --- src/mlnet/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlnet/Program.cs b/src/mlnet/Program.cs index 957437b1b0..3f622eb2ca 100644 --- a/src/mlnet/Program.cs +++ b/src/mlnet/Program.cs @@ -70,7 +70,7 @@ public static void Main(string[] args) logger.Log(LogLevel.Debug, e.ToString()); logger.Log(LogLevel.Info, Strings.LookIntoLogFile); logger.Log(LogLevel.Error, Strings.Exiting); - exitCode = -1; + exitCode = 1; } }); From ab2688f32d99297853c6b490aa2fbadf9aeea5f0 Mon Sep 17 00:00:00 2001 From: Srujan Saggam Date: Thu, 2 May 2019 10:38:05 -0700 Subject: [PATCH 15/15] reverse the declaration --- src/mlnet/Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlnet/Program.cs b/src/mlnet/Program.cs index 3f622eb2ca..568eaa4d4a 100644 --- a/src/mlnet/Program.cs +++ b/src/mlnet/Program.cs @@ -23,7 +23,7 @@ class Program public static void Main(string[] args) { var telemetry = new MlTelemetry(); - int exitCode = 0; + int exitCode = 1; // Create handler outside so that commandline and the handler is decoupled and testable. var handler = CommandHandler.Create( (options) => @@ -63,6 +63,7 @@ public static void Main(string[] args) // Execute the command command.Execute(); + exitCode = 0; } catch (Exception e) { @@ -70,7 +71,6 @@ public static void Main(string[] args) logger.Log(LogLevel.Debug, e.ToString()); logger.Log(LogLevel.Info, Strings.LookIntoLogFile); logger.Log(LogLevel.Error, Strings.Exiting); - exitCode = 1; } });