Skip to content

[AutoML] Fix error handling in CLI. #3618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 29, 2019
Merged

[AutoML] Fix error handling in CLI. #3618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a5008a8
fix error handling
srsaggam Apr 29, 2019
c40cf93
renaming variables
srsaggam Apr 29, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 62 additions & 14 deletions src/mlnet/CodeGenerator/CodeGenerationHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,17 @@
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.ExceptionServices;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.ML.AutoML;
using Microsoft.ML.CLI.CodeGenerator.CSharp;
using Microsoft.ML.CLI.Data;
using Microsoft.ML.CLI.ShellProgressBar;
using Microsoft.ML.CLI.Utilities;
using Microsoft.ML.Data;
using NLog;
using NLog.Targets;

namespace Microsoft.ML.CLI.CodeGenerator
{
Expand Down Expand Up @@ -81,7 +77,7 @@ public void GenerateCode()
// i.e there is no common class/interface to handle all three tasks together.

ExperimentResult<BinaryClassificationMetrics> binaryExperimentResult = default;
ExperimentResult<MulticlassClassificationMetrics> multiExperimentResult = default;
ExperimentResult<MulticlassClassificationMetrics> multiclassExperimentResult = default;
ExperimentResult<RegressionMetrics> regressionExperimentResult = default;
if (verboseLevel > LogLevel.Trace)
{
Expand Down Expand Up @@ -111,20 +107,22 @@ public void GenerateCode()

if (verboseLevel > LogLevel.Trace && !Console.IsOutputRedirected)
{
Exception ex = null;
using (var pbar = new FixedDurationBar(wait, "", options))
{
pbar.Message = Strings.WaitingForFirstIteration;
Thread t = default;
switch (taskKind)
{
// TODO: It may be a good idea to convert the below Threads to Tasks or get rid of this progress bar all together and use an existing one in opensource.
case TaskKind.BinaryClassification:
t = new Thread(() => binaryExperimentResult = automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, pbar));
t = new Thread(() => SafeExecute(() => automlEngine.ExploreBinaryClassificationModels(context, trainData, validationData, columnInformation, new BinaryExperimentSettings().OptimizingMetric, pbar), out ex, out binaryExperimentResult, pbar));
break;
case TaskKind.Regression:
t = new Thread(() => regressionExperimentResult = automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, pbar));
t = new Thread(() => SafeExecute(() => automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric, pbar), out ex, out regressionExperimentResult, pbar));
break;
case TaskKind.MulticlassClassification:
t = new Thread(() => multiExperimentResult = automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, pbar));
t = new Thread(() => SafeExecute(() => automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric, pbar), out ex, out multiclassExperimentResult, pbar));
break;
default:
logger.Log(LogLevel.Error, Strings.UnsupportedMlTask);
Expand All @@ -147,6 +145,10 @@ public void GenerateCode()
pbar.Message = originalMessage;
}
}
if (ex != null)
{
throw ex;
}
}
}
else
Expand All @@ -160,7 +162,7 @@ public void GenerateCode()
regressionExperimentResult = automlEngine.ExploreRegressionModels(context, trainData, validationData, columnInformation, new RegressionExperimentSettings().OptimizingMetric);
break;
case TaskKind.MulticlassClassification:
multiExperimentResult = automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric);
multiclassExperimentResult = automlEngine.ExploreMultiClassificationModels(context, trainData, validationData, columnInformation, new MulticlassExperimentSettings().OptimizingMetric);
break;
default:
logger.Log(LogLevel.Error, Strings.UnsupportedMlTask);
Expand Down Expand Up @@ -204,11 +206,11 @@ public void GenerateCode()
ConsolePrinter.PrintIterationSummary(regressionExperimentResult.RunDetails, new RegressionExperimentSettings().OptimizingMetric, 5);
break;
case TaskKind.MulticlassClassification:
var bestMultiIteration = multiExperimentResult.BestRun;
bestPipeline = bestMultiIteration.Pipeline;
bestModel = bestMultiIteration.Model;
ConsolePrinter.ExperimentResultsHeader(LogLevel.Info, settings.MlTask, settings.Dataset.Name, columnInformation.LabelColumnName, elapsedTime.ToString("F2"), multiExperimentResult.RunDetails.Count());
ConsolePrinter.PrintIterationSummary(multiExperimentResult.RunDetails, new MulticlassExperimentSettings().OptimizingMetric, 5);
var bestMulticlassIteration = multiclassExperimentResult.BestRun;
bestPipeline = bestMulticlassIteration.Pipeline;
bestModel = bestMulticlassIteration.Model;
ConsolePrinter.ExperimentResultsHeader(LogLevel.Info, settings.MlTask, settings.Dataset.Name, columnInformation.LabelColumnName, elapsedTime.ToString("F2"), multiclassExperimentResult.RunDetails.Count());
ConsolePrinter.PrintIterationSummary(multiclassExperimentResult.RunDetails, new MulticlassExperimentSettings().OptimizingMetric, 5);
break;
}
}
Expand Down Expand Up @@ -278,5 +280,51 @@ private void ConsumeAutoMLSDKLogs(MLContext context)
}
};
}

private void SafeExecute(Func<ExperimentResult<BinaryClassificationMetrics>> p, out Exception ex, out ExperimentResult<BinaryClassificationMetrics> binaryExperimentResult, FixedDurationBar pbar)
{
try
{
binaryExperimentResult = p.Invoke();
ex = null;
}
catch (Exception e)
{
ex = e;
binaryExperimentResult = null;
return;
}
}

private void SafeExecute(Func<ExperimentResult<RegressionMetrics>> p, out Exception ex, out ExperimentResult<RegressionMetrics> regressionExperimentResult, FixedDurationBar pbar)
{
try
{
regressionExperimentResult = p.Invoke();
ex = null;
}
catch (Exception e)
{
ex = e;
regressionExperimentResult = null;
return;
}
}

private void SafeExecute(Func<ExperimentResult<MulticlassClassificationMetrics>> p, out Exception ex, out ExperimentResult<MulticlassClassificationMetrics> multiClassExperimentResult, FixedDurationBar pbar)
{
try
{
multiClassExperimentResult = p.Invoke();
ex = null;
}
catch (Exception e)
{
ex = e;
multiClassExperimentResult = null;
pbar.Dispose(); // or ((ManualResetEvent)pbar.CompletedHandle).Set();
return;
}
}
}
}
1 change: 1 addition & 0 deletions src/mlnet/ProgressBar/ProgressBar.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,7 @@ public void Dispose()
_timer?.Dispose();
_timer = null;
foreach (var c in this.Children) c.Dispose();
OnDone();
}
}
}