diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PermutationFeatureImportanceLoadFromDisk.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PermutationFeatureImportanceLoadFromDisk.cs
new file mode 100644
index 0000000000..3a94a113c4
--- /dev/null
+++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/PermutationFeatureImportanceLoadFromDisk.cs
@@ -0,0 +1,138 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML;
+using Microsoft.ML.Calibrators;
+using Microsoft.ML.Data;
+using Microsoft.ML.Trainers;
+
+namespace Samples.Dynamic.Trainers.MulticlassClassification
+{
+ public static class PermutationFeatureImportanceLoadFromDisk
+ {
+ public static void Example()
+ {
+ // Create a new context for ML.NET operations. It can be used for
+ // exception tracking and logging, as a catalog of available operations
+ // and as the source of randomness.
+ var mlContext = new MLContext(seed: 1);
+
+ // Create sample data.
+ var samples = GenerateData();
+
+ // Load the sample data as an IDataView.
+ var data = mlContext.Data.LoadFromEnumerable(samples);
+
+ // Define a training pipeline that concatenates features into a vector,
+ // normalizes them, and then trains a linear model.
+ var featureColumns =
+ new string[] { nameof(Data.Feature1), nameof(Data.Feature2) };
+
+ var pipeline = mlContext.Transforms
+ .Concatenate("Features", featureColumns)
+ .Append(mlContext.Transforms.Conversion.MapValueToKey("Label"))
+ .Append(mlContext.Transforms.NormalizeMinMax("Features"))
+ .Append(mlContext.MulticlassClassification.Trainers
+ .SdcaMaximumEntropy());
+
+ // Fit the pipeline to the data and save the model
+ var model0 = pipeline.Fit(data);
+ var modelPath = "./model0.zip";
+ mlContext.Model.Save(model0, data.Schema, modelPath);
+
+ // Load the model
+ var model = mlContext.Model.Load(modelPath, out var schema);
+
+ // Transform the dataset.
+ var transformedData = model.Transform(data);
+
+ // Extract the predictor.
+ var linearPredictor = (model as TransformerChain<ITransformer>).LastTransformer as MulticlassPredictionTransformer<MaximumEntropyModelParameters>;
+
+ // Compute the permutation metrics for the linear model using the
+ // normalized data.
+ var permutationMetrics = mlContext.MulticlassClassification
+ .PermutationFeatureImportance(linearPredictor, transformedData,
+ permutationCount: 30);
+
+ // Now let's look at which features are most important to the model
+ // overall. Get the feature indices sorted by their impact on
+ // microaccuracy.
+ var sortedIndices = permutationMetrics
+ .Select((metrics, index) => new { index, metrics.MicroAccuracy })
+ .OrderByDescending(feature => Math.Abs(feature.MicroAccuracy.Mean))
+ .Select(feature => feature.index);
+
+ Console.WriteLine("Feature\tChange in MicroAccuracy\t95% Confidence in "
+ + "the Mean Change in MicroAccuracy");
+
+ var microAccuracy = permutationMetrics.Select(x => x.MicroAccuracy)
+ .ToArray();
+
+ foreach (int i in sortedIndices)
+ {
+ Console.WriteLine("{0}\t{1:G4}\t{2:G4}",
+ featureColumns[i],
+ microAccuracy[i].Mean,
+ 1.96 * microAccuracy[i].StandardError);
+ }
+
+ // Expected output:
+ //Feature Change in MicroAccuracy 95% Confidence in the Mean Change in MicroAccuracy
+ //Feature2 -0.1396 0.0008036
+ //Feature1 -0.05421 0.0006154
+
+ }
+
+ private class Data
+ {
+ public float Label { get; set; }
+
+ public float Feature1 { get; set; }
+
+ public float Feature2 { get; set; }
+ }
+
+ /// <summary>
+ /// Generate an enumerable of Data objects, creating the label as a simple
+ /// linear combination of the features.
+ /// </summary>
+ /// <param name="nExamples">The number of examples.</param>
+ /// <param name="bias">The bias, or offset, in the calculation of the
+ /// label.</param>
+ /// <param name="weight1">The weight to multiply the first feature with to
+ /// compute the label.</param>
+ /// <param name="weight2">The weight to multiply the second feature with to
+ /// compute the label.</param>
+ /// <param name="seed">The seed for generating feature values and label
+ /// noise.</param>
+ /// <returns>An enumerable of Data objects.</returns>
+ private static IEnumerable<Data> GenerateData(int nExamples = 10000,
+ double bias = 0, double weight1 = 1, double weight2 = 2, int seed = 1)
+ {
+ var rng = new Random(seed);
+ var max = bias + 4.5 * weight1 + 4.5 * weight2 + 0.5;
+ for (int i = 0; i < nExamples; i++)
+ {
+ var data = new Data
+ {
+ Feature1 = (float)(rng.Next(10) * (rng.NextDouble() - 0.5)),
+ Feature2 = (float)(rng.Next(10) * (rng.NextDouble() - 0.5)),
+ };
+
+ // Create a noisy label.
+ var value = (float)
+ (bias + weight1 * data.Feature1 + weight2 * data.Feature2 +
+ rng.NextDouble() - 0.5);
+
+ if (value < max / 3)
+ data.Label = 0;
+ else if (value < 2 * max / 3)
+ data.Label = 1;
+ else
+ data.Label = 2;
+ yield return data;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/PermutationFeatureImportanceLoadFromDisk.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/PermutationFeatureImportanceLoadFromDisk.cs
new file mode 100644
index 0000000000..e12e17554b
--- /dev/null
+++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Ranking/PermutationFeatureImportanceLoadFromDisk.cs
@@ -0,0 +1,144 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using Microsoft.ML.Trainers.FastTree;
+
+namespace Samples.Dynamic.Trainers.Ranking
+{
+ public static class PermutationFeatureImportanceLoadFromDisk
+ {
+ public static void Example()
+ {
+ // Create a new context for ML.NET operations. It can be used for
+ // exception tracking and logging, as a catalog of available operations
+ // and as the source of randomness.
+ var mlContext = new MLContext(seed: 1);
+
+ // Create sample data.
+ var samples = GenerateData();
+
+ // Load the sample data as an IDataView.
+ var data = mlContext.Data.LoadFromEnumerable(samples);
+
+ // Define a training pipeline that concatenates features into a vector,
+ // normalizes them, and then trains a linear model.
+ var featureColumns = new string[] { nameof(Data.Feature1), nameof(
+ Data.Feature2) };
+ var pipeline = mlContext.Transforms.Concatenate("Features",
+ featureColumns)
+ .Append(mlContext.Transforms.Conversion.MapValueToKey("Label"))
+ .Append(mlContext.Transforms.Conversion.MapValueToKey(
+ "GroupId"))
+ .Append(mlContext.Transforms.NormalizeMinMax("Features"))
+ .Append(mlContext.Ranking.Trainers.FastTree());
+
+ // Train the model and save to disk
+ var model0 = pipeline.Fit(data);
+ var modelPath = "./model0.zip";
+ mlContext.Model.Save(model0, data.Schema, modelPath);
+
+ // Load model
+ var model = mlContext.Model.Load(modelPath, out var schema);
+
+ // Transform Data
+ var transformedData = model.Transform(data);
+
+ // Extract the predictor
+ var linearPredictor = (model as TransformerChain<ITransformer>).LastTransformer as RankingPredictionTransformer<FastTreeRankingModelParameters>;
+
+ // Compute the permutation metrics for the linear model using the
+ // normalized data.
+ var permutationMetrics = mlContext.Ranking.PermutationFeatureImportance(
+ linearPredictor, transformedData, permutationCount: 30);
+
+ // Now let's look at which features are most important to the model
+ // overall. Get the feature indices sorted by their impact on NDCG@1.
+ var sortedIndices = permutationMetrics.Select((metrics, index) => new {
+ index,
+ metrics.NormalizedDiscountedCumulativeGains
+ })
+ .OrderByDescending(feature => Math.Abs(
+ feature.NormalizedDiscountedCumulativeGains[0].Mean))
+
+ .Select(feature => feature.index);
+
+ Console.WriteLine("Feature\tChange in NDCG@1\t95% Confidence in the" +
+ "Mean Change in NDCG@1");
+ var ndcg = permutationMetrics.Select(
+ x => x.NormalizedDiscountedCumulativeGains).ToArray();
+ foreach (int i in sortedIndices)
+ {
+ Console.WriteLine("{0}\t{1:G4}\t{2:G4}",
+ featureColumns[i],
+ ndcg[i][0].Mean,
+ 1.96 * ndcg[i][0].StandardError);
+ }
+
+ // Expected output:
+ // Feature Change in NDCG@1 95% Confidence in the Mean Change in NDCG@1
+ // Feature2 -0.2432 0.001762
+ // Feature1 -0.05235 0.001116
+ }
+
+ private class Data
+ {
+ public float Label { get; set; }
+
+ public int GroupId { get; set; }
+
+ public float Feature1 { get; set; }
+
+ public float Feature2 { get; set; }
+ }
+
+ /// <summary>
+ /// Generate an enumerable of Data objects, creating the label as a simple
+ /// linear combination of the features.
+ /// </summary>
+ ///
+ /// <param name="nExamples">The number of examples.</param>
+ ///
+ /// <param name="bias">The bias, or offset, in the calculation of the label.
+ /// </param>
+ ///
+ /// <param name="weight1">The weight to multiply the first feature with to
+ /// compute the label.</param>
+ ///
+ /// <param name="weight2">The weight to multiply the second feature with to
+ /// compute the label.</param>
+ ///
+ /// <param name="seed">The seed for generating feature values and label
+ /// noise.</param>
+ ///
+ /// <returns>An enumerable of Data objects.</returns>
+ private static IEnumerable<Data> GenerateData(int nExamples = 10000,
+ double bias = 0, double weight1 = 1, double weight2 = 2, int seed = 1,
+ int groupSize = 5)
+ {
+ var rng = new Random(seed);
+ var max = bias + 4.5 * weight1 + 4.5 * weight2 + 0.5;
+ for (int i = 0; i < nExamples; i++)
+ {
+ var data = new Data
+ {
+ GroupId = i / groupSize,
+ Feature1 = (float)(rng.Next(10) * (rng.NextDouble() - 0.5)),
+ Feature2 = (float)(rng.Next(10) * (rng.NextDouble() - 0.5)),
+ };
+
+ // Create a noisy label.
+ var value = (float)(bias + weight1 * data.Feature1 + weight2 *
+ data.Feature2 + rng.NextDouble() - 0.5);
+ if (value < max / 3)
+ data.Label = 0;
+ else if (value < 2 * max / 3)
+ data.Label = 1;
+ else
+ data.Label = 2;
+ yield return data;
+ }
+ }
+ }
+}
diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/PermutationFeatureImportanceLoadFromDisk.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/PermutationFeatureImportanceLoadFromDisk.cs
new file mode 100644
index 0000000000..1a783b6c2a
--- /dev/null
+++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/PermutationFeatureImportanceLoadFromDisk.cs
@@ -0,0 +1,132 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using Microsoft.ML.Trainers;
+
+namespace Samples.Dynamic.Trainers.Regression
+{
+ public static class PermutationFeatureImportanceLoadFromDisk
+ {
+ public static void Example()
+ {
+ // Create a new context for ML.NET operations. It can be used for
+ // exception tracking and logging, as a catalog of available operations
+ // and as the source of randomness.
+ var mlContext = new MLContext(seed: 1);
+
+ // Create sample data.
+ var samples = GenerateData();
+
+ // Load the sample data as an IDataView.
+ var data = mlContext.Data.LoadFromEnumerable(samples);
+
+ // Define a training pipeline that concatenates features into a vector,
+ // normalizes them, and then trains a linear model.
+ var featureColumns = new string[] { nameof(Data.Feature1),
+ nameof(Data.Feature2) };
+
+ var pipeline = mlContext.Transforms.Concatenate(
+ "Features",
+ featureColumns)
+ .Append(mlContext.Transforms.NormalizeMinMax("Features"))
+ .Append(mlContext.Regression.Trainers.Ols());
+
+ // Train the model and save to disk
+ var model0 = pipeline.Fit(data);
+ var modelPath = "./model0.zip";
+ mlContext.Model.Save(model0, data.Schema, modelPath);
+
+ // Load model
+ var model = mlContext.Model.Load(modelPath, out var schema);
+
+ // Transform Data
+ var transformedData = model.Transform(data);
+
+ // Extract the predictor.
+ var linearPredictor = (model as TransformerChain<ITransformer>).LastTransformer as RegressionPredictionTransformer<OlsModelParameters>;
+
+ // Compute the permutation metrics for the linear model using the
+ // normalized data.
+ var permutationMetrics = mlContext.Regression
+ .PermutationFeatureImportance(
+ linearPredictor, transformedData, permutationCount: 30);
+
+ // Now let's look at which features are most important to the model
+ // overall. Get the feature indices sorted by their impact on RMSE.
+ var sortedIndices = permutationMetrics
+ .Select((metrics, index) => new {
+ index,
+ metrics.RootMeanSquaredError
+ })
+
+ .OrderByDescending(feature => Math.Abs(
+ feature.RootMeanSquaredError.Mean))
+
+ .Select(feature => feature.index);
+
+ Console.WriteLine("Feature\tModel Weight\tChange in RMSE\t95%" +
+ "Confidence in the Mean Change in RMSE");
+
+ var rmse = permutationMetrics.Select(x => x.RootMeanSquaredError)
+ .ToArray();
+
+ foreach (int i in sortedIndices)
+ {
+ Console.WriteLine("{0}\t{1:0.00}\t{2:G4}\t{3:G4}",
+ featureColumns[i],
+ linearPredictor.Model.Weights[i],
+ rmse[i].Mean,
+ 1.96 * rmse[i].StandardError);
+ }
+
+ // Expected output:
+ // Feature Model Weight Change in RMSE 95% Confidence in the Mean Change in RMSE
+ // Feature2 9.00 4.01 0.006723
+ // Feature1 4.48 1.901 0.003235
+ }
+
+ private class Data
+ {
+ public float Label { get; set; }
+
+ public float Feature1 { get; set; }
+
+ public float Feature2 { get; set; }
+ }
+
+ /// <summary>
+ /// Generate an enumerable of Data objects, creating the label as a simple
+ /// linear combination of the features.
+ /// </summary>
+ /// <param name="nExamples">The number of examples.</param>
+ /// <param name="bias">The bias, or offset, in the calculation of the label.
+ /// </param>
+ /// <param name="weight1">The weight to multiply the first feature with to
+ /// compute the label.</param>
+ /// <param name="weight2">The weight to multiply the second feature with to
+ /// compute the label.</param>
+ /// <param name="seed">The seed for generating feature values and label
+ /// noise.</param>
+ /// <returns>An enumerable of Data objects.</returns>
+ private static IEnumerable<Data> GenerateData(int nExamples = 10000,
+ double bias = 0, double weight1 = 1, double weight2 = 2, int seed = 1)
+ {
+ var rng = new Random(seed);
+ for (int i = 0; i < nExamples; i++)
+ {
+ var data = new Data
+ {
+ Feature1 = (float)(rng.Next(10) * (rng.NextDouble() - 0.5)),
+ Feature2 = (float)(rng.Next(10) * (rng.NextDouble() - 0.5)),
+ };
+
+ // Create a noisy label.
+ data.Label = (float)(bias + weight1 * data.Feature1 + weight2 *
+ data.Feature2 + rng.NextDouble() - 0.5);
+ yield return data;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs b/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs
index a3b3ee10c1..53751614c2 100644
--- a/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs
+++ b/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs
@@ -2,22 +2,24 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
+using System;
using System.IO;
+using System.Reflection;
using Microsoft.ML;
using Microsoft.ML.Data;
using Microsoft.ML.Data.IO;
using Microsoft.ML.Runtime;
-[assembly: LoadableClass(typeof(BinaryPredictionTransformer<IPredictorProducing<float>>), typeof(BinaryPredictionTransformer), null, typeof(SignatureLoadModel),
+[assembly: LoadableClass(typeof(ISingleFeaturePredictionTransformer<object>), typeof(BinaryPredictionTransformer), null, typeof(SignatureLoadModel),
"", BinaryPredictionTransformer.LoaderSignature)]
-[assembly: LoadableClass(typeof(MulticlassPredictionTransformer<IPredictorProducing<VBuffer<float>>>), typeof(MulticlassPredictionTransformer), null, typeof(SignatureLoadModel),
+[assembly: LoadableClass(typeof(ISingleFeaturePredictionTransformer<object>), typeof(MulticlassPredictionTransformer), null, typeof(SignatureLoadModel),
"", MulticlassPredictionTransformer.LoaderSignature)]
-[assembly: LoadableClass(typeof(RegressionPredictionTransformer<IPredictorProducing<float>>), typeof(RegressionPredictionTransformer), null, typeof(SignatureLoadModel),
+[assembly: LoadableClass(typeof(ISingleFeaturePredictionTransformer<object>), typeof(RegressionPredictionTransformer), null, typeof(SignatureLoadModel),
"", RegressionPredictionTransformer.LoaderSignature)]
-[assembly: LoadableClass(typeof(RankingPredictionTransformer<IPredictorProducing<float>>), typeof(RankingPredictionTransformer), null, typeof(SignatureLoadModel),
+[assembly: LoadableClass(typeof(ISingleFeaturePredictionTransformer<object>), typeof(RankingPredictionTransformer), null, typeof(SignatureLoadModel),
"", RankingPredictionTransformer.LoaderSignature)]
[assembly: LoadableClass(typeof(AnomalyPredictionTransformer<IPredictorProducing<float>>), typeof(AnomalyPredictionTransformer), null, typeof(SignatureLoadModel),
@@ -28,6 +30,10 @@
namespace Microsoft.ML.Data
{
+ internal static class PredictionTransformerBase
+ {
+ internal const string DirModel = "Model";
+ }
/// <summary>
/// Base class for transformers with no feature column, or more than one feature columns.
@@ -44,7 +50,7 @@ public abstract class PredictionTransformerBase<TModel> : IPredictionTransformer
private protected IPredictor ModelAsPredictor => (IPredictor)Model;
[BestFriend]
- private protected const string DirModel = "Model";
+ private protected const string DirModel = PredictionTransformerBase.DirModel;
[BestFriend]
private protected const string DirTransSchema = "TrainSchema";
[BestFriend]
@@ -91,12 +97,27 @@ private protected PredictionTransformerBase(IHost host, ModelLoadContext ctx)
// *** Binary format ***
// model: prediction model.
- // stream: empty data view that contains train schema.
- // id of string: feature column.
ctx.LoadModel<TModel, SignatureLoadModel>(host, out TModel model, DirModel);
Model = model;
+ InitializeLogic(host, ctx);
+ }
+
+ [BestFriend]
+ private protected PredictionTransformerBase(IHost host, ModelLoadContext ctx, TModel model)
+ {
+ Host = host;
+ Model = model; // prediction model
+ InitializeLogic(host, ctx);
+ }
+
+ private void InitializeLogic(IHost host, ModelLoadContext ctx)
+ {
+ // *** Binary format ***
+ // stream: empty data view that contains train schema.
+ // id of string: feature column.
+
// Clone the stream with the schema into memory.
var ms = new MemoryStream();
ctx.TryLoadBinaryStream(DirTransSchema, reader =>
@@ -215,6 +236,21 @@ private protected SingleFeaturePredictionTransformerBase(IHost host, ModelLoadCo
BindableMapper = ScoreUtils.GetSchemaBindableMapper(Host, ModelAsPredictor);
}
+ private protected SingleFeaturePredictionTransformerBase(IHost host, ModelLoadContext ctx, TModel model)
+ : base(host, ctx, model)
+ {
+ FeatureColumnName = ctx.LoadStringOrNull();
+
+ if (FeatureColumnName == null)
+ FeatureColumnType = null;
+ else if (!TrainSchema.TryGetColumnIndex(FeatureColumnName, out int col))
+ throw Host.ExceptSchemaMismatch(nameof(FeatureColumnName), "feature", FeatureColumnName);
+ else
+ FeatureColumnType = TrainSchema[col].Type;
+
+ BindableMapper = ScoreUtils.GetSchemaBindableMapper(Host, ModelAsPredictor);
+ }
+
/// <summary>
/// Schema propagation for this prediction transformer.
/// </summary>
@@ -348,14 +384,25 @@ internal BinaryPredictionTransformer(IHostEnvironment env, TModel model, DataVie
internal BinaryPredictionTransformer(IHostEnvironment env, ModelLoadContext ctx)
: base(Contracts.CheckRef(env, nameof(env)).Register(nameof(BinaryPredictionTransformer<TModel>)), ctx)
+ {
+ InitializationLogic(ctx, out Threshold, out ThresholdColumn);
+ }
+
+ internal BinaryPredictionTransformer(IHostEnvironment env, ModelLoadContext ctx, IHost host, TModel model)
+ : base(host, ctx, model)
+ {
+ InitializationLogic(ctx, out Threshold, out ThresholdColumn);
+ }
+
+ private void InitializationLogic(ModelLoadContext ctx, out float threshold, out string thresholdcolumn)
{
// *** Binary format ***
// <base info>
// float: scorer threshold
// id of string: scorer threshold column
- Threshold = ctx.Reader.ReadSingle();
- ThresholdColumn = ctx.LoadString();
+ threshold = ctx.Reader.ReadSingle();
+ thresholdcolumn = ctx.LoadString();
SetScorer();
}
@@ -414,12 +461,24 @@ internal MulticlassPredictionTransformer(IHostEnvironment env, TModel model, Dat
internal MulticlassPredictionTransformer(IHostEnvironment env, ModelLoadContext ctx)
: base(Contracts.CheckRef(env, nameof(env)).Register(nameof(MulticlassPredictionTransformer<TModel>)), ctx)
+ {
+ InitializationLogic(ctx, out _trainLabelColumn);
+ }
+
+ internal MulticlassPredictionTransformer(IHostEnvironment env, ModelLoadContext ctx, IHost host, TModel model)
+ : base(host, ctx, model)
+ {
+
+ InitializationLogic(ctx, out _trainLabelColumn);
+ }
+
+ private void InitializationLogic(ModelLoadContext ctx, out string trainLabelColumn)
{
// *** Binary format ***
// <base info>
// id of string: train label column
- _trainLabelColumn = ctx.LoadStringOrNull();
+ trainLabelColumn = ctx.LoadStringOrNull();
SetScorer();
}
@@ -475,6 +534,12 @@ internal RegressionPredictionTransformer(IHostEnvironment env, ModelLoadContext
Scorer = GetGenericScorer();
}
+ internal RegressionPredictionTransformer(IHostEnvironment env, ModelLoadContext ctx, IHost host, TModel model)
+ : base(host, ctx, model)
+ {
+ Scorer = GetGenericScorer();
+ }
+
private protected override void SaveCore(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
@@ -517,6 +582,12 @@ internal RankingPredictionTransformer(IHostEnvironment env, ModelLoadContext ctx
Scorer = GetGenericScorer();
}
+ internal RankingPredictionTransformer(IHostEnvironment env, ModelLoadContext ctx, IHost host, TModel model)
+ : base(host, ctx, model)
+ {
+ Scorer = GetGenericScorer();
+ }
+
private protected override void SaveCore(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
@@ -595,33 +666,109 @@ private static VersionInfo GetVersionInfo()
internal static class BinaryPredictionTransformer
{
public const string LoaderSignature = "BinaryPredXfer";
+ private const string DirModel = PredictionTransformerBase.DirModel;
- public static BinaryPredictionTransformer<IPredictorProducing<float>> Create(IHostEnvironment env, ModelLoadContext ctx)
- => new BinaryPredictionTransformer<IPredictorProducing<float>>(env, ctx);
+ public static ISingleFeaturePredictionTransformer<object> Create(IHostEnvironment env, ModelLoadContext ctx)
+ {
+ // Load internal model to be used as TModel of BinaryPredictionTransformer<TModel>
+ var host = Contracts.CheckRef(env, nameof(env)).Register(nameof(BinaryPredictionTransformer<IPredictorProducing<float>>));
+ ctx.LoadModel<IPredictorProducing<float>, SignatureLoadModel>(host, out IPredictorProducing<float> model, DirModel);
+
+ Type generic = typeof(BinaryPredictionTransformer<>);
+ return (ISingleFeaturePredictionTransformer<object>) CreatePredictionTransformer.Create(env, ctx, host, model, generic);
+ }
}
internal static class MulticlassPredictionTransformer
{
public const string LoaderSignature = "MulticlassPredXfer";
+ private const string DirModel = PredictionTransformerBase.DirModel;
- public static MulticlassPredictionTransformer<IPredictorProducing<VBuffer<float>>> Create(IHostEnvironment env, ModelLoadContext ctx)
- => new MulticlassPredictionTransformer<IPredictorProducing<VBuffer<float>>>(env, ctx);
+ public static ISingleFeaturePredictionTransformer<object> Create(IHostEnvironment env, ModelLoadContext ctx)
+ {
+ // Load internal model to be used as TModel of MulticlassPredictionTransformer<TModel>
+ var host = Contracts.CheckRef(env, nameof(env)).Register(nameof(MulticlassPredictionTransformer<IPredictorProducing<VBuffer<float>>>));
+ ctx.LoadModel<IPredictorProducing<VBuffer<float>>, SignatureLoadModel>(host, out IPredictorProducing<VBuffer<float>> model, DirModel);
+
+ Type generic = typeof(MulticlassPredictionTransformer<>);
+ return (ISingleFeaturePredictionTransformer<object>) CreatePredictionTransformer.Create(env, ctx, host, model, generic);
+ }
}
internal static class RegressionPredictionTransformer
{
public const string LoaderSignature = "RegressionPredXfer";
+ private const string DirModel = PredictionTransformerBase.DirModel;
+
+ public static ISingleFeaturePredictionTransformer<object> Create(IHostEnvironment env, ModelLoadContext ctx)
+ {
+ // Load internal model to be used as TModel of RegressionPredictionTransformer<TModel>
+ var host = Contracts.CheckRef(env, nameof(env)).Register(nameof(RegressionPredictionTransformer<IPredictorProducing<float>>));
+ ctx.LoadModel<IPredictorProducing<float>, SignatureLoadModel>(host, out IPredictorProducing<float> model, DirModel);
- public static RegressionPredictionTransformer<IPredictorProducing<float>> Create(IHostEnvironment env, ModelLoadContext ctx)
- => new RegressionPredictionTransformer<IPredictorProducing<float>>(env, ctx);
+ Type generic = typeof(RegressionPredictionTransformer<>);
+ return (ISingleFeaturePredictionTransformer<object>) CreatePredictionTransformer.Create(env, ctx, host, model, generic);
+
+ }
}
internal static class RankingPredictionTransformer
{
public const string LoaderSignature = "RankingPredXfer";
+ private const string DirModel = PredictionTransformerBase.DirModel;
+
+ public static ISingleFeaturePredictionTransformer<object> Create(IHostEnvironment env, ModelLoadContext ctx)
+ {
+ // Load internal model to be used as TModel of RankingPredictionTransformer<TModel>
+ var host = Contracts.CheckRef(env, nameof(env)).Register(nameof(RankingPredictionTransformer<IPredictorProducing<float>>));
+ ctx.LoadModel<IPredictorProducing<float>, SignatureLoadModel>(host, out IPredictorProducing<float> model, DirModel);
+
+ Type generic = typeof(RankingPredictionTransformer<>);
+ return (ISingleFeaturePredictionTransformer<object>) CreatePredictionTransformer.Create(env, ctx, host, model, generic);
+ }
+ }
+
+ internal static class CreatePredictionTransformer
+ {
+ internal static object Create(IHostEnvironment env, ModelLoadContext ctx, IHost host, IPredictorProducing<float> model, Type generic)
+ {
+ // Create generic type of the prediction transformer using the correct TModel.
+ // Return an instance of that type, passing the previously loaded model to the constructor
+ Type[] genericTypeArgs = { model.GetType() };
+ Type constructed = generic.MakeGenericType(genericTypeArgs);
- public static RankingPredictionTransformer<IPredictorProducing<float>> Create(IHostEnvironment env, ModelLoadContext ctx)
- => new RankingPredictionTransformer<IPredictorProducing<float>>(env, ctx);
+ Type[] constructorArgs = {
+ typeof(IHostEnvironment),
+ typeof(ModelLoadContext),
+ typeof(IHost),
+ model.GetType()
+ };
+
+ var genericCtor = constructed.GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, constructorArgs, null);
+ var genericInstance = genericCtor.Invoke(new object[] { env, ctx, host, model });
+
+ return genericInstance;
+ }
+
+ internal static object Create(IHostEnvironment env, ModelLoadContext ctx, IHost host, IPredictorProducing<VBuffer<float>> model, Type generic)
+ {
+ // Create generic type of the prediction transformer using the correct TModel.
+ // Return an instance of that type, passing the previously loaded model to the constructor
+ Type[] genericTypeArgs = { model.GetType() };
+ Type constructed = generic.MakeGenericType(genericTypeArgs);
+
+ Type[] constructorArgs = {
+ typeof(IHostEnvironment),
+ typeof(ModelLoadContext),
+ typeof(IHost),
+ model.GetType()
+ };
+
+ var genericCtor = constructed.GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, constructorArgs, null);
+ var genericInstance = genericCtor.Invoke(new object[] { env, ctx, host, model });
+
+ return genericInstance;
+ }
}
internal static class AnomalyPredictionTransformer
diff --git a/test/Microsoft.ML.Functional.Tests/Explainability.cs b/test/Microsoft.ML.Functional.Tests/Explainability.cs
index 11729c3959..a63da1abea 100644
--- a/test/Microsoft.ML.Functional.Tests/Explainability.cs
+++ b/test/Microsoft.ML.Functional.Tests/Explainability.cs
@@ -6,6 +6,7 @@
using Microsoft.ML.Functional.Tests.Datasets;
using Microsoft.ML.RunTests;
using Microsoft.ML.TestFramework;
+using Microsoft.ML.Trainers;
using Microsoft.ML.Trainers.FastTree;
using Xunit;
using Xunit.Abstractions;
@@ -24,8 +25,10 @@ public Explainability(ITestOutputHelper output) : base(output)
/// <summary>
/// GlobalFeatureImportance: PFI can be used to compute global feature importance.
/// </summary>
- [Fact]
- public void GlobalFeatureImportanceWithPermutationFeatureImportance()
+ [Theory]
+ [InlineData(true)]
+ [InlineData(false)]
+ public void GlobalFeatureImportanceWithPermutationFeatureImportance(bool saveModel)
{
var mlContext = new MLContext(seed: 1);
@@ -36,12 +39,38 @@ public void GlobalFeatureImportanceWithPermutationFeatureImportance()
var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features)
.Append(mlContext.Regression.Trainers.Sdca());
- // Fit the pipeline and transform the data.
+ // Fit the pipeline
var model = pipeline.Fit(data);
- var transformedData = model.Transform(data);
+
+ IDataView transformedData;
+ RegressionPredictionTransformer<LinearRegressionModelParameters> linearPredictor;
+
+ if(saveModel)
+ {
+ ITransformer loadedModel;
+
+ // Load and save the model
+ var modelAndSchemaPath = GetOutputPath("TestFunctionalTestPFI.zip");
+ mlContext.Model.Save(model, data.Schema, modelAndSchemaPath);
+ loadedModel = mlContext.Model.Load(modelAndSchemaPath, out var schema);
+
+ // Transform the data
+ transformedData = loadedModel.Transform(data);
+
+ // Extract linear predictor
+ linearPredictor = (loadedModel as TransformerChain<ITransformer>).LastTransformer as RegressionPredictionTransformer<LinearRegressionModelParameters>;
+ }
+ else
+ {
+ // Transform the data
+ transformedData = model.Transform(data);
+
+ // Extract linear predictor
+ linearPredictor = model.LastTransformer;
+ }
// Compute the permutation feature importance to look at global feature importance.
- var permutationMetrics = mlContext.Regression.PermutationFeatureImportance(model.LastTransformer, transformedData);
+ var permutationMetrics = mlContext.Regression.PermutationFeatureImportance(linearPredictor, transformedData);
// Make sure the correct number of features came back.
Assert.Equal(HousingRegression.Features.Length, permutationMetrics.Length);
diff --git a/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs b/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs
index ac86a8703f..ac04e5452a 100644
--- a/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs
+++ b/test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs
@@ -4,11 +4,13 @@
using System;
using System.Collections.Immutable;
+using System.IO;
using System.Linq;
using Microsoft.ML.Data;
using Microsoft.ML.Internal.Utilities;
using Microsoft.ML.RunTests;
using Microsoft.ML.Trainers;
+using Microsoft.ML.Trainers.FastTree;
using Xunit;
using Xunit.Abstractions;
@@ -25,12 +27,28 @@ public PermutationFeatureImportanceTests(ITestOutputHelper output) : base(output
/// <summary>
/// Test PFI Regression for Dense Features
/// </summary>
- [Fact]
- public void TestPfiRegressionOnDenseFeatures()
+ [Theory]
+ [InlineData(true)]
+ [InlineData(false)]
+ public void TestPfiRegressionOnDenseFeatures(bool saveModel)
{
var data = GetDenseDataset();
var model = ML.Regression.Trainers.OnlineGradientDescent().Fit(data);
- var pfi = ML.Regression.PermutationFeatureImportance(model, data);
+
+ ImmutableArray<RegressionMetricsStatistics> pfi;
+ if(saveModel)
+ {
+ var modelAndSchemaPath = GetOutputPath("TestPfiRegressionOnDenseFeatures.zip");
+ ML.Model.Save(model, data.Schema, modelAndSchemaPath);
+
+ var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema);
+ var castedModel = loadedModel as RegressionPredictionTransformer<LinearRegressionModelParameters>;
+ pfi = ML.Regression.PermutationFeatureImportance(castedModel, data);
+ }
+ else
+ {
+ pfi = ML.Regression.PermutationFeatureImportance(model, data);
+ }
// Pfi Indices:
// X1: 0
@@ -58,12 +76,30 @@ public void TestPfiRegressionOnDenseFeatures()
/// <summary>
/// Test PFI Regression Standard Deviation and Standard Error for Dense Features
/// </summary>
- [Fact]
- public void TestPfiRegressionStandardDeviationAndErrorOnDenseFeatures()
+ [Theory]
+ [InlineData(true)]
+ [InlineData(false)]
+ public void TestPfiRegressionStandardDeviationAndErrorOnDenseFeatures(bool saveModel)
{
var data = GetDenseDataset();
var model = ML.Regression.Trainers.OnlineGradientDescent().Fit(data);
- var pfi = ML.Regression.PermutationFeatureImportance(model, data, permutationCount: 20);
+
+ ImmutableArray<RegressionMetricsStatistics> pfi;
+
+ if(saveModel)
+ {
+ var modelAndSchemaPath = GetOutputPath("TestPfiRegressionStandardDeviationAndErrorOnDenseFeatures.zip");
+ ML.Model.Save(model, data.Schema, modelAndSchemaPath);
+
+ var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema);
+ var castedModel = loadedModel as RegressionPredictionTransformer<LinearRegressionModelParameters>;
+ pfi = ML.Regression.PermutationFeatureImportance(castedModel, data, permutationCount: 20);
+ }
+ else
+ {
+ pfi = ML.Regression.PermutationFeatureImportance(model, data, permutationCount: 20);
+ }
+
// Keep the permutation count high so fluctuations are kept to a minimum
// but not high enough to slow down the tests
// (fluctuations lead to random test failures)
@@ -110,12 +146,28 @@ public void TestPfiRegressionStandardDeviationAndErrorOnDenseFeatures()
/// <summary>
/// Test PFI Regression for Sparse Features
/// </summary>
- [Fact]
- public void TestPfiRegressionOnSparseFeatures()
+ [Theory]
+ [InlineData(true)]
+ [InlineData(false)]
+ public void TestPfiRegressionOnSparseFeatures(bool saveModel)
{
var data = GetSparseDataset();
var model = ML.Regression.Trainers.OnlineGradientDescent().Fit(data);
- var results = ML.Regression.PermutationFeatureImportance(model, data);
+
+ ImmutableArray<RegressionMetricsStatistics> results;
+ if(saveModel)
+ {
+ var modelAndSchemaPath = GetOutputPath("TestPfiRegressionOnSparseFeatures.zip");
+ ML.Model.Save(model, data.Schema, modelAndSchemaPath);
+
+ var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema);
+ var castedModel = loadedModel as RegressionPredictionTransformer<LinearRegressionModelParameters>;
+ results = ML.Regression.PermutationFeatureImportance(castedModel, data);
+ }
+ else
+ {
+ results = ML.Regression.PermutationFeatureImportance(model, data);
+ }
// Pfi Indices:
// X1: 0
@@ -227,12 +279,28 @@ public void TestPfiBinaryClassificationOnSparseFeatures()
/// <summary>
/// Test PFI Multiclass Classification for Dense Features
/// </summary>
- [Fact]
- public void TestPfiMulticlassClassificationOnDenseFeatures()
+ [Theory]
+ [InlineData(true)]
+ [InlineData(false)]
+ public void TestPfiMulticlassClassificationOnDenseFeatures(bool saveModel)
{
var data = GetDenseDataset(TaskType.MulticlassClassification);
var model = ML.MulticlassClassification.Trainers.LbfgsMaximumEntropy().Fit(data);
- var pfi = ML.MulticlassClassification.PermutationFeatureImportance(model, data);
+
+ ImmutableArray<MulticlassClassificationMetricsStatistics> pfi;
+ if(saveModel)
+ {
+ var modelAndSchemaPath = GetOutputPath("TestPfiMulticlassClassificationOnDenseFeatures.zip");
+ ML.Model.Save(model, data.Schema, modelAndSchemaPath);
+
+ var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema);
+ var castedModel = loadedModel as MulticlassPredictionTransformer<MaximumEntropyModelParameters>;
+ pfi = ML.MulticlassClassification.PermutationFeatureImportance(castedModel, data);
+ }
+ else
+ {
+ pfi = ML.MulticlassClassification.PermutationFeatureImportance(model, data);
+ }
// Pfi Indices:
// X1: 0
@@ -264,14 +332,29 @@ public void TestPfiMulticlassClassificationOnDenseFeatures()
/// <summary>
/// Test PFI Multiclass Classification for Sparse Features
/// </summary>
- [Fact]
- public void TestPfiMulticlassClassificationOnSparseFeatures()
+ [Theory]
+ [InlineData(true)]
+ [InlineData(false)]
+ public void TestPfiMulticlassClassificationOnSparseFeatures(bool saveModel)
{
var data = GetSparseDataset(TaskType.MulticlassClassification);
var model = ML.MulticlassClassification.Trainers.LbfgsMaximumEntropy(
new LbfgsMaximumEntropyMulticlassTrainer.Options { MaximumNumberOfIterations = 1000 }).Fit(data);
- var pfi = ML.MulticlassClassification.PermutationFeatureImportance(model, data);
+ ImmutableArray<MulticlassClassificationMetricsStatistics> pfi;
+ if(saveModel)
+ {
+ var modelAndSchemaPath = GetOutputPath("TestPfiMulticlassClassificationOnSparseFeatures.zip");
+ ML.Model.Save(model, data.Schema, modelAndSchemaPath);
+
+ var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema);
+ var castedModel = loadedModel as MulticlassPredictionTransformer<MaximumEntropyModelParameters>;
+ pfi = ML.MulticlassClassification.PermutationFeatureImportance(castedModel, data);
+ }
+ else
+ {
+ pfi = ML.MulticlassClassification.PermutationFeatureImportance(model, data);
+ }
// Pfi Indices:
// X1: 0
@@ -305,14 +388,31 @@ public void TestPfiMulticlassClassificationOnSparseFeatures()
#region Ranking Tests
/// <summary>
- /// Test PFI Multiclass Classification for Dense Features
+ /// Test PFI Ranking Classification for Dense Features
/// </summary>
- [Fact]
- public void TestPfiRankingOnDenseFeatures()
+ [Theory]
+ [InlineData(true)]
+ [InlineData(false)]
+ public void TestPfiRankingOnDenseFeatures(bool saveModel)
{
var data = GetDenseDataset(TaskType.Ranking);
var model = ML.Ranking.Trainers.FastTree().Fit(data);
- var pfi = ML.Ranking.PermutationFeatureImportance(model, data);
+
+ ImmutableArray<RankingMetricsStatistics> pfi;
+ if(saveModel)
+ {
+ var modelAndSchemaPath = GetOutputPath("TestPfiRankingOnDenseFeatures.zip");
+ ML.Model.Save(model, data.Schema, modelAndSchemaPath);
+
+ var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema);
+ var castedModel = loadedModel as RankingPredictionTransformer<FastTreeRankingModelParameters>;
+ pfi = ML.Ranking.PermutationFeatureImportance(castedModel, data);
+ }
+ else
+ {
+ pfi = ML.Ranking.PermutationFeatureImportance(model, data);
+ }
+
// Pfi Indices:
// X1: 0 // For Ranking, this column won't result in misorderings
@@ -335,15 +435,32 @@ public void TestPfiRankingOnDenseFeatures()
Done();
}
+
/// <summary>
/// Test PFI Multiclass Classification for Sparse Features
/// </summary>
- [Fact]
- public void TestPfiRankingOnSparseFeatures()
+ [Theory]
+ [InlineData(true)]
+ [InlineData(false)]
+ public void TestPfiRankingOnSparseFeatures(bool saveModel)
{
var data = GetSparseDataset(TaskType.Ranking);
var model = ML.Ranking.Trainers.FastTree().Fit(data);
- var pfi = ML.Ranking.PermutationFeatureImportance(model, data);
+
+ ImmutableArray<RankingMetricsStatistics> pfi;
+ if(saveModel)
+ {
+ var modelAndSchemaPath = GetOutputPath("TestPfiRankingOnSparseFeatures.zip");
+ ML.Model.Save(model, data.Schema, modelAndSchemaPath);
+
+ var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema);
+ var castedModel = loadedModel as RankingPredictionTransformer<FastTreeRankingModelParameters>;
+ pfi = ML.Ranking.PermutationFeatureImportance(castedModel, data);
+ }
+ else
+ {
+ pfi = ML.Ranking.PermutationFeatureImportance(model, data);
+ }
// Pfi Indices:
// X1: 0
diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/LbfgsTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/LbfgsTests.cs
index 1174c87aca..f80ee68d2d 100644
--- a/test/Microsoft.ML.Tests/TrainerEstimators/LbfgsTests.cs
+++ b/test/Microsoft.ML.Tests/TrainerEstimators/LbfgsTests.cs
@@ -125,8 +125,8 @@ public void TestLRWithStats()
using (var fs = File.OpenRead(modelAndSchemaPath))
transformerChain = ML.Model.Load(fs, out var schema);
- var lastTransformer = ((TransformerChain<ITransformer>)transformerChain).LastTransformer as BinaryPredictionTransformer<IPredictorProducing<float>>;
- var model = lastTransformer.Model as ParameterMixingCalibratedModelParameters<IPredictorProducing<float>, ICalibrator>;
+ var lastTransformer = ((TransformerChain<ITransformer>)transformerChain).LastTransformer as BinaryPredictionTransformer<ParameterMixingCalibratedModelParameters<IPredictorProducing<float>, ICalibrator>>;
+ var model = lastTransformer.Model;
linearModel = model.SubModel as LinearBinaryModelParameters;
@@ -215,8 +215,8 @@ public void TestMLRWithStats()
using (var fs = File.OpenRead(modelAndSchemaPath))
transformerChain = ML.Model.Load(fs, out var schema);
- var lastTransformer = ((TransformerChain<ITransformer>)transformerChain).LastTransformer as MulticlassPredictionTransformer<IPredictorProducing<VBuffer<float>>>;
- model = lastTransformer.Model as MaximumEntropyModelParameters;
+ var lastTransformer = ((TransformerChain<ITransformer>)transformerChain).LastTransformer as MulticlassPredictionTransformer<MaximumEntropyModelParameters>;
+ model = lastTransformer.Model;
validateStats(model);