Fixes #4292 about using PFI with BPT and CMPB

docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/PermutationFeatureImportanceLoadFromDisk.cs

@@ -0,0 +1,106 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML;
+using Microsoft.ML.Calibrators;
+using Microsoft.ML.Data;
+using Microsoft.ML.Trainers;
+
+namespace Samples.Dynamic.Trainers.BinaryClassification
+{
+    public static class PermutationFeatureImportanceLoadFromDisk
+    {
+        public static void Example()
+        {
+
+            var mlContext = new MLContext(seed: 1);
+            var samples = GenerateData();
+            var data = mlContext.Data.LoadFromEnumerable(samples);
+
+            // Create pipeline
+            var featureColumns =
+                new string[] { nameof(Data.Feature1), nameof(Data.Feature2) };
+            var pipeline = mlContext.Transforms
+                .Concatenate("Features", featureColumns)
+                .Append(mlContext.Transforms.NormalizeMinMax("Features"))
+                .Append(mlContext.BinaryClassification.Trainers.SdcaLogisticRegression()
+                );
+
+            // Create and save model
+            var model0 = pipeline.Fit(data);
+            var lt = model0.LastTransformer;
+            var modelPath = "./model.zip";
+            mlContext.Model.Save(model0, data.Schema, modelPath);
+
+            // Load model
+            var model = mlContext.Model.Load(modelPath, out var schema);
+
+            // Transform the dataset.
+            var transformedData = model.Transform(data);
+
+            var linearPredictor = (model as TransformerChain<ITransformer>).LastTransformer as BinaryPredictionTransformer<CalibratedModelParametersBase<LinearBinaryModelParameters, PlattCalibrator>>;
+
+            // Execute PFI with the linearPredictor
+            var permutationMetrics = mlContext.BinaryClassification
+                .PermutationFeatureImportance(linearPredictor, transformedData,
+                permutationCount: 30);
+
+            // Sort indices according to PFI results
+            var sortedIndices = permutationMetrics
+                .Select((metrics, index) => new { index, metrics.AreaUnderRocCurve })
+                .OrderByDescending(
+                feature => Math.Abs(feature.AreaUnderRocCurve.Mean))
+                .Select(feature => feature.index);
+
+            Console.WriteLine("Feature\tModel Weight\tChange in AUC"
+                + "\t95% Confidence in the Mean Change in AUC");
+            var auc = permutationMetrics.Select(x => x.AreaUnderRocCurve).ToArray();
+            foreach (int i in sortedIndices)
+            {
+                Console.WriteLine("{0}\t{1:0.00}\t{2:G4}\t{3:G4}",
+                    featureColumns[i],
+                    linearPredictor.Model.SubModel.Weights[i], // this way we can access the weights inside the submodel
+                    auc[i].Mean,
+                    1.96 * auc[i].StandardError);
+            }
+
+            // Expected output:
+            //  Feature     Model Weight Change in AUC  95% Confidence in the Mean Change in AUC
+            //  Feature2        35.15     -0.387        0.002015
+            //  Feature1        17.94     -0.1514       0.0008963
+        }
+
+        private class Data
+        {
+            public bool Label { get; set; }
+
+            public float Feature1 { get; set; }
+
+            public float Feature2 { get; set; }
+        }
+
+        /// Generate Data
+        private static IEnumerable<Data> GenerateData(int nExamples = 10000,
+            double bias = 0, double weight1 = 1, double weight2 = 2, int seed = 1)
+        {
+            var rng = new Random(seed);
+            for (int i = 0; i < nExamples; i++)
+            {
+                var data = new Data
+                {
+                    Feature1 = (float)(rng.Next(10) * (rng.NextDouble() - 0.5)),
+                    Feature2 = (float)(rng.Next(10) * (rng.NextDouble() - 0.5)),
+                };
+
+                // Create a noisy label.
+                var value = (float)(bias + weight1 * data.Feature1 + weight2 *
+                    data.Feature2 + rng.NextDouble() - 0.5);
+
+                data.Label = Sigmoid(value) > 0.5;
+                yield return data;
+            }
+        }
+
+        private static double Sigmoid(double x) => 1.0 / (1.0 + Math.Exp(-1 * x));
+    }
+}

src/Microsoft.ML.Data/Prediction/Calibrator.cs

@@ -8,6 +8,7 @@
 using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
+using System.Reflection;
 using Microsoft.ML;
 using Microsoft.ML.Calibrators;
 using Microsoft.ML.CommandLine;
@@ -396,6 +397,7 @@ bool ISingleCanSaveOnnx.SaveAsOnnx(OnnxContext ctx, string[] outputNames, string
     }
 
     [BestFriend]
+    [PredictionTransformerLoadType(typeof(CalibratedModelParametersBase<,>))]
     internal sealed class ValueMapperCalibratedModelParameters<TSubModel, TCalibrator> :
         ValueMapperCalibratedModelParametersBase<TSubModel, TCalibrator>, ICanSaveModel
         where TSubModel : class
@@ -430,8 +432,8 @@ private static VersionInfo GetVersionInfoBulk()
                 loaderAssemblyName: typeof(ValueMapperCalibratedModelParameters<TSubModel, TCalibrator>).Assembly.FullName);
         }
 
-        private ValueMapperCalibratedModelParameters(IHostEnvironment env, ModelLoadContext ctx)
-            : base(env, RegistrationName, GetPredictor(env, ctx), GetCalibrator(env, ctx))
+        private ValueMapperCalibratedModelParameters(IHostEnvironment env, ModelLoadContext ctx, TSubModel predictor, TCalibrator calibrator)
+            : base(env, RegistrationName, predictor, calibrator)
         {
         }
 
@@ -443,7 +445,16 @@ private static CalibratedModelParametersBase Create(IHostEnvironment env, ModelL
             var ver2 = GetVersionInfoBulk();
             var ver = ctx.Header.ModelSignature == ver2.ModelSignature ? ver2 : ver1;
             ctx.CheckAtModel(ver);
-            return new ValueMapperCalibratedModelParameters<TSubModel, TCalibrator>(env, ctx);
+
+            // Load first the predictor and calibrator
+            var predictor = GetPredictor(env, ctx);
+            var calibrator = GetCalibrator(env, ctx);
+
+            // Create a generic type using the correct parameter types of predictor and calibrator
+            Type genericType = typeof(ValueMapperCalibratedModelParameters<,>);
+            var genericInstance = CreateCalibratedModelParameters.Create(env, ctx, predictor, calibrator, genericType);
+
+            return (CalibratedModelParametersBase)genericInstance;
         }
 
         void ICanSaveModel.Save(ModelSaveContext ctx)
@@ -456,6 +467,7 @@ void ICanSaveModel.Save(ModelSaveContext ctx)
     }
 
     [BestFriend]
+    [PredictionTransformerLoadType(typeof(CalibratedModelParametersBase<,>))]
     internal sealed class FeatureWeightsCalibratedModelParameters<TSubModel, TCalibrator> :
         ValueMapperCalibratedModelParametersBase<TSubModel, TCalibrator>,
         IPredictorWithFeatureWeights<float>,
@@ -487,8 +499,9 @@ private static VersionInfo GetVersionInfo()
                 loaderAssemblyName: typeof(FeatureWeightsCalibratedModelParameters<TSubModel, TCalibrator>).Assembly.FullName);
         }
 
-        private FeatureWeightsCalibratedModelParameters(IHostEnvironment env, ModelLoadContext ctx)
-            : base(env, RegistrationName, GetPredictor(env, ctx), GetCalibrator(env, ctx))
+        private FeatureWeightsCalibratedModelParameters(IHostEnvironment env, ModelLoadContext ctx,
+            TSubModel predictor, TCalibrator calibrator)
+            : base(env, RegistrationName, predictor, calibrator)
         {
             Host.Check(SubModel is IPredictorWithFeatureWeights<float>, "Predictor does not implement " + nameof(IPredictorWithFeatureWeights<float>));
             _featureWeights = (IPredictorWithFeatureWeights<float>)SubModel;
@@ -499,7 +512,16 @@ private static CalibratedModelParametersBase Create(IHostEnvironment env, ModelL
             Contracts.CheckValue(env, nameof(env));
             env.CheckValue(ctx, nameof(ctx));
             ctx.CheckAtModel(GetVersionInfo());
-            return new FeatureWeightsCalibratedModelParameters<TSubModel, TCalibrator>(env, ctx);
+
+            // Load first the predictor and calibrator
+            var predictor = GetPredictor(env, ctx);
+            var calibrator = GetCalibrator(env, ctx);
+
+            // Create a generic type using the correct parameter types of predictor and calibrator
+            Type genericType = typeof(FeatureWeightsCalibratedModelParameters<,>);
+            var genericInstance = CreateCalibratedModelParameters.Create(env, ctx, predictor, calibrator, genericType);
+
+            return (CalibratedModelParametersBase) genericInstance;
         }
 
         void ICanSaveModel.Save(ModelSaveContext ctx)
@@ -520,6 +542,7 @@ public void GetFeatureWeights(ref VBuffer<float> weights)
     /// Encapsulates a predictor and a calibrator that implement <see cref="IParameterMixer"/>.
     /// Its implementation of <see cref="IParameterMixer.CombineParameters"/> combines both the predictors and the calibrators.
     /// </summary>
+    [PredictionTransformerLoadType(typeof(CalibratedModelParametersBase <,>))]
     internal sealed class ParameterMixingCalibratedModelParameters<TSubModel, TCalibrator> :
         ValueMapperCalibratedModelParametersBase<TSubModel, TCalibrator>,
         IParameterMixer<float>,
@@ -553,8 +576,8 @@ private static VersionInfo GetVersionInfo()
                 loaderAssemblyName: typeof(ParameterMixingCalibratedModelParameters<TSubModel, TCalibrator>).Assembly.FullName);
         }
 
-        private ParameterMixingCalibratedModelParameters(IHostEnvironment env, ModelLoadContext ctx)
-            : base(env, RegistrationName, GetPredictor(env, ctx), GetCalibrator(env, ctx))
+        private ParameterMixingCalibratedModelParameters(IHostEnvironment env, ModelLoadContext ctx, TSubModel predictor, TCalibrator calibrator)
+            : base(env, RegistrationName, predictor, calibrator)
         {
             Host.Check(SubModel is IParameterMixer<float>, "Predictor does not implement " + nameof(IParameterMixer));
             Host.Check(SubModel is IPredictorWithFeatureWeights<float>, "Predictor does not implement " + nameof(IPredictorWithFeatureWeights<float>));
@@ -566,7 +589,16 @@ private static CalibratedModelParametersBase Create(IHostEnvironment env, ModelL
             Contracts.CheckValue(env, nameof(env));
             env.CheckValue(ctx, nameof(ctx));
             ctx.CheckAtModel(GetVersionInfo());
-            return new ParameterMixingCalibratedModelParameters<TSubModel, TCalibrator>(env, ctx);
+
+            // Load first the predictor and calibrator
+            var predictor = GetPredictor(env, ctx);
+            var calibrator = GetCalibrator(env, ctx);
+
+            // Create a generic type using the correct parameter types of predictor and calibrator
+            Type genericType = typeof(ParameterMixingCalibratedModelParameters<,>);
+            object genericInstance = CreateCalibratedModelParameters.Create(env, ctx, predictor, calibrator, genericType);
+
+            return (CalibratedModelParametersBase) genericInstance;
         }
 
         void ICanSaveModel.Save(ModelSaveContext ctx)
@@ -777,6 +809,28 @@ ValueMapper<TSrc, VBuffer<float>> IFeatureContributionMapper.GetFeatureContribut
         }
     }
 
+    internal static class CreateCalibratedModelParameters
+    {
+        internal static object Create(IHostEnvironment env, ModelLoadContext ctx, object predictor, ICalibrator calibrator, Type calibratedModelParametersType)
+        {
+            Type[] genericTypeArgs = { predictor.GetType(), calibrator.GetType() };
+            Type constructed = calibratedModelParametersType.MakeGenericType(genericTypeArgs);
+
+            Type[] constructorArgs = {
+                typeof(IHostEnvironment),
+                typeof(ModelLoadContext),
+                predictor.GetType(),
+                calibrator.GetType()
+            };
+
+            // Call the appropiate constructor of the created generic type passing on the previously loaded predictor and calibrator
+            var genericCtor = constructed.GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, constructorArgs, null);
+            object genericInstance = genericCtor.Invoke(new object[] { env, ctx, predictor, calibrator });
+
+            return genericInstance;
+        }
+    }
+
     [BestFriend]
     internal static class CalibratorUtils
     {