[Do Not Review] :Upgrade to ML.Scoring v 1.2.0, with bug fixes and GPU/CUDA support

build/Dependencies.props

@@ -16,7 +16,7 @@
   <PropertyGroup>
     <GoogleProtobufPackageVersion>3.5.1</GoogleProtobufPackageVersion>
     <LightGBMPackageVersion>2.2.1.1</LightGBMPackageVersion>
-    <MicrosoftMLScoring>1.1.0</MicrosoftMLScoring>
+    <MicrosoftMLScoring>1.2.0</MicrosoftMLScoring>
     <MlNetMklDepsPackageVersion>0.0.0.7</MlNetMklDepsPackageVersion>
     <ParquetDotNetPackageVersion>2.1.3</ParquetDotNetPackageVersion>
     <SystemDrawingCommonPackageVersion>4.5.0</SystemDrawingCommonPackageVersion>

docs/samples/Microsoft.ML.Samples/Dynamic/OnnxTransform.cs

@@ -0,0 +1,121 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Transforms;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Runtime.ImageAnalytics;
+using Microsoft.ML;
+using System;
+using System.IO;
+
+namespace Microsoft.ML.Samples.Dynamic
+{
+    class OnnxTransformExample
+    {
+        public static void OnnxTransformSample(string[] args)
+        {
+            // Download the squeeznet image model from ONNX model zoo, version 1.2
+            // https://github.com/onnx/models/tree/master/squeezenet
+            var model_location = @"squeezenet\model.onnx";
+
+            var env = new MLContext();
+
+            // Use the utility functions to inspect models inputs, outputs, shape and type
+            // Load the model using the OnnxModel class
+            var onnxModel = new OnnxModel(model_location);
+
+            // This model has only 1 input, so inspect 0th index for input node metadata
+            var inputSchema = onnxModel.ModelInfo.InputsInfo[0];
+            var inputName = inputSchema.Name;
+            var inputShape = inputSchema.Shape;
+
+            // Deduce image dimensions from inputShape
+            var numChannels = (int) inputShape[1];
+            var imageHeight = (int) inputShape[2];
+            var imageWidth =  (int) inputShape[3];
+
+            // Similarly, get output node metadata
+            var outputSchema = onnxModel.ModelInfo.OutputsInfo[0];
+            var outputName = outputSchema.Name;
+            var outputShape = outputSchema.Shape;
+
+            var dataFile = @"test\data\images\images.tsv";
+            var imageFolder = Path.GetDirectoryName(dataFile);
+
+            // Use Textloader to load the text file which references the images to load
+            // Preview ...
+            // banana.jpg banana
+            // hotdog.jpg hotdog
+            // tomato.jpg tomato
+            var data = TextLoader.Create(env, new TextLoader.Arguments()
+            {
+                Column = new[]
+                {
+                    new TextLoader.Column("ImagePath", DataKind.TX, 0),
+                    new TextLoader.Column("Name", DataKind.TX, 1),
+                 }
+            }, new MultiFileSource(dataFile));
+
+            // Load the images referenced in the text file
+            var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
+            {
+                Column = new ImageLoaderTransform.Column[1]
+                {
+                        new ImageLoaderTransform.Column() { Source=  "ImagePath", Name="ImageReal" }
+                },
+                ImageFolder = imageFolder
+            }, data);
+
+            // Resize the images to match model dimensions
+            var cropped = ImageResizerTransform.Create(env, new ImageResizerTransform.Arguments()
+            {
+                Column = new ImageResizerTransform.Column[1]{
+                        new ImageResizerTransform.Column(){ Source = "ImageReal", Name= "ImageCropped", ImageHeight =imageHeight, ImageWidth = imageWidth, Resizing = ImageResizerTransform.ResizingKind.IsoCrop}}
+            }, images);
+
+            // Extract out the RBG pixel values. 
+            // InterleaveArgb = true makes the values RGBRGBRGB. Otherwise it's RRR...GGG...BBB.
+            var pixels = ImagePixelExtractorTransform.Create(env, new ImagePixelExtractorTransform.Arguments()
+            {
+                Column = new ImagePixelExtractorTransform.Column[1]{
+                        new ImagePixelExtractorTransform.Column() {  Source= "ImageCropped", Name = inputName, InterleaveArgb=true}
+                    }
+            }, cropped);
+
+            // Create OnnxTransform, passing in the input and output names the model expects.
+            IDataView trans = OnnxTransform.Create(env, pixels, model_location, new[] { inputName }, new[] { outputName });
+
+            trans.Schema.TryGetColumnIndex(outputName, out int output);
+            using (var cursor = trans.GetRowCursor(col => col == output))
+            {
+                var numRows = 0;
+                var buffer = default(VBuffer<float>);
+                var getter = cursor.GetGetter<VBuffer<float>>(output);
+                // For each image, retrieve the model scores
+                while (cursor.MoveNext())
+                {
+                    int i = 0;
+                    getter(ref buffer);
+                    // print scores for first 3 classes
+                    foreach(var score in buffer.GetValues())
+                    {
+                        Console.WriteLine(String.Format("Example # {0} :Score for class {1} = {2} ",numRows, i, score));
+                        if (++i > 2) break;
+                    }
+                    numRows += 1;
+                }
+            }
+            // Results look like below...
+            // Example # 0 :Score for class 0 = 1.133263E-06
+            // Example # 0 :Score for class 1 = 1.80478E-07
+            // Example # 0 :Score for class 2 = 1.595297E-07
+            // Example # 1 :Score for class 0 = 1.805106E-05
+            // Example # 1 :Score for class 1 = 1.257452E-05
+            // Example # 1 :Score for class 2 = 2.412128E-06
+            // Example # 2 :Score for class 0 = 1.346096E-06
+            // Example # 2 :Score for class 1 = 1.918751E-07
+            // Example # 2 :Score for class 2 = 7.203341E-08
+        }
+    }
+}

docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj

@@ -13,6 +13,8 @@
     <ProjectReference Include="..\..\..\src\Microsoft.ML.LightGBM\Microsoft.ML.LightGBM.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.Recommender\Microsoft.ML.Recommender.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.TimeSeries\Microsoft.ML.TimeSeries.csproj" />
+    <ProjectReference Include="..\..\..\src\Microsoft.ML.ImageAnalytics\Microsoft.ML.ImageAnalytics.csproj" />
+    <ProjectReference Include="..\..\..\src\/Microsoft.ML.OnnxTransform\Microsoft.ML.OnnxTransform.csproj" />
 
     <NativeAssemblyReference Include="CpuMathNative" />
     <NativeAssemblyReference Include="FastTreeNative" />

src/Microsoft.ML.OnnxTransform/OnnxTransform.cs

@@ -36,6 +36,29 @@
 
 namespace Microsoft.ML.Transforms
 {
+    /// <summary>
+    /// <p>A transform for scoring ONNX models in the ML.NET framework.</p>
+    /// <format type="text/markdown">
+    /// <![CDATA[
+    /// [!code-csharp[MF](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/OnnxTransform.cs)]
+    /// ]]>
+    /// </format>
+    /// </summary>
+    /// <remarks>
+    /// <p>Supports inferencing of models in 1.2 and 1.3 format, using the
+    /// <a href='https://www.nuget.org/packages/Microsoft.ML.Scoring/'>Microsoft.ML.Scoring</a> library
+    /// </p>
+    /// <p>The inputs and outputs of the onnx models must of of Tensors. Sequence and Maps are not yet supported.</p>
+    /// <p>Supports optional GPU execution via the CUDA 9.2 libraries. The
+    /// <a href='https://www.nuget.org/packages/Microsoft.ML.Scoring/'>CUDA 9.2 Toolkit</a>
+    /// and
+    /// <a href='https://www.nuget.org/packages/Microsoft.ML.Scoring/'>cuDNN</a>
+    /// libraries need to be installed separately. By default models are run on CPU. To run on GPU if available,
+    /// set the parameter 'gpuDeviceID' to a valid non-negative number.
+    /// </p>
+    /// <p>Visit https://github.com/onnx/models to see a list of readily available models to get started with.</p>
+    /// <p>Refer to http://onnx.ai' for more information about ONNX.</p>
+    /// </remarks>
     public sealed class OnnxTransform : RowToRowTransformerBase
     {
         public sealed class Arguments : TransformInputBase
@@ -48,6 +71,9 @@ public sealed class Arguments : TransformInputBase
 
             [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Name of the output column.", SortOrder = 2)]
             public string[] OutputColumns;
+
+            [Argument(ArgumentType.AtMostOnce | ArgumentType.Required, HelpText = "GPU device id to run on. Typically 0,1 etc. Default of -1 runs on CPU. Requires CUDA 9.2.", SortOrder = 3)]
+            public int GpuDeviceId = -1;
         }
 
         private readonly Arguments _args;
@@ -77,7 +103,13 @@ private static VersionInfo GetVersionInfo()
 
         public static IDataTransform Create(IHostEnvironment env, IDataView input, string modelFile, string[] inputColumns, string[] outputColumns)
         {
-            var args = new Arguments { ModelFile = modelFile, InputColumns = inputColumns, OutputColumns = outputColumns };
+            var args = new Arguments { ModelFile = modelFile, InputColumns = inputColumns, OutputColumns = outputColumns, GpuDeviceId = -1 };
+            return Create(env, args, input);
+        }
+
+        public static IDataTransform Create(IHostEnvironment env, IDataView input, string modelFile, string[] inputColumns, string[] outputColumns, int gpuDeviceId = -1)
+        {
+            var args = new Arguments { ModelFile = modelFile, InputColumns = inputColumns, OutputColumns = outputColumns, GpuDeviceId = gpuDeviceId };
             return Create(env, args, input);
         }
 
@@ -141,10 +173,10 @@ private OnnxTransform(IHostEnvironment env, Arguments args, byte[] modelBytes =
             {
                 Host.CheckNonWhiteSpace(args.ModelFile, nameof(args.ModelFile));
                 Host.CheckUserArg(File.Exists(args.ModelFile), nameof(args.ModelFile));
-                Model = new OnnxModel(args.ModelFile);
+                Model = new OnnxModel(args.ModelFile, args.GpuDeviceId);
             }
             else
-                Model = OnnxModel.CreateFromBytes(modelBytes);
+                Model = OnnxModel.CreateFromBytes(modelBytes, args.GpuDeviceId);
 
             var modelInfo = Model.ModelInfo;
             Inputs = args.InputColumns;
@@ -165,13 +197,13 @@ private OnnxTransform(IHostEnvironment env, Arguments args, byte[] modelBytes =
             _args = args;
         }
 
-        public OnnxTransform(IHostEnvironment env, string modelFile, string inputColumn, string outputColumn)
-            : this(env, new Arguments() { ModelFile = modelFile, InputColumns = new[] { inputColumn }, OutputColumns = new[] { outputColumn } })
+        public OnnxTransform(IHostEnvironment env, string modelFile, string inputColumn, string outputColumn, int gpuDeviceId = -1)
+            : this(env, new Arguments() { ModelFile = modelFile, InputColumns = new[] { inputColumn }, OutputColumns = new[] { outputColumn }, GpuDeviceId = gpuDeviceId })
         {
         }
 
-        public OnnxTransform(IHostEnvironment env, string modelFile, string[] inputColumns, string[] outputColumns)
-            : this(env, new Arguments() { ModelFile = modelFile, InputColumns = inputColumns, OutputColumns = outputColumns })
+        public OnnxTransform(IHostEnvironment env, string modelFile, string[] inputColumns, string[] outputColumns, int gpuDeviceId = -1)
+            : this(env, new Arguments() { ModelFile = modelFile, InputColumns = inputColumns, OutputColumns = outputColumns, GpuDeviceId = gpuDeviceId })
         {
         }
 
@@ -418,10 +450,14 @@ public Tensor GetTensor()
             }
         }
     }
+
+    /// <summary>
+    /// A class implementing the estimator interface of the OnnxTransform.
+    /// </summary>
     public sealed class OnnxScoringEstimator : TrivialEstimator<OnnxTransform>
     {
-        public OnnxScoringEstimator(IHostEnvironment env, string modelFile, string[] inputs, string[] outputs)
-           : this(env, new OnnxTransform(env, modelFile, inputs, outputs))
+        public OnnxScoringEstimator(IHostEnvironment env, string modelFile, string[] inputs, string[] outputs, int gpuDeviceId = -1)
+           : this(env, new OnnxTransform(env, modelFile, inputs, outputs, gpuDeviceId))
         {
         }
 
@@ -459,7 +495,7 @@ public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
             {
                 resultDic[Transformer.Outputs[i]] = new SchemaShape.Column(Transformer.Outputs[i],
                     Transformer.OutputTypes[i].IsKnownSizeVector ? SchemaShape.Column.VectorKind.Vector
-                    : SchemaShape.Column.VectorKind.VariableVector, NumberType.R4, false);
+                    : SchemaShape.Column.VectorKind.VariableVector, Transformer.OutputTypes[i].ItemType, false);
             }
             return new SchemaShape(resultDic.Values);
         }

src/Microsoft.ML.OnnxTransform/OnnxUtils.cs

@@ -18,12 +18,12 @@
 namespace Microsoft.ML.Transforms
 {
     /// <summary>
-    /// OnnxModel is a facad for ModelManager. ModelManager is provided by Sonoma API,
-    /// and it has a lot of functionality (multiple models, multiple versions) that are not
-    /// needed by Onnx transform, which only needs a single model. This facad simplifies the
-    /// usage of onnx model.
+    /// OnnxModel is a utility class to load ONNX models, and retrieve metadata
+    /// for inputs and outputs. The metadata includes the names, shapes and types
+    /// It provides API to open a session, score tensors and return
+    /// the results.
     /// </summary>
-    internal sealed class OnnxModel
+    public sealed class OnnxModel
     {
         /// <summary>
         /// OnnxModelInfo contains the data that we should get from
@@ -47,8 +47,17 @@ public OnnxModelInfo(OnnxNodeInfo[] inputsInfo, OnnxNodeInfo[] outputsInfo)
         /// </summary>
         public class OnnxNodeInfo
         {
+            /// <summary>
+            /// The Name of the input node
+            /// </summary>
             public readonly string Name;
+            /// <summary>
+            /// The shape of the input node
+            /// </summary>
             public readonly OnnxShape Shape;
+            /// <summary>
+            /// The type of the input node
+            /// </summary>
             public readonly DataType Type;
 
             public OnnxNodeInfo(string name, OnnxShape shape, DataType type)
@@ -68,36 +77,70 @@ public OnnxNodeInfo(string name, OnnxShape shape, DataType type)
         public readonly List<string> InputNames;
         public readonly List<string> OutputNames;
 
-        public OnnxModel(string modelFile)
+        /// <summary>
+        /// Constructs OnnxModel object from file.
+        /// </summary>
+        /// <param name="modelFile">File path to onnx model</param>
+        public OnnxModel(string modelFile) : this(modelFile, -1)
+        {
+        }
+
+        /// <summary>
+        /// Constructs OnnxModel object from file.
+        /// </summary>
+        /// <param name="modelFile">File path to onnx model</param>
+        /// <param name="gpuDeviceId">-1 for CPU execution. A non-negative value runs execution on the specified GPU</param>
+        public OnnxModel(string modelFile, int gpuDeviceId)
         {
             _modelFile = modelFile;
 
             // Load the onnx model
             var modelFileInfo = new FileInfo(modelFile);
             _modelName = Path.GetFileNameWithoutExtension(modelFileInfo.Name);
             _modelManager = new ModelManager(modelFileInfo.Directory.FullName, true);
-            _modelManager.InitOnnxModel(_modelName, _ignoredVersion);
+            _modelManager.InitOnnxModel(_modelName, _ignoredVersion, gpuDeviceId);
 
             ModelInfo = new OnnxModelInfo(GetInputsInfo(), GetOutputsInfo());
             InputNames = ModelInfo.InputsInfo.Select(i => i.Name).ToList();
             OutputNames = ModelInfo.OutputsInfo.Select(i => i.Name).ToList();
         }
 
+        /// <summary>
+        /// Create an OnnxModel from a byte[]
+        /// </summary>
+        /// <param name="modelBytes">A byte array containing the serialized model</param>
+        /// <returns></returns>
         public static OnnxModel CreateFromBytes(byte[] modelBytes)
+        {
+            return CreateFromBytes(modelBytes, -1);
+        }
+
+        /// <summary>
+        /// Create an OnnxModel from a byte[]
+        /// </summary>
+        /// <param name="modelBytes">A byte array containing the serialized model</param>
+        /// <param name="gpuDeviceId">Default =-1 for CPU execution. Specify non-negative device ID for GPU execution</param>
+        /// <returns>OnnxModel</returns>
+        public static OnnxModel CreateFromBytes(byte[] modelBytes, int gpuDeviceId=-1)
         {
             var tempModelDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
             Directory.CreateDirectory(tempModelDir);
 
             var tempModelFile = Path.Combine(tempModelDir, "model.onnx");
             File.WriteAllBytes(tempModelFile, modelBytes);
-            return new OnnxModel(tempModelFile);
+            return new OnnxModel(tempModelFile, gpuDeviceId);
 
             // TODO:
             // tempModelFile is needed in case the model needs to be saved
             // Either have to save the modelbytes and delete the temp dir/file,
             // or keep the dir/file and write proper cleanup when application closes
         }
 
+        /// <summary>
+        /// Uses an already open session to score a list of Tensors/NamedOnnxValues.
+        /// </summary>
+        /// <param name="inputTensors">The NamedOnnxValues/Tensors to score</param>
+        /// <returns>A list of NamedOnnxValues/Tensors</returns>
         public List<Tensor> Run(List<Tensor> inputTensors)
         {
             var outputTensors = _modelManager.RunModel(
@@ -106,12 +149,20 @@ public List<Tensor> Run(List<Tensor> inputTensors)
             return outputTensors;
         }
 
+        /// <summary>
+        /// Convert the model to a byte array.
+        /// </summary>
+        /// <returns>byte[]</returns>
         public byte[] ToByteArray()
         {
             return File.ReadAllBytes(_modelFile);
         }
 
-        private OnnxNodeInfo[] GetInputsInfo()
+        /// <summary>
+        /// Returns input metadata of the ONNX model.
+        /// </summary>
+        /// <returns>OnnxNodeInfo[]</returns>
+        public OnnxNodeInfo[] GetInputsInfo()
         {
             return DictToNodesInfo(
                     _modelManager.GetInputTypeDict(_modelName, _ignoredVersion),