further cleanup and fix build

artidoro · artidoro · commit 50505e39b04d · 2019-02-03T14:20:34.000-08:00
diff --git a/src/Microsoft.ML.Data/Transforms/ColumnSelecting.cs b/src/Microsoft.ML.Data/Transforms/ColumnSelecting.cs
@@ -40,7 +40,8 @@ namespace Microsoft.ML.Transforms
     /// </summary>
     public sealed class ColumnSelectingEstimator : TrivialEstimator<ColumnSelectingTransformer>
     {
-        public static class Defaults
+        [BestFriend]
+        internal static class Defaults
         {
             public const bool KeepHidden = false;
             public const bool IgnoreMissing = false;
diff --git a/src/Microsoft.ML.OnnxTransform/Microsoft.ML.OnnxTransform.csproj b/src/Microsoft.ML.OnnxTransform/Microsoft.ML.OnnxTransform.csproj
@@ -4,6 +4,8 @@
     <TargetFramework>netstandard2.0</TargetFramework>
     <IncludeInPackage>Microsoft.ML.OnnxTransform</IncludeInPackage>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <ApplicationIcon />
+    <Win32Resource />
   </PropertyGroup>
 
   <ItemGroup>
diff --git a/src/Microsoft.ML.OnnxTransform/OnnxTransform.cs b/src/Microsoft.ML.OnnxTransform/OnnxTransform.cs
@@ -88,9 +88,9 @@ internal sealed class Options : TransformInputBase
         internal const string ShortName = "Onnx";
         internal const string LoaderSignature = "OnnxTransform";
 
-        public readonly string[] Inputs;
-        public readonly string[] Outputs;
-        public readonly ColumnType[] OutputTypes;
+        internal readonly string[] Inputs;
+        internal readonly string[] Outputs;
+        internal readonly ColumnType[] OutputTypes;
 
         private static VersionInfo GetVersionInfo()
         {
@@ -512,7 +512,7 @@ public NamedOnnxValue GetNamedOnnxValue()
     }
 
     /// <summary>
-    /// A class implementing the estimator interface of the OnnxTransform.
+    /// A class implementing the estimator interface of the <see cref="OnnxTransformer"/>.
     /// </summary>
     public sealed class OnnxScoringEstimator : TrivialEstimator<OnnxTransformer>
     {
diff --git a/src/Microsoft.ML.OnnxTransform/Properties/AssemblyInfo.cs b/src/Microsoft.ML.OnnxTransform/Properties/AssemblyInfo.cs
@@ -0,0 +1,13 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Runtime.CompilerServices;
+using Microsoft.ML;
+
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.DnnImageFeaturizer.AlexNet" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.DnnImageFeaturizer.ResNet101" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.DnnImageFeaturizer.ResNet18" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.DnnImageFeaturizer.ResNet50" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.OnnxTransform.StaticPipe" + PublicKey.Value)]
+
diff --git a/src/Microsoft.ML.TensorFlow/Properties/AssemblyInfo.cs b/src/Microsoft.ML.TensorFlow/Properties/AssemblyInfo.cs
@@ -0,0 +1,9 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Runtime.CompilerServices;
+using Microsoft.ML;
+
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.TensorFlow.StaticPipe" + PublicKey.Value)]
+
diff --git a/src/Microsoft.ML.TensorFlow/TensorflowCatalog.cs b/src/Microsoft.ML.TensorFlow/TensorflowCatalog.cs
@@ -11,6 +11,26 @@ namespace Microsoft.ML
     /// <include file='doc.xml' path='doc/members/member[@name="TensorflowTransformer"]/*' />
     public static class TensorflowCatalog
     {
+        /// <summary>
+        /// Scores a dataset using a pre-traiend TensorFlow model located in <paramref name="modelLocation"/>.
+        /// </summary>
+        /// <param name="catalog">The transform's catalog.</param>
+        /// <param name="modelLocation">Location of the TensorFlow model.</param>
+        /// <param name="inputColumnName"> The name of the model inputs.</param>
+        /// <param name="outputColumnName">The name of the requested model outputs.</param>
+        /// <example>
+        /// <format type="text/markdown">
+        /// <![CDATA[
+        /// [!code-csharp[ScoreTensorFlowModel](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlowTransform.cs)]
+        /// ]]>
+        /// </format>
+        /// </example>
+        public static TensorFlowEstimator ScoreTensorFlowModel(this TransformsCatalog catalog,
+            string modelLocation,
+            string outputColumnName,
+            string inputColumnName)
+            => new TensorFlowEstimator(CatalogUtils.GetEnvironment(catalog), new[] { outputColumnName }, new[] { inputColumnName }, modelLocation);
+
         /// <summary>
         /// Scores a dataset using a pre-traiend TensorFlow model located in <paramref name="modelLocation"/>.
         /// </summary>
@@ -31,6 +51,19 @@ public static TensorFlowEstimator ScoreTensorFlowModel(this TransformsCatalog ca
             string[] inputColumnNames)
             => new TensorFlowEstimator(CatalogUtils.GetEnvironment(catalog), outputColumnNames, inputColumnNames, modelLocation);
 
+        /// <summary>
+        /// Scores a dataset using a pre-traiend TensorFlow model specified via <paramref name="tensorFlowModel"/>.
+        /// </summary>
+        /// <param name="catalog">The transform's catalog.</param>
+        /// <param name="tensorFlowModel">The pre-trained TensorFlow model.</param>
+        /// <param name="inputColumnName"> The name of the model inputs.</param>
+        /// <param name="outputColumnName">The name of the requested model outputs.</param>
+        public static TensorFlowEstimator ScoreTensorFlowModel(this TransformsCatalog catalog,
+            TensorFlowModelInfo tensorFlowModel,
+            string outputColumnName,
+            string inputColumnName)
+            => new TensorFlowEstimator(CatalogUtils.GetEnvironment(catalog), new[] { outputColumnName }, new[] { inputColumnName }, tensorFlowModel);
+
         /// <summary>
         /// Scores a dataset using a pre-traiend TensorFlow model specified via <paramref name="tensorFlowModel"/>.
         /// </summary>
@@ -49,20 +82,20 @@ public static TensorFlowEstimator ScoreTensorFlowModel(this TransformsCatalog ca
         /// The model is specified in the <see cref="TensorFlowTransformer.Options.ModelLocation"/>.
         /// </summary>
         /// <param name="catalog">The transform's catalog.</param>
-        /// <param name="args">The <see cref="TensorFlowTransformer.Options"/> specifying the inputs and the settings of the <see cref="TensorFlowEstimator"/>.</param>
+        /// <param name="options">The <see cref="TensorFlowTransformer.Options"/> specifying the inputs and the settings of the <see cref="TensorFlowEstimator"/>.</param>
         public static TensorFlowEstimator TensorFlow(this TransformsCatalog catalog,
-            TensorFlowTransformer.Options args)
-            => new TensorFlowEstimator(CatalogUtils.GetEnvironment(catalog), args);
+            TensorFlowTransformer.Options options)
+            => new TensorFlowEstimator(CatalogUtils.GetEnvironment(catalog), options);
 
         /// <summary>
         /// Scores or retrains (based on setting of the <see cref="TensorFlowTransformer.Options.ReTrain"/>) a pre-traiend TensorFlow model specified via <paramref name="tensorFlowModel"/>.
         /// </summary>
         /// <param name="catalog">The transform's catalog.</param>
-        /// <param name="args">The <see cref="TensorFlowTransformer.Options"/> specifying the inputs and the settings of the <see cref="TensorFlowEstimator"/>.</param>
+        /// <param name="options">The <see cref="TensorFlowTransformer.Options"/> specifying the inputs and the settings of the <see cref="TensorFlowEstimator"/>.</param>
         /// <param name="tensorFlowModel">The pre-trained TensorFlow model.</param>
         public static TensorFlowEstimator TensorFlow(this TransformsCatalog catalog,
-            TensorFlowTransformer.Options args,
+            TensorFlowTransformer.Options options,
             TensorFlowModelInfo tensorFlowModel)
-            => new TensorFlowEstimator(CatalogUtils.GetEnvironment(catalog), args, tensorFlowModel);
+            => new TensorFlowEstimator(CatalogUtils.GetEnvironment(catalog), options, tensorFlowModel);
     }
 }
diff --git a/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs b/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs
@@ -37,6 +37,9 @@ namespace Microsoft.ML.Transforms
     /// <include file='doc.xml' path='doc/members/member[@name="TensorflowTransformer"]/*' />
     public sealed class TensorFlowTransformer : RowToRowTransformerBase
     {
+        /// <summary>
+        /// The settings for the <see cref="TensorFlowTransformer"/>.
+        /// </summary>
         public sealed class Options : TransformInputBase
         {
             /// <summary>
@@ -148,8 +151,8 @@ public sealed class Options : TransformInputBase
         internal readonly TFShape[] TFInputShapes;
         internal TFGraph Graph => Session.Graph;
 
-        public readonly string[] Inputs;
-        public readonly string[] Outputs;
+        internal readonly string[] Inputs;
+        internal readonly string[] Outputs;
 
         internal static int BatchSize = 1;
         internal const string Summary = "Transforms the data using the TensorFlow model.";
@@ -1077,6 +1080,9 @@ public TFTensor GetBufferedBatchTensor()
         }
     }
 
+    /// <summary>
+    /// Estimator for the <see cref="TensorFlowTransformer"/>.
+    /// </summary>
     public sealed class TensorFlowEstimator : IEstimator<TensorFlowTransformer>
     {
         private readonly IHost _host;
@@ -1153,7 +1159,7 @@ public SchemaShape GetOutputSchema(SchemaShape inputSchema)
         }
 
         /// <summary>
-        /// Train and return a <see cref="ITransformer"/>.
+        /// Trains and returns a <see cref="TensorFlowTransformer"/>.
         /// </summary>
         public TensorFlowTransformer Fit(IDataView input)
         {
diff --git a/src/Microsoft.ML.Transforms/KeyToVectorMapping.cs b/src/Microsoft.ML.Transforms/KeyToVectorMapping.cs
@@ -29,6 +29,9 @@
 
 namespace Microsoft.ML.Transforms.Conversions
 {
+    /// <summary>
+    ///  Converts the key types back to binary verctors.
+    /// </summary>
     public sealed class KeyToBinaryVectorMappingTransformer : OneToOneTransformerBase
     {
         internal sealed class Options
@@ -421,6 +424,9 @@ private void EncodeValueToBinary(BufferBuilder<float> bldr, uint value, int bits
         }
     }
 
+    /// <summary>
+    ///  Converts the key types back to binary verctors.
+    /// </summary>
     public sealed class KeyToBinaryVectorMappingEstimator : TrivialEstimator<KeyToBinaryVectorMappingTransformer>
     {
         internal KeyToBinaryVectorMappingEstimator(IHostEnvironment env, params (string outputColumnName, string inputColumnName)[] columns)
diff --git a/test/Microsoft.ML.OnnxTransformTest/OnnxTransformTests.cs b/test/Microsoft.ML.OnnxTransformTest/OnnxTransformTests.cs
@@ -120,7 +120,7 @@ void TestSimpleCase()
             var xyData = new List<TestDataXY> { new TestDataXY() { A = new float[inputSize] } };
             var stringData = new List<TestDataDifferntType> { new TestDataDifferntType() { data_0 = new string[inputSize] } };
             var sizeData = new List<TestDataSize> { new TestDataSize() { data_0 = new float[2] } };
-            var pipe = new OnnxScoringEstimator(Env, new[] { "softmaxout_1" }, new[] { "data_0" }, modelFile);
+            var pipe = ML.Transforms.ApplyOnnxModel(modelFile, new[] { "softmaxout_1" }, new[] { "data_0" });
 
             var invalidDataWrongNames = ML.Data.ReadFromEnumerable(xyData);
             var invalidDataWrongTypes = ML.Data.ReadFromEnumerable(stringData);
@@ -160,7 +160,7 @@ void TestOldSavingAndLoading(int? gpuDeviceId, bool fallbackToCpu)
 
             var inputNames = new[] { "data_0" };
             var outputNames = new[] { "softmaxout_1" };
-            var est = new OnnxScoringEstimator(Env, outputNames, inputNames, modelFile, gpuDeviceId, fallbackToCpu);
+            var est = ML.Transforms.ApplyOnnxModel(modelFile, outputNames, inputNames, gpuDeviceId, fallbackToCpu);
             var transformer = est.Fit(dataView);
             var result = transformer.Transform(dataView);
             var resultRoles = new RoleMappedData(result);
@@ -266,7 +266,7 @@ public void OnnxModelScenario()
                         }
                     });
 
-                var onnx = new OnnxTransformer(env, "softmaxout_1", modelFile, "data_0").Transform(dataView);
+                var onnx = ML.Transforms.ApplyOnnxModel(modelFile, "softmaxout_1", "data_0").Fit(dataView).Transform(dataView);
 
                 onnx.Schema.TryGetColumnIndex("softmaxout_1", out int score);
 
@@ -299,7 +299,7 @@ public void OnnxModelMultiInput()
                             inb = new float[] {1,2,3,4,5}
                         }
                     });
-                var onnx = new OnnxTransformer(env, new[] { "outa", "outb" }, new[] { "ina", "inb" }, modelFile).Transform(dataView);
+                var onnx = ML.Transforms.ApplyOnnxModel(modelFile, new[] { "outa", "outb" }, new[] { "ina", "inb" }).Fit(dataView).Transform(dataView);
 
                 onnx.Schema.TryGetColumnIndex("outa", out int scoresa);
                 onnx.Schema.TryGetColumnIndex("outb", out int scoresb);
@@ -337,7 +337,7 @@ public void TestUnknownDimensions()
                     new TestDataUnknownDimensions(){input = new float[] {-1.1f, -1.3f, 1.2f }},
                 };
             var idv = mlContext.Data.ReadFromEnumerable(data);
-            var pipeline = new OnnxScoringEstimator(mlContext, modelFile);
+            var pipeline = ML.Transforms.ApplyOnnxModel(modelFile);
             var transformedValues = pipeline.Fit(idv).Transform(idv);
             var predictions = mlContext.CreateEnumerable<PredictionUnknownDimensions>(transformedValues, reuseRowObject: false).ToArray();
 
diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs
@@ -72,7 +72,7 @@ public void SimpleEndToEndOnnxConversionTest()
                 // Step 3: Evaluate the saved ONNX model using the data used to train the ML.NET pipeline.
                 string[] inputNames = onnxModel.Graph.Input.Select(valueInfoProto => valueInfoProto.Name).ToArray();
                 string[] outputNames = onnxModel.Graph.Output.Select(valueInfoProto => valueInfoProto.Name).ToArray();
-                var onnxEstimator = new OnnxScoringEstimator(mlContext, outputNames, inputNames, onnxModelPath);
+                var onnxEstimator = mlContext.Transforms.ApplyOnnxModel(onnxModelPath, outputNames, inputNames);
                 var onnxTransformer = onnxEstimator.Fit(data);
                 var onnxResult = onnxTransformer.Transform(data);
 
@@ -156,7 +156,7 @@ public void KmeansOnnxConversionTest()
                 // Evaluate the saved ONNX model using the data used to train the ML.NET pipeline.
                 string[] inputNames = onnxModel.Graph.Input.Select(valueInfoProto => valueInfoProto.Name).ToArray();
                 string[] outputNames = onnxModel.Graph.Output.Select(valueInfoProto => valueInfoProto.Name).ToArray();
-                var onnxEstimator = new OnnxScoringEstimator(mlContext, outputNames, inputNames, onnxModelPath);
+                var onnxEstimator = mlContext.Transforms.ApplyOnnxModel(onnxModelPath, outputNames, inputNames);
                 var onnxTransformer = onnxEstimator.Fit(data);
                 var onnxResult = onnxTransformer.Transform(data);
                 CompareSelectedR4VectorColumns("Score", "Score0", transformedData, onnxResult, 3);
diff --git a/test/Microsoft.ML.Tests/Scenarios/TensorflowTests.cs b/test/Microsoft.ML.Tests/Scenarios/TensorflowTests.cs
@@ -37,7 +37,7 @@ public void TensorFlowTransforCifarEndToEndTest()
             var pipeEstimator = new ImageLoadingEstimator(mlContext, imageFolder, ("ImageReal", "ImagePath"))
                     .Append(new ImageResizingEstimator(mlContext, "ImageCropped", imageHeight, imageWidth, "ImageReal"))
                     .Append(new ImagePixelExtractingEstimator(mlContext, "Input", "ImageCropped", interleave: true))
-                    .Append(new TensorFlowEstimator(mlContext, new[] { "Output" }, new[] { "Input" }, model_location))
+                    .Append(mlContext.Transforms.ScoreTensorFlowModel(model_location, new string[] { "Output" }, new[] { "Input" }))
                     .Append(new ColumnConcatenatingEstimator(mlContext, "Features", "Output"))
                     .Append(new ValueToKeyMappingEstimator(mlContext, "Label"))
                     .AppendCacheCheckpoint(mlContext)
diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs
diff --git a/test/Microsoft.ML.Tests/TensorFlowEstimatorTests.cs b/test/Microsoft.ML.Tests/TensorFlowEstimatorTests.cs

Original file line number	Diff line number	Diff line change
`@@ -40,7 +40,8 @@ namespace Microsoft.ML.Transforms`
`40`	`40`	`/// </summary>`
`41`	`41`	`public sealed class ColumnSelectingEstimator : TrivialEstimator<ColumnSelectingTransformer>`
`42`	`42`	`{`
`43`		`- public static class Defaults`
	`43`	`+ [BestFriend]`
	`44`	`+ internal static class Defaults`
`44`	`45`	`{`
`45`	`46`	`public const bool KeepHidden = false;`
`46`	`47`	`public const bool IgnoreMissing = false;`
Original file line number	Diff line number	Diff line change
`@@ -88,9 +88,9 @@ internal sealed class Options : TransformInputBase`
`88`	`88`	`internal const string ShortName = "Onnx";`
`89`	`89`	`internal const string LoaderSignature = "OnnxTransform";`
`90`	`90`
`91`		`- public readonly string[] Inputs;`
`92`		`- public readonly string[] Outputs;`
`93`		`- public readonly ColumnType[] OutputTypes;`
	`91`	`+ internal readonly string[] Inputs;`
	`92`	`+ internal readonly string[] Outputs;`
	`93`	`+ internal readonly ColumnType[] OutputTypes;`
`94`	`94`
`95`	`95`	`private static VersionInfo GetVersionInfo()`
`96`	`96`	`{`
`@@ -512,7 +512,7 @@ public NamedOnnxValue GetNamedOnnxValue()`
`512`	`512`	`}`
`513`	`513`
`514`	`514`	`/// <summary>`
`515`		`- /// A class implementing the estimator interface of the OnnxTransform.`
	`515`	`+ /// A class implementing the estimator interface of the <see cref="OnnxTransformer"/>.`
`516`	`516`	`/// </summary>`
`517`	`517`	`public sealed class OnnxScoringEstimator : TrivialEstimator<OnnxTransformer>`
`518`	`518`	`{`
Original file line number	Diff line number	Diff line change
`@@ -37,6 +37,9 @@ namespace Microsoft.ML.Transforms`
`37`	`37`	`/// <include file='doc.xml' path='doc/members/member[@name="TensorflowTransformer"]/*' />`
`38`	`38`	`public sealed class TensorFlowTransformer : RowToRowTransformerBase`
`39`	`39`	`{`
	`40`	`+ /// <summary>`
	`41`	`+ /// The settings for the <see cref="TensorFlowTransformer"/>.`
	`42`	`+ /// </summary>`
`40`	`43`	`public sealed class Options : TransformInputBase`
`41`	`44`	`{`
`42`	`45`	`/// <summary>`
`@@ -148,8 +151,8 @@ public sealed class Options : TransformInputBase`
`148`	`151`	`internal readonly TFShape[] TFInputShapes;`
`149`	`152`	`internal TFGraph Graph => Session.Graph;`
`150`	`153`
`151`		`- public readonly string[] Inputs;`
`152`		`- public readonly string[] Outputs;`
	`154`	`+ internal readonly string[] Inputs;`
	`155`	`+ internal readonly string[] Outputs;`
`153`	`156`
`154`	`157`	`internal static int BatchSize = 1;`
`155`	`158`	`internal const string Summary = "Transforms the data using the TensorFlow model.";`
`@@ -1077,6 +1080,9 @@ public TFTensor GetBufferedBatchTensor()`
`1077`	`1080`	`}`
`1078`	`1081`	`}`
`1079`	`1082`
	`1083`	`+ /// <summary>`
	`1084`	`+ /// Estimator for the <see cref="TensorFlowTransformer"/>.`
	`1085`	`+ /// </summary>`
`1080`	`1086`	`public sealed class TensorFlowEstimator : IEstimator<TensorFlowTransformer>`
`1081`	`1087`	`{`
`1082`	`1088`	`private readonly IHost _host;`
`@@ -1153,7 +1159,7 @@ public SchemaShape GetOutputSchema(SchemaShape inputSchema)`
`1153`	`1159`	`}`
`1154`	`1160`
`1155`	`1161`	`/// <summary>`
`1156`		`- /// Train and return a <see cref="ITransformer"/>.`
	`1162`	`+ /// Trains and returns a <see cref="TensorFlowTransformer"/>.`
`1157`	`1163`	`/// </summary>`
`1158`	`1164`	`public TensorFlowTransformer Fit(IDataView input)`
`1159`	`1165`	`{`
Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,9 @@`
`29`	`29`
`30`	`30`	`namespace Microsoft.ML.Transforms.Conversions`
`31`	`31`	`{`
	`32`	`+ /// <summary>`
	`33`	`+ /// Converts the key types back to binary verctors.`
	`34`	`+ /// </summary>`
`32`	`35`	`public sealed class KeyToBinaryVectorMappingTransformer : OneToOneTransformerBase`
`33`	`36`	`{`
`34`	`37`	`internal sealed class Options`
`@@ -421,6 +424,9 @@ private void EncodeValueToBinary(BufferBuilder<float> bldr, uint value, int bits`
`421`	`424`	`}`
`422`	`425`	`}`
`423`	`426`
	`427`	`+ /// <summary>`
	`428`	`+ /// Converts the key types back to binary verctors.`
	`429`	`+ /// </summary>`
`424`	`430`	`public sealed class KeyToBinaryVectorMappingEstimator : TrivialEstimator<KeyToBinaryVectorMappingTransformer>`
`425`	`431`	`{`
`426`	`432`	`internal KeyToBinaryVectorMappingEstimator(IHostEnvironment env, params (string outputColumnName, string inputColumnName)[] columns)`