diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index bad42afd5d..6b76249bd5 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -133,6 +133,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.SamplesUtils", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Recommender", "src\Microsoft.ML.Recommender\Microsoft.ML.Recommender.csproj", "{C8E1772B-DFD9-4A4D-830D-6AAB1C668BB3}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.DnnImageFeaturizer.ResNet18", "src\Microsoft.ML.DnnImageFeaturizer.ResNet18\Microsoft.ML.DnnImageFeaturizer.ResNet18.csproj", "{9222FC9D-599A-49A5-B685-08CC9A5C81D7}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.DnnImageFeaturizer.AlexNet", "src\Microsoft.ML.DnnImageFeaturizer.AlexNet\Microsoft.ML.DnnImageFeaturizer.AlexNet.csproj", "{6C29AA9B-054B-4762-BEA5-D305B932AA80}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.DnnImageFeaturizer.ResNet50", "src\Microsoft.ML.DnnImageFeaturizer.ResNet50\Microsoft.ML.DnnImageFeaturizer.ResNet50.csproj", "{4805129D-78C8-46D4-9519-0AD9B0574D6D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.DnnImageFeaturizer.ResNet101", "src\Microsoft.ML.DnnImageFeaturizer.ResNet101\Microsoft.ML.DnnImageFeaturizer.ResNet101.csproj", "{DB7CEB5E-8BE6-48A7-87BE-B91D9AE96F71}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -501,6 +509,38 @@ Global {C8E1772B-DFD9-4A4D-830D-6AAB1C668BB3}.Release|Any CPU.Build.0 = Release|Any CPU {C8E1772B-DFD9-4A4D-830D-6AAB1C668BB3}.Release-Intrinsics|Any CPU.ActiveCfg = Release-Intrinsics|Any CPU {C8E1772B-DFD9-4A4D-830D-6AAB1C668BB3}.Release-Intrinsics|Any CPU.Build.0 = Release-Intrinsics|Any CPU + {9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug-Intrinsics|Any CPU + {9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Debug-Intrinsics|Any CPU.Build.0 = Debug-Intrinsics|Any CPU + {9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Release|Any CPU.Build.0 = Release|Any CPU + {9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Release-Intrinsics|Any CPU.ActiveCfg = Release-Intrinsics|Any CPU + {9222FC9D-599A-49A5-B685-08CC9A5C81D7}.Release-Intrinsics|Any CPU.Build.0 = Release-Intrinsics|Any CPU + {6C29AA9B-054B-4762-BEA5-D305B932AA80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6C29AA9B-054B-4762-BEA5-D305B932AA80}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6C29AA9B-054B-4762-BEA5-D305B932AA80}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug-Intrinsics|Any CPU + {6C29AA9B-054B-4762-BEA5-D305B932AA80}.Debug-Intrinsics|Any CPU.Build.0 = Debug-Intrinsics|Any CPU + {6C29AA9B-054B-4762-BEA5-D305B932AA80}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6C29AA9B-054B-4762-BEA5-D305B932AA80}.Release|Any CPU.Build.0 = Release|Any CPU + {6C29AA9B-054B-4762-BEA5-D305B932AA80}.Release-Intrinsics|Any CPU.ActiveCfg = Release-Intrinsics|Any CPU + {6C29AA9B-054B-4762-BEA5-D305B932AA80}.Release-Intrinsics|Any CPU.Build.0 = Release-Intrinsics|Any CPU + {4805129D-78C8-46D4-9519-0AD9B0574D6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4805129D-78C8-46D4-9519-0AD9B0574D6D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4805129D-78C8-46D4-9519-0AD9B0574D6D}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug-Intrinsics|Any CPU + {4805129D-78C8-46D4-9519-0AD9B0574D6D}.Debug-Intrinsics|Any CPU.Build.0 = Debug-Intrinsics|Any CPU + {4805129D-78C8-46D4-9519-0AD9B0574D6D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4805129D-78C8-46D4-9519-0AD9B0574D6D}.Release|Any CPU.Build.0 = Release|Any CPU + {4805129D-78C8-46D4-9519-0AD9B0574D6D}.Release-Intrinsics|Any CPU.ActiveCfg = Release-Intrinsics|Any CPU + {4805129D-78C8-46D4-9519-0AD9B0574D6D}.Release-Intrinsics|Any CPU.Build.0 = Release-Intrinsics|Any CPU + {DB7CEB5E-8BE6-48A7-87BE-B91D9AE96F71}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DB7CEB5E-8BE6-48A7-87BE-B91D9AE96F71}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DB7CEB5E-8BE6-48A7-87BE-B91D9AE96F71}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug-Intrinsics|Any CPU + {DB7CEB5E-8BE6-48A7-87BE-B91D9AE96F71}.Debug-Intrinsics|Any CPU.Build.0 = Debug-Intrinsics|Any CPU + {DB7CEB5E-8BE6-48A7-87BE-B91D9AE96F71}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DB7CEB5E-8BE6-48A7-87BE-B91D9AE96F71}.Release|Any CPU.Build.0 = Release|Any CPU + {DB7CEB5E-8BE6-48A7-87BE-B91D9AE96F71}.Release-Intrinsics|Any CPU.ActiveCfg = Release-Intrinsics|Any CPU + {DB7CEB5E-8BE6-48A7-87BE-B91D9AE96F71}.Release-Intrinsics|Any CPU.Build.0 = Release-Intrinsics|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -556,6 +596,10 @@ Global {ECB71297-9DF1-48CE-B93A-CD969221F9B6} = {DA452A53-2E94-4433-B08C-041EDEC729E6} {11A5210E-2EA7-42F1-80DB-827762E9C781} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {C8E1772B-DFD9-4A4D-830D-6AAB1C668BB3} = {09EADF06-BE25-4228-AB53-95AE3E15B530} + {9222FC9D-599A-49A5-B685-08CC9A5C81D7} = {09EADF06-BE25-4228-AB53-95AE3E15B530} + {6C29AA9B-054B-4762-BEA5-D305B932AA80} = {09EADF06-BE25-4228-AB53-95AE3E15B530} + {4805129D-78C8-46D4-9519-0AD9B0574D6D} = {09EADF06-BE25-4228-AB53-95AE3E15B530} + {DB7CEB5E-8BE6-48A7-87BE-B91D9AE96F71} = {09EADF06-BE25-4228-AB53-95AE3E15B530} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D} diff --git a/pkg/Microsoft.ML.DnnImageFeaturizer.AlexNet/Microsoft.ML.DnnImageFeaturizer.AlexNet.nupkgproj b/pkg/Microsoft.ML.DnnImageFeaturizer.AlexNet/Microsoft.ML.DnnImageFeaturizer.AlexNet.nupkgproj new file mode 100644 index 0000000000..a19ff5245c --- /dev/null +++ b/pkg/Microsoft.ML.DnnImageFeaturizer.AlexNet/Microsoft.ML.DnnImageFeaturizer.AlexNet.nupkgproj @@ -0,0 +1,19 @@ + + + + netstandard2.0 + ML.NET component for pretrained AlexNet image featurization + + + + + + + + + + + + + + diff --git a/pkg/Microsoft.ML.DnnImageFeaturizer.AlexNet/Microsoft.ML.DnnImageFeaturizer.AlexNet.symbols.nupkgproj b/pkg/Microsoft.ML.DnnImageFeaturizer.AlexNet/Microsoft.ML.DnnImageFeaturizer.AlexNet.symbols.nupkgproj new file mode 100644 index 0000000000..8c6a7fcc4c --- /dev/null +++ b/pkg/Microsoft.ML.DnnImageFeaturizer.AlexNet/Microsoft.ML.DnnImageFeaturizer.AlexNet.symbols.nupkgproj @@ -0,0 +1,5 @@ + + + + + diff --git a/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet101/Microsoft.ML.DnnImageFeaturizer.ResNet101.nupkgproj b/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet101/Microsoft.ML.DnnImageFeaturizer.ResNet101.nupkgproj new file mode 100644 index 0000000000..58714cd0e3 --- /dev/null +++ b/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet101/Microsoft.ML.DnnImageFeaturizer.ResNet101.nupkgproj @@ -0,0 +1,18 @@ + + + + netstandard2.0 + ML.NET component for pretrained ResNet101 image featurization + + + + + + + + + + + + + diff --git a/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet101/Microsoft.ML.DnnImageFeaturizer.ResNet101.symbols.nupkgproj b/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet101/Microsoft.ML.DnnImageFeaturizer.ResNet101.symbols.nupkgproj new file mode 100644 index 0000000000..7035bef747 --- /dev/null +++ b/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet101/Microsoft.ML.DnnImageFeaturizer.ResNet101.symbols.nupkgproj @@ -0,0 +1,5 @@ + + + + + diff --git a/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet18/Microsoft.ML.DnnImageFeaturizer.ResNet18.nupkgproj b/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet18/Microsoft.ML.DnnImageFeaturizer.ResNet18.nupkgproj new file mode 100644 index 0000000000..1d5b96fb0d --- /dev/null +++ b/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet18/Microsoft.ML.DnnImageFeaturizer.ResNet18.nupkgproj @@ -0,0 +1,18 @@ + + + + netstandard2.0 + ML.NET component for pretrained ResNet18 image featurization + + + + + + + + + + + + + diff --git a/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet18/Microsoft.ML.DnnImageFeaturizer.ResNet18.symbols.nupkgproj b/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet18/Microsoft.ML.DnnImageFeaturizer.ResNet18.symbols.nupkgproj new file mode 100644 index 0000000000..9fb3f5ca75 --- /dev/null +++ b/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet18/Microsoft.ML.DnnImageFeaturizer.ResNet18.symbols.nupkgproj @@ -0,0 +1,5 @@ + + + + + diff --git a/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet50/Microsoft.ML.DnnImageFeaturizer.ResNet50.nupkgproj b/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet50/Microsoft.ML.DnnImageFeaturizer.ResNet50.nupkgproj new file mode 100644 index 0000000000..e9d6025592 --- /dev/null +++ b/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet50/Microsoft.ML.DnnImageFeaturizer.ResNet50.nupkgproj @@ -0,0 +1,18 @@ + + + + netstandard2.0 + ML.NET component for pretrained ResNet50 image featurization + + + + + + + + + + + + + diff --git a/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet50/Microsoft.ML.DnnImageFeaturizer.ResNet50.symbols.nupkgproj b/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet50/Microsoft.ML.DnnImageFeaturizer.ResNet50.symbols.nupkgproj new file mode 100644 index 0000000000..2b04e494f9 --- /dev/null +++ b/pkg/Microsoft.ML.DnnImageFeaturizer.ResNet50/Microsoft.ML.DnnImageFeaturizer.ResNet50.symbols.nupkgproj @@ -0,0 +1,5 @@ + + + + + diff --git a/pkg/common/DnnImageFeaturizer.props b/pkg/common/DnnImageFeaturizer.props new file mode 100644 index 0000000000..b8b445eebe --- /dev/null +++ b/pkg/common/DnnImageFeaturizer.props @@ -0,0 +1,9 @@ + + + + + DnnImageModels\%(RecursiveDir)%(Filename)%(Extension) + PreserveNewest + + + \ No newline at end of file diff --git a/src/Microsoft.ML.DnnImageFeaturizer.AlexNet/AlexNetExtension.cs b/src/Microsoft.ML.DnnImageFeaturizer.AlexNet/AlexNetExtension.cs new file mode 100644 index 0000000000..5d216e10b3 --- /dev/null +++ b/src/Microsoft.ML.DnnImageFeaturizer.AlexNet/AlexNetExtension.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Data; +using System.IO; +using System.Reflection; + +namespace Microsoft.ML.Transforms +{ + /// + /// This is an extension method to be used with the in order to use a pretrained AlexNet model. + /// The NuGet containing this extension is also guaranteed to include the binary model file. + /// + public static class AlexNetExtension + { + /// + /// Returns an estimator chain with the two corresponding models (a preprocessing one and a main one) required for the AlexNet pipeline. + /// Also includes the renaming ColumnsCopyingTransforms required to be able to use arbitrary input and output column names. + /// This assumes both of the models are in the same location as the file containing this method, which they will be if used through the NuGet. + /// This should be the default way to use AlexNet if importing the model from a NuGet. + /// + public static EstimatorChain AlexNet(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string inputColumn, string outputColumn) + { + return AlexNet(dnnModelContext, env, inputColumn, outputColumn, Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "DnnImageModels")); + } + + /// + /// This allows a custom model location to be specified. This is useful is a custom model is specified, + /// or if the model is desired to be placed or shipped separately in a different folder from the main application. Note that because Onnx models + /// must be in a directory all by themsleves for the OnnxTransform to work, this method appends a AlexNetOnnx/AlexNetPrepOnnx subdirectory + /// to the passed in directory to prevent having to make that directory manually each time. + /// + public static EstimatorChain AlexNet(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string inputColumn, string outputColumn, string modelDir) + { + var modelChain = new EstimatorChain(); + + var inputRename = new ColumnsCopyingEstimator(env, new[] { (inputColumn, "OriginalInput") }); + var midRename = new ColumnsCopyingEstimator(env, new[] { ("PreprocessedInput", "Input140") }); + var endRename = new ColumnsCopyingEstimator(env, new[] { ("Dropout234_Output_0", outputColumn) }); + + // There are two estimators created below. The first one is for image preprocessing and the second one is the actual DNN model. + var prepEstimator = new OnnxScoringEstimator(env, Path.Combine(modelDir, "AlexNetPrepOnnx", "AlexNetPreprocess.onnx"), new[] { "OriginalInput" }, new[] { "PreprocessedInput" }); + var mainEstimator = new OnnxScoringEstimator(env, Path.Combine(modelDir, "AlexNetOnnx", "AlexNet.onnx"), new[] { "Input140" }, new[] { "Dropout234_Output_0" }); + modelChain = modelChain.Append(inputRename); + var modelChain2 = modelChain.Append(prepEstimator); + modelChain = modelChain2.Append(midRename); + modelChain2 = modelChain.Append(mainEstimator); + modelChain = modelChain2.Append(endRename); + return modelChain; + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.DnnImageFeaturizer.AlexNet/Microsoft.ML.DnnImageFeaturizer.AlexNet.csproj b/src/Microsoft.ML.DnnImageFeaturizer.AlexNet/Microsoft.ML.DnnImageFeaturizer.AlexNet.csproj new file mode 100644 index 0000000000..46391d7d6b --- /dev/null +++ b/src/Microsoft.ML.DnnImageFeaturizer.AlexNet/Microsoft.ML.DnnImageFeaturizer.AlexNet.csproj @@ -0,0 +1,12 @@ + + + + netstandard2.0 + Microsoft.ML.DnnImageFeaturizer.AlexNet + + + + + + + \ No newline at end of file diff --git a/src/Microsoft.ML.DnnImageFeaturizer.ResNet101/Microsoft.ML.DnnImageFeaturizer.ResNet101.csproj b/src/Microsoft.ML.DnnImageFeaturizer.ResNet101/Microsoft.ML.DnnImageFeaturizer.ResNet101.csproj new file mode 100644 index 0000000000..c9eb12268d --- /dev/null +++ b/src/Microsoft.ML.DnnImageFeaturizer.ResNet101/Microsoft.ML.DnnImageFeaturizer.ResNet101.csproj @@ -0,0 +1,12 @@ + + + + netstandard2.0 + Microsoft.ML.DnnImageFeaturizer.ResNet101 + + + + + + + \ No newline at end of file diff --git a/src/Microsoft.ML.DnnImageFeaturizer.ResNet101/ResNet101Extension.cs b/src/Microsoft.ML.DnnImageFeaturizer.ResNet101/ResNet101Extension.cs new file mode 100644 index 0000000000..4f5c825c46 --- /dev/null +++ b/src/Microsoft.ML.DnnImageFeaturizer.ResNet101/ResNet101Extension.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Data; +using System.IO; +using System.Reflection; + +namespace Microsoft.ML.Transforms +{ + /// + /// This is an extension method to be used with the in order to use a pretrained ResNet101 model. + /// The NuGet containing this extension is also guaranteed to include the binary model file. + /// + public static class ResNet101Extension + { + /// + /// Returns an estimator chain with the two corresponding models (a preprocessing one and a main one) required for the ResNet pipeline. + /// Also includes the renaming ColumnsCopyingTransforms required to be able to use arbitrary input and output column names. + /// This assumes both of the models are in the same location as the file containing this method, which they will be if used through the NuGet. + /// This should be the default way to use ResNet101 if importing the model from a NuGet. + /// + public static EstimatorChain ResNet101(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string inputColumn, string outputColumn) + { + return ResNet101(dnnModelContext, env, inputColumn, outputColumn, Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "DnnImageModels")); + } + + /// + /// This allows a custom model location to be specified. This is useful is a custom model is specified, + /// or if the model is desired to be placed or shipped separately in a different folder from the main application. Note that because Onnx models + /// must be in a directory all by themsleves for the OnnxTransform to work, this method appends a ResNet101Onnx/ResNetPrepOnnx subdirectory + /// to the passed in directory to prevent having to make that directory manually each time. + /// + public static EstimatorChain ResNet101(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string inputColumn, string outputColumn, string modelDir) + { + var modelChain = new EstimatorChain(); + + var inputRename = new ColumnsCopyingEstimator(env, new[] { (inputColumn, "OriginalInput") }); + var midRename = new ColumnsCopyingEstimator(env, new[] { ("PreprocessedInput", "Input1600") }); + var endRename = new ColumnsCopyingEstimator(env, new[] { ("Pooling2286_Output_0", outputColumn) }); + + // There are two estimators created below. The first one is for image preprocessing and the second one is the actual DNN model. + var prepEstimator = new OnnxScoringEstimator(env, Path.Combine(modelDir, "ResNetPrepOnnx", "ResNetPreprocess.onnx"), new[] { "OriginalInput" }, new[] { "PreprocessedInput" }); + var mainEstimator = new OnnxScoringEstimator(env, Path.Combine(modelDir, "ResNet101Onnx", "ResNet101.onnx"), new[] { "Input1600" }, new[] { "Pooling2286_Output_0" }); + modelChain = modelChain.Append(inputRename); + var modelChain2 = modelChain.Append(prepEstimator); + modelChain = modelChain2.Append(midRename); + modelChain2 = modelChain.Append(mainEstimator); + modelChain = modelChain2.Append(endRename); + return modelChain; + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.DnnImageFeaturizer.ResNet18/Microsoft.ML.DnnImageFeaturizer.ResNet18.csproj b/src/Microsoft.ML.DnnImageFeaturizer.ResNet18/Microsoft.ML.DnnImageFeaturizer.ResNet18.csproj new file mode 100644 index 0000000000..6ae6417758 --- /dev/null +++ b/src/Microsoft.ML.DnnImageFeaturizer.ResNet18/Microsoft.ML.DnnImageFeaturizer.ResNet18.csproj @@ -0,0 +1,12 @@ + + + + netstandard2.0 + Microsoft.ML.DnnImageFeaturizer.ResNet18 + + + + + + + \ No newline at end of file diff --git a/src/Microsoft.ML.DnnImageFeaturizer.ResNet18/ResNet18Extension.cs b/src/Microsoft.ML.DnnImageFeaturizer.ResNet18/ResNet18Extension.cs new file mode 100644 index 0000000000..6a04f768fe --- /dev/null +++ b/src/Microsoft.ML.DnnImageFeaturizer.ResNet18/ResNet18Extension.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Data; +using System.IO; +using System.Reflection; + +namespace Microsoft.ML.Transforms +{ + /// + /// This is an extension method to be used with the in order to use a pretrained ResNet18 model. + /// The NuGet containing this extension is also guaranteed to include the binary model file. + /// + public static class ResNet18Extension + { + /// + /// Returns an estimator chain with the two corresponding models (a preprocessing one and a main one) required for the ResNet pipeline. + /// Also includes the renaming ColumnsCopyingTransforms required to be able to use arbitrary input and output column names. + /// This assumes both of the models are in the same location as the file containing this method, which they will be if used through the NuGet. + /// This should be the default way to use ResNet18 if importing the model from a NuGet. + /// + public static EstimatorChain ResNet18(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string inputColumn, string outputColumn) + { + return ResNet18(dnnModelContext, env, inputColumn, outputColumn, Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "DnnImageModels")); + } + + /// + /// This allows a custom model location to be specified. This is useful is a custom model is specified, + /// or if the model is desired to be placed or shipped separately in a different folder from the main application. Note that because Onnx models + /// must be in a directory all by themsleves for the OnnxTransform to work, this method appends a ResNet18Onnx/ResNetPrepOnnx subdirectory + /// to the passed in directory to prevent having to make that directory manually each time. + /// + public static EstimatorChain ResNet18(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string inputColumn, string outputColumn, string modelDir) + { + var modelChain = new EstimatorChain(); + + var inputRename = new ColumnsCopyingEstimator(env, new[] { (inputColumn, "OriginalInput") }); + var midRename = new ColumnsCopyingEstimator(env, new[] { ("PreprocessedInput", "Input247") }); + var endRename = new ColumnsCopyingEstimator(env, new[] { ("Pooling395_Output_0", outputColumn) }); + + // There are two estimators created below. The first one is for image preprocessing and the second one is the actual DNN model. + var prepEstimator = new OnnxScoringEstimator(env, Path.Combine(modelDir, "ResNetPrepOnnx", "ResNetPreprocess.onnx"), new[] { "OriginalInput" }, new[] { "PreprocessedInput" }); + var mainEstimator = new OnnxScoringEstimator(env, Path.Combine(modelDir, "ResNet18Onnx", "ResNet18.onnx"), new[] { "Input247" }, new[] { "Pooling395_Output_0" }); + modelChain = modelChain.Append(inputRename); + var modelChain2 = modelChain.Append(prepEstimator); + modelChain = modelChain2.Append(midRename); + modelChain2 = modelChain.Append(mainEstimator); + modelChain = modelChain2.Append(endRename); + return modelChain; + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.DnnImageFeaturizer.ResNet50/Microsoft.ML.DnnImageFeaturizer.ResNet50.csproj b/src/Microsoft.ML.DnnImageFeaturizer.ResNet50/Microsoft.ML.DnnImageFeaturizer.ResNet50.csproj new file mode 100644 index 0000000000..9f3ef3e729 --- /dev/null +++ b/src/Microsoft.ML.DnnImageFeaturizer.ResNet50/Microsoft.ML.DnnImageFeaturizer.ResNet50.csproj @@ -0,0 +1,12 @@ + + + + netstandard2.0 + Microsoft.ML.DnnImageFeaturizer.ResNet50 + + + + + + + \ No newline at end of file diff --git a/src/Microsoft.ML.DnnImageFeaturizer.ResNet50/ResNet50Extension.cs b/src/Microsoft.ML.DnnImageFeaturizer.ResNet50/ResNet50Extension.cs new file mode 100644 index 0000000000..acf9cfa033 --- /dev/null +++ b/src/Microsoft.ML.DnnImageFeaturizer.ResNet50/ResNet50Extension.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Data; +using System.IO; +using System.Reflection; + +namespace Microsoft.ML.Transforms +{ + /// + /// This is an extension method to be used with the in order to use a pretrained ResNet50 model. + /// The NuGet containing this extension is also guaranteed to include the binary model file. + /// + public static class ResNet50Extension + { + /// + /// Returns an estimator chain with the two corresponding models (a preprocessing one and a main one) required for the ResNet pipeline. + /// Also includes the renaming ColumnsCopyingTransforms required to be able to use arbitrary input and output column names. + /// This assumes both of the models are in the same location as the file containing this method, which they will be if used through the NuGet. + /// This should be the default way to use ResNet50 if importing the model from a NuGet. + /// + public static EstimatorChain ResNet50(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string inputColumn, string outputColumn) + { + return ResNet50(dnnModelContext, env, inputColumn, outputColumn, Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "DnnImageModels")); + } + + /// + /// This allows a custom model location to be specified. This is useful is a custom model is specified, + /// or if the model is desired to be placed or shipped separately in a different folder from the main application. Note that because Onnx models + /// must be in a directory all by themsleves for the OnnxTransform to work, this method appends a ResNet50Onnx/ResNetPrepOnnx subdirectory + /// to the passed in directory to prevent having to make that directory manually each time. + /// + public static EstimatorChain ResNet50(this DnnImageModelSelector dnnModelContext, IHostEnvironment env, string inputColumn, string outputColumn, string modelDir) + { + var modelChain = new EstimatorChain(); + + var inputRename = new ColumnsCopyingEstimator(env, new[] { (inputColumn, "OriginalInput") }); + var midRename = new ColumnsCopyingEstimator(env, new[] { ("PreprocessedInput", "Input750") }); + var endRename = new ColumnsCopyingEstimator(env, new[] { ("Pooling1096_Output_0", outputColumn) }); + + // There are two estimators created below. The first one is for image preprocessing and the second one is the actual DNN model. + var prepEstimator = new OnnxScoringEstimator(env, Path.Combine(modelDir, "ResNetPrepOnnx", "ResNetPreprocess.onnx"), new[] { "OriginalInput" }, new[] { "PreprocessedInput" }); + var mainEstimator = new OnnxScoringEstimator(env, Path.Combine(modelDir, "ResNet50Onnx", "ResNet50.onnx"), new[] { "Input750" }, new[] { "Pooling1096_Output_0" }); + modelChain = modelChain.Append(inputRename); + var modelChain2 = modelChain.Append(prepEstimator); + modelChain = modelChain2.Append(midRename); + modelChain2 = modelChain.Append(mainEstimator); + modelChain = modelChain2.Append(endRename); + return modelChain; + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.OnnxTransform/DnnImageFeaturizerTransform.cs b/src/Microsoft.ML.OnnxTransform/DnnImageFeaturizerTransform.cs new file mode 100644 index 0000000000..f7189127ba --- /dev/null +++ b/src/Microsoft.ML.OnnxTransform/DnnImageFeaturizerTransform.cs @@ -0,0 +1,140 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Core.Data; +using Microsoft.ML.Data; +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.Internal.Utilities; +using Microsoft.ML.StaticPipe; +using Microsoft.ML.StaticPipe.Runtime; +using System; +using System.Collections.Generic; + +namespace Microsoft.ML.Transforms +{ + /// + /// This is a helper class that is required to use the . + /// Note that by default, it is not usable as it does not have any valid methods that return an + /// that is used by the DnnImageFeaturizeEstimator. + /// In order to use this, at least one model project with the corresponding extension methods must by included. + /// See Microsoft.ML.DNNImageFeaturizer.ResNet18 for an example. + /// + public sealed class DnnImageModelSelector + { + } + + /// + /// This is a helper class used to store all the inputs to an extension method on a DnnImageModelSelector required to return + /// a chain of two s. + /// + public sealed class DnnImageFeaturizerInput + { + public IHostEnvironment Environment { get; } + public string InputColumn { get; } + public DnnImageModelSelector ModelSelector { get; } + public string OutputColumn { get; } + + public DnnImageFeaturizerInput(IHostEnvironment env, string inputColumn, string outputColumn, DnnImageModelSelector modelSelector) + { + Environment = env; + InputColumn = inputColumn; + OutputColumn = outputColumn; + ModelSelector = modelSelector; + } + } + + /// + /// The Dnn Image Featurizer is just a wrapper around two s and three + /// with present pretrained DNN models. The ColumnsCopying are there to allow arbitrary column input and output names, as by default + /// the ONNXTransform requires the names of the columns to be identical to the names of the ONNX model nodes. + /// Note that because of this, it only works on Windows machines as that is a constraint of the OnnxTransform. + /// + public sealed class DnnImageFeaturizerEstimator : IEstimator> + { + private readonly EstimatorChain _modelChain; + + /// + /// Constructor for the estimator for a DnnImageFeaturizer transform. + /// + /// Host environment. + /// An extension method on the that creates a chain of two + /// s (one for preprocessing and one with a pretrained image DNN) with specific models + /// included in a package together with that extension method. It also contains three s + /// to allow arbitrary column naming, as the ONNXEstimators require very specific naming based on the models. + /// For an example, see Microsoft.ML.DnnImageFeaturizer.ResNet18 + /// inputColumn column name. + /// Output column name. + public DnnImageFeaturizerEstimator(IHostEnvironment env, Func> modelFactory, string inputColumn, string outputColumn) + { + _modelChain = modelFactory( new DnnImageFeaturizerInput(env, inputColumn, outputColumn, new DnnImageModelSelector())); + } + + /// + /// Note that OnnxEstimator which this is based on is a trivial estimator, so this does not do any actual training, + /// just verifies the schema. + /// + public TransformerChain Fit(IDataView input) + { + return _modelChain.Fit(input); + } + + public SchemaShape GetOutputSchema(SchemaShape inputSchema) + { + return _modelChain.GetOutputSchema(inputSchema); + } + } + + public static class DnnImageFeaturizerStaticExtensions + { + private sealed class OutColumn : Vector + { + public PipelineColumn Input { get; } + + public OutColumn(Vector input, Func> modelFactory) + : base(new Reconciler(modelFactory), input) + { + Input = input; + } + } + + private sealed class Reconciler : EstimatorReconciler + { + private readonly Func> _modelFactory; + + public Reconciler(Func> modelFactory) + { + _modelFactory = modelFactory; + } + + public override IEstimator Reconcile(IHostEnvironment env, + PipelineColumn[] toOutput, + IReadOnlyDictionary inputNames, + IReadOnlyDictionary outputNames, + IReadOnlyCollection usedNames) + { + Contracts.Assert(toOutput.Length == 1); + + var outCol = (OutColumn)toOutput[0]; + return new DnnImageFeaturizerEstimator(env, _modelFactory, inputNames[outCol.Input], outputNames[outCol]); + } + } + + /// + /// Creates and applies a DnnImageFeaturizer transform to be used by the static API. + /// for more information about how the transformation works. + /// + /// Vector of image pixel weights. + /// An extension method on the that creates a chain of two + /// s (one for preprocessing and one with a pretrained image DNN) with specific models + /// included in a package together with that extension method. + /// For an example, see Microsoft.ML.DnnImageFeaturizer.ResNet18 + /// A vector of float feature weights based on the input image. + public static Vector DnnImageFeaturizer(this Vector input, Func> modelFactory) + { + Contracts.CheckValue(input, nameof(input)); + return new OutColumn(input, modelFactory); + } + } +} diff --git a/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj b/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj new file mode 100644 index 0000000000..267603f7d5 --- /dev/null +++ b/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj @@ -0,0 +1,67 @@ + + + + + netstandard2.0 + + + + + + $(ObjDir)DnnImageModels + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/Microsoft.ML.OnnxTransformTest/DnnImageFeaturizerTest.cs b/test/Microsoft.ML.OnnxTransformTest/DnnImageFeaturizerTest.cs new file mode 100644 index 0000000000..fbd8b36b19 --- /dev/null +++ b/test/Microsoft.ML.OnnxTransformTest/DnnImageFeaturizerTest.cs @@ -0,0 +1,199 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Core.Data; +using Microsoft.ML.Runtime.Api; +using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.RunTests; +using Microsoft.ML.Runtime.ImageAnalytics; +using Microsoft.ML.Transforms; +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; +using Xunit; +using Xunit.Abstractions; +using System.Reflection; +using Microsoft.ML.Runtime.Model; + +namespace Microsoft.ML.Tests +{ + public class DnnImageFeaturizerTests : TestDataPipeBase + { + private const int inputSize = 3 * 224 * 224; + + private class TestData + { + [VectorType(inputSize)] + public float[] data_0; + } + private class TestDataSize + { + [VectorType(2)] + public float[] data_0; + } + private class TestDataXY + { + [VectorType(inputSize)] + public float[] A; + } + private class TestDataDifferntType + { + [VectorType(inputSize)] + public string[] data_0; + } + + private float[] getSampleArrayData() + { + var samplevector = new float[inputSize]; + for (int i = 0; i < inputSize; i++) + samplevector[i] = (i / ((float) inputSize)); + return samplevector; + } + + public DnnImageFeaturizerTests(ITestOutputHelper helper) : base(helper) + { + } + + // Onnx is only supported on x64 Windows + [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + void TestDnnImageFeaturizer() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + return; + + + var samplevector = getSampleArrayData(); + + var dataView = ComponentCreation.CreateDataView(Env, + new TestData[] { + new TestData() + { + data_0 = samplevector + }, + }); + + var xyData = new List { new TestDataXY() { A = new float[inputSize] } }; + var stringData = new List { new TestDataDifferntType() { data_0 = new string[inputSize] } }; + var sizeData = new List { new TestDataSize() { data_0 = new float[2] } }; + var pipe = new DnnImageFeaturizerEstimator(Env, m => m.ModelSelector.ResNet18(m.Environment, m.InputColumn, m.OutputColumn), "data_0", "output_1"); + + var invalidDataWrongNames = ComponentCreation.CreateDataView(Env, xyData); + var invalidDataWrongTypes = ComponentCreation.CreateDataView(Env, stringData); + var invalidDataWrongVectorSize = ComponentCreation.CreateDataView(Env, sizeData); + TestEstimatorCore(pipe, dataView, invalidInput: invalidDataWrongNames); + TestEstimatorCore(pipe, dataView, invalidInput: invalidDataWrongTypes); + pipe.GetOutputSchema(SchemaShape.Create(invalidDataWrongVectorSize.Schema)); + try + { + pipe.Fit(invalidDataWrongVectorSize); + Assert.False(true); + } + catch (ArgumentOutOfRangeException) { } + catch (InvalidOperationException) { } + } + + // Onnx is only supported on x64 Windows + [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + public void OnnxStatic() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + return; + + var env = new MLContext(null, 1); + var imageHeight = 224; + var imageWidth = 224; + var dataFile = GetDataPath("images/images.tsv"); + var imageFolder = Path.GetDirectoryName(dataFile); + + var data = TextLoader.CreateReader(env, ctx => ( + imagePath: ctx.LoadText(0), + name: ctx.LoadText(1))) + .Read(dataFile); + + var pipe = data.MakeNewEstimator() + .Append(row => ( + row.name, + data_0: row.imagePath.LoadAsImage(imageFolder).Resize(imageHeight, imageWidth).ExtractPixels(interleaveArgb: true))) + .Append(row => (row.name, output_1: row.data_0.DnnImageFeaturizer(m => m.ModelSelector.ResNet18(m.Environment, m.InputColumn, m.OutputColumn)))); + + TestEstimatorCore(pipe.AsDynamic, data.AsDynamic); + + var result = pipe.Fit(data).Transform(data).AsDynamic; + result.Schema.TryGetColumnIndex("output_1", out int output); + using (var cursor = result.GetRowCursor(col => col == output)) + { + var buffer = default(VBuffer); + var getter = cursor.GetGetter>(output); + var numRows = 0; + while (cursor.MoveNext()) + { + getter(ref buffer); + Assert.Equal(512, buffer.Length); + numRows += 1; + } + Assert.Equal(3, numRows); + } + } + + // Onnx is only supported on x64 Windows + [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + public void TestOldSavingAndLoading() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + return; + + + var samplevector = getSampleArrayData(); + + var dataView = ComponentCreation.CreateDataView(Env, + new TestData[] { + new TestData() + { + data_0 = samplevector + } + }); + + var inputNames = "data_0"; + var outputNames = "output_1"; + var est = new DnnImageFeaturizerEstimator(Env, m => m.ModelSelector.ResNet18(m.Environment, m.InputColumn, m.OutputColumn), inputNames, outputNames); + var transformer = est.Fit(dataView); + var result = transformer.Transform(dataView); + var resultRoles = new RoleMappedData(result); + using (var ms = new MemoryStream()) + { + TrainUtils.SaveModel(Env, Env.Start("saving"), ms, null, resultRoles); + ms.Position = 0; + var loadedView = ModelFileUtils.LoadTransforms(Env, dataView, ms); + + loadedView.Schema.TryGetColumnIndex(outputNames, out int softMaxOut1); + using (var cursor = loadedView.GetRowCursor(col => col == softMaxOut1)) + { + VBuffer softMaxValue = default; + var softMaxGetter = cursor.GetGetter>(softMaxOut1); + float sum = 0f; + int i = 0; + while (cursor.MoveNext()) + { + softMaxGetter(ref softMaxValue); + var values = softMaxValue.DenseValues(); + foreach (var val in values) + { + sum += val; + if (i == 0) + Assert.InRange(val, 0.0, 0.00001); + if (i == 7) + Assert.InRange(val, 0.62935, 0.62940); + if (i == 500) + Assert.InRange(val, 0.15521, 0.155225); + i++; + } + } + Assert.InRange(sum, 83.50, 84.50); + } + } + } + } +} diff --git a/test/Microsoft.ML.OnnxTransformTest/Microsoft.ML.OnnxTransformTest.csproj b/test/Microsoft.ML.OnnxTransformTest/Microsoft.ML.OnnxTransformTest.csproj index 19715d8e2c..37df5ac124 100644 --- a/test/Microsoft.ML.OnnxTransformTest/Microsoft.ML.OnnxTransformTest.csproj +++ b/test/Microsoft.ML.OnnxTransformTest/Microsoft.ML.OnnxTransformTest.csproj @@ -5,10 +5,26 @@ + + + + + DnnImageModels\ResNetPrepOnnx\ResNetPreprocess.onnx + PreserveNewest + + + + + + DnnImageModels\ResNet18Onnx\ResNet18.onnx + PreserveNewest + + +