Image support

Microsoft.ML.sln

@@ -97,6 +97,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer",
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer.Tests", "test\Microsoft.ML.CodeAnalyzer.Tests\Microsoft.ML.CodeAnalyzer.Tests.csproj", "{3E4ABF07-7970-4BE6-B45B-A13D3C397545}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.ImageAnalytics", "src\Microsoft.ML.ImageAnalytics\Microsoft.ML.ImageAnalytics.csproj", "{00E38F77-1E61-4CDF-8F97-1417D4E85053}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -329,6 +331,14 @@ Global
 		{3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release|Any CPU.Build.0 = Release|Any CPU
 		{3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
 		{3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release|Any CPU.Build.0 = Release|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -367,6 +377,7 @@ Global
 		{BF66A305-DF10-47E4-8D81-42049B149D2B} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
 		{B4E55B2D-2A92-46E7-B72F-E76D6FD83440} = {7F13E156-3EBA-4021-84A5-CD56BA72F99E}
 		{3E4ABF07-7970-4BE6-B45B-A13D3C397545} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{00E38F77-1E61-4CDF-8F97-1417D4E85053} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

build/Dependencies.props

@@ -9,5 +9,6 @@
     <PublishSymbolsPackageVersion>1.0.0-beta-62824-02</PublishSymbolsPackageVersion>
     <LightGBMPackageVersion>2.1.2.2</LightGBMPackageVersion>
     <MlNetMklDepsPackageVersion>0.0.0.1</MlNetMklDepsPackageVersion>
+    <SystemDrawingCommonPackageVersion>4.5.0</SystemDrawingCommonPackageVersion>
   </PropertyGroup>
 </Project>

pkg/Microsoft.ML.ImageAnalytics/Microsoft.ML.ImageAnalytics.nupkgproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk" DefaultTargets="Pack">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <PackageDescription>ML.NET component for Image support</PackageDescription>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="../Microsoft.ML/Microsoft.ML.nupkgproj" />
+    <PackageReference Include="System.Drawing.Common" Version="$(SystemDrawingCommonPackageVersion)" />
+  </ItemGroup>
+
+</Project>

pkg/Microsoft.ML.ImageAnalytics/Microsoft.ML.ImageAnalytics.symbols.nupkgproj

@@ -0,0 +1,5 @@
+<Project DefaultTargets="Pack">
+
+  <Import Project="Microsoft.ML.ImageAnalytics.nupkgproj" />
+
+</Project>

src/Microsoft.ML.ImageAnalytics/EntryPoints/ImageAnalytics.cs

@@ -0,0 +1,79 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.EntryPoints;
+using Microsoft.ML.Runtime.ImageAnalytics.EntryPoints;
+
+[assembly: LoadableClass(typeof(void), typeof(ImageAnalytics), null, typeof(SignatureEntryPointModule), "ImageAnalytics")]
+namespace Microsoft.ML.Runtime.ImageAnalytics.EntryPoints
+{
+    public static class ImageAnalytics
+    {
+        [TlcModule.EntryPoint(Name = "Transforms.ImageLoader", Desc = ImageLoaderTransform.Summary,
+            UserName = ImageLoaderTransform.UserName, ShortName = ImageLoaderTransform.LoaderSignature)]
+        public static CommonOutputs.TransformOutput ImageLoader(IHostEnvironment env, ImageLoaderTransform.Arguments input)
+        {
+            var h = EntryPointUtils.CheckArgsAndCreateHost(env, "ImageLoaderTransform", input);
+            var xf = new ImageLoaderTransform(h, input, input.Data);
+            return new CommonOutputs.TransformOutput()
+            {
+                Model = new TransformModel(h, xf, input.Data),
+                OutputData = xf
+            };
+        }
+
+        [TlcModule.EntryPoint(Name = "Transforms.ImageResizer", Desc = ImageResizerTransform.Summary,
+            UserName = ImageResizerTransform.UserName, ShortName = ImageResizerTransform.LoaderSignature)]
+        public static CommonOutputs.TransformOutput ImageResizer(IHostEnvironment env, ImageResizerTransform.Arguments input)
+        {
+            var h = EntryPointUtils.CheckArgsAndCreateHost(env, "ImageResizerTransform", input);
+            var xf = new ImageResizerTransform(h, input, input.Data);
+            return new CommonOutputs.TransformOutput()
+            {
+                Model = new TransformModel(h, xf, input.Data),
+                OutputData = xf
+            };
+        }
+
+        [TlcModule.EntryPoint(Name = "Transforms.ImagePixelExtractor", Desc = ImagePixelExtractorTransform.Summary,
+            UserName = ImagePixelExtractorTransform.UserName, ShortName = ImagePixelExtractorTransform.LoaderSignature)]
+        public static CommonOutputs.TransformOutput ImagePixelExtractor(IHostEnvironment env, ImagePixelExtractorTransform.Arguments input)
+        {
+            var h = EntryPointUtils.CheckArgsAndCreateHost(env, "ImagePixelExtractorTransform", input);
+            var xf = new ImagePixelExtractorTransform(h, input, input.Data);
+            return new CommonOutputs.TransformOutput()
+            {
+                Model = new TransformModel(h, xf, input.Data),
+                OutputData = xf
+            };
+        }
+
+        [TlcModule.EntryPoint(Name = "Transforms.ImageGrayscale", Desc = ImageGrayscaleTransform.Summary,
+            UserName = ImageGrayscaleTransform.UserName, ShortName = ImageGrayscaleTransform.LoaderSignature)]
+        public static CommonOutputs.TransformOutput ImageGrayscale(IHostEnvironment env, ImageGrayscaleTransform.Arguments input)
+        {
+            var h = EntryPointUtils.CheckArgsAndCreateHost(env, "ImageGrayscaleTransform", input);
+            var xf = new ImageGrayscaleTransform(h, input, input.Data);
+            return new CommonOutputs.TransformOutput()
+            {
+                Model = new TransformModel(h, xf, input.Data),
+                OutputData = xf
+            };
+        }
+
+        [TlcModule.EntryPoint(Name = "Transforms.VectorToImage", Desc = VectorToImageTransform.Summary,
+            UserName = VectorToImageTransform.UserName, ShortName = VectorToImageTransform.LoaderSignature)]
+        public static CommonOutputs.TransformOutput VectorToImage(IHostEnvironment env, VectorToImageTransform.Arguments input)
+        {
+            var h = EntryPointUtils.CheckArgsAndCreateHost(env, "VectorToImageTransform", input);
+            var xf = new VectorToImageTransform(h, input, input.Data);
+            return new CommonOutputs.TransformOutput()
+            {
+                Model = new TransformModel(h, xf, input.Data),
+                OutputData = xf
+            };
+        }
+    }
+}

src/Microsoft.ML.ImageAnalytics/ImageGrayscaleTransform.cs

@@ -0,0 +1,171 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Drawing;
+using System.Drawing.Imaging;
+using System.Text;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.CommandLine;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Runtime.EntryPoints;
+using Microsoft.ML.Runtime.Internal.Utilities;
+using Microsoft.ML.Runtime.Model;
+using Microsoft.ML.Runtime.ImageAnalytics;
+
+[assembly: LoadableClass(ImageGrayscaleTransform.Summary, typeof(ImageGrayscaleTransform), typeof(ImageGrayscaleTransform.Arguments), typeof(SignatureDataTransform),
+    ImageGrayscaleTransform.UserName, "ImageGrayscaleTransform", "ImageGrayscale")]
+
+[assembly: LoadableClass(ImageGrayscaleTransform.Summary, typeof(ImageGrayscaleTransform), null, typeof(SignatureLoadDataTransform),
+    ImageGrayscaleTransform.UserName, ImageGrayscaleTransform.LoaderSignature)]
+
+namespace Microsoft.ML.Runtime.ImageAnalytics
+{
+    // REVIEW: Rewrite as LambdaTransform to simplify.
+    // REVIEW: Should it be separate transform or part of ImageResizerTransform?
+    /// <summary>
+    /// Transform which takes one or many columns of <see cref="ImageType"/> type in IDataView and
+    /// convert them to greyscale representation of the same image.
+    /// </summary>
+    public sealed class ImageGrayscaleTransform : OneToOneTransformBase
+    {
+        public sealed class Column : OneToOneColumn
+        {
+            public static Column Parse(string str)
+            {
+                var res = new Column();
+                if (res.TryParse(str))
+                    return res;
+                return null;
+            }
+
+            public bool TryUnparse(StringBuilder sb)
+            {
+                Contracts.AssertValue(sb);
+                return TryUnparseCore(sb);
+            }
+        }
+
+        public class Arguments : TransformInputBase
+        {
+            [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)]
+            public Column[] Column;
+        }
+
+        internal const string Summary = "Convert image into grayscale.";
+
+        internal const string UserName = "Image Greyscale Transform";
+        public const string LoaderSignature = "ImageGrayscaleTransform";
+        private static VersionInfo GetVersionInfo()
+        {
+            return new VersionInfo(
+                modelSignature: "IMGGRAYT",
+                verWrittenCur: 0x00010001, // Initial
+                verReadableCur: 0x00010001,
+                verWeCanReadBack: 0x00010001,
+                loaderSignature: LoaderSignature);
+        }
+
+        private const string RegistrationName = "ImageGrayscale";
+
+        // Public constructor corresponding to SignatureDataTransform.
+        public ImageGrayscaleTransform(IHostEnvironment env, Arguments args, IDataView input)
+            : base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, t => t is ImageType ? null : "Expected Image type")
+        {
+            Host.AssertNonEmpty(Infos);
+            Host.Assert(Infos.Length == Utils.Size(args.Column));
+            Metadata.Seal();
+        }
+
+        private ImageGrayscaleTransform(IHost host, ModelLoadContext ctx, IDataView input)
+            : base(host, ctx, input, t => t is ImageType ? null : "Expected Image type")
+        {
+            Host.AssertValue(ctx);
+            // *** Binary format ***
+            // <base>
+            Host.AssertNonEmpty(Infos);
+            Metadata.Seal();
+        }
+
+        public static ImageGrayscaleTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
+        {
+            Contracts.CheckValue(env, nameof(env));
+            var h = env.Register(RegistrationName);
+            h.CheckValue(ctx, nameof(ctx));
+            h.CheckValue(input, nameof(input));
+            ctx.CheckAtModel(GetVersionInfo());
+            return h.Apply("Loading Model", ch => new ImageGrayscaleTransform(h, ctx, input));
+        }
+
+        public override void Save(ModelSaveContext ctx)
+        {
+            Host.CheckValue(ctx, nameof(ctx));
+            ctx.CheckAtModel();
+            ctx.SetVersionInfo(GetVersionInfo());
+
+            // *** Binary format ***
+            // <base>
+            SaveBase(ctx);
+        }
+
+        protected override ColumnType GetColumnTypeCore(int iinfo)
+        {
+            Host.Assert(0 <= iinfo & iinfo < Infos.Length);
+            return Infos[iinfo].TypeSrc;
+        }
+
+        private static readonly ColorMatrix _grayscaleColorMatrix = new ColorMatrix(
+                new float[][]
+                {
+                    new float[] {.3f, .3f, .3f, 0, 0},
+                    new float[] {.59f, .59f, .59f, 0, 0},
+                    new float[] {.11f, .11f, .11f, 0, 0},
+                    new float[] {0, 0, 0, 1, 0},
+                    new float[] {0, 0, 0, 0, 1}
+                });
+
+        protected override Delegate GetGetterCore(IChannel ch, IRow input, int iinfo, out Action disposer)
+        {
+            Host.AssertValueOrNull(ch);
+            Host.AssertValue(input);
+            Host.Assert(0 <= iinfo && iinfo < Infos.Length);
+
+            var src = default(Bitmap);
+            var getSrc = GetSrcGetter<Bitmap>(input, iinfo);
+
+            disposer =
+                () =>
+                {
+                    if (src != null)
+                    {
+                        src.Dispose();
+                        src = null;
+                    }
+                };
+
+            ValueGetter<Bitmap> del =
+                (ref Bitmap dst) =>
+                {
+                    if (dst != null)
+                        dst.Dispose();
+
+                    getSrc(ref src);
+                    if (src == null || src.Height <= 0 || src.Width <= 0)
+                        return;
+
+                    dst = new Bitmap(src.Width, src.Height);
+                    ImageAttributes attributes = new ImageAttributes();
+                    attributes.SetColorMatrix(_grayscaleColorMatrix);
+                    var srcRectangle = new Rectangle(0, 0, src.Width, src.Height);
+                    using (var g = Graphics.FromImage(dst))
+                    {
+                        g.DrawImage(src, srcRectangle, 0, 0, src.Width, src.Height, GraphicsUnit.Pixel, attributes);
+                    }
+                    Host.Assert(dst.Width == src.Width && dst.Height == src.Height);
+                };
+
+            return del;
+        }
+    }
+}