diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index 58e24041f1..28d1528d83 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -97,6 +97,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CodeAnalyzer.Tests", "test\Microsoft.ML.CodeAnalyzer.Tests\Microsoft.ML.CodeAnalyzer.Tests.csproj", "{3E4ABF07-7970-4BE6-B45B-A13D3C397545}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.ImageAnalytics", "src\Microsoft.ML.ImageAnalytics\Microsoft.ML.ImageAnalytics.csproj", "{00E38F77-1E61-4CDF-8F97-1417D4E85053}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -329,6 +331,14 @@ Global {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release|Any CPU.Build.0 = Release|Any CPU {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU {3E4ABF07-7970-4BE6-B45B-A13D3C397545}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU + {00E38F77-1E61-4CDF-8F97-1417D4E85053}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {00E38F77-1E61-4CDF-8F97-1417D4E85053}.Debug|Any CPU.Build.0 = Debug|Any CPU + {00E38F77-1E61-4CDF-8F97-1417D4E85053}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU + {00E38F77-1E61-4CDF-8F97-1417D4E85053}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU + {00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release|Any CPU.ActiveCfg = Release|Any CPU + {00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release|Any CPU.Build.0 = Release|Any CPU + {00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU + {00E38F77-1E61-4CDF-8F97-1417D4E85053}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -367,6 +377,7 @@ Global {BF66A305-DF10-47E4-8D81-42049B149D2B} = {D3D38B03-B557-484D-8348-8BADEE4DF592} {B4E55B2D-2A92-46E7-B72F-E76D6FD83440} = {7F13E156-3EBA-4021-84A5-CD56BA72F99E} {3E4ABF07-7970-4BE6-B45B-A13D3C397545} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} + {00E38F77-1E61-4CDF-8F97-1417D4E85053} = {09EADF06-BE25-4228-AB53-95AE3E15B530} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D} diff --git a/build/Dependencies.props b/build/Dependencies.props index 5aafe118e5..9667ddec04 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -9,5 +9,6 @@ 1.0.0-beta-62824-02 2.1.2.2 0.0.0.1 + 4.5.0 diff --git a/pkg/Microsoft.ML.ImageAnalytics/Microsoft.ML.ImageAnalytics.nupkgproj b/pkg/Microsoft.ML.ImageAnalytics/Microsoft.ML.ImageAnalytics.nupkgproj new file mode 100644 index 0000000000..8bdef45d07 --- /dev/null +++ b/pkg/Microsoft.ML.ImageAnalytics/Microsoft.ML.ImageAnalytics.nupkgproj @@ -0,0 +1,13 @@ + + + + netstandard2.0 + ML.NET component for Image support + + + + + + + + diff --git a/pkg/Microsoft.ML.ImageAnalytics/Microsoft.ML.ImageAnalytics.symbols.nupkgproj b/pkg/Microsoft.ML.ImageAnalytics/Microsoft.ML.ImageAnalytics.symbols.nupkgproj new file mode 100644 index 0000000000..b36800ea0b --- /dev/null +++ b/pkg/Microsoft.ML.ImageAnalytics/Microsoft.ML.ImageAnalytics.symbols.nupkgproj @@ -0,0 +1,5 @@ + + + + + diff --git a/src/Microsoft.ML.ImageAnalytics/EntryPoints/ImageAnalytics.cs b/src/Microsoft.ML.ImageAnalytics/EntryPoints/ImageAnalytics.cs new file mode 100644 index 0000000000..97c613485f --- /dev/null +++ b/src/Microsoft.ML.ImageAnalytics/EntryPoints/ImageAnalytics.cs @@ -0,0 +1,79 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.EntryPoints; +using Microsoft.ML.Runtime.ImageAnalytics.EntryPoints; + +[assembly: LoadableClass(typeof(void), typeof(ImageAnalytics), null, typeof(SignatureEntryPointModule), "ImageAnalytics")] +namespace Microsoft.ML.Runtime.ImageAnalytics.EntryPoints +{ + public static class ImageAnalytics + { + [TlcModule.EntryPoint(Name = "Transforms.ImageLoader", Desc = ImageLoaderTransform.Summary, + UserName = ImageLoaderTransform.UserName, ShortName = ImageLoaderTransform.LoaderSignature)] + public static CommonOutputs.TransformOutput ImageLoader(IHostEnvironment env, ImageLoaderTransform.Arguments input) + { + var h = EntryPointUtils.CheckArgsAndCreateHost(env, "ImageLoaderTransform", input); + var xf = new ImageLoaderTransform(h, input, input.Data); + return new CommonOutputs.TransformOutput() + { + Model = new TransformModel(h, xf, input.Data), + OutputData = xf + }; + } + + [TlcModule.EntryPoint(Name = "Transforms.ImageResizer", Desc = ImageResizerTransform.Summary, + UserName = ImageResizerTransform.UserName, ShortName = ImageResizerTransform.LoaderSignature)] + public static CommonOutputs.TransformOutput ImageResizer(IHostEnvironment env, ImageResizerTransform.Arguments input) + { + var h = EntryPointUtils.CheckArgsAndCreateHost(env, "ImageResizerTransform", input); + var xf = new ImageResizerTransform(h, input, input.Data); + return new CommonOutputs.TransformOutput() + { + Model = new TransformModel(h, xf, input.Data), + OutputData = xf + }; + } + + [TlcModule.EntryPoint(Name = "Transforms.ImagePixelExtractor", Desc = ImagePixelExtractorTransform.Summary, + UserName = ImagePixelExtractorTransform.UserName, ShortName = ImagePixelExtractorTransform.LoaderSignature)] + public static CommonOutputs.TransformOutput ImagePixelExtractor(IHostEnvironment env, ImagePixelExtractorTransform.Arguments input) + { + var h = EntryPointUtils.CheckArgsAndCreateHost(env, "ImagePixelExtractorTransform", input); + var xf = new ImagePixelExtractorTransform(h, input, input.Data); + return new CommonOutputs.TransformOutput() + { + Model = new TransformModel(h, xf, input.Data), + OutputData = xf + }; + } + + [TlcModule.EntryPoint(Name = "Transforms.ImageGrayscale", Desc = ImageGrayscaleTransform.Summary, + UserName = ImageGrayscaleTransform.UserName, ShortName = ImageGrayscaleTransform.LoaderSignature)] + public static CommonOutputs.TransformOutput ImageGrayscale(IHostEnvironment env, ImageGrayscaleTransform.Arguments input) + { + var h = EntryPointUtils.CheckArgsAndCreateHost(env, "ImageGrayscaleTransform", input); + var xf = new ImageGrayscaleTransform(h, input, input.Data); + return new CommonOutputs.TransformOutput() + { + Model = new TransformModel(h, xf, input.Data), + OutputData = xf + }; + } + + [TlcModule.EntryPoint(Name = "Transforms.VectorToImage", Desc = VectorToImageTransform.Summary, + UserName = VectorToImageTransform.UserName, ShortName = VectorToImageTransform.LoaderSignature)] + public static CommonOutputs.TransformOutput VectorToImage(IHostEnvironment env, VectorToImageTransform.Arguments input) + { + var h = EntryPointUtils.CheckArgsAndCreateHost(env, "VectorToImageTransform", input); + var xf = new VectorToImageTransform(h, input, input.Data); + return new CommonOutputs.TransformOutput() + { + Model = new TransformModel(h, xf, input.Data), + OutputData = xf + }; + } + } +} diff --git a/src/Microsoft.ML.ImageAnalytics/ImageGrayscaleTransform.cs b/src/Microsoft.ML.ImageAnalytics/ImageGrayscaleTransform.cs new file mode 100644 index 0000000000..7a267cf1b8 --- /dev/null +++ b/src/Microsoft.ML.ImageAnalytics/ImageGrayscaleTransform.cs @@ -0,0 +1,171 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Drawing; +using System.Drawing.Imaging; +using System.Text; +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.CommandLine; +using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.EntryPoints; +using Microsoft.ML.Runtime.Internal.Utilities; +using Microsoft.ML.Runtime.Model; +using Microsoft.ML.Runtime.ImageAnalytics; + +[assembly: LoadableClass(ImageGrayscaleTransform.Summary, typeof(ImageGrayscaleTransform), typeof(ImageGrayscaleTransform.Arguments), typeof(SignatureDataTransform), + ImageGrayscaleTransform.UserName, "ImageGrayscaleTransform", "ImageGrayscale")] + +[assembly: LoadableClass(ImageGrayscaleTransform.Summary, typeof(ImageGrayscaleTransform), null, typeof(SignatureLoadDataTransform), + ImageGrayscaleTransform.UserName, ImageGrayscaleTransform.LoaderSignature)] + +namespace Microsoft.ML.Runtime.ImageAnalytics +{ + // REVIEW: Rewrite as LambdaTransform to simplify. + // REVIEW: Should it be separate transform or part of ImageResizerTransform? + /// + /// Transform which takes one or many columns of type in IDataView and + /// convert them to greyscale representation of the same image. + /// + public sealed class ImageGrayscaleTransform : OneToOneTransformBase + { + public sealed class Column : OneToOneColumn + { + public static Column Parse(string str) + { + var res = new Column(); + if (res.TryParse(str)) + return res; + return null; + } + + public bool TryUnparse(StringBuilder sb) + { + Contracts.AssertValue(sb); + return TryUnparseCore(sb); + } + } + + public class Arguments : TransformInputBase + { + [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)] + public Column[] Column; + } + + internal const string Summary = "Convert image into grayscale."; + + internal const string UserName = "Image Greyscale Transform"; + public const string LoaderSignature = "ImageGrayscaleTransform"; + private static VersionInfo GetVersionInfo() + { + return new VersionInfo( + modelSignature: "IMGGRAYT", + verWrittenCur: 0x00010001, // Initial + verReadableCur: 0x00010001, + verWeCanReadBack: 0x00010001, + loaderSignature: LoaderSignature); + } + + private const string RegistrationName = "ImageGrayscale"; + + // Public constructor corresponding to SignatureDataTransform. + public ImageGrayscaleTransform(IHostEnvironment env, Arguments args, IDataView input) + : base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, t => t is ImageType ? null : "Expected Image type") + { + Host.AssertNonEmpty(Infos); + Host.Assert(Infos.Length == Utils.Size(args.Column)); + Metadata.Seal(); + } + + private ImageGrayscaleTransform(IHost host, ModelLoadContext ctx, IDataView input) + : base(host, ctx, input, t => t is ImageType ? null : "Expected Image type") + { + Host.AssertValue(ctx); + // *** Binary format *** + // + Host.AssertNonEmpty(Infos); + Metadata.Seal(); + } + + public static ImageGrayscaleTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) + { + Contracts.CheckValue(env, nameof(env)); + var h = env.Register(RegistrationName); + h.CheckValue(ctx, nameof(ctx)); + h.CheckValue(input, nameof(input)); + ctx.CheckAtModel(GetVersionInfo()); + return h.Apply("Loading Model", ch => new ImageGrayscaleTransform(h, ctx, input)); + } + + public override void Save(ModelSaveContext ctx) + { + Host.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(); + ctx.SetVersionInfo(GetVersionInfo()); + + // *** Binary format *** + // + SaveBase(ctx); + } + + protected override ColumnType GetColumnTypeCore(int iinfo) + { + Host.Assert(0 <= iinfo & iinfo < Infos.Length); + return Infos[iinfo].TypeSrc; + } + + private static readonly ColorMatrix _grayscaleColorMatrix = new ColorMatrix( + new float[][] + { + new float[] {.3f, .3f, .3f, 0, 0}, + new float[] {.59f, .59f, .59f, 0, 0}, + new float[] {.11f, .11f, .11f, 0, 0}, + new float[] {0, 0, 0, 1, 0}, + new float[] {0, 0, 0, 0, 1} + }); + + protected override Delegate GetGetterCore(IChannel ch, IRow input, int iinfo, out Action disposer) + { + Host.AssertValueOrNull(ch); + Host.AssertValue(input); + Host.Assert(0 <= iinfo && iinfo < Infos.Length); + + var src = default(Bitmap); + var getSrc = GetSrcGetter(input, iinfo); + + disposer = + () => + { + if (src != null) + { + src.Dispose(); + src = null; + } + }; + + ValueGetter del = + (ref Bitmap dst) => + { + if (dst != null) + dst.Dispose(); + + getSrc(ref src); + if (src == null || src.Height <= 0 || src.Width <= 0) + return; + + dst = new Bitmap(src.Width, src.Height); + ImageAttributes attributes = new ImageAttributes(); + attributes.SetColorMatrix(_grayscaleColorMatrix); + var srcRectangle = new Rectangle(0, 0, src.Width, src.Height); + using (var g = Graphics.FromImage(dst)) + { + g.DrawImage(src, srcRectangle, 0, 0, src.Width, src.Height, GraphicsUnit.Pixel, attributes); + } + Host.Assert(dst.Width == src.Width && dst.Height == src.Height); + }; + + return del; + } + } +} diff --git a/src/Microsoft.ML.ImageAnalytics/ImageLoaderTransform.cs b/src/Microsoft.ML.ImageAnalytics/ImageLoaderTransform.cs new file mode 100644 index 0000000000..488c710743 --- /dev/null +++ b/src/Microsoft.ML.ImageAnalytics/ImageLoaderTransform.cs @@ -0,0 +1,178 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Drawing; +using System.IO; +using System.Text; +using Microsoft.ML.Runtime.ImageAnalytics; +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.CommandLine; +using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.EntryPoints; +using Microsoft.ML.Runtime.Internal.Utilities; +using Microsoft.ML.Runtime.Model; + +[assembly: LoadableClass(ImageLoaderTransform.Summary, typeof(ImageLoaderTransform), typeof(ImageLoaderTransform.Arguments), typeof(SignatureDataTransform), + ImageLoaderTransform.UserName, "ImageLoaderTransform", "ImageLoader")] + +[assembly: LoadableClass(ImageLoaderTransform.Summary, typeof(ImageLoaderTransform), null, typeof(SignatureLoadDataTransform), + ImageLoaderTransform.UserName, ImageLoaderTransform.LoaderSignature)] + +namespace Microsoft.ML.Runtime.ImageAnalytics +{ + // REVIEW: Rewrite as LambdaTransform to simplify. + /// + /// Transform which takes one or many columns of type and loads them as + /// + public sealed class ImageLoaderTransform : OneToOneTransformBase + { + public sealed class Column : OneToOneColumn + { + public static Column Parse(string str) + { + Contracts.AssertNonEmpty(str); + + var res = new Column(); + if (res.TryParse(str)) + return res; + return null; + } + + public bool TryUnparse(StringBuilder sb) + { + Contracts.AssertValue(sb); + return TryUnparseCore(sb); + } + } + + public sealed class Arguments : TransformInputBase + { + [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", + ShortName = "col", SortOrder = 1)] + public Column[] Column; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Folder where to search for images", ShortName = "folder")] + public string ImageFolder; + } + + internal const string Summary = "Load images from files."; + internal const string UserName = "Image Loader Transform"; + public const string LoaderSignature = "ImageLoaderTransform"; + + private static VersionInfo GetVersionInfo() + { + return new VersionInfo( + modelSignature: "IMGLOADT", + //verWrittenCur: 0x00010001, // Initial + verWrittenCur: 0x00010002, // Swith from OpenCV to Bitmap + verReadableCur: 0x00010002, + verWeCanReadBack: 0x00010002, + loaderSignature: LoaderSignature); + } + + private readonly ImageType _type; + private readonly string _imageFolder; + + private const string RegistrationName = "ImageLoader"; + + // Public constructor corresponding to SignatureDataTransform. + public ImageLoaderTransform(IHostEnvironment env, Arguments args, IDataView input) + : base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, TestIsText) + { + Host.AssertNonEmpty(Infos); + _imageFolder = args.ImageFolder; + Host.Assert(Infos.Length == Utils.Size(args.Column)); + _type = new ImageType(); + Metadata.Seal(); + } + + private ImageLoaderTransform(IHost host, ModelLoadContext ctx, IDataView input) + : base(host, ctx, input, TestIsText) + { + Host.AssertValue(ctx); + + // *** Binary format *** + // + _imageFolder = ctx.Reader.ReadString(); + _type = new ImageType(); + Metadata.Seal(); + } + + public static ImageLoaderTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) + { + Contracts.CheckValue(env, nameof(env)); + var h = env.Register(RegistrationName); + h.CheckValue(ctx, nameof(ctx)); + h.CheckValue(input, nameof(input)); + ctx.CheckAtModel(GetVersionInfo()); + return h.Apply("Loading Model", ch => new ImageLoaderTransform(h, ctx, input)); + } + + public override void Save(ModelSaveContext ctx) + { + Host.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(); + ctx.SetVersionInfo(GetVersionInfo()); + + // *** Binary format *** + // + ctx.Writer.Write(_imageFolder); + SaveBase(ctx); + } + + protected override ColumnType GetColumnTypeCore(int iinfo) + { + Host.Check(0 <= iinfo && iinfo < Infos.Length); + return _type; + } + + protected override Delegate GetGetterCore(IChannel ch, IRow input, int iinfo, out Action disposer) + { + Host.AssertValue(ch, nameof(ch)); + Host.AssertValue(input); + Host.Assert(0 <= iinfo && iinfo < Infos.Length); + disposer = null; + + var getSrc = GetSrcGetter(input, iinfo); + DvText src = default; + ValueGetter del = + (ref Bitmap dst) => + { + if (dst != null) + { + dst.Dispose(); + dst = null; + } + + getSrc(ref src); + + if (src.Length > 0) + { + // Catch exceptions and pass null through. Should also log failures... + try + { + string path = src.ToString(); + if (!string.IsNullOrWhiteSpace(_imageFolder)) + path = Path.Combine(_imageFolder, path); + dst = new Bitmap(path); + } + catch (Exception e) + { + // REVIEW: We catch everything since the documentation for new Bitmap(string) + // appears to be incorrect. When the file isn't found, it throws an ArgumentException, + // while the documentation says FileNotFoundException. Not sure what it will throw + // in other cases, like corrupted file, etc. + + // REVIEW : Log failures. + ch.Info(e.Message); + ch.Info(e.StackTrace); + dst = null; + } + } + }; + return del; + } + } +} diff --git a/src/Microsoft.ML.ImageAnalytics/ImagePixelExtractorTransform.cs b/src/Microsoft.ML.ImageAnalytics/ImagePixelExtractorTransform.cs new file mode 100644 index 0000000000..de0aa98124 --- /dev/null +++ b/src/Microsoft.ML.ImageAnalytics/ImagePixelExtractorTransform.cs @@ -0,0 +1,541 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Drawing; +using System.Text; +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.CommandLine; +using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.EntryPoints; +using Microsoft.ML.Runtime.ImageAnalytics; +using Microsoft.ML.Runtime.Internal.Utilities; +using Microsoft.ML.Runtime.Model; + +[assembly: LoadableClass(ImagePixelExtractorTransform.Summary, typeof(ImagePixelExtractorTransform), typeof(ImagePixelExtractorTransform.Arguments), typeof(SignatureDataTransform), + ImagePixelExtractorTransform.UserName, "ImagePixelExtractorTransform", "ImagePixelExtractor")] + +[assembly: LoadableClass(ImagePixelExtractorTransform.Summary, typeof(ImagePixelExtractorTransform), null, typeof(SignatureLoadDataTransform), + ImagePixelExtractorTransform.UserName, ImagePixelExtractorTransform.LoaderSignature)] + +namespace Microsoft.ML.Runtime.ImageAnalytics +{ + // REVIEW: Rewrite as LambdaTransform to simplify. + /// + /// Transform which takes one or many columns of and convert them into vector representation. + /// + public sealed class ImagePixelExtractorTransform : OneToOneTransformBase + { + public class Column : OneToOneColumn + { + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use alpha channel", ShortName = "alpha")] + public bool? UseAlpha; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use red channel", ShortName = "red")] + public bool? UseRed; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use green channel", ShortName = "green")] + public bool? UseGreen; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use blue channel", ShortName = "blue")] + public bool? UseBlue; + + // REVIEW: Consider turning this into an enum that allows for pixel, line, or planar interleaving. + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to separate each channel or interleave in ARGB order", ShortName = "interleave")] + public bool? InterleaveArgb; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to convert to floating point", ShortName = "conv")] + public bool? Convert; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Offset (pre-scale)")] + public Single? Offset; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Scale factor")] + public Single? Scale; + + public static Column Parse(string str) + { + Contracts.AssertNonEmpty(str); + + var res = new Column(); + if (res.TryParse(str)) + return res; + return null; + } + + public bool TryUnparse(StringBuilder sb) + { + Contracts.AssertValue(sb); + if (UseAlpha != null || UseRed != null || UseGreen != null || UseBlue != null || Convert != null || + Offset != null || Scale != null || InterleaveArgb != null) + { + return false; + } + return TryUnparseCore(sb); + } + } + + public class Arguments : TransformInputBase + { + [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)] + public Column[] Column; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use alpha channel", ShortName = "alpha")] + public bool UseAlpha = false; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use red channel", ShortName = "red")] + public bool UseRed = true; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use green channel", ShortName = "green")] + public bool UseGreen = true; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use blue channel", ShortName = "blue")] + public bool UseBlue = true; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to separate each channel or interleave in ARGB order", ShortName = "interleave")] + public bool InterleaveArgb = false; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to convert to floating point", ShortName = "conv")] + public bool Convert = true; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Offset (pre-scale)")] + public Single? Offset; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Scale factor")] + public Single? Scale; + } + + /// + /// Which color channels are extracted. Note that these values are serialized so should not be modified. + /// + [Flags] + private enum ColorBits : byte + { + Alpha = 0x01, + Red = 0x02, + Green = 0x04, + Blue = 0x08, + + All = Alpha | Red | Green | Blue + } + + private sealed class ColInfoEx + { + public readonly ColorBits Colors; + public readonly byte Planes; + + public readonly bool Convert; + public readonly Single Offset; + public readonly Single Scale; + public readonly bool Interleave; + + public bool Alpha { get { return (Colors & ColorBits.Alpha) != 0; } } + public bool Red { get { return (Colors & ColorBits.Red) != 0; } } + public bool Green { get { return (Colors & ColorBits.Green) != 0; } } + public bool Blue { get { return (Colors & ColorBits.Blue) != 0; } } + + public ColInfoEx(Column item, Arguments args) + { + if (item.UseAlpha ?? args.UseAlpha) { Colors |= ColorBits.Alpha; Planes++; } + if (item.UseRed ?? args.UseRed) { Colors |= ColorBits.Red; Planes++; } + if (item.UseGreen ?? args.UseGreen) { Colors |= ColorBits.Green; Planes++; } + if (item.UseBlue ?? args.UseBlue) { Colors |= ColorBits.Blue; Planes++; } + Contracts.CheckUserArg(Planes > 0, nameof(item.UseRed), "Need to use at least one color plane"); + + Interleave = item.InterleaveArgb ?? args.InterleaveArgb; + + Convert = item.Convert ?? args.Convert; + if (!Convert) + { + Offset = 0; + Scale = 1; + } + else + { + Offset = item.Offset ?? args.Offset ?? 0; + Scale = item.Scale ?? args.Scale ?? 1; + Contracts.CheckUserArg(FloatUtils.IsFinite(Offset), nameof(item.Offset)); + Contracts.CheckUserArg(FloatUtils.IsFiniteNonZero(Scale), nameof(item.Scale)); + } + } + + public ColInfoEx(ModelLoadContext ctx) + { + Contracts.AssertValue(ctx); + + // *** Binary format *** + // byte: colors + // byte: convert + // Float: offset + // Float: scale + // byte: separateChannels + Colors = (ColorBits)ctx.Reader.ReadByte(); + Contracts.CheckDecode(Colors != 0); + Contracts.CheckDecode((Colors & ColorBits.All) == Colors); + + // Count the planes. + int planes = (int)Colors; + planes = (planes & 0x05) + ((planes >> 1) & 0x05); + planes = (planes & 0x03) + ((planes >> 2) & 0x03); + Planes = (byte)planes; + Contracts.Assert(0 < Planes & Planes <= 4); + + Convert = ctx.Reader.ReadBoolByte(); + Offset = ctx.Reader.ReadFloat(); + Contracts.CheckDecode(FloatUtils.IsFinite(Offset)); + Scale = ctx.Reader.ReadFloat(); + Contracts.CheckDecode(FloatUtils.IsFiniteNonZero(Scale)); + Contracts.CheckDecode(Convert || Offset == 0 && Scale == 1); + Interleave = ctx.Reader.ReadBoolByte(); + } + + public void Save(ModelSaveContext ctx) + { + Contracts.AssertValue(ctx); + +#if DEBUG + // This code is used in deserialization - assert that it matches what we computed above. + int planes = (int)Colors; + planes = (planes & 0x05) + ((planes >> 1) & 0x05); + planes = (planes & 0x03) + ((planes >> 2) & 0x03); + Contracts.Assert(planes == Planes); +#endif + + // *** Binary format *** + // byte: colors + // byte: convert + // Float: offset + // Float: scale + // byte: separateChannels + Contracts.Assert(Colors != 0); + Contracts.Assert((Colors & ColorBits.All) == Colors); + ctx.Writer.Write((byte)Colors); + ctx.Writer.WriteBoolByte(Convert); + Contracts.Assert(FloatUtils.IsFinite(Offset)); + ctx.Writer.Write(Offset); + Contracts.Assert(FloatUtils.IsFiniteNonZero(Scale)); + Contracts.Assert(Convert || Offset == 0 && Scale == 1); + ctx.Writer.Write(Scale); + ctx.Writer.WriteBoolByte(Interleave); + } + } + + internal const string Summary = "Extract color plane(s) from an image. Options include scaling, offset and conversion to floating point."; + internal const string UserName = "Image Pixel Extractor Transform"; + public const string LoaderSignature = "ImagePixelExtractor"; + private static VersionInfo GetVersionInfo() + { + return new VersionInfo( + modelSignature: "IMGPXEXT", + //verWrittenCur: 0x00010001, // Initial + verWrittenCur: 0x00010002, // Swith from OpenCV to Bitmap + verReadableCur: 0x00010002, + verWeCanReadBack: 0x00010002, + loaderSignature: LoaderSignature); + } + + private const string RegistrationName = "ImagePixelExtractor"; + + private readonly ColInfoEx[] _exes; + private readonly VectorType[] _types; + + // Public constructor corresponding to SignatureDataTransform. + public ImagePixelExtractorTransform(IHostEnvironment env, Arguments args, IDataView input) + : base(env, RegistrationName, Contracts.CheckRef(args, nameof(args)).Column, input, + t => t is ImageType ? null : "Expected Image type") + { + Host.AssertNonEmpty(Infos); + Host.Assert(Infos.Length == Utils.Size(args.Column)); + + _exes = new ColInfoEx[Infos.Length]; + for (int i = 0; i < _exes.Length; i++) + { + var item = args.Column[i]; + _exes[i] = new ColInfoEx(item, args); + } + + _types = ConstructTypes(true); + } + + private ImagePixelExtractorTransform(IHost host, ModelLoadContext ctx, IDataView input) + : base(host, ctx, input, t => t is ImageType ? null : "Expected Image type") + { + Host.AssertValue(ctx); + + // *** Binary format *** + // + // + // foreach added column + // ColInfoEx + Host.AssertNonEmpty(Infos); + _exes = new ColInfoEx[Infos.Length]; + for (int i = 0; i < _exes.Length; i++) + _exes[i] = new ColInfoEx(ctx); + + _types = ConstructTypes(false); + } + + public static ImagePixelExtractorTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) + { + Contracts.CheckValue(env, nameof(env)); + var h = env.Register(RegistrationName); + h.CheckValue(ctx, nameof(ctx)); + h.CheckValue(input, nameof(input)); + ctx.CheckAtModel(GetVersionInfo()); + + return h.Apply("Loading Model", + ch => + { + // *** Binary format *** + // int: sizeof(Float) + // + int cbFloat = ctx.Reader.ReadInt32(); + ch.CheckDecode(cbFloat == sizeof(Single)); + return new ImagePixelExtractorTransform(h, ctx, input); + }); + } + + public override void Save(ModelSaveContext ctx) + { + Host.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(); + ctx.SetVersionInfo(GetVersionInfo()); + + // *** Binary format *** + // int: sizeof(Float) + // + // foreach added column + // ColInfoEx + ctx.Writer.Write(sizeof(Single)); + SaveBase(ctx); + + Host.Assert(_exes.Length == Infos.Length); + for (int i = 0; i < _exes.Length; i++) + _exes[i].Save(ctx); + } + + private VectorType[] ConstructTypes(bool user) + { + var types = new VectorType[Infos.Length]; + for (int i = 0; i < Infos.Length; i++) + { + var info = Infos[i]; + var ex = _exes[i]; + Host.Assert(ex.Planes > 0); + + var type = Source.Schema.GetColumnType(info.Source) as ImageType; + Host.Assert(type != null); + if (type.Height <= 0 || type.Width <= 0) + { + // REVIEW: Could support this case by making the destination column be variable sized. + // However, there's no mechanism to communicate the dimensions through with the pixel data. + string name = Source.Schema.GetColumnName(info.Source); + throw user ? + Host.ExceptUserArg(nameof(Arguments.Column), "Column '{0}' does not have known size", name) : + Host.Except("Column '{0}' does not have known size", name); + } + int height = type.Height; + int width = type.Width; + Host.Assert(height > 0); + Host.Assert(width > 0); + Host.Assert((long)height * width <= int.MaxValue / 4); + + if (ex.Interleave) + types[i] = new VectorType(ex.Convert ? NumberType.Float : NumberType.U1, height, width, ex.Planes); + else + types[i] = new VectorType(ex.Convert ? NumberType.Float : NumberType.U1, ex.Planes, height, width); + } + Metadata.Seal(); + return types; + } + + protected override ColumnType GetColumnTypeCore(int iinfo) + { + Host.Assert(0 <= iinfo & iinfo < Infos.Length); + return _types[iinfo]; + } + + protected override Delegate GetGetterCore(IChannel ch, IRow input, int iinfo, out Action disposer) + { + Host.AssertValueOrNull(ch); + Host.AssertValue(input); + Host.Assert(0 <= iinfo && iinfo < Infos.Length); + + if (_exes[iinfo].Convert) + return GetGetterCore(input, iinfo, out disposer); + return GetGetterCore(input, iinfo, out disposer); + } + + //REVIEW Rewrite it to where TValue : IConvertible + private ValueGetter> GetGetterCore(IRow input, int iinfo, out Action disposer) + { + var type = _types[iinfo]; + Host.Assert(type.DimCount == 3); + + var ex = _exes[iinfo]; + + int planes = ex.Interleave ? type.GetDim(2) : type.GetDim(0); + int height = ex.Interleave ? type.GetDim(0) : type.GetDim(1); + int width = ex.Interleave ? type.GetDim(1) : type.GetDim(2); + + int size = type.ValueCount; + Host.Assert(size > 0); + Host.Assert(size == planes * height * width); + int cpix = height * width; + + var getSrc = GetSrcGetter(input, iinfo); + var src = default(Bitmap); + + disposer = + () => + { + if (src != null) + { + src.Dispose(); + src = null; + } + }; + + return + (ref VBuffer dst) => + { + getSrc(ref src); + Contracts.AssertValueOrNull(src); + + if (src == null) + { + dst = new VBuffer(size, 0, dst.Values, dst.Indices); + return; + } + + Host.Check(src.PixelFormat == System.Drawing.Imaging.PixelFormat.Format32bppArgb); + Host.Check(src.Height == height && src.Width == width); + + var values = dst.Values; + if (Utils.Size(values) < size) + values = new TValue[size]; + + Single offset = ex.Offset; + Single scale = ex.Scale; + Host.Assert(scale != 0); + + var vf = values as Single[]; + var vb = values as byte[]; + Host.Assert(vf != null || vb != null); + bool needScale = offset != 0 || scale != 1; + Host.Assert(!needScale || vf != null); + + bool a = ex.Alpha; + bool r = ex.Red; + bool g = ex.Green; + bool b = ex.Blue; + + int h = height; + int w = width; + + if (ex.Interleave) + { + int idst = 0; + for (int y = 0; y < h; ++y) + for (int x = 0; x < w; x++) + { + var pb = src.GetPixel(y, x); + if (vb != null) + { + if (a) { vb[idst++] = (byte)0; } + if (r) { vb[idst++] = pb.R; } + if (g) { vb[idst++] = pb.G; } + if (b) { vb[idst++] = pb.B; } + } + else if (!needScale) + { + if (a) { vf[idst++] = 0.0f; } + if (r) { vf[idst++] = pb.R; } + if (g) { vf[idst++] = pb.G; } + if (b) { vf[idst++] = pb.B; } + } + else + { + if (a) { vf[idst++] = 0.0f; } + if (r) { vf[idst++] = (pb.R - offset) * scale; } + if (g) { vf[idst++] = (pb.B - offset) * scale; } + if (b) { vf[idst++] = (pb.G - offset) * scale; } + } + } + Host.Assert(idst == size); + } + else + { + int idstMin = 0; + if (ex.Alpha) + { + // The image only has rgb but we need to supply alpha as well, so fake it up, + // assuming that it is 0xFF. + if (vf != null) + { + Single v = (0xFF - offset) * scale; + for (int i = 0; i < cpix; i++) + vf[i] = v; + } + else + { + for (int i = 0; i < cpix; i++) + vb[i] = 0xFF; + } + idstMin = cpix; + + // We've preprocessed alpha, avoid it in the + // scan operation below. + a = false; + } + + for (int y = 0; y < h; ++y) + { + int idstBase = idstMin + y * w; + + // Note that the bytes are in order BGR[A]. We arrange the layers in order ARGB. + if (vb != null) + { + for (int x = 0; x < w; x++, idstBase++) + { + var pb = src.GetPixel(x, y); + int idst = idstBase; + if (a) { vb[idst] = pb.A; idst += cpix; } + if (r) { vb[idst] = pb.R; idst += cpix; } + if (g) { vb[idst] = pb.G; idst += cpix; } + if (b) { vb[idst] = pb.B; idst += cpix; } + } + } + else if (!needScale) + { + for (int x = 0; x < w; x++, idstBase++) + { + var pb = src.GetPixel(x, y); + int idst = idstBase; + if (a) { vf[idst] = pb.A; idst += cpix; } + if (r) { vf[idst] = pb.R; idst += cpix; } + if (g) { vf[idst] = pb.G; idst += cpix; } + if (b) { vf[idst] = pb.B; idst += cpix; } + } + } + else + { + for (int x = 0; x < w; x++, idstBase++) + { + var pb = src.GetPixel(x, y); + int idst = idstBase; + if (a) { vf[idst] = (pb.A - offset) * scale; idst += cpix; } + if (r) { vf[idst] = (pb.R - offset) * scale; idst += cpix; } + if (g) { vf[idst] = (pb.G - offset) * scale; idst += cpix; } + if (b) { vf[idst] = (pb.B - offset) * scale; idst += cpix; } + } + } + } + } + + dst = new VBuffer(size, values, dst.Indices); + }; + } + } +} diff --git a/src/Microsoft.ML.ImageAnalytics/ImageResizerTransform.cs b/src/Microsoft.ML.ImageAnalytics/ImageResizerTransform.cs new file mode 100644 index 0000000000..dd1abc9181 --- /dev/null +++ b/src/Microsoft.ML.ImageAnalytics/ImageResizerTransform.cs @@ -0,0 +1,370 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Drawing; +using System.Text; +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.CommandLine; +using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.EntryPoints; +using Microsoft.ML.Runtime.ImageAnalytics; +using Microsoft.ML.Runtime.Internal.Internallearn; +using Microsoft.ML.Runtime.Internal.Utilities; +using Microsoft.ML.Runtime.Model; + +[assembly: LoadableClass(ImageResizerTransform.Summary, typeof(ImageResizerTransform), typeof(ImageResizerTransform.Arguments), + typeof(SignatureDataTransform), ImageResizerTransform.UserName, "ImageResizerTransform", "ImageResizer")] + +[assembly: LoadableClass(ImageResizerTransform.Summary, typeof(ImageResizerTransform), null, typeof(SignatureLoadDataTransform), + ImageResizerTransform.UserName, ImageResizerTransform.LoaderSignature)] + +namespace Microsoft.ML.Runtime.ImageAnalytics +{ + // REVIEW: Rewrite as LambdaTransform to simplify. + /// + /// Transform which takes one or many columns of and resize them to provided height and width. + /// + public sealed class ImageResizerTransform : OneToOneTransformBase + { + public enum ResizingKind : byte + { + [TGUI(Label = "Isotropic with Padding")] + IsoPad = 0, + + [TGUI(Label = "Isotropic with Cropping")] + IsoCrop = 1 + } + + public enum Anchor : byte + { + Right = 0, + Left = 1, + Top = 2, + Bottom = 3, + Center = 4 + } + + public sealed class Column : OneToOneColumn + { + [Argument(ArgumentType.AtMostOnce, HelpText = "Width of the resized image", ShortName = "width")] + public int? ImageWidth; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Height of the resized image", ShortName = "height")] + public int? ImageHeight; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Resizing method", ShortName = "scale")] + public ResizingKind? Resizing; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Anchor for cropping", ShortName = "anchor")] + public Anchor? CropAnchor; + + public static Column Parse(string str) + { + Contracts.AssertNonEmpty(str); + + var res = new Column(); + if (res.TryParse(str)) + return res; + return null; + } + + public bool TryUnparse(StringBuilder sb) + { + Contracts.AssertValue(sb); + if (ImageWidth != null || ImageHeight != null || Resizing != null || CropAnchor != null) + return false; + return TryUnparseCore(sb); + } + } + + public class Arguments : TransformInputBase + { + [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)] + public Column[] Column; + + [Argument(ArgumentType.Required, HelpText = "Resized width of the image", ShortName = "width")] + public int ImageWidth; + + [Argument(ArgumentType.Required, HelpText = "Resized height of the image", ShortName = "height")] + public int ImageHeight; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Resizing method", ShortName = "scale")] + public ResizingKind Resizing = ResizingKind.IsoCrop; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Anchor for cropping", ShortName = "anchor")] + public Anchor CropAnchor = Anchor.Center; + } + + /// + /// Extra information for each column (in addition to ColumnInfo). + /// + private sealed class ColInfoEx + { + public readonly int Width; + public readonly int Height; + public readonly ResizingKind Scale; + public readonly Anchor Anchor; + public readonly ColumnType Type; + + public ColInfoEx(int width, int height, ResizingKind scale, Anchor anchor) + { + Contracts.CheckUserArg(width > 0, nameof(Column.ImageWidth)); + Contracts.CheckUserArg(height > 0, nameof(Column.ImageHeight)); + Contracts.CheckUserArg(Enum.IsDefined(typeof(ResizingKind), scale), nameof(Column.Resizing)); + Contracts.CheckUserArg(Enum.IsDefined(typeof(Anchor), anchor), nameof(Column.CropAnchor)); + + Width = width; + Height = height; + Scale = scale; + Anchor = anchor; + Type = new ImageType(Height, Width); + } + } + + internal const string Summary = "Scales an image to specified dimensions using one of the three scale types: isotropic with padding, " + + "isotropic with cropping or anisotropic. In case of isotropic padding, transparent color is used to pad resulting image."; + + internal const string UserName = "Image Resizer Transform"; + public const string LoaderSignature = "ImageScalerTransform"; + private static VersionInfo GetVersionInfo() + { + return new VersionInfo( + modelSignature: "IMGSCALF", + //verWrittenCur: 0x00010001, // Initial + verWrittenCur: 0x00010002, // Swith from OpenCV to Bitmap + verReadableCur: 0x00010002, + verWeCanReadBack: 0x00010002, + loaderSignature: LoaderSignature); + } + + private const string RegistrationName = "ImageScaler"; + + // This is parallel to Infos. + private readonly ColInfoEx[] _exes; + + // Public constructor corresponding to SignatureDataTransform. + public ImageResizerTransform(IHostEnvironment env, Arguments args, IDataView input) + : base(env, RegistrationName, env.CheckRef(args, nameof(args)).Column, input, t => t is ImageType ? null : "Expected Image type") + { + Host.AssertNonEmpty(Infos); + Host.Assert(Infos.Length == Utils.Size(args.Column)); + + _exes = new ColInfoEx[Infos.Length]; + for (int i = 0; i < _exes.Length; i++) + { + var item = args.Column[i]; + _exes[i] = new ColInfoEx( + item.ImageWidth ?? args.ImageWidth, + item.ImageHeight ?? args.ImageHeight, + item.Resizing ?? args.Resizing, + item.CropAnchor ?? args.CropAnchor); + } + Metadata.Seal(); + } + + private ImageResizerTransform(IHost host, ModelLoadContext ctx, IDataView input) + : base(host, ctx, input, t => t is ImageType ? null : "Expected Image type") + { + Host.AssertValue(ctx); + + // *** Binary format *** + // + // + // for each added column + // int: width + // int: height + // byte: scaling kind + Host.AssertNonEmpty(Infos); + + _exes = new ColInfoEx[Infos.Length]; + for (int i = 0; i < _exes.Length; i++) + { + int width = ctx.Reader.ReadInt32(); + Host.CheckDecode(width > 0); + int height = ctx.Reader.ReadInt32(); + Host.CheckDecode(height > 0); + var scale = (ResizingKind)ctx.Reader.ReadByte(); + Host.CheckDecode(Enum.IsDefined(typeof(ResizingKind), scale)); + var anchor = (Anchor)ctx.Reader.ReadByte(); + Host.CheckDecode(Enum.IsDefined(typeof(Anchor), anchor)); + _exes[i] = new ColInfoEx(width, height, scale, anchor); + } + Metadata.Seal(); + } + + public static ImageResizerTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) + { + Contracts.CheckValue(env, nameof(env)); + var h = env.Register(RegistrationName); + h.CheckValue(ctx, nameof(ctx)); + h.CheckValue(input, nameof(input)); + ctx.CheckAtModel(GetVersionInfo()); + return h.Apply("Loading Model", + ch => + { + // *** Binary format *** + // int: sizeof(Float) + // + int cbFloat = ctx.Reader.ReadInt32(); + ch.CheckDecode(cbFloat == sizeof(Single)); + return new ImageResizerTransform(h, ctx, input); + }); + } + + public override void Save(ModelSaveContext ctx) + { + Host.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(); + ctx.SetVersionInfo(GetVersionInfo()); + + // *** Binary format *** + // int: sizeof(Float) + // + // for each added column + // int: width + // int: height + // byte: scaling kind + ctx.Writer.Write(sizeof(Single)); + SaveBase(ctx); + + Host.Assert(_exes.Length == Infos.Length); + for (int i = 0; i < _exes.Length; i++) + { + var ex = _exes[i]; + ctx.Writer.Write(ex.Width); + ctx.Writer.Write(ex.Height); + Host.Assert((ResizingKind)(byte)ex.Scale == ex.Scale); + ctx.Writer.Write((byte)ex.Scale); + Host.Assert((Anchor)(byte)ex.Anchor == ex.Anchor); + ctx.Writer.Write((byte)ex.Anchor); + } + } + + protected override ColumnType GetColumnTypeCore(int iinfo) + { + Host.Check(0 <= iinfo && iinfo < Infos.Length); + return _exes[iinfo].Type; + } + + protected override Delegate GetGetterCore(IChannel ch, IRow input, int iinfo, out Action disposer) + { + Host.AssertValueOrNull(ch); + Host.AssertValue(input); + Host.Assert(0 <= iinfo && iinfo < Infos.Length); + + var src = default(Bitmap); + var getSrc = GetSrcGetter(input, iinfo); + var ex = _exes[iinfo]; + + disposer = + () => + { + if (src != null) + { + src.Dispose(); + src = null; + } + }; + + ValueGetter del = + (ref Bitmap dst) => + { + if (dst != null) + dst.Dispose(); + + getSrc(ref src); + if (src == null || src.Height <= 0 || src.Width <= 0) + return; + if (src.Height == ex.Height && src.Width == ex.Width) + { + dst = src; + return; + } + + int sourceWidth = src.Width; + int sourceHeight = src.Height; + int sourceX = 0; + int sourceY = 0; + int destX = 0; + int destY = 0; + int destWidth = 0; + int destHeight = 0; + float aspect = 0; + float widthAspect = 0; + float heightAspect = 0; + + widthAspect = (float)ex.Width / sourceWidth; + heightAspect = (float)ex.Height / sourceHeight; + + if (ex.Scale == ResizingKind.IsoPad) + { + widthAspect = (float)ex.Width / sourceWidth; + heightAspect = (float)ex.Height / sourceHeight; + if (heightAspect < widthAspect) + { + aspect = heightAspect; + destX = (int)((ex.Width - (sourceWidth * aspect)) / 2); + } + else + { + aspect = widthAspect; + destY = (int)((ex.Height - (sourceHeight * aspect)) / 2); + } + + destWidth = (int)(sourceWidth * aspect); + destHeight = (int)(sourceHeight * aspect); + } + else + { + if (heightAspect < widthAspect) + { + aspect = widthAspect; + switch (ex.Anchor) + { + case Anchor.Top: + destY = 0; + break; + case Anchor.Bottom: + destY = (int)(ex.Height - (sourceHeight * aspect)); + break; + default: + destY = (int)((ex.Height - (sourceHeight * aspect)) / 2); + break; + } + } + else + { + aspect = heightAspect; + switch (ex.Anchor) + { + case Anchor.Left: + destX = 0; + break; + case Anchor.Right: + destX = (int)(ex.Width - (sourceWidth * aspect)); + break; + default: + destX = (int)((ex.Width - (sourceWidth * aspect)) / 2); + break; + } + } + + destWidth = (int)(sourceWidth * aspect); + destHeight = (int)(sourceHeight * aspect); + } + dst = new Bitmap(ex.Width, ex.Height); + var srcRectangle = new Rectangle(sourceX, sourceY, sourceWidth, sourceHeight); + var destRectangle = new Rectangle(destX, destY, destWidth, destHeight); + using (var g = Graphics.FromImage(dst)) + { + g.DrawImage(src, destRectangle, srcRectangle, GraphicsUnit.Pixel); + } + Host.Assert(dst.Width == ex.Width && dst.Height == ex.Height); + }; + + return del; + } + } +} diff --git a/src/Microsoft.ML.ImageAnalytics/ImageType.cs b/src/Microsoft.ML.ImageAnalytics/ImageType.cs new file mode 100644 index 0000000000..bc822f80b0 --- /dev/null +++ b/src/Microsoft.ML.ImageAnalytics/ImageType.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Drawing; +using Microsoft.ML.Runtime.Data; + +namespace Microsoft.ML.Runtime.ImageAnalytics +{ + public sealed class ImageType : StructuredType + { + public readonly int Height; + public readonly int Width; + public ImageType(int height, int width) + : base(typeof(Bitmap)) + { + Contracts.CheckParam(height > 0, nameof(height)); + Contracts.CheckParam(width > 0, nameof(width)); + Contracts.CheckParam((long)height * width <= int.MaxValue / 4, nameof(height), nameof(height) + " * " + nameof(width) + " is too large"); + Height = height; + Width = width; + } + + public ImageType() : base(typeof(Image)) + { + } + + public override bool Equals(ColumnType other) + { + if (other == this) + return true; + if (!(other is ImageType tmp)) + return false; + if (Height != tmp.Height) + return false; + return Width != tmp.Width; + } + + public override string ToString() + { + if (Height == 0 && Width == 0) + return "Image"; + return string.Format("Image<{0}, {1}>", Height, Width); + } + } +} diff --git a/src/Microsoft.ML.ImageAnalytics/Microsoft.ML.ImageAnalytics.csproj b/src/Microsoft.ML.ImageAnalytics/Microsoft.ML.ImageAnalytics.csproj new file mode 100644 index 0000000000..1a4fa6b66d --- /dev/null +++ b/src/Microsoft.ML.ImageAnalytics/Microsoft.ML.ImageAnalytics.csproj @@ -0,0 +1,18 @@ + + + + netstandard2.0 + Microsoft.ML.Runtime.ImageAnalytics + Microsoft.ML.Runtime.ImageAnalytics + + + + + + + + + + + + diff --git a/src/Microsoft.ML.ImageAnalytics/VectorToImageTransform.cs b/src/Microsoft.ML.ImageAnalytics/VectorToImageTransform.cs new file mode 100644 index 0000000000..b9d35a6cdc --- /dev/null +++ b/src/Microsoft.ML.ImageAnalytics/VectorToImageTransform.cs @@ -0,0 +1,419 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Drawing; +using System.Text; +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.CommandLine; +using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.EntryPoints; +using Microsoft.ML.Runtime.ImageAnalytics; +using Microsoft.ML.Runtime.Internal.Utilities; +using Microsoft.ML.Runtime.Model; + +[assembly: LoadableClass(VectorToImageTransform.Summary, typeof(VectorToImageTransform), typeof(VectorToImageTransform.Arguments), + typeof(SignatureDataTransform), VectorToImageTransform.UserName, "VectorToImageTransform", "VectorToImage")] + +[assembly: LoadableClass(VectorToImageTransform.Summary, typeof(VectorToImageTransform), null, typeof(SignatureLoadDataTransform), + VectorToImageTransform.UserName, VectorToImageTransform.LoaderSignature)] + +namespace Microsoft.ML.Runtime.ImageAnalytics +{ + // REVIEW: Rewrite as LambdaTransform to simplify. + + /// + /// Transform which takes one or many columns with vectors in them and transform them to representation. + /// + public sealed class VectorToImageTransform : OneToOneTransformBase + { + public class Column : OneToOneColumn + { + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use alpha channel", ShortName = "alpha")] + public bool? ContainsAlpha; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use red channel", ShortName = "red")] + public bool? ContainsRed; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use green channel", ShortName = "green")] + public bool? ContainsGreen; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use blue channel", ShortName = "blue")] + public bool? ContainsBlue; + + // REVIEW: Consider turning this into an enum that allows for pixel, line, or planar interleaving. + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to separate each channel or interleave in ARGB order", ShortName = "interleave")] + public bool? InterleaveArgb; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Width of the image", ShortName = "width")] + public int? ImageWidth; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Height of the image", ShortName = "height")] + public int? ImageHeight; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Offset (pre-scale)")] + public Single? Offset; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Scale factor")] + public Single? Scale; + + public static Column Parse(string str) + { + Contracts.AssertNonEmpty(str); + + var res = new Column(); + if (res.TryParse(str)) + return res; + return null; + } + + public bool TryUnparse(StringBuilder sb) + { + Contracts.AssertValue(sb); + if (ContainsAlpha != null || ContainsRed != null || ContainsGreen != null || ContainsBlue != null || ImageWidth != null || + ImageHeight != null || Offset != null || Scale != null || InterleaveArgb != null) + { + return false; + } + return TryUnparseCore(sb); + } + } + + public class Arguments : TransformInputBase + { + [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", ShortName = "col", SortOrder = 1)] + public Column[] Column; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use alpha channel", ShortName = "alpha")] + public bool ContainsAlpha = false; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use red channel", ShortName = "red")] + public bool ContainsRed = true; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use green channel", ShortName = "green")] + public bool ContainsGreen = true; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use blue channel", ShortName = "blue")] + public bool ContainsBlue = true; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to separate each channel or interleave in ARGB order", ShortName = "interleave")] + public bool InterleaveArgb = false; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Width of the image", ShortName = "width")] + public int ImageWidth; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Height of the image", ShortName = "height")] + public int ImageHeight; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Offset (pre-scale)")] + public Single? Offset; + + [Argument(ArgumentType.AtMostOnce, HelpText = "Scale factor")] + public Single? Scale; + } + + /// + /// Which color channels are extracted. Note that these values are serialized so should not be modified. + /// + [Flags] + private enum ColorBits : byte + { + Alpha = 0x01, + Red = 0x02, + Green = 0x04, + Blue = 0x08, + + All = Alpha | Red | Green | Blue + } + + private sealed class ColInfoEx + { + public readonly ColorBits Colors; + public readonly byte Planes; + + public readonly int Width; + public readonly int Height; + public readonly Single Offset; + public readonly Single Scale; + public readonly bool Interleave; + + public bool Alpha { get { return (Colors & ColorBits.Alpha) != 0; } } + public bool Red { get { return (Colors & ColorBits.Red) != 0; } } + public bool Green { get { return (Colors & ColorBits.Green) != 0; } } + public bool Blue { get { return (Colors & ColorBits.Blue) != 0; } } + + public ColInfoEx(Column item, Arguments args) + { + if (item.ContainsAlpha ?? args.ContainsAlpha) { Colors |= ColorBits.Alpha; Planes++; } + if (item.ContainsRed ?? args.ContainsRed) { Colors |= ColorBits.Red; Planes++; } + if (item.ContainsGreen ?? args.ContainsGreen) { Colors |= ColorBits.Green; Planes++; } + if (item.ContainsBlue ?? args.ContainsBlue) { Colors |= ColorBits.Blue; Planes++; } + Contracts.CheckUserArg(Planes > 0, nameof(item.ContainsRed), "Need to use at least one color plane"); + + Interleave = item.InterleaveArgb ?? args.InterleaveArgb; + + Width = item.ImageWidth ?? args.ImageWidth; + Height = item.ImageHeight ?? args.ImageHeight; + Offset = item.Offset ?? args.Offset ?? 0; + Scale = item.Scale ?? args.Scale ?? 1; + Contracts.CheckUserArg(FloatUtils.IsFinite(Offset), nameof(item.Offset)); + Contracts.CheckUserArg(FloatUtils.IsFiniteNonZero(Scale), nameof(item.Scale)); + } + + public ColInfoEx(ModelLoadContext ctx) + { + Contracts.AssertValue(ctx); + + // *** Binary format *** + // byte: colors + // int: widht + // int: height + // Float: offset + // Float: scale + // byte: separateChannels + Colors = (ColorBits)ctx.Reader.ReadByte(); + Contracts.CheckDecode(Colors != 0); + Contracts.CheckDecode((Colors & ColorBits.All) == Colors); + + // Count the planes. + int planes = (int)Colors; + planes = (planes & 0x05) + ((planes >> 1) & 0x05); + planes = (planes & 0x03) + ((planes >> 2) & 0x03); + Planes = (byte)planes; + Contracts.Assert(0 < Planes & Planes <= 4); + + Width = ctx.Reader.ReadInt32(); + Contracts.CheckDecode(Width > 0); + Height = ctx.Reader.ReadInt32(); + Contracts.CheckDecode(Height > 0); + Offset = ctx.Reader.ReadFloat(); + Contracts.CheckDecode(FloatUtils.IsFinite(Offset)); + Scale = ctx.Reader.ReadFloat(); + Contracts.CheckDecode(FloatUtils.IsFiniteNonZero(Scale)); + Interleave = ctx.Reader.ReadBoolByte(); + } + + public void Save(ModelSaveContext ctx) + { + Contracts.AssertValue(ctx); + +#if DEBUG + // This code is used in deserialization - assert that it matches what we computed above. + int planes = (int)Colors; + planes = (planes & 0x05) + ((planes >> 1) & 0x05); + planes = (planes & 0x03) + ((planes >> 2) & 0x03); + Contracts.Assert(planes == Planes); +#endif + + // *** Binary format *** + // byte: colors + // byte: convert + // Float: offset + // Float: scale + // byte: separateChannels + Contracts.Assert(Colors != 0); + Contracts.Assert((Colors & ColorBits.All) == Colors); + ctx.Writer.Write((byte)Colors); + ctx.Writer.Write(Width); + ctx.Writer.Write(Height); + Contracts.Assert(FloatUtils.IsFinite(Offset)); + ctx.Writer.Write(Offset); + Contracts.Assert(FloatUtils.IsFiniteNonZero(Scale)); + ctx.Writer.Write(Scale); + ctx.Writer.WriteBoolByte(Interleave); + } + } + + public const string Summary = "Converts vector array into image type."; + public const string UserName = "Vector To Image Transform"; + public const string LoaderSignature = "VectorToImageConverter"; + private static VersionInfo GetVersionInfo() + { + return new VersionInfo( + modelSignature: "VECTOIMG", + //verWrittenCur: 0x00010001, // Initial + verWrittenCur: 0x00010002, // Swith from OpenCV to Bitmap + verReadableCur: 0x00010002, + verWeCanReadBack: 0x00010002, + loaderSignature: LoaderSignature); + } + + private const string RegistrationName = "VectorToImageConverter"; + + private readonly ColInfoEx[] _exes; + private readonly ImageType[] _types; + + // Public constructor corresponding to SignatureDataTransform. + public VectorToImageTransform(IHostEnvironment env, Arguments args, IDataView input) + : base(env, RegistrationName, Contracts.CheckRef(args, nameof(args)).Column, input, + t => t is VectorType ? null : "Expected VectorType type") + { + Host.AssertNonEmpty(Infos); + Host.Assert(Infos.Length == Utils.Size(args.Column)); + + _exes = new ColInfoEx[Infos.Length]; + _types = new ImageType[Infos.Length]; + for (int i = 0; i < _exes.Length; i++) + { + var item = args.Column[i]; + _exes[i] = new ColInfoEx(item, args); + _types[i] = new ImageType(_exes[i].Height, _exes[i].Width); + } + Metadata.Seal(); + } + + private VectorToImageTransform(IHost host, ModelLoadContext ctx, IDataView input) + : base(host, ctx, input, t => t is VectorType ? null : "Expected VectorType type") + { + Host.AssertValue(ctx); + + // *** Binary format *** + // + // + // foreach added column + // ColInfoEx + Host.AssertNonEmpty(Infos); + _exes = new ColInfoEx[Infos.Length]; + _types = new ImageType[Infos.Length]; + for (int i = 0; i < _exes.Length; i++) + { + _exes[i] = new ColInfoEx(ctx); + _types[i] = new ImageType(_exes[i].Height, _exes[i].Width); + } + Metadata.Seal(); + } + + public static VectorToImageTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) + { + Contracts.CheckValue(env, nameof(env)); + var h = env.Register(RegistrationName); + h.CheckValue(ctx, nameof(ctx)); + h.CheckValue(input, nameof(input)); + ctx.CheckAtModel(GetVersionInfo()); + + return h.Apply("Loading Model", + ch => + { + // *** Binary format *** + // int: sizeof(Float) + // + int cbFloat = ctx.Reader.ReadInt32(); + ch.CheckDecode(cbFloat == sizeof(Single)); + return new VectorToImageTransform(h, ctx, input); + }); + } + + public override void Save(ModelSaveContext ctx) + { + Host.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(); + ctx.SetVersionInfo(GetVersionInfo()); + + // *** Binary format *** + // int: sizeof(Float) + // + // foreach added column + // ColInfoEx + ctx.Writer.Write(sizeof(Single)); + SaveBase(ctx); + + Host.Assert(_exes.Length == Infos.Length); + for (int i = 0; i < _exes.Length; i++) + _exes[i].Save(ctx); + } + + protected override ColumnType GetColumnTypeCore(int iinfo) + { + Host.Assert(0 <= iinfo & iinfo < Infos.Length); + return _types[iinfo]; + } + + protected override Delegate GetGetterCore(IChannel ch, IRow input, int iinfo, out Action disposer) + { + Host.AssertValueOrNull(ch); + Host.AssertValue(input); + Host.Assert(0 <= iinfo && iinfo < Infos.Length); + + var type = _types[iinfo]; + var ex = _exes[iinfo]; + bool needScale = ex.Offset != 0 || ex.Scale != 1; + disposer = null; + var sourceType = Schema.GetColumnType(Infos[iinfo].Source); + if (sourceType.ItemType == NumberType.R4 || sourceType.ItemType == NumberType.R8) + return GetterFromType(input, iinfo, ex, needScale); + else + if (sourceType.ItemType == NumberType.U1) + return GetterFromType(input, iinfo, ex, false); + else + throw Contracts.Except("We only support float or byte arrays"); + + } + + private ValueGetter GetterFromType(IRow input, int iinfo, ColInfoEx ex, bool needScale) where TValue : IConvertible + { + var getSrc = GetSrcGetter>(input, iinfo); + var src = default(VBuffer); + int width = ex.Width; + int height = ex.Height; + float offset = ex.Offset; + float scale = ex.Scale; + + return + (ref Bitmap dst) => + { + getSrc(ref src); + if (src.Count == 0) + { + dst = null; + return; + } + VBuffer dense = default; + src.CopyToDense(ref dense); + var values = dense.Values; + dst = new Bitmap(width, height); + dst.SetResolution(width, height); + int cpix = height * width; + int planes = dense.Count / cpix; + int position = 0; + + for (int x = 0; x < width; x++) + for (int y = 0; y < height; ++y) + { + float red = 0; + float green = 0; + float blue = 0; + float alpha = 0; + if (ex.Interleave) + { + if (ex.Alpha) position++; + if (ex.Red) red = Convert.ToSingle(values[position++]); + if (ex.Green) green = Convert.ToSingle(values[position++]); + if (ex.Blue) blue = Convert.ToSingle(values[position++]); + } + else + { + position = y * width + x; + if (ex.Alpha) { alpha = Convert.ToSingle(values[position]); position += cpix; } + if (ex.Red) { red = Convert.ToSingle(values[position]); position += cpix; } + if (ex.Green) { green = Convert.ToSingle(values[position]); position += cpix; } + if (ex.Blue) { blue = Convert.ToSingle(values[position]); position += cpix; } + } + Color pixel; + if (!needScale) + pixel = Color.FromArgb((int)alpha, (int)red, (int)green, (int)blue); + else + { + pixel = Color.FromArgb( + (int)((alpha - offset) * scale), + (int)((red - offset) * scale), + (int)((green - offset) * scale), + (int)((blue - offset) * scale)); + } + dst.SetPixel(x, y, pixel); + } + }; + } + } +} + diff --git a/src/Microsoft.ML/CSharpApi.cs b/src/Microsoft.ML/CSharpApi.cs index c32aa23694..b4a4c4bb94 100644 --- a/src/Microsoft.ML/CSharpApi.cs +++ b/src/Microsoft.ML/CSharpApi.cs @@ -1102,6 +1102,54 @@ public void Add(Microsoft.ML.Transforms.HashConverter input, Microsoft.ML.Transf _jsonNodes.Add(Serialize("Transforms.HashConverter", input, output)); } + public Microsoft.ML.Transforms.ImageGrayscale.Output Add(Microsoft.ML.Transforms.ImageGrayscale input) + { + var output = new Microsoft.ML.Transforms.ImageGrayscale.Output(); + Add(input, output); + return output; + } + + public void Add(Microsoft.ML.Transforms.ImageGrayscale input, Microsoft.ML.Transforms.ImageGrayscale.Output output) + { + _jsonNodes.Add(Serialize("Transforms.ImageGrayscale", input, output)); + } + + public Microsoft.ML.Transforms.ImageLoader.Output Add(Microsoft.ML.Transforms.ImageLoader input) + { + var output = new Microsoft.ML.Transforms.ImageLoader.Output(); + Add(input, output); + return output; + } + + public void Add(Microsoft.ML.Transforms.ImageLoader input, Microsoft.ML.Transforms.ImageLoader.Output output) + { + _jsonNodes.Add(Serialize("Transforms.ImageLoader", input, output)); + } + + public Microsoft.ML.Transforms.ImagePixelExtractor.Output Add(Microsoft.ML.Transforms.ImagePixelExtractor input) + { + var output = new Microsoft.ML.Transforms.ImagePixelExtractor.Output(); + Add(input, output); + return output; + } + + public void Add(Microsoft.ML.Transforms.ImagePixelExtractor input, Microsoft.ML.Transforms.ImagePixelExtractor.Output output) + { + _jsonNodes.Add(Serialize("Transforms.ImagePixelExtractor", input, output)); + } + + public Microsoft.ML.Transforms.ImageResizer.Output Add(Microsoft.ML.Transforms.ImageResizer input) + { + var output = new Microsoft.ML.Transforms.ImageResizer.Output(); + Add(input, output); + return output; + } + + public void Add(Microsoft.ML.Transforms.ImageResizer input, Microsoft.ML.Transforms.ImageResizer.Output output) + { + _jsonNodes.Add(Serialize("Transforms.ImageResizer", input, output)); + } + public Microsoft.ML.Transforms.KeyToTextConverter.Output Add(Microsoft.ML.Transforms.KeyToTextConverter input) { var output = new Microsoft.ML.Transforms.KeyToTextConverter.Output(); @@ -1534,6 +1582,18 @@ public void Add(Microsoft.ML.Transforms.TwoHeterogeneousModelCombiner input, Mic _jsonNodes.Add(Serialize("Transforms.TwoHeterogeneousModelCombiner", input, output)); } + public Microsoft.ML.Transforms.VectorToImage.Output Add(Microsoft.ML.Transforms.VectorToImage input) + { + var output = new Microsoft.ML.Transforms.VectorToImage.Output(); + Add(input, output); + return output; + } + + public void Add(Microsoft.ML.Transforms.VectorToImage input, Microsoft.ML.Transforms.VectorToImage.Output output) + { + _jsonNodes.Add(Serialize("Transforms.VectorToImage", input, output)); + } + public Microsoft.ML.Transforms.WordEmbeddings.Output Add(Microsoft.ML.Transforms.WordEmbeddings input) { var output = new Microsoft.ML.Transforms.WordEmbeddings.Output(); @@ -11986,7 +12046,7 @@ public HashConverterPipelineStep(Output output) namespace Transforms { - public sealed partial class KeyToValueTransformColumn : OneToOneColumn, IOneToOneColumn + public sealed partial class ImageGrayscaleTransformColumn : OneToOneColumn, IOneToOneColumn { /// /// Name of the new column @@ -12000,15 +12060,17 @@ public sealed partial class KeyToValueTransformColumn : OneToOneColumn - public sealed partial class KeyToTextConverter : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem + /// + /// Convert image into grayscale. + /// + public sealed partial class ImageGrayscale : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem { - public KeyToTextConverter() + public ImageGrayscale() { } - public KeyToTextConverter(params string[] inputColumns) + public ImageGrayscale(params string[] inputColumns) { if (inputColumns != null) { @@ -12019,7 +12081,7 @@ public KeyToTextConverter(params string[] inputColumns) } } - public KeyToTextConverter(params (string inputColumn, string outputColumn)[] inputOutputColumns) + public ImageGrayscale(params (string inputColumn, string outputColumn)[] inputOutputColumns) { if (inputOutputColumns != null) { @@ -12032,15 +12094,15 @@ public KeyToTextConverter(params (string inputColumn, string outputColumn)[] inp public void AddColumn(string inputColumn) { - var list = Column == null ? new List() : new List(Column); - list.Add(OneToOneColumn.Create(inputColumn)); + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(inputColumn)); Column = list.ToArray(); } public void AddColumn(string outputColumn, string inputColumn) { - var list = Column == null ? new List() : new List(Column); - list.Add(OneToOneColumn.Create(outputColumn, inputColumn)); + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(outputColumn, inputColumn)); Column = list.ToArray(); } @@ -12048,7 +12110,7 @@ public void AddColumn(string outputColumn, string inputColumn) /// /// New column definition(s) (optional form: name:src) /// - public KeyToValueTransformColumn[] Column { get; set; } + public ImageGrayscaleTransformColumn[] Column { get; set; } /// /// Input dataset @@ -12077,18 +12139,18 @@ public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Exper { if (!(previousStep is ILearningPipelineDataStep dataStep)) { - throw new InvalidOperationException($"{ nameof(KeyToTextConverter)} only supports an { nameof(ILearningPipelineDataStep)} as an input."); + throw new InvalidOperationException($"{ nameof(ImageGrayscale)} only supports an { nameof(ILearningPipelineDataStep)} as an input."); } Data = dataStep.Data; } Output output = experiment.Add(this); - return new KeyToTextConverterPipelineStep(output); + return new ImageGrayscalePipelineStep(output); } - private class KeyToTextConverterPipelineStep : ILearningPipelineDataStep + private class ImageGrayscalePipelineStep : ILearningPipelineDataStep { - public KeyToTextConverterPipelineStep(Output output) + public ImageGrayscalePipelineStep(Output output) { Data = output.OutputData; Model = output.Model; @@ -12103,22 +12165,76 @@ public KeyToTextConverterPipelineStep(Output output) namespace Transforms { + public sealed partial class ImageLoaderTransformColumn : OneToOneColumn, IOneToOneColumn + { + /// + /// Name of the new column + /// + public string Name { get; set; } + + /// + /// Name of the source column + /// + public string Source { get; set; } + + } + /// - /// Transforms the label to either key or bool (if needed) to make it suitable for classification. + /// Load images from files. /// - public sealed partial class LabelColumnKeyBooleanConverter : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem + public sealed partial class ImageLoader : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem { + public ImageLoader() + { + } + + public ImageLoader(params string[] inputColumns) + { + if (inputColumns != null) + { + foreach (string input in inputColumns) + { + AddColumn(input); + } + } + } + + public ImageLoader(params (string inputColumn, string outputColumn)[] inputOutputColumns) + { + if (inputOutputColumns != null) + { + foreach (var inputOutput in inputOutputColumns) + { + AddColumn(inputOutput.outputColumn, inputOutput.inputColumn); + } + } + } + + public void AddColumn(string inputColumn) + { + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(inputColumn)); + Column = list.ToArray(); + } + + public void AddColumn(string outputColumn, string inputColumn) + { + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(outputColumn, inputColumn)); + Column = list.ToArray(); + } + /// - /// Convert the key values to text + /// New column definition(s) (optional form: name:src) /// - public bool TextKeyValues { get; set; } = true; + public ImageLoaderTransformColumn[] Column { get; set; } /// - /// The label column + /// Folder where to search for images /// - public string LabelColumn { get; set; } + public string ImageFolder { get; set; } /// /// Input dataset @@ -12147,18 +12263,18 @@ public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Exper { if (!(previousStep is ILearningPipelineDataStep dataStep)) { - throw new InvalidOperationException($"{ nameof(LabelColumnKeyBooleanConverter)} only supports an { nameof(ILearningPipelineDataStep)} as an input."); + throw new InvalidOperationException($"{ nameof(ImageLoader)} only supports an { nameof(ILearningPipelineDataStep)} as an input."); } Data = dataStep.Data; } Output output = experiment.Add(this); - return new LabelColumnKeyBooleanConverterPipelineStep(output); + return new ImageLoaderPipelineStep(output); } - private class LabelColumnKeyBooleanConverterPipelineStep : ILearningPipelineDataStep + private class ImageLoaderPipelineStep : ILearningPipelineDataStep { - public LabelColumnKeyBooleanConverterPipelineStep(Output output) + public ImageLoaderPipelineStep(Output output) { Data = output.OutputData; Model = output.Model; @@ -12173,12 +12289,47 @@ public LabelColumnKeyBooleanConverterPipelineStep(Output output) namespace Transforms { - public sealed partial class LabelIndicatorTransformColumn : OneToOneColumn, IOneToOneColumn + public sealed partial class ImagePixelExtractorTransformColumn : OneToOneColumn, IOneToOneColumn { /// - /// The positive example class for binary classification. + /// Whether to use alpha channel /// - public int? ClassIndex { get; set; } + public bool? UseAlpha { get; set; } + + /// + /// Whether to use red channel + /// + public bool? UseRed { get; set; } + + /// + /// Whether to use green channel + /// + public bool? UseGreen { get; set; } + + /// + /// Whether to use blue channel + /// + public bool? UseBlue { get; set; } + + /// + /// Whether to separate each channel or interleave in ARGB order + /// + public bool? InterleaveArgb { get; set; } + + /// + /// Whether to convert to floating point + /// + public bool? Convert { get; set; } + + /// + /// Offset (pre-scale) + /// + public float? Offset { get; set; } + + /// + /// Scale factor + /// + public float? Scale { get; set; } /// /// Name of the new column @@ -12193,16 +12344,16 @@ public sealed partial class LabelIndicatorTransformColumn : OneToOneColumn - public sealed partial class LabelIndicator : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem + public sealed partial class ImagePixelExtractor : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem { - public LabelIndicator() + public ImagePixelExtractor() { } - public LabelIndicator(params string[] inputColumns) + public ImagePixelExtractor(params string[] inputColumns) { if (inputColumns != null) { @@ -12213,7 +12364,7 @@ public LabelIndicator(params string[] inputColumns) } } - public LabelIndicator(params (string inputColumn, string outputColumn)[] inputOutputColumns) + public ImagePixelExtractor(params (string inputColumn, string outputColumn)[] inputOutputColumns) { if (inputOutputColumns != null) { @@ -12226,15 +12377,15 @@ public LabelIndicator(params (string inputColumn, string outputColumn)[] inputOu public void AddColumn(string inputColumn) { - var list = Column == null ? new List() : new List(Column); - list.Add(OneToOneColumn.Create(inputColumn)); + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(inputColumn)); Column = list.ToArray(); } public void AddColumn(string outputColumn, string inputColumn) { - var list = Column == null ? new List() : new List(Column); - list.Add(OneToOneColumn.Create(outputColumn, inputColumn)); + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(outputColumn, inputColumn)); Column = list.ToArray(); } @@ -12242,77 +12393,47 @@ public void AddColumn(string outputColumn, string inputColumn) /// /// New column definition(s) (optional form: name:src) /// - public LabelIndicatorTransformColumn[] Column { get; set; } + public ImagePixelExtractorTransformColumn[] Column { get; set; } /// - /// Label of the positive class. + /// Whether to use alpha channel /// - public int ClassIndex { get; set; } + public bool UseAlpha { get; set; } = false; /// - /// Input dataset + /// Whether to use red channel /// - public Var Data { get; set; } = new Var(); - - - public sealed class Output : Microsoft.ML.Runtime.EntryPoints.CommonOutputs.ITransformOutput - { - /// - /// Transformed dataset - /// - public Var OutputData { get; set; } = new Var(); - - /// - /// Transform model - /// - public Var Model { get; set; } = new Var(); - - } - public Var GetInputData() => Data; - - public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment) - { - if (previousStep != null) - { - if (!(previousStep is ILearningPipelineDataStep dataStep)) - { - throw new InvalidOperationException($"{ nameof(LabelIndicator)} only supports an { nameof(ILearningPipelineDataStep)} as an input."); - } - - Data = dataStep.Data; - } - Output output = experiment.Add(this); - return new LabelIndicatorPipelineStep(output); - } + public bool UseRed { get; set; } = true; - private class LabelIndicatorPipelineStep : ILearningPipelineDataStep - { - public LabelIndicatorPipelineStep(Output output) - { - Data = output.OutputData; - Model = output.Model; - } + /// + /// Whether to use green channel + /// + public bool UseGreen { get; set; } = true; - public Var Data { get; } - public Var Model { get; } - } - } - } + /// + /// Whether to use blue channel + /// + public bool UseBlue { get; set; } = true; - namespace Transforms - { + /// + /// Whether to separate each channel or interleave in ARGB order + /// + public bool InterleaveArgb { get; set; } = false; - /// - /// Transforms the label to float to make it suitable for regression. - /// - public sealed partial class LabelToFloatConverter : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem - { + /// + /// Whether to convert to floating point + /// + public bool Convert { get; set; } = true; + /// + /// Offset (pre-scale) + /// + public float? Offset { get; set; } /// - /// The label column + /// Scale factor /// - public string LabelColumn { get; set; } + public float? Scale { get; set; } /// /// Input dataset @@ -12341,18 +12462,18 @@ public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Exper { if (!(previousStep is ILearningPipelineDataStep dataStep)) { - throw new InvalidOperationException($"{ nameof(LabelToFloatConverter)} only supports an { nameof(ILearningPipelineDataStep)} as an input."); + throw new InvalidOperationException($"{ nameof(ImagePixelExtractor)} only supports an { nameof(ILearningPipelineDataStep)} as an input."); } Data = dataStep.Data; } Output output = experiment.Add(this); - return new LabelToFloatConverterPipelineStep(output); + return new ImagePixelExtractorPipelineStep(output); } - private class LabelToFloatConverterPipelineStep : ILearningPipelineDataStep + private class ImagePixelExtractorPipelineStep : ILearningPipelineDataStep { - public LabelToFloatConverterPipelineStep(Output output) + public ImagePixelExtractorPipelineStep(Output output) { Data = output.OutputData; Model = output.Model; @@ -12366,56 +12487,611 @@ public LabelToFloatConverterPipelineStep(Output output) namespace Transforms { + public enum ImageResizerTransformResizingKind : byte + { + IsoPad = 0, + IsoCrop = 1 + } - public sealed partial class LdaTransformColumn : OneToOneColumn, IOneToOneColumn + public enum ImageResizerTransformAnchor : byte { - /// - /// The number of topics in the LDA - /// - public int? NumTopic { get; set; } + Right = 0, + Left = 1, + Top = 2, + Bottom = 3, + Center = 4 + } - /// - /// Dirichlet prior on document-topic vectors - /// - public float? AlphaSum { get; set; } + public sealed partial class ImageResizerTransformColumn : OneToOneColumn, IOneToOneColumn + { /// - /// Dirichlet prior on vocab-topic vectors + /// Width of the resized image /// - public float? Beta { get; set; } + public int? ImageWidth { get; set; } /// - /// Number of Metropolis Hasting step + /// Height of the resized image /// - public int? Mhstep { get; set; } + public int? ImageHeight { get; set; } /// - /// Number of iterations + /// Resizing method /// - public int? NumIterations { get; set; } + public ImageResizerTransformResizingKind? Resizing { get; set; } /// - /// Compute log likelihood over local dataset on this iteration interval + /// Anchor for cropping /// - public int? LikelihoodInterval { get; set; } + public ImageResizerTransformAnchor? CropAnchor { get; set; } /// - /// The number of training threads + /// Name of the new column /// - public int? NumThreads { get; set; } + public string Name { get; set; } /// - /// The threshold of maximum count of tokens per doc + /// Name of the source column /// - public int? NumMaxDocToken { get; set; } + public string Source { get; set; } - /// - /// The number of words to summarize the topic - /// - public int? NumSummaryTermPerTopic { get; set; } + } - /// - /// The number of burn-in iterations + /// + /// Scales an image to specified dimensions using one of the three scale types: isotropic with padding, isotropic with cropping or anisotropic. In case of isotropic padding, transparent color is used to pad resulting image. + /// + public sealed partial class ImageResizer : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem + { + + public ImageResizer() + { + } + + public ImageResizer(params string[] inputColumns) + { + if (inputColumns != null) + { + foreach (string input in inputColumns) + { + AddColumn(input); + } + } + } + + public ImageResizer(params (string inputColumn, string outputColumn)[] inputOutputColumns) + { + if (inputOutputColumns != null) + { + foreach (var inputOutput in inputOutputColumns) + { + AddColumn(inputOutput.outputColumn, inputOutput.inputColumn); + } + } + } + + public void AddColumn(string inputColumn) + { + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(inputColumn)); + Column = list.ToArray(); + } + + public void AddColumn(string outputColumn, string inputColumn) + { + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(outputColumn, inputColumn)); + Column = list.ToArray(); + } + + + /// + /// New column definition(s) (optional form: name:src) + /// + public ImageResizerTransformColumn[] Column { get; set; } + + /// + /// Resized width of the image + /// + public int ImageWidth { get; set; } + + /// + /// Resized height of the image + /// + public int ImageHeight { get; set; } + + /// + /// Resizing method + /// + public ImageResizerTransformResizingKind Resizing { get; set; } = ImageResizerTransformResizingKind.IsoCrop; + + /// + /// Anchor for cropping + /// + public ImageResizerTransformAnchor CropAnchor { get; set; } = ImageResizerTransformAnchor.Center; + + /// + /// Input dataset + /// + public Var Data { get; set; } = new Var(); + + + public sealed class Output : Microsoft.ML.Runtime.EntryPoints.CommonOutputs.ITransformOutput + { + /// + /// Transformed dataset + /// + public Var OutputData { get; set; } = new Var(); + + /// + /// Transform model + /// + public Var Model { get; set; } = new Var(); + + } + public Var GetInputData() => Data; + + public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment) + { + if (previousStep != null) + { + if (!(previousStep is ILearningPipelineDataStep dataStep)) + { + throw new InvalidOperationException($"{ nameof(ImageResizer)} only supports an { nameof(ILearningPipelineDataStep)} as an input."); + } + + Data = dataStep.Data; + } + Output output = experiment.Add(this); + return new ImageResizerPipelineStep(output); + } + + private class ImageResizerPipelineStep : ILearningPipelineDataStep + { + public ImageResizerPipelineStep(Output output) + { + Data = output.OutputData; + Model = output.Model; + } + + public Var Data { get; } + public Var Model { get; } + } + } + } + + namespace Transforms + { + + public sealed partial class KeyToValueTransformColumn : OneToOneColumn, IOneToOneColumn + { + /// + /// Name of the new column + /// + public string Name { get; set; } + + /// + /// Name of the source column + /// + public string Source { get; set; } + + } + + /// + public sealed partial class KeyToTextConverter : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem + { + + public KeyToTextConverter() + { + } + + public KeyToTextConverter(params string[] inputColumns) + { + if (inputColumns != null) + { + foreach (string input in inputColumns) + { + AddColumn(input); + } + } + } + + public KeyToTextConverter(params (string inputColumn, string outputColumn)[] inputOutputColumns) + { + if (inputOutputColumns != null) + { + foreach (var inputOutput in inputOutputColumns) + { + AddColumn(inputOutput.outputColumn, inputOutput.inputColumn); + } + } + } + + public void AddColumn(string inputColumn) + { + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(inputColumn)); + Column = list.ToArray(); + } + + public void AddColumn(string outputColumn, string inputColumn) + { + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(outputColumn, inputColumn)); + Column = list.ToArray(); + } + + + /// + /// New column definition(s) (optional form: name:src) + /// + public KeyToValueTransformColumn[] Column { get; set; } + + /// + /// Input dataset + /// + public Var Data { get; set; } = new Var(); + + + public sealed class Output : Microsoft.ML.Runtime.EntryPoints.CommonOutputs.ITransformOutput + { + /// + /// Transformed dataset + /// + public Var OutputData { get; set; } = new Var(); + + /// + /// Transform model + /// + public Var Model { get; set; } = new Var(); + + } + public Var GetInputData() => Data; + + public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment) + { + if (previousStep != null) + { + if (!(previousStep is ILearningPipelineDataStep dataStep)) + { + throw new InvalidOperationException($"{ nameof(KeyToTextConverter)} only supports an { nameof(ILearningPipelineDataStep)} as an input."); + } + + Data = dataStep.Data; + } + Output output = experiment.Add(this); + return new KeyToTextConverterPipelineStep(output); + } + + private class KeyToTextConverterPipelineStep : ILearningPipelineDataStep + { + public KeyToTextConverterPipelineStep(Output output) + { + Data = output.OutputData; + Model = output.Model; + } + + public Var Data { get; } + public Var Model { get; } + } + } + } + + namespace Transforms + { + + /// + /// Transforms the label to either key or bool (if needed) to make it suitable for classification. + /// + public sealed partial class LabelColumnKeyBooleanConverter : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem + { + + + /// + /// Convert the key values to text + /// + public bool TextKeyValues { get; set; } = true; + + /// + /// The label column + /// + public string LabelColumn { get; set; } + + /// + /// Input dataset + /// + public Var Data { get; set; } = new Var(); + + + public sealed class Output : Microsoft.ML.Runtime.EntryPoints.CommonOutputs.ITransformOutput + { + /// + /// Transformed dataset + /// + public Var OutputData { get; set; } = new Var(); + + /// + /// Transform model + /// + public Var Model { get; set; } = new Var(); + + } + public Var GetInputData() => Data; + + public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment) + { + if (previousStep != null) + { + if (!(previousStep is ILearningPipelineDataStep dataStep)) + { + throw new InvalidOperationException($"{ nameof(LabelColumnKeyBooleanConverter)} only supports an { nameof(ILearningPipelineDataStep)} as an input."); + } + + Data = dataStep.Data; + } + Output output = experiment.Add(this); + return new LabelColumnKeyBooleanConverterPipelineStep(output); + } + + private class LabelColumnKeyBooleanConverterPipelineStep : ILearningPipelineDataStep + { + public LabelColumnKeyBooleanConverterPipelineStep(Output output) + { + Data = output.OutputData; + Model = output.Model; + } + + public Var Data { get; } + public Var Model { get; } + } + } + } + + namespace Transforms + { + + public sealed partial class LabelIndicatorTransformColumn : OneToOneColumn, IOneToOneColumn + { + /// + /// The positive example class for binary classification. + /// + public int? ClassIndex { get; set; } + + /// + /// Name of the new column + /// + public string Name { get; set; } + + /// + /// Name of the source column + /// + public string Source { get; set; } + + } + + /// + /// Label remapper used by OVA + /// + public sealed partial class LabelIndicator : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem + { + + public LabelIndicator() + { + } + + public LabelIndicator(params string[] inputColumns) + { + if (inputColumns != null) + { + foreach (string input in inputColumns) + { + AddColumn(input); + } + } + } + + public LabelIndicator(params (string inputColumn, string outputColumn)[] inputOutputColumns) + { + if (inputOutputColumns != null) + { + foreach (var inputOutput in inputOutputColumns) + { + AddColumn(inputOutput.outputColumn, inputOutput.inputColumn); + } + } + } + + public void AddColumn(string inputColumn) + { + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(inputColumn)); + Column = list.ToArray(); + } + + public void AddColumn(string outputColumn, string inputColumn) + { + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(outputColumn, inputColumn)); + Column = list.ToArray(); + } + + + /// + /// New column definition(s) (optional form: name:src) + /// + public LabelIndicatorTransformColumn[] Column { get; set; } + + /// + /// Label of the positive class. + /// + public int ClassIndex { get; set; } + + /// + /// Input dataset + /// + public Var Data { get; set; } = new Var(); + + + public sealed class Output : Microsoft.ML.Runtime.EntryPoints.CommonOutputs.ITransformOutput + { + /// + /// Transformed dataset + /// + public Var OutputData { get; set; } = new Var(); + + /// + /// Transform model + /// + public Var Model { get; set; } = new Var(); + + } + public Var GetInputData() => Data; + + public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment) + { + if (previousStep != null) + { + if (!(previousStep is ILearningPipelineDataStep dataStep)) + { + throw new InvalidOperationException($"{ nameof(LabelIndicator)} only supports an { nameof(ILearningPipelineDataStep)} as an input."); + } + + Data = dataStep.Data; + } + Output output = experiment.Add(this); + return new LabelIndicatorPipelineStep(output); + } + + private class LabelIndicatorPipelineStep : ILearningPipelineDataStep + { + public LabelIndicatorPipelineStep(Output output) + { + Data = output.OutputData; + Model = output.Model; + } + + public Var Data { get; } + public Var Model { get; } + } + } + } + + namespace Transforms + { + + /// + /// Transforms the label to float to make it suitable for regression. + /// + public sealed partial class LabelToFloatConverter : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem + { + + + /// + /// The label column + /// + public string LabelColumn { get; set; } + + /// + /// Input dataset + /// + public Var Data { get; set; } = new Var(); + + + public sealed class Output : Microsoft.ML.Runtime.EntryPoints.CommonOutputs.ITransformOutput + { + /// + /// Transformed dataset + /// + public Var OutputData { get; set; } = new Var(); + + /// + /// Transform model + /// + public Var Model { get; set; } = new Var(); + + } + public Var GetInputData() => Data; + + public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment) + { + if (previousStep != null) + { + if (!(previousStep is ILearningPipelineDataStep dataStep)) + { + throw new InvalidOperationException($"{ nameof(LabelToFloatConverter)} only supports an { nameof(ILearningPipelineDataStep)} as an input."); + } + + Data = dataStep.Data; + } + Output output = experiment.Add(this); + return new LabelToFloatConverterPipelineStep(output); + } + + private class LabelToFloatConverterPipelineStep : ILearningPipelineDataStep + { + public LabelToFloatConverterPipelineStep(Output output) + { + Data = output.OutputData; + Model = output.Model; + } + + public Var Data { get; } + public Var Model { get; } + } + } + } + + namespace Transforms + { + + public sealed partial class LdaTransformColumn : OneToOneColumn, IOneToOneColumn + { + /// + /// The number of topics in the LDA + /// + public int? NumTopic { get; set; } + + /// + /// Dirichlet prior on document-topic vectors + /// + public float? AlphaSum { get; set; } + + /// + /// Dirichlet prior on vocab-topic vectors + /// + public float? Beta { get; set; } + + /// + /// Number of Metropolis Hasting step + /// + public int? Mhstep { get; set; } + + /// + /// Number of iterations + /// + public int? NumIterations { get; set; } + + /// + /// Compute log likelihood over local dataset on this iteration interval + /// + public int? LikelihoodInterval { get; set; } + + /// + /// The number of training threads + /// + public int? NumThreads { get; set; } + + /// + /// The threshold of maximum count of tokens per doc + /// + public int? NumMaxDocToken { get; set; } + + /// + /// The number of words to summarize the topic + /// + public int? NumSummaryTermPerTopic { get; set; } + + /// + /// The number of burn-in iterations /// public int? NumBurninIterations { get; set; } = 10; @@ -15542,6 +16218,215 @@ public sealed class Output } } + namespace Transforms + { + + public sealed partial class VectorToImageTransformColumn : OneToOneColumn, IOneToOneColumn + { + /// + /// Whether to use alpha channel + /// + public bool? ContainsAlpha { get; set; } + + /// + /// Whether to use red channel + /// + public bool? ContainsRed { get; set; } + + /// + /// Whether to use green channel + /// + public bool? ContainsGreen { get; set; } + + /// + /// Whether to use blue channel + /// + public bool? ContainsBlue { get; set; } + + /// + /// Whether to separate each channel or interleave in ARGB order + /// + public bool? InterleaveArgb { get; set; } + + /// + /// Width of the image + /// + public int? ImageWidth { get; set; } + + /// + /// Height of the image + /// + public int? ImageHeight { get; set; } + + /// + /// Offset (pre-scale) + /// + public float? Offset { get; set; } + + /// + /// Scale factor + /// + public float? Scale { get; set; } + + /// + /// Name of the new column + /// + public string Name { get; set; } + + /// + /// Name of the source column + /// + public string Source { get; set; } + + } + + /// + /// Converts vector array into image type. + /// + public sealed partial class VectorToImage : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem + { + + public VectorToImage() + { + } + + public VectorToImage(params string[] inputColumns) + { + if (inputColumns != null) + { + foreach (string input in inputColumns) + { + AddColumn(input); + } + } + } + + public VectorToImage(params (string inputColumn, string outputColumn)[] inputOutputColumns) + { + if (inputOutputColumns != null) + { + foreach (var inputOutput in inputOutputColumns) + { + AddColumn(inputOutput.outputColumn, inputOutput.inputColumn); + } + } + } + + public void AddColumn(string inputColumn) + { + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(inputColumn)); + Column = list.ToArray(); + } + + public void AddColumn(string outputColumn, string inputColumn) + { + var list = Column == null ? new List() : new List(Column); + list.Add(OneToOneColumn.Create(outputColumn, inputColumn)); + Column = list.ToArray(); + } + + + /// + /// New column definition(s) (optional form: name:src) + /// + public VectorToImageTransformColumn[] Column { get; set; } + + /// + /// Whether to use alpha channel + /// + public bool ContainsAlpha { get; set; } = false; + + /// + /// Whether to use red channel + /// + public bool ContainsRed { get; set; } = true; + + /// + /// Whether to use green channel + /// + public bool ContainsGreen { get; set; } = true; + + /// + /// Whether to use blue channel + /// + public bool ContainsBlue { get; set; } = true; + + /// + /// Whether to separate each channel or interleave in ARGB order + /// + public bool InterleaveArgb { get; set; } = false; + + /// + /// Width of the image + /// + public int ImageWidth { get; set; } + + /// + /// Height of the image + /// + public int ImageHeight { get; set; } + + /// + /// Offset (pre-scale) + /// + public float? Offset { get; set; } + + /// + /// Scale factor + /// + public float? Scale { get; set; } + + /// + /// Input dataset + /// + public Var Data { get; set; } = new Var(); + + + public sealed class Output : Microsoft.ML.Runtime.EntryPoints.CommonOutputs.ITransformOutput + { + /// + /// Transformed dataset + /// + public Var OutputData { get; set; } = new Var(); + + /// + /// Transform model + /// + public Var Model { get; set; } = new Var(); + + } + public Var GetInputData() => Data; + + public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment) + { + if (previousStep != null) + { + if (!(previousStep is ILearningPipelineDataStep dataStep)) + { + throw new InvalidOperationException($"{ nameof(VectorToImage)} only supports an { nameof(ILearningPipelineDataStep)} as an input."); + } + + Data = dataStep.Data; + } + Output output = experiment.Add(this); + return new VectorToImagePipelineStep(output); + } + + private class VectorToImagePipelineStep : ILearningPipelineDataStep + { + public VectorToImagePipelineStep(Output output) + { + Data = output.OutputData; + Model = output.Model; + } + + public Var Data { get; } + public Var Model { get; } + } + } + } + namespace Transforms { public enum WordEmbeddingsTransformPretrainedModelKind diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv index 7ab4900d95..ba4eceb25f 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv +++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv @@ -88,6 +88,10 @@ Transforms.FeatureSelectorByCount Selects the slots for which the count of non-d Transforms.FeatureSelectorByMutualInformation Selects the top k slots across all specified columns ordered by their mutual information with the label column. Microsoft.ML.Runtime.EntryPoints.SelectFeatures MutualInformationSelect Microsoft.ML.Runtime.Data.MutualInformationFeatureSelectionTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.GlobalContrastNormalizer Performs a global contrast normalization on input values: Y = (s * X - M) / D, where s is a scale, M is mean and D is either L2 norm or standard deviation. Microsoft.ML.Runtime.Data.LpNormalization GcNormalize Microsoft.ML.Runtime.Data.LpNormNormalizerTransform+GcnArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.HashConverter Converts column values into hashes. This transform accepts both numeric and text inputs, both single and vector-valued columns. This is a part of the Dracula transform. Microsoft.ML.Runtime.Data.HashJoin Apply Microsoft.ML.Runtime.Data.HashJoinTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput +Transforms.ImageGrayscale Convert image into grayscale. Microsoft.ML.Runtime.ImageAnalytics.EntryPoints.ImageAnalytics ImageGrayscale Microsoft.ML.Runtime.ImageAnalytics.ImageGrayscaleTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput +Transforms.ImageLoader Load images from files. Microsoft.ML.Runtime.ImageAnalytics.EntryPoints.ImageAnalytics ImageLoader Microsoft.ML.Runtime.ImageAnalytics.ImageLoaderTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput +Transforms.ImagePixelExtractor Extract color plane(s) from an image. Options include scaling, offset and conversion to floating point. Microsoft.ML.Runtime.ImageAnalytics.EntryPoints.ImageAnalytics ImagePixelExtractor Microsoft.ML.Runtime.ImageAnalytics.ImagePixelExtractorTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput +Transforms.ImageResizer Scales an image to specified dimensions using one of the three scale types: isotropic with padding, isotropic with cropping or anisotropic. In case of isotropic padding, transparent color is used to pad resulting image. Microsoft.ML.Runtime.ImageAnalytics.EntryPoints.ImageAnalytics ImageResizer Microsoft.ML.Runtime.ImageAnalytics.ImageResizerTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.KeyToTextConverter KeyToValueTransform utilizes KeyValues metadata to map key indices to the corresponding values in the KeyValues metadata. Microsoft.ML.Runtime.Data.Categorical KeyToText Microsoft.ML.Runtime.Data.KeyToValueTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LabelColumnKeyBooleanConverter Transforms the label to either key or bool (if needed) to make it suitable for classification. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner PrepareClassificationLabel Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+ClassificationLabelInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LabelIndicator Label remapper used by OVA Microsoft.ML.Runtime.Data.LabelIndicatorTransform LabelIndicator Microsoft.ML.Runtime.Data.LabelIndicatorTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput @@ -124,5 +128,6 @@ Transforms.TextToKeyConverter Converts input values (words, numbers, etc.) to in Transforms.TrainTestDatasetSplitter Split the dataset into train and test sets Microsoft.ML.Runtime.EntryPoints.TrainTestSplit Split Microsoft.ML.Runtime.EntryPoints.TrainTestSplit+Input Microsoft.ML.Runtime.EntryPoints.TrainTestSplit+Output Transforms.TreeLeafFeaturizer Trains a tree ensemble, or loads it from a file, then maps a numeric feature vector to three outputs: 1. A vector containing the individual tree outputs of the tree ensemble. 2. A vector indicating the leaves that the feature vector falls on in the tree ensemble. 3. A vector indicating the paths that the feature vector falls on in the tree ensemble. If a both a model file and a trainer are specified - will use the model file. If neither are specified, will train a default FastTree model. This can handle key labels by training a regression model towards their optionally permuted indices. Microsoft.ML.Runtime.Data.TreeFeaturize Featurizer Microsoft.ML.Runtime.Data.TreeEnsembleFeaturizerTransform+ArgumentsForEntryPoint Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.TwoHeterogeneousModelCombiner Combines a TransformModel and a PredictorModel into a single PredictorModel. Microsoft.ML.Runtime.EntryPoints.ModelOperations CombineTwoModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+SimplePredictorModelInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelOutput +Transforms.VectorToImage Converts vector array into image type. Microsoft.ML.Runtime.ImageAnalytics.EntryPoints.ImageAnalytics VectorToImage Microsoft.ML.Runtime.ImageAnalytics.VectorToImageTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.WordEmbeddings Word Embeddings transform is a text featurizer which converts vectors of text tokens into sentence vectors using a pre-trained model Microsoft.ML.Runtime.Transforms.TextAnalytics WordEmbeddings Microsoft.ML.Runtime.Data.WordEmbeddingsTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.WordTokenizer The input to this transform is text, and the output is a vector of text containing the words (tokens) in the original text. The separator is space, but can be specified as any other character (or multiple characters) if needed. Microsoft.ML.Runtime.Transforms.TextAnalytics DelimitedTokenizeTransform Microsoft.ML.Runtime.Data.DelimitedTokenizeTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index a8aac09113..35b554c6ed 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -17983,6 +17983,628 @@ "ITransformOutput" ] }, + { + "Name": "Transforms.ImageGrayscale", + "Desc": "Convert image into grayscale.", + "FriendlyName": "Image Greyscale Transform", + "ShortName": "ImageGrayscaleTransform", + "Inputs": [ + { + "Name": "Column", + "Type": { + "Kind": "Array", + "ItemType": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Name", + "Type": "String", + "Desc": "Name of the new column", + "Aliases": [ + "name" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "Source", + "Type": "String", + "Desc": "Name of the source column", + "Aliases": [ + "src" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + } + ] + } + }, + "Desc": "New column definition(s) (optional form: name:src)", + "Aliases": [ + "col" + ], + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "OutputData", + "Type": "DataView", + "Desc": "Transformed dataset" + }, + { + "Name": "Model", + "Type": "TransformModel", + "Desc": "Transform model" + } + ], + "InputKind": [ + "ITransformInput" + ], + "OutputKind": [ + "ITransformOutput" + ] + }, + { + "Name": "Transforms.ImageLoader", + "Desc": "Load images from files.", + "FriendlyName": "Image Loader Transform", + "ShortName": "ImageLoaderTransform", + "Inputs": [ + { + "Name": "Column", + "Type": { + "Kind": "Array", + "ItemType": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Name", + "Type": "String", + "Desc": "Name of the new column", + "Aliases": [ + "name" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "Source", + "Type": "String", + "Desc": "Name of the source column", + "Aliases": [ + "src" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + } + ] + } + }, + "Desc": "New column definition(s) (optional form: name:src)", + "Aliases": [ + "col" + ], + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "ImageFolder", + "Type": "String", + "Desc": "Folder where to search for images", + "Aliases": [ + "folder" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + } + ], + "Outputs": [ + { + "Name": "OutputData", + "Type": "DataView", + "Desc": "Transformed dataset" + }, + { + "Name": "Model", + "Type": "TransformModel", + "Desc": "Transform model" + } + ], + "InputKind": [ + "ITransformInput" + ], + "OutputKind": [ + "ITransformOutput" + ] + }, + { + "Name": "Transforms.ImagePixelExtractor", + "Desc": "Extract color plane(s) from an image. Options include scaling, offset and conversion to floating point.", + "FriendlyName": "Image Pixel Extractor Transform", + "ShortName": "ImagePixelExtractor", + "Inputs": [ + { + "Name": "Column", + "Type": { + "Kind": "Array", + "ItemType": { + "Kind": "Struct", + "Fields": [ + { + "Name": "UseAlpha", + "Type": "Bool", + "Desc": "Whether to use alpha channel", + "Aliases": [ + "alpha" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "UseRed", + "Type": "Bool", + "Desc": "Whether to use red channel", + "Aliases": [ + "red" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "UseGreen", + "Type": "Bool", + "Desc": "Whether to use green channel", + "Aliases": [ + "green" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "UseBlue", + "Type": "Bool", + "Desc": "Whether to use blue channel", + "Aliases": [ + "blue" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "InterleaveArgb", + "Type": "Bool", + "Desc": "Whether to separate each channel or interleave in ARGB order", + "Aliases": [ + "interleave" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Convert", + "Type": "Bool", + "Desc": "Whether to convert to floating point", + "Aliases": [ + "conv" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Offset", + "Type": "Float", + "Desc": "Offset (pre-scale)", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Scale", + "Type": "Float", + "Desc": "Scale factor", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Name", + "Type": "String", + "Desc": "Name of the new column", + "Aliases": [ + "name" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "Source", + "Type": "String", + "Desc": "Name of the source column", + "Aliases": [ + "src" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + } + ] + } + }, + "Desc": "New column definition(s) (optional form: name:src)", + "Aliases": [ + "col" + ], + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "UseAlpha", + "Type": "Bool", + "Desc": "Whether to use alpha channel", + "Aliases": [ + "alpha" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "UseRed", + "Type": "Bool", + "Desc": "Whether to use red channel", + "Aliases": [ + "red" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "UseGreen", + "Type": "Bool", + "Desc": "Whether to use green channel", + "Aliases": [ + "green" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "UseBlue", + "Type": "Bool", + "Desc": "Whether to use blue channel", + "Aliases": [ + "blue" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "InterleaveArgb", + "Type": "Bool", + "Desc": "Whether to separate each channel or interleave in ARGB order", + "Aliases": [ + "interleave" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "Convert", + "Type": "Bool", + "Desc": "Whether to convert to floating point", + "Aliases": [ + "conv" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "Offset", + "Type": "Float", + "Desc": "Offset (pre-scale)", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Scale", + "Type": "Float", + "Desc": "Scale factor", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + } + ], + "Outputs": [ + { + "Name": "OutputData", + "Type": "DataView", + "Desc": "Transformed dataset" + }, + { + "Name": "Model", + "Type": "TransformModel", + "Desc": "Transform model" + } + ], + "InputKind": [ + "ITransformInput" + ], + "OutputKind": [ + "ITransformOutput" + ] + }, + { + "Name": "Transforms.ImageResizer", + "Desc": "Scales an image to specified dimensions using one of the three scale types: isotropic with padding, isotropic with cropping or anisotropic. In case of isotropic padding, transparent color is used to pad resulting image.", + "FriendlyName": "Image Resizer Transform", + "ShortName": "ImageScalerTransform", + "Inputs": [ + { + "Name": "Column", + "Type": { + "Kind": "Array", + "ItemType": { + "Kind": "Struct", + "Fields": [ + { + "Name": "ImageWidth", + "Type": "Int", + "Desc": "Width of the resized image", + "Aliases": [ + "width" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "ImageHeight", + "Type": "Int", + "Desc": "Height of the resized image", + "Aliases": [ + "height" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Resizing", + "Type": { + "Kind": "Enum", + "Values": [ + "IsoPad", + "IsoCrop" + ] + }, + "Desc": "Resizing method", + "Aliases": [ + "scale" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "CropAnchor", + "Type": { + "Kind": "Enum", + "Values": [ + "Right", + "Left", + "Top", + "Bottom", + "Center" + ] + }, + "Desc": "Anchor for cropping", + "Aliases": [ + "anchor" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Name", + "Type": "String", + "Desc": "Name of the new column", + "Aliases": [ + "name" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "Source", + "Type": "String", + "Desc": "Name of the source column", + "Aliases": [ + "src" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + } + ] + } + }, + "Desc": "New column definition(s) (optional form: name:src)", + "Aliases": [ + "col" + ], + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "ImageWidth", + "Type": "Int", + "Desc": "Resized width of the image", + "Aliases": [ + "width" + ], + "Required": true, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 0 + }, + { + "Name": "ImageHeight", + "Type": "Int", + "Desc": "Resized height of the image", + "Aliases": [ + "height" + ], + "Required": true, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 0 + }, + { + "Name": "Resizing", + "Type": { + "Kind": "Enum", + "Values": [ + "IsoPad", + "IsoCrop" + ] + }, + "Desc": "Resizing method", + "Aliases": [ + "scale" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": "IsoCrop" + }, + { + "Name": "CropAnchor", + "Type": { + "Kind": "Enum", + "Values": [ + "Right", + "Left", + "Top", + "Bottom", + "Center" + ] + }, + "Desc": "Anchor for cropping", + "Aliases": [ + "anchor" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": "Center" + } + ], + "Outputs": [ + { + "Name": "OutputData", + "Type": "DataView", + "Desc": "Transformed dataset" + }, + { + "Name": "Model", + "Type": "TransformModel", + "Desc": "Transform model" + } + ], + "InputKind": [ + "ITransformInput" + ], + "OutputKind": [ + "ITransformOutput" + ] + }, { "Name": "Transforms.KeyToTextConverter", "Desc": "KeyToValueTransform utilizes KeyValues metadata to map key indices to the corresponding values in the KeyValues metadata.", @@ -21593,6 +22215,286 @@ } ] }, + { + "Name": "Transforms.VectorToImage", + "Desc": "Converts vector array into image type.", + "FriendlyName": "Vector To Image Transform", + "ShortName": "VectorToImageConverter", + "Inputs": [ + { + "Name": "Column", + "Type": { + "Kind": "Array", + "ItemType": { + "Kind": "Struct", + "Fields": [ + { + "Name": "ContainsAlpha", + "Type": "Bool", + "Desc": "Whether to use alpha channel", + "Aliases": [ + "alpha" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "ContainsRed", + "Type": "Bool", + "Desc": "Whether to use red channel", + "Aliases": [ + "red" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "ContainsGreen", + "Type": "Bool", + "Desc": "Whether to use green channel", + "Aliases": [ + "green" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "ContainsBlue", + "Type": "Bool", + "Desc": "Whether to use blue channel", + "Aliases": [ + "blue" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "InterleaveArgb", + "Type": "Bool", + "Desc": "Whether to separate each channel or interleave in ARGB order", + "Aliases": [ + "interleave" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "ImageWidth", + "Type": "Int", + "Desc": "Width of the image", + "Aliases": [ + "width" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "ImageHeight", + "Type": "Int", + "Desc": "Height of the image", + "Aliases": [ + "height" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Offset", + "Type": "Float", + "Desc": "Offset (pre-scale)", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Scale", + "Type": "Float", + "Desc": "Scale factor", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Name", + "Type": "String", + "Desc": "Name of the new column", + "Aliases": [ + "name" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "Source", + "Type": "String", + "Desc": "Name of the source column", + "Aliases": [ + "src" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + } + ] + } + }, + "Desc": "New column definition(s) (optional form: name:src)", + "Aliases": [ + "col" + ], + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "ContainsAlpha", + "Type": "Bool", + "Desc": "Whether to use alpha channel", + "Aliases": [ + "alpha" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "ContainsRed", + "Type": "Bool", + "Desc": "Whether to use red channel", + "Aliases": [ + "red" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "ContainsGreen", + "Type": "Bool", + "Desc": "Whether to use green channel", + "Aliases": [ + "green" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "ContainsBlue", + "Type": "Bool", + "Desc": "Whether to use blue channel", + "Aliases": [ + "blue" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "InterleaveArgb", + "Type": "Bool", + "Desc": "Whether to separate each channel or interleave in ARGB order", + "Aliases": [ + "interleave" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "ImageWidth", + "Type": "Int", + "Desc": "Width of the image", + "Aliases": [ + "width" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 0 + }, + { + "Name": "ImageHeight", + "Type": "Int", + "Desc": "Height of the image", + "Aliases": [ + "height" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 0 + }, + { + "Name": "Offset", + "Type": "Float", + "Desc": "Offset (pre-scale)", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Scale", + "Type": "Float", + "Desc": "Scale factor", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + } + ], + "Outputs": [ + { + "Name": "OutputData", + "Type": "DataView", + "Desc": "Transformed dataset" + }, + { + "Name": "Model", + "Type": "TransformModel", + "Desc": "Transform model" + } + ], + "InputKind": [ + "ITransformInput" + ], + "OutputKind": [ + "ITransformOutput" + ] + }, { "Name": "Transforms.WordEmbeddings", "Desc": "Word Embeddings transform is a text featurizer which converts vectors of text tokens into sentence vectors using a pre-trained model", diff --git a/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj b/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj index ea92975b35..a9b1b991ae 100644 --- a/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj +++ b/test/Microsoft.ML.Core.Tests/Microsoft.ML.Core.Tests.csproj @@ -9,6 +9,7 @@ + diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index d9db7c432b..c44678bf07 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -265,6 +265,53 @@ private string GetBuildPrefix() #endif } + [Fact(Skip = "Execute this test if you want to regenerate ep-list and _manifest.json")] + public void RegenerateEntryPointCatalog() + { + var buildPrefix = GetBuildPrefix(); + var epListFile = buildPrefix + "_ep-list.tsv"; + var manifestFile = buildPrefix + "_manifest.json"; + + var entryPointsSubDir = Path.Combine("..", "Common", "EntryPoints"); + var catalog = ModuleCatalog.CreateInstance(Env); + var epListPath = GetBaselinePath(entryPointsSubDir, epListFile); + DeleteOutputPath(epListPath); + + var regex = new Regex(@"\r\n?|\n", RegexOptions.Compiled); + File.WriteAllLines(epListPath, catalog.AllEntryPoints() + .Select(x => string.Join("\t", + x.Name, + regex.Replace(x.Description, ""), + x.Method.DeclaringType, + x.Method.Name, + x.InputType, + x.OutputType) + .Replace(Environment.NewLine, "")) + .OrderBy(x => x)); + + + var jObj = JsonManifestUtils.BuildAllManifests(Env, catalog); + + //clean up the description from the new line characters + if (jObj[FieldNames.TopEntryPoints] != null && jObj[FieldNames.TopEntryPoints] is JArray) + { + foreach (JToken entry in jObj[FieldNames.TopEntryPoints].Children()) + if (entry[FieldNames.Desc] != null) + entry[FieldNames.Desc] = regex.Replace(entry[FieldNames.Desc].ToString(), ""); + } + var manifestPath = GetBaselinePath(entryPointsSubDir, manifestFile); + DeleteOutputPath(manifestPath); + + using (var file = File.OpenWrite(manifestPath)) + using (var writer = new StreamWriter(file)) + using (var jw = new JsonTextWriter(writer)) + { + jw.Formatting = Formatting.Indented; + jObj.WriteTo(jw); + } + } + + [Fact] public void EntryPointCatalog() { diff --git a/test/Microsoft.ML.Tests/ImagesTests.cs b/test/Microsoft.ML.Tests/ImagesTests.cs new file mode 100644 index 0000000000..a12032400a --- /dev/null +++ b/test/Microsoft.ML.Tests/ImagesTests.cs @@ -0,0 +1,183 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime.Api; +using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.ImageAnalytics; +using Microsoft.ML.TestFramework; +using System.Drawing; +using System.IO; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.ML.Tests +{ + public class ImageTests : BaseTestClass + { + public ImageTests(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public void TestSaveImages() + { + using (var env = new TlcEnvironment()) + { + var dataFile = GetDataPath("images/images.tsv"); + var imageFolder = Path.GetDirectoryName(dataFile); + var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile)); + var images = new ImageLoaderTransform(env, new ImageLoaderTransform.Arguments() + { + Column = new ImageLoaderTransform.Column[1] + { + new ImageLoaderTransform.Column() { Source= "ImagePath", Name="ImageReal" } + }, + ImageFolder = imageFolder + }, data); + + var cropped = new ImageResizerTransform(env, new ImageResizerTransform.Arguments() + { + Column = new ImageResizerTransform.Column[1]{ + new ImageResizerTransform.Column() { Name= "ImageCropped", Source = "ImageReal", ImageHeight =100, ImageWidth = 100, Resizing = ImageResizerTransform.ResizingKind.IsoPad} + } + }, images); + + cropped.Schema.TryGetColumnIndex("ImagePath", out int pathColumn); + cropped.Schema.TryGetColumnIndex("ImageCropped", out int cropBitmapColumn); + using (var cursor = cropped.GetRowCursor((x) => true)) + { + var pathGetter = cursor.GetGetter(pathColumn); + DvText path = default; + var bitmapCropGetter = cursor.GetGetter(cropBitmapColumn); + Bitmap bitmap = default; + while (cursor.MoveNext()) + { + pathGetter(ref path); + bitmapCropGetter(ref bitmap); + Assert.NotNull(bitmap); + var fileToSave = GetOutputPath(Path.GetFileNameWithoutExtension(path.ToString()) + ".cropped.jpg"); + bitmap.Save(fileToSave, System.Drawing.Imaging.ImageFormat.Jpeg); + } + } + } + } + + [Fact] + public void TestGreyscaleTransformImages() + { + using (var env = new TlcEnvironment()) + { + var imageHeight = 150; + var imageWidth = 100; + var dataFile = GetDataPath("images/images.tsv"); + var imageFolder = Path.GetDirectoryName(dataFile); + var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile)); + var images = new ImageLoaderTransform(env, new ImageLoaderTransform.Arguments() + { + Column = new ImageLoaderTransform.Column[1] + { + new ImageLoaderTransform.Column() { Source= "ImagePath", Name="ImageReal" } + }, + ImageFolder = imageFolder + }, data); + var cropped = new ImageResizerTransform(env, new ImageResizerTransform.Arguments() + { + Column = new ImageResizerTransform.Column[1]{ + new ImageResizerTransform.Column() { Name= "ImageCropped", Source = "ImageReal", ImageHeight =imageHeight, ImageWidth = imageWidth, Resizing = ImageResizerTransform.ResizingKind.IsoCrop} + } + }, images); + + var grey = new ImageGrayscaleTransform(env, new ImageGrayscaleTransform.Arguments() + { + Column = new ImageGrayscaleTransform.Column[1]{ + new ImageGrayscaleTransform.Column() { Name= "ImageGrey", Source = "ImageCropped"} + } + }, cropped); + + grey.Schema.TryGetColumnIndex("ImageGrey", out int greyColumn); + using (var cursor = grey.GetRowCursor((x) => true)) + { + var bitmapGetter = cursor.GetGetter(greyColumn); + Bitmap bitmap = default; + while (cursor.MoveNext()) + { + bitmapGetter(ref bitmap); + Assert.NotNull(bitmap); + for (int x = 0; x < imageWidth; x++) + for (int y = 0; y < imageHeight; y++) + { + var pixel = bitmap.GetPixel(x, y); + // greyscale image has same values for R,G and B + Assert.True(pixel.R == pixel.G && pixel.G == pixel.B); + } + } + } + } + } + + [Fact] + public void TestBackAndForthConversion() + { + using (var env = new TlcEnvironment()) + { + var imageHeight = 100; + var imageWidth = 130; + var dataFile = GetDataPath("images/images.tsv"); + var imageFolder = Path.GetDirectoryName(dataFile); + var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile)); + var images = new ImageLoaderTransform(env, new ImageLoaderTransform.Arguments() + { + Column = new ImageLoaderTransform.Column[1] + { + new ImageLoaderTransform.Column() { Source= "ImagePath", Name="ImageReal" } + }, + ImageFolder = imageFolder + }, data); + var cropped = new ImageResizerTransform(env, new ImageResizerTransform.Arguments() + { + Column = new ImageResizerTransform.Column[1]{ + new ImageResizerTransform.Column() { Source = "ImageReal", Name= "ImageCropped", ImageHeight =imageHeight, ImageWidth = imageWidth, Resizing = ImageResizerTransform.ResizingKind.IsoCrop} + } + }, images); + + var pixels = new ImagePixelExtractorTransform(env, new ImagePixelExtractorTransform.Arguments() + { + Column = new ImagePixelExtractorTransform.Column[1]{ + new ImagePixelExtractorTransform.Column() { Source= "ImageCropped", Name = "ImagePixels", UseAlpha=true} + } + }, cropped); + + var backToBitmaps = new VectorToImageTransform(env, new VectorToImageTransform.Arguments() + { + Column = new VectorToImageTransform.Column[1]{ + new VectorToImageTransform.Column() { Source= "ImagePixels", Name = "ImageRestored" , ImageHeight=imageHeight, ImageWidth=imageWidth, ContainsAlpha=true} + } + }, pixels); + + backToBitmaps.Schema.TryGetColumnIndex("ImageRestored", out int bitmapColumn); + backToBitmaps.Schema.TryGetColumnIndex("ImageCropped", out int cropBitmapColumn); + using (var cursor = backToBitmaps.GetRowCursor((x) => true)) + { + var bitmapGetter = cursor.GetGetter(bitmapColumn); + Bitmap restoredBitmap = default; + + var bitmapCropGetter = cursor.GetGetter(cropBitmapColumn); + Bitmap croppedBitmap = default; + while (cursor.MoveNext()) + { + bitmapGetter(ref restoredBitmap); + Assert.NotNull(restoredBitmap); + bitmapCropGetter(ref croppedBitmap); + Assert.NotNull(croppedBitmap); + for (int x = 0; x < imageWidth; x++) + for (int y = 0; y < imageHeight; y++) + { + Assert.True(croppedBitmap.GetPixel(x, y) == restoredBitmap.GetPixel(x, y)); + } + } + } + } + } + } +} diff --git a/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj b/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj index 2a2ea8bca1..b088cfb05e 100644 --- a/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj +++ b/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj @@ -5,6 +5,7 @@ + diff --git a/test/data/images/banana.jpg b/test/data/images/banana.jpg new file mode 100644 index 0000000000..45cb14b607 Binary files /dev/null and b/test/data/images/banana.jpg differ diff --git a/test/data/images/hotdog.jpg b/test/data/images/hotdog.jpg new file mode 100644 index 0000000000..94c918c7dc Binary files /dev/null and b/test/data/images/hotdog.jpg differ diff --git a/test/data/images/images.tsv b/test/data/images/images.tsv new file mode 100644 index 0000000000..0315f7865b --- /dev/null +++ b/test/data/images/images.tsv @@ -0,0 +1,3 @@ +banana.jpg banana +hotdog.jpg hotdog +tomato.jpg tomato \ No newline at end of file diff --git a/test/data/images/tomato.jpg b/test/data/images/tomato.jpg new file mode 100644 index 0000000000..b101856e14 Binary files /dev/null and b/test/data/images/tomato.jpg differ