diff --git a/src/Microsoft.ML.ImageAnalytics/EntryPoints/ImageAnalytics.cs b/src/Microsoft.ML.ImageAnalytics/EntryPoints/ImageAnalytics.cs index 522ebd0347..f5404445f4 100644 --- a/src/Microsoft.ML.ImageAnalytics/EntryPoints/ImageAnalytics.cs +++ b/src/Microsoft.ML.ImageAnalytics/EntryPoints/ImageAnalytics.cs @@ -63,12 +63,12 @@ public static CommonOutputs.TransformOutput ImageGrayscale(IHostEnvironment env, }; } - [TlcModule.EntryPoint(Name = "Transforms.VectorToImage", Desc = VectorToImageTransform.Summary, - UserName = VectorToImageTransform.UserName, ShortName = VectorToImageTransform.LoaderSignature)] - public static CommonOutputs.TransformOutput VectorToImage(IHostEnvironment env, VectorToImageTransform.Arguments input) + [TlcModule.EntryPoint(Name = "Transforms.VectorToImage", Desc = VectorToImageConvertingTransformer.Summary, + UserName = VectorToImageConvertingTransformer.UserName, ShortName = VectorToImageConvertingTransformer.LoaderSignature)] + public static CommonOutputs.TransformOutput VectorToImage(IHostEnvironment env, VectorToImageConvertingTransformer.Options input) { var h = EntryPointUtils.CheckArgsAndCreateHost(env, "VectorToImageTransform", input); - var xf = new VectorToImageTransform(h, input, input.Data); + var xf = VectorToImageConvertingTransformer.Create(h, input, input.Data); return new CommonOutputs.TransformOutput() { Model = new TransformModelImpl(h, xf, input.Data), diff --git a/src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs b/src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs index b44afeb6a3..40cf183604 100644 --- a/src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs @@ -72,7 +72,7 @@ public static ImagePixelExtractingEstimator ExtractPixels(this TransformsCatalog /// /// The transform's catalog. - /// The name of the columns containing the image paths, and per-column configurations. + /// The name of the columns containing the images, and per-column configurations. public static ImagePixelExtractingEstimator ExtractPixels(this TransformsCatalog catalog, params ImagePixelExtractingEstimator.ColumnInfo[] columns) => new ImagePixelExtractingEstimator(CatalogUtils.GetEnvironment(catalog), columns); @@ -133,5 +133,33 @@ public static ImageResizingEstimator ResizeImages(this TransformsCatalog catalog /// public static ImageResizingEstimator ResizeImages(this TransformsCatalog catalog, params ImageResizingEstimator.ColumnInfo[] columns) => new ImageResizingEstimator(CatalogUtils.GetEnvironment(catalog), columns); + + /// + /// Converts vectors of pixels into representation. + /// + /// The transform's catalog. + /// The name of the columns containing the pixels, and per-column configurations. + public static VectorToImageConvertingEstimator ConvertToImage(this TransformsCatalog catalog, params VectorToImageConvertingEstimator.ColumnInfo[] columns) + => new VectorToImageConvertingEstimator(CatalogUtils.GetEnvironment(catalog), columns); + + /// + /// Converts vectors of pixels into representation. + /// + /// The transforms' catalog. + /// The height of the output images. + /// The width of the output images. + /// Name of the column resulting from the transformation of . + /// Name of column to transform. If set to , the value of the will be used as source. + /// Specifies which are in the input pixel vectors. The order of colors is: Alpha, Red, Green Blue. + /// Whether the pixels are interleaved, meaning whether they are in `ARGB ARGB` order, or separated in the planar form, where the colors are specified one by one + /// alpha, red, green, blue for all the pixels of the image. + /// The values are scaled by this value before being converted to pixels. + /// The offset is subtracted (before scaling) before converting the values to pixels. + public static VectorToImageConvertingEstimator ConvertToImage(this TransformsCatalog catalog, int height, int width, string outputColumnName, string inputColumnName = null, + ImagePixelExtractingEstimator.ColorBits colors = VectorToImageConvertingTransformer.Defaults.Colors, + bool interleave = VectorToImageConvertingTransformer.Defaults.InterleaveArgb, + float scale = VectorToImageConvertingTransformer.Defaults.Scale, + float offset = VectorToImageConvertingTransformer.Defaults.Offset) + => new VectorToImageConvertingEstimator(CatalogUtils.GetEnvironment(catalog), height, width, outputColumnName, inputColumnName, colors, interleave, scale, offset); } } diff --git a/src/Microsoft.ML.ImageAnalytics/VectorToImageTransform.cs b/src/Microsoft.ML.ImageAnalytics/VectorToImageTransform.cs index d53eee3aeb..a4e7b0f6c5 100644 --- a/src/Microsoft.ML.ImageAnalytics/VectorToImageTransform.cs +++ b/src/Microsoft.ML.ImageAnalytics/VectorToImageTransform.cs @@ -3,7 +3,9 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using System.Drawing; +using System.Linq; using System.Text; using Microsoft.Data.DataView; using Microsoft.ML; @@ -14,22 +16,31 @@ using Microsoft.ML.Internal.Utilities; using Microsoft.ML.Model; -[assembly: LoadableClass(VectorToImageTransform.Summary, typeof(VectorToImageTransform), typeof(VectorToImageTransform.Arguments), - typeof(SignatureDataTransform), VectorToImageTransform.UserName, "VectorToImageTransform", "VectorToImage")] +[assembly: LoadableClass(VectorToImageConvertingTransformer.Summary, typeof(IDataTransform), typeof(VectorToImageConvertingTransformer), typeof(VectorToImageConvertingTransformer.Options), typeof(SignatureDataTransform), + ImagePixelExtractingTransformer.UserName, "VectorToImageTransform", "VectorToImage")] -[assembly: LoadableClass(VectorToImageTransform.Summary, typeof(VectorToImageTransform), null, typeof(SignatureLoadDataTransform), - VectorToImageTransform.UserName, VectorToImageTransform.LoaderSignature)] +[assembly: LoadableClass(VectorToImageConvertingTransformer.Summary, typeof(IDataTransform), typeof(VectorToImageConvertingTransformer), null, typeof(SignatureLoadDataTransform), + VectorToImageConvertingTransformer.UserName, VectorToImageConvertingTransformer.LoaderSignature)] + +[assembly: LoadableClass(typeof(VectorToImageConvertingTransformer), null, typeof(SignatureLoadModel), + VectorToImageConvertingTransformer.UserName, VectorToImageConvertingTransformer.LoaderSignature)] + +[assembly: LoadableClass(typeof(IRowMapper), typeof(VectorToImageConvertingTransformer), null, typeof(SignatureLoadRowMapper), + VectorToImageConvertingTransformer.UserName, VectorToImageConvertingTransformer.LoaderSignature)] namespace Microsoft.ML.ImageAnalytics { - // REVIEW: Rewrite as LambdaTransform to simplify. - /// - /// Transform which takes one or many columns with vectors in them and transform them to representation. + /// produced by fitting the to an . /// - internal sealed class VectorToImageTransform : OneToOneTransformBase + /// + /// + /// + /// + /// + public sealed class VectorToImageConvertingTransformer : OneToOneTransformerBase { - public class Column : OneToOneColumn + internal class Column : OneToOneColumn { [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use alpha channel", ShortName = "alpha")] public bool? ContainsAlpha; @@ -81,25 +92,25 @@ internal bool TryUnparse(StringBuilder sb) } } - public class Arguments : TransformInputBase + internal class Options: TransformInputBase { [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "New column definition(s) (optional form: name:src)", Name = "Column", ShortName = "col", SortOrder = 1)] public Column[] Columns; [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use alpha channel", ShortName = "alpha")] - public bool ContainsAlpha = false; + public bool ContainsAlpha = (Defaults.Colors & ImagePixelExtractingEstimator.ColorBits.Alpha) > 0; [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use red channel", ShortName = "red")] - public bool ContainsRed = true; + public bool ContainsRed = (Defaults.Colors & ImagePixelExtractingEstimator.ColorBits.Red) > 0; [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use green channel", ShortName = "green")] - public bool ContainsGreen = true; + public bool ContainsGreen = (Defaults.Colors & ImagePixelExtractingEstimator.ColorBits.Green) > 0; [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to use blue channel", ShortName = "blue")] - public bool ContainsBlue = true; + public bool ContainsBlue = (Defaults.Colors & ImagePixelExtractingEstimator.ColorBits.Blue) > 0; [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to separate each channel or interleave in ARGB order", ShortName = "interleave")] - public bool InterleaveArgb = false; + public bool InterleaveArgb = Defaults.InterleaveArgb; [Argument(ArgumentType.AtMostOnce, HelpText = "Width of the image", ShortName = "width")] public int ImageWidth; @@ -108,29 +119,289 @@ public class Arguments : TransformInputBase public int ImageHeight; [Argument(ArgumentType.AtMostOnce, HelpText = "Offset (pre-scale)")] - public Single? Offset; + public Single Offset = Defaults.Offset; [Argument(ArgumentType.AtMostOnce, HelpText = "Scale factor")] - public Single? Scale; + public Single Scale = Defaults.Scale; + } + + internal static class Defaults + { + public const ImagePixelExtractingEstimator.ColorBits Colors = ImagePixelExtractingEstimator.ColorBits.Rgb; + public const bool InterleaveArgb = false; + public const Single Offset = 0; + public const Single Scale = 1; + } + + internal const string Summary = "Converts vector array into image type."; + internal const string UserName = "Vector To Image Transform"; + internal const string LoaderSignature = "VectorToImageConverter"; + private static VersionInfo GetVersionInfo() + { + return new VersionInfo( + modelSignature: "VECTOIMG", + //verWrittenCur: 0x00010001, // Initial + //verWrittenCur: 0x00010002, // Swith from OpenCV to Bitmap + verWrittenCur: 0x00010003, // don't serialize sizeof(Single) + verReadableCur: 0x00010003, + verWeCanReadBack: 0x00010003, + loaderSignature: LoaderSignature, + loaderAssemblyName: typeof(VectorToImageConvertingTransformer).Assembly.FullName); } + private const string RegistrationName = "VectorToImageConverter"; + + private readonly VectorToImageConvertingEstimator.ColumnInfo[] _columns; + /// - /// Which color channels are extracted. Note that these values are serialized so should not be modified. + /// The columns passed to this . /// - [Flags] - private enum ColorBits : byte + public IReadOnlyCollection Columns => _columns.AsReadOnly(); + + internal VectorToImageConvertingTransformer(IHostEnvironment env, params VectorToImageConvertingEstimator.ColumnInfo[] columns) + :base(Contracts.CheckRef(env, nameof(env)).Register(RegistrationName), GetColumnPairs(columns)) + { + Host.AssertNonEmpty(columns); + + _columns = columns.ToArray(); + } + + internal VectorToImageConvertingTransformer(IHostEnvironment env, string outputColumnName, string inputColumnName, int imageHeight, int imageWidth, ImagePixelExtractingEstimator.ColorBits colors, bool interleave, float scale, float offset) + : this(env, new VectorToImageConvertingEstimator.ColumnInfo(outputColumnName, inputColumnName, imageHeight, imageWidth, colors, interleave, scale, offset)) + { + } + + // Constructor corresponding to SignatureDataTransform. + internal static IDataTransform Create(IHostEnvironment env, Options args, IDataView input) + { + Contracts.CheckValue(env, nameof(env)); + env.CheckValue(args, nameof(args)); + env.CheckValue(input, nameof(input)); + + env.CheckValue(args.Columns, nameof(args.Columns)); + + var columns = new VectorToImageConvertingEstimator.ColumnInfo[args.Columns.Length]; + for (int i = 0; i < columns.Length; i++) + { + var item = args.Columns[i]; + columns[i] = new VectorToImageConvertingEstimator.ColumnInfo(item, args); + } + + var transformer = new VectorToImageConvertingTransformer(env, columns); + return new RowToRowMapperTransform(env, input, transformer.MakeRowMapper(input.Schema), transformer.MakeRowMapper); + } + + private VectorToImageConvertingTransformer(IHost host, ModelLoadContext ctx) + : base(host, ctx) + { + Host.AssertValue(ctx); + + // *** Binary format *** + // + // foreach added column + // ColumnInfo + + _columns = new VectorToImageConvertingEstimator.ColumnInfo[ColumnPairs.Length]; + for (int i = 0; i < _columns.Length; i++) + _columns[i] = new VectorToImageConvertingEstimator.ColumnInfo(ColumnPairs[i].outputColumnName, ColumnPairs[i].inputColumnName, ctx); + } + + private static VectorToImageConvertingTransformer Create(IHostEnvironment env, ModelLoadContext ctx) + { + Contracts.CheckValue(env, nameof(env)); + var h = env.Register(RegistrationName); + h.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(GetVersionInfo()); + + return h.Apply("Loading Model", + ch => + { + return new VectorToImageConvertingTransformer(h, ctx); + }); + } + + // Factory method for SignatureLoadDataTransform. + private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) + => Create(env, ctx).MakeDataTransform(input); + + // Factory method for SignatureLoadRowMapper. + private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, DataViewSchema inputSchema) + => Create(env, ctx).MakeRowMapper(inputSchema); + + private protected override void SaveModel(ModelSaveContext ctx) + { + Host.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(); + ctx.SetVersionInfo(GetVersionInfo()); + + // *** Binary format *** + // + // foreach added column + // ColInfoEx + base.SaveColumns(ctx); + + for (int i = 0; i < _columns.Length; i++) + _columns[i].Save(ctx); + } + + private static (string outputColumnName, string inputColumnName)[] GetColumnPairs(VectorToImageConvertingEstimator.ColumnInfo[] columns) + { + Contracts.CheckValue(columns, nameof(columns)); + return columns.Select(x => (x.Name, x.InputColumnName)).ToArray(); + } + + private protected override IRowMapper MakeRowMapper(DataViewSchema schema) => new Mapper(this, schema); + + protected override void CheckInputColumn(DataViewSchema inputSchema, int col, int srcCol) { - Alpha = 0x01, - Red = 0x02, - Green = 0x04, - Blue = 0x08, + var inputColName = _columns[col].InputColumnName; + var vectorType = inputSchema[srcCol].Type as VectorType; + if (vectorType == null) + throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", inputColName, "image", inputSchema[srcCol].Type.ToString()); - All = Alpha | Red | Green | Blue + if (vectorType.GetValueCount() != _columns[col].Height * _columns[col].Width * _columns[col].Planes) + throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", inputColName, new VectorType(vectorType.ItemType, _columns[col].Height, _columns[col].Width, _columns[col].Planes).ToString(), vectorType.ToString()); } - private sealed class ColInfoEx + private sealed class Mapper : OneToOneMapperBase { - public readonly ColorBits Colors; + private readonly VectorToImageConvertingTransformer _parent; + private readonly ImageType[] _types; + + public Mapper(VectorToImageConvertingTransformer parent, DataViewSchema inputSchema) + : base(parent.Host.Register(nameof(Mapper)), parent, inputSchema) + { + _parent = parent; + _types = ConstructTypes(_parent._columns); + } + + protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore() + => _parent._columns.Select((x, idx) => new DataViewSchema.DetachedColumn(x.Name, _types[idx], null)).ToArray(); + + protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func activeOutput, out Action disposer) + { + Host.AssertValue(input); + Host.Assert(0 <= iinfo && iinfo < _parent._columns.Length); + + var type = _types[iinfo]; + var ex = _parent._columns[iinfo]; + bool needScale = ex.Offset != 0 || ex.Scale != 1; + disposer = null; + var sourceType = InputSchema[ColMapNewToOld[iinfo]].Type; + var sourceItemType = sourceType.GetItemType(); + if (sourceItemType == NumberDataViewType.Single || sourceItemType == NumberDataViewType.Double) + return GetterFromType(NumberDataViewType.Single, input, iinfo, ex, needScale); + else + if (sourceItemType == NumberDataViewType.Byte) + return GetterFromType(NumberDataViewType.Byte, input, iinfo, ex, false); + else + throw Contracts.Except("We only support float, double or byte arrays"); + } + + private ValueGetter GetterFromType(PrimitiveDataViewType srcType, DataViewRow input, int iinfo, + VectorToImageConvertingEstimator.ColumnInfo ex, bool needScale) where TValue : IConvertible + { + Contracts.Assert(typeof(TValue) == srcType.RawType); + var getSrc = RowCursorUtils.GetVecGetterAs(srcType, input, ColMapNewToOld[iinfo]); + var src = default(VBuffer); + int width = ex.Width; + int height = ex.Height; + float offset = ex.Offset; + float scale = ex.Scale; + + return + (ref Bitmap dst) => + { + getSrc(ref src); + if (src.GetValues().Length == 0) + { + dst = null; + return; + } + VBuffer dense = default; + src.CopyToDense(ref dense); + var values = dense.GetValues(); + dst = new Bitmap(width, height); + dst.SetResolution(width, height); + int cpix = height * width; + int position = 0; + + for (int y = 0; y < height; ++y) + for (int x = 0; x < width; x++) + { + float red = 0; + float green = 0; + float blue = 0; + float alpha = 0; + if (ex.Interleave) + { + if (ex.Alpha) + alpha = Convert.ToSingle(values[position++]); + if (ex.Red) + red = Convert.ToSingle(values[position++]); + if (ex.Green) + green = Convert.ToSingle(values[position++]); + if (ex.Blue) + blue = Convert.ToSingle(values[position++]); + } + else + { + position = y * width + x; + if (ex.Alpha) + { alpha = Convert.ToSingle(values[position]); position += cpix; } + if (ex.Red) + { red = Convert.ToSingle(values[position]); position += cpix; } + if (ex.Green) + { green = Convert.ToSingle(values[position]); position += cpix; } + if (ex.Blue) + { blue = Convert.ToSingle(values[position]); position += cpix; } + } + Color pixel; + if (!needScale) + pixel = Color.FromArgb((int)alpha, (int)red, (int)green, (int)blue); + else + { + pixel = Color.FromArgb( + ex.Alpha ? (int)Math.Round((alpha - offset) * scale) : 0, + (int)Math.Round((red - offset) * scale), + (int)Math.Round((green - offset) * scale), + (int)Math.Round((blue - offset) * scale)); + } + dst.SetPixel(x, y, pixel); + } + }; + } + + private static ImageType[] ConstructTypes(VectorToImageConvertingEstimator.ColumnInfo[] columns) + { + return columns.Select(c => new ImageType(c.Height, c.Width)).ToArray(); + } + } + } + + /// + /// that converts vectors containing pixel representations of images in to representation. + /// + /// + /// Calling in this estimator, produces an . + /// + /// + /// + /// + public sealed class VectorToImageConvertingEstimator : TrivialEstimator + { + /// + /// Describes how the transformer handles one image pixel extraction column pair. + /// + public sealed class ColumnInfo + { + /// Name of the column resulting from the transformation of . + public readonly string Name; + + /// Name of column to transform. + public readonly string InputColumnName; + + public readonly ImagePixelExtractingEstimator.ColorBits Colors; public readonly byte Planes; public readonly int Width; @@ -139,33 +410,48 @@ private sealed class ColInfoEx public readonly Single Scale; public readonly bool Interleave; - public bool Alpha { get { return (Colors & ColorBits.Alpha) != 0; } } - public bool Red { get { return (Colors & ColorBits.Red) != 0; } } - public bool Green { get { return (Colors & ColorBits.Green) != 0; } } - public bool Blue { get { return (Colors & ColorBits.Blue) != 0; } } + public bool Alpha => (Colors & ImagePixelExtractingEstimator.ColorBits.Alpha) != 0; + public bool Red => (Colors & ImagePixelExtractingEstimator.ColorBits.Red) != 0; + public bool Green => (Colors & ImagePixelExtractingEstimator.ColorBits.Green) != 0; + public bool Blue => (Colors & ImagePixelExtractingEstimator.ColorBits.Blue) != 0; - public ColInfoEx(Column item, Arguments args) + internal ColumnInfo(VectorToImageConvertingTransformer.Column item, VectorToImageConvertingTransformer.Options args) { - if (item.ContainsAlpha ?? args.ContainsAlpha) { Colors |= ColorBits.Alpha; Planes++; } - if (item.ContainsRed ?? args.ContainsRed) { Colors |= ColorBits.Red; Planes++; } - if (item.ContainsGreen ?? args.ContainsGreen) { Colors |= ColorBits.Green; Planes++; } - if (item.ContainsBlue ?? args.ContainsBlue) { Colors |= ColorBits.Blue; Planes++; } + Contracts.CheckValue(item, nameof(item)); + Contracts.CheckValue(args, nameof(args)); + + Name = item.Name; + InputColumnName = item.Source ?? item.Name; + + if (item.ContainsAlpha ?? args.ContainsAlpha) + { Colors |= ImagePixelExtractingEstimator.ColorBits.Alpha; Planes++; } + if (item.ContainsRed ?? args.ContainsRed) + { Colors |= ImagePixelExtractingEstimator.ColorBits.Red; Planes++; } + if (item.ContainsGreen ?? args.ContainsGreen) + { Colors |= ImagePixelExtractingEstimator.ColorBits.Green; Planes++; } + if (item.ContainsBlue ?? args.ContainsBlue) + { Colors |= ImagePixelExtractingEstimator.ColorBits.Blue; Planes++; } Contracts.CheckUserArg(Planes > 0, nameof(item.ContainsRed), "Need to use at least one color plane"); Interleave = item.InterleaveArgb ?? args.InterleaveArgb; Width = item.ImageWidth ?? args.ImageWidth; Height = item.ImageHeight ?? args.ImageHeight; - Offset = item.Offset ?? args.Offset ?? 0; - Scale = item.Scale ?? args.Scale ?? 1; + Offset = item.Offset ?? args.Offset; + Scale = item.Scale ?? args.Scale; Contracts.CheckUserArg(FloatUtils.IsFinite(Offset), nameof(item.Offset)); Contracts.CheckUserArg(FloatUtils.IsFiniteNonZero(Scale), nameof(item.Scale)); } - public ColInfoEx(ModelLoadContext ctx) + public ColumnInfo(string outputColumnName, string inputColumnName, ModelLoadContext ctx) { + Contracts.AssertNonEmpty(outputColumnName); + Contracts.AssertNonEmpty(inputColumnName); Contracts.AssertValue(ctx); + Name = outputColumnName; + InputColumnName = inputColumnName; + // *** Binary format *** // byte: colors // int: widht @@ -173,9 +459,9 @@ public ColInfoEx(ModelLoadContext ctx) // Float: offset // Float: scale // byte: separateChannels - Colors = (ColorBits)ctx.Reader.ReadByte(); + Colors = (ImagePixelExtractingEstimator.ColorBits)ctx.Reader.ReadByte(); Contracts.CheckDecode(Colors != 0); - Contracts.CheckDecode((Colors & ColorBits.All) == Colors); + Contracts.CheckDecode((Colors & ImagePixelExtractingEstimator.ColorBits.All) == Colors); // Count the planes. int planes = (int)Colors; @@ -195,6 +481,36 @@ public ColInfoEx(ModelLoadContext ctx) Interleave = ctx.Reader.ReadBoolByte(); } + public ColumnInfo(string outputColumnName, string inputColumnName, + int imageHeight, int imageWidth, ImagePixelExtractingEstimator.ColorBits colors, bool interleave, float scale, float offset) + { + Contracts.CheckNonWhiteSpace(outputColumnName, nameof(InputColumnName)); + + Name = outputColumnName; + InputColumnName = inputColumnName ?? outputColumnName; + Colors = colors; + if ((byte)(Colors & ImagePixelExtractingEstimator.ColorBits.Alpha) > 0) + Planes++; + if ((byte)(Colors & ImagePixelExtractingEstimator.ColorBits.Red) > 0) + Planes++; + if ((byte)(Colors & ImagePixelExtractingEstimator.ColorBits.Green) > 0) + Planes++; + if ((byte)(Colors & ImagePixelExtractingEstimator.ColorBits.Blue) > 0) + Planes++; + Contracts.CheckParam(Planes > 0, nameof(colors), "Need to use at least one color plane"); + + Interleave = interleave; + + Contracts.CheckParam(imageWidth > 0, nameof(imageWidth), "Image width must be greater than zero"); + Contracts.CheckParam(imageHeight > 0, nameof(imageHeight), "Image height must be greater than zero"); + Contracts.CheckParam(FloatUtils.IsFinite(offset), nameof(offset)); + Contracts.CheckParam(FloatUtils.IsFiniteNonZero(scale), nameof(scale)); + Width = imageWidth; + Height = imageHeight; + Offset = offset; + Scale = scale; + } + internal void Save(ModelSaveContext ctx) { Contracts.AssertValue(ctx); @@ -214,7 +530,7 @@ internal void Save(ModelSaveContext ctx) // Float: scale // byte: separateChannels Contracts.Assert(Colors != 0); - Contracts.Assert((Colors & ColorBits.All) == Colors); + Contracts.Assert((Colors & ImagePixelExtractingEstimator.ColorBits.All) == Colors); ctx.Writer.Write((byte)Colors); ctx.Writer.Write(Width); ctx.Writer.Write(Height); @@ -226,195 +542,63 @@ internal void Save(ModelSaveContext ctx) } } - public const string Summary = "Converts vector array into image type."; - public const string UserName = "Vector To Image Transform"; - public const string LoaderSignature = "VectorToImageConverter"; - private static VersionInfo GetVersionInfo() + /// + /// Convert pixels values into an image. + /// + /// The host environment. + /// The height of the image + /// The width of the image + /// Name of the column resulting from the transformation of . Null means is replaced. + /// Name of the input column. + /// What colors to extract. + /// Whether to interleave the pixels, meaning keep them in the `RGB RGB` order, or leave them in the plannar form: of all red pixels, + /// than all green, than all blue. + /// Scale color pixel value by this amount. + /// Offset color pixel value by this amount. + [BestFriend] + internal VectorToImageConvertingEstimator(IHostEnvironment env, + int imageHeight, + int imageWidth, + string outputColumnName, + string inputColumnName = null, + ImagePixelExtractingEstimator.ColorBits colors = ImagePixelExtractingEstimator.ColorBits.Rgb, + bool interleave = false, + float scale = 1, + float offset = 0) + : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(VectorToImageConvertingEstimator)), new VectorToImageConvertingTransformer(env, outputColumnName, inputColumnName, imageHeight, imageWidth, colors, interleave, scale, offset)) { - return new VersionInfo( - modelSignature: "VECTOIMG", - //verWrittenCur: 0x00010001, // Initial - verWrittenCur: 0x00010002, // Swith from OpenCV to Bitmap - verReadableCur: 0x00010002, - verWeCanReadBack: 0x00010002, - loaderSignature: LoaderSignature, - loaderAssemblyName: typeof(VectorToImageTransform).Assembly.FullName); } - private const string RegistrationName = "VectorToImageConverter"; - - private readonly ColInfoEx[] _exes; - private readonly ImageType[] _types; - - // Public constructor corresponding to SignatureDataTransform. - public VectorToImageTransform(IHostEnvironment env, Arguments args, IDataView input) - : base(env, RegistrationName, Contracts.CheckRef(args, nameof(args)).Columns, input, - t => t is VectorType ? null : "Expected VectorType type") + /// + /// Extract pixels values from image and produce array of values. + /// + /// The host environment. + /// Describes the parameters of pixel extraction for each column pair. + internal VectorToImageConvertingEstimator(IHostEnvironment env, params ColumnInfo[] columns) + : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(VectorToImageConvertingEstimator)), new VectorToImageConvertingTransformer(env, columns)) { - Host.AssertNonEmpty(Infos); - Host.Assert(Infos.Length == Utils.Size(args.Columns)); - - _exes = new ColInfoEx[Infos.Length]; - _types = new ImageType[Infos.Length]; - for (int i = 0; i < _exes.Length; i++) - { - var item = args.Columns[i]; - _exes[i] = new ColInfoEx(item, args); - _types[i] = new ImageType(_exes[i].Height, _exes[i].Width); - } - Metadata.Seal(); } - private VectorToImageTransform(IHost host, ModelLoadContext ctx, IDataView input) - : base(host, ctx, input, t => t is VectorType ? null : "Expected VectorType type") + /// + /// Returns the of the schema which will be produced by the transformer. + /// Used for schema propagation and verification in a pipeline. + /// + public override SchemaShape GetOutputSchema(SchemaShape inputSchema) { - Host.AssertValue(ctx); - - // *** Binary format *** - // - // - // foreach added column - // ColInfoEx - Host.AssertNonEmpty(Infos); - _exes = new ColInfoEx[Infos.Length]; - _types = new ImageType[Infos.Length]; - for (int i = 0; i < _exes.Length; i++) + Host.CheckValue(inputSchema, nameof(inputSchema)); + var result = inputSchema.ToDictionary(x => x.Name); + foreach (var colInfo in Transformer.Columns) { - _exes[i] = new ColInfoEx(ctx); - _types[i] = new ImageType(_exes[i].Height, _exes[i].Width); - } - Metadata.Seal(); - } + if (!inputSchema.TryFindColumn(colInfo.InputColumnName, out var col)) + throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", colInfo.InputColumnName); + if (col.Kind != SchemaShape.Column.VectorKind.Vector || (col.ItemType != NumberDataViewType.Single && col.ItemType != NumberDataViewType.Double && col.ItemType != NumberDataViewType.Byte)) + throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", colInfo.InputColumnName, "known-size vector of type float, double or byte", col.GetTypeString()); - public static VectorToImageTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input) - { - Contracts.CheckValue(env, nameof(env)); - var h = env.Register(RegistrationName); - h.CheckValue(ctx, nameof(ctx)); - h.CheckValue(input, nameof(input)); - ctx.CheckAtModel(GetVersionInfo()); - - return h.Apply("Loading Model", - ch => - { - // *** Binary format *** - // int: sizeof(Float) - // - int cbFloat = ctx.Reader.ReadInt32(); - ch.CheckDecode(cbFloat == sizeof(Single)); - return new VectorToImageTransform(h, ctx, input); - }); - } - - private protected override void SaveModel(ModelSaveContext ctx) - { - Host.CheckValue(ctx, nameof(ctx)); - ctx.CheckAtModel(); - ctx.SetVersionInfo(GetVersionInfo()); - - // *** Binary format *** - // int: sizeof(Float) - // - // foreach added column - // ColInfoEx - ctx.Writer.Write(sizeof(Single)); - SaveBase(ctx); - - Host.Assert(_exes.Length == Infos.Length); - for (int i = 0; i < _exes.Length; i++) - _exes[i].Save(ctx); - } - - protected override DataViewType GetColumnTypeCore(int iinfo) - { - Host.Assert(0 <= iinfo & iinfo < Infos.Length); - return _types[iinfo]; - } - - protected override Delegate GetGetterCore(IChannel ch, DataViewRow input, int iinfo, out Action disposer) - { - Host.AssertValueOrNull(ch); - Host.AssertValue(input); - Host.Assert(0 <= iinfo && iinfo < Infos.Length); - - var type = _types[iinfo]; - var ex = _exes[iinfo]; - bool needScale = ex.Offset != 0 || ex.Scale != 1; - disposer = null; - var sourceType = InputSchema[Infos[iinfo].Source].Type; - var sourceItemType = sourceType.GetItemType(); - if (sourceItemType == NumberDataViewType.Single || sourceItemType == NumberDataViewType.Double) - return GetterFromType(input, iinfo, ex, needScale); - else - if (sourceItemType == NumberDataViewType.Byte) - return GetterFromType(input, iinfo, ex, false); - else - throw Contracts.Except("We only support float or byte arrays"); - - } + var itemType = new ImageType(colInfo.Height, colInfo.Width); + result[colInfo.Name] = new SchemaShape.Column(colInfo.Name, SchemaShape.Column.VectorKind.Scalar, itemType, false); + } - private ValueGetter GetterFromType(DataViewRow input, int iinfo, ColInfoEx ex, bool needScale) where TValue : IConvertible - { - var getSrc = GetSrcGetter>(input, iinfo); - var src = default(VBuffer); - int width = ex.Width; - int height = ex.Height; - float offset = ex.Offset; - float scale = ex.Scale; - - return - (ref Bitmap dst) => - { - getSrc(ref src); - if (src.GetValues().Length == 0) - { - dst = null; - return; - } - VBuffer dense = default; - src.CopyToDense(ref dense); - var values = dense.GetValues(); - dst = new Bitmap(width, height); - dst.SetResolution(width, height); - int cpix = height * width; - int position = 0; - - for (int y = 0; y < height; ++y) - for (int x = 0; x < width; x++) - { - float red = 0; - float green = 0; - float blue = 0; - float alpha = 0; - if (ex.Interleave) - { - if (ex.Alpha) alpha = Convert.ToSingle(values[position++]); - if (ex.Red) red = Convert.ToSingle(values[position++]); - if (ex.Green) green = Convert.ToSingle(values[position++]); - if (ex.Blue) blue = Convert.ToSingle(values[position++]); - } - else - { - position = y * width + x; - if (ex.Alpha) { alpha = Convert.ToSingle(values[position]); position += cpix; } - if (ex.Red) { red = Convert.ToSingle(values[position]); position += cpix; } - if (ex.Green) { green = Convert.ToSingle(values[position]); position += cpix; } - if (ex.Blue) { blue = Convert.ToSingle(values[position]); position += cpix; } - } - Color pixel; - if (!needScale) - pixel = Color.FromArgb((int)alpha, (int)red, (int)green, (int)blue); - else - { - pixel = Color.FromArgb( - ex.Alpha ? (int)Math.Round((alpha - offset) * scale) : 0, - (int)Math.Round((red - offset) * scale), - (int)Math.Round((green - offset) * scale), - (int)Math.Round((blue - offset) * scale)); - } - dst.SetPixel(x, y, pixel); - } - }; + return new SchemaShape(result.Values); } } } diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv index 32f0f88d9a..bb683a564a 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv +++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv @@ -132,6 +132,6 @@ Transforms.TextToKeyConverter Converts input values (words, numbers, etc.) to in Transforms.TrainTestDatasetSplitter Split the dataset into train and test sets Microsoft.ML.EntryPoints.TrainTestSplit Split Microsoft.ML.EntryPoints.TrainTestSplit+Input Microsoft.ML.EntryPoints.TrainTestSplit+Output Transforms.TreeLeafFeaturizer Trains a tree ensemble, or loads it from a file, then maps a numeric feature vector to three outputs: 1. A vector containing the individual tree outputs of the tree ensemble. 2. A vector indicating the leaves that the feature vector falls on in the tree ensemble. 3. A vector indicating the paths that the feature vector falls on in the tree ensemble. If a both a model file and a trainer are specified - will use the model file. If neither are specified, will train a default FastTree model. This can handle key labels by training a regression model towards their optionally permuted indices. Microsoft.ML.Data.TreeFeaturize Featurizer Microsoft.ML.Data.TreeEnsembleFeaturizerTransform+ArgumentsForEntryPoint Microsoft.ML.EntryPoints.CommonOutputs+TransformOutput Transforms.TwoHeterogeneousModelCombiner Combines a TransformModel and a PredictorModel into a single PredictorModel. Microsoft.ML.EntryPoints.ModelOperations CombineTwoModels Microsoft.ML.EntryPoints.ModelOperations+SimplePredictorModelInput Microsoft.ML.EntryPoints.ModelOperations+PredictorModelOutput -Transforms.VectorToImage Converts vector array into image type. Microsoft.ML.ImageAnalytics.EntryPoints.ImageAnalytics VectorToImage Microsoft.ML.ImageAnalytics.VectorToImageTransform+Arguments Microsoft.ML.EntryPoints.CommonOutputs+TransformOutput +Transforms.VectorToImage Converts vector array into image type. Microsoft.ML.ImageAnalytics.EntryPoints.ImageAnalytics VectorToImage Microsoft.ML.ImageAnalytics.VectorToImageConvertingTransformer+Options Microsoft.ML.EntryPoints.CommonOutputs+TransformOutput Transforms.WordEmbeddings Word Embeddings transform is a text featurizer which converts vectors of text tokens into sentence vectors using a pre-trained model Microsoft.ML.Transforms.Text.TextAnalytics WordEmbeddings Microsoft.ML.Transforms.Text.WordEmbeddingsExtractingTransformer+Options Microsoft.ML.EntryPoints.CommonOutputs+TransformOutput Transforms.WordTokenizer The input to this transform is text, and the output is a vector of text containing the words (tokens) in the original text. The separator is space, but can be specified as any other character (or multiple characters) if needed. Microsoft.ML.Transforms.Text.TextAnalytics DelimitedTokenizeTransform Microsoft.ML.Transforms.Text.WordTokenizingTransformer+Options Microsoft.ML.EntryPoints.CommonOutputs+TransformOutput diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index 9fb6b0a308..f557cce156 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -23212,8 +23212,8 @@ "Desc": "Offset (pre-scale)", "Required": false, "SortOrder": 150.0, - "IsNullable": true, - "Default": null + "IsNullable": false, + "Default": 0.0 }, { "Name": "Scale", @@ -23221,8 +23221,8 @@ "Desc": "Scale factor", "Required": false, "SortOrder": 150.0, - "IsNullable": true, - "Default": null + "IsNullable": false, + "Default": 1.0 } ], "Outputs": [ diff --git a/test/Microsoft.ML.Tests/ImagesTests.cs b/test/Microsoft.ML.Tests/ImagesTests.cs index b2f1c26155..901f508b6d 100644 --- a/test/Microsoft.ML.Tests/ImagesTests.cs +++ b/test/Microsoft.ML.Tests/ImagesTests.cs @@ -201,15 +201,8 @@ public void TestBackAndForthConversionWithAlphaInterleave() var cropped = new ImageResizingTransformer(env, "ImageCropped", imageWidth, imageHeight, "ImageReal").Transform(images); var pixels = new ImagePixelExtractingTransformer(env, "ImagePixels", "ImageCropped", ImagePixelExtractingEstimator.ColorBits.All, true, 2f / 255, 127.5f).Transform(cropped); - IDataView backToBitmaps = new VectorToImageTransform(env, new VectorToImageTransform.Arguments() - { - InterleaveArgb = true, - Offset = -1f, - Scale = 255f / 2, - Columns = new VectorToImageTransform.Column[1]{ - new VectorToImageTransform.Column() { Name = "ImageRestored" , Source= "ImagePixels", ImageHeight=imageHeight, ImageWidth=imageWidth, ContainsAlpha=true} - } - }, pixels); + IDataView backToBitmaps = new VectorToImageConvertingTransformer(env, "ImageRestored", "ImagePixels", + imageHeight, imageWidth, ImagePixelExtractingEstimator.ColorBits.All, true, 255f / 2, -1f).Transform(pixels); var fname = nameof(TestBackAndForthConversionWithAlphaInterleave) + "_model.zip"; @@ -268,15 +261,8 @@ public void TestBackAndForthConversionWithoutAlphaInterleave() var cropped = new ImageResizingTransformer(env, "ImageCropped", imageWidth, imageHeight, "ImageReal").Transform(images); var pixels = new ImagePixelExtractingTransformer(env, "ImagePixels", "ImageCropped", ImagePixelExtractingEstimator.ColorBits.Rgb, true, 2f / 255, 127.5f).Transform(cropped); - IDataView backToBitmaps = new VectorToImageTransform(env, new VectorToImageTransform.Arguments() - { - InterleaveArgb = true, - Offset = -1f, - Scale = 255f / 2, - Columns = new VectorToImageTransform.Column[1]{ - new VectorToImageTransform.Column() { Name = "ImageRestored" , Source= "ImagePixels", ImageHeight=imageHeight, ImageWidth=imageWidth, ContainsAlpha=false} - } - }, pixels); + IDataView backToBitmaps = new VectorToImageConvertingTransformer(env, "ImageRestored", "ImagePixels", + imageHeight, imageWidth, ImagePixelExtractingEstimator.ColorBits.Rgb, true, 255f / 2, -1f).Transform(pixels); var fname = nameof(TestBackAndForthConversionWithoutAlphaInterleave) + "_model.zip"; @@ -335,15 +321,8 @@ public void TestBackAndForthConversionWithAlphaNoInterleave() var cropped = new ImageResizingTransformer(env, "ImageCropped", imageWidth, imageHeight, "ImageReal").Transform(images); var pixels = new ImagePixelExtractingTransformer(env, "ImagePixels", "ImageCropped", ImagePixelExtractingEstimator.ColorBits.All, false, 2f / 255, 127.5f).Transform(cropped); - IDataView backToBitmaps = new VectorToImageTransform(env, new VectorToImageTransform.Arguments() - { - InterleaveArgb = false, - Offset = -1f, - Scale = 255f / 2, - Columns = new VectorToImageTransform.Column[1]{ - new VectorToImageTransform.Column() { Name = "ImageRestored" , Source= "ImagePixels", ImageHeight=imageHeight, ImageWidth=imageWidth, ContainsAlpha=true} - } - }, pixels); + IDataView backToBitmaps = new VectorToImageConvertingTransformer(env, "ImageRestored", "ImagePixels", + imageHeight, imageWidth, ImagePixelExtractingEstimator.ColorBits.All, false, 255f / 2, -1f).Transform(pixels); var fname = nameof(TestBackAndForthConversionWithAlphaNoInterleave) + "_model.zip"; @@ -402,15 +381,8 @@ public void TestBackAndForthConversionWithoutAlphaNoInterleave() var cropped = new ImageResizingTransformer(env, "ImageCropped", imageWidth, imageHeight, "ImageReal").Transform(images); var pixels = new ImagePixelExtractingTransformer(env, "ImagePixels", "ImageCropped", ImagePixelExtractingEstimator.ColorBits.Rgb, false, 2f / 255, 127.5f).Transform(cropped); - IDataView backToBitmaps = new VectorToImageTransform(env, new VectorToImageTransform.Arguments() - { - InterleaveArgb = false, - Offset = -1f, - Scale = 255f / 2, - Columns = new VectorToImageTransform.Column[1]{ - new VectorToImageTransform.Column() { Name = "ImageRestored" , Source= "ImagePixels", ImageHeight=imageHeight, ImageWidth=imageWidth, ContainsAlpha=false} - } - }, pixels); + IDataView backToBitmaps = new VectorToImageConvertingTransformer(env, "ImageRestored", "ImagePixels", + imageHeight, imageWidth, ImagePixelExtractingEstimator.ColorBits.Rgb, false, 255f / 2, -1f).Transform(pixels); var fname = nameof(TestBackAndForthConversionWithoutAlphaNoInterleave) + "_model.zip"; @@ -470,13 +442,8 @@ public void TestBackAndForthConversionWithAlphaInterleaveNoOffset() var pixels = new ImagePixelExtractingTransformer(env, "ImagePixels", "ImageCropped", ImagePixelExtractingEstimator.ColorBits.All, true).Transform(cropped); - IDataView backToBitmaps = new VectorToImageTransform(env, new VectorToImageTransform.Arguments() - { - InterleaveArgb = true, - Columns = new VectorToImageTransform.Column[1]{ - new VectorToImageTransform.Column() { Name = "ImageRestored" , Source= "ImagePixels", ImageHeight=imageHeight, ImageWidth=imageWidth, ContainsAlpha=true} - } - }, pixels); + IDataView backToBitmaps = new VectorToImageConvertingTransformer(env, "ImageRestored", "ImagePixels", + imageHeight, imageWidth, ImagePixelExtractingEstimator.ColorBits.All, true, 1, 0).Transform(pixels); var fname = nameof(TestBackAndForthConversionWithAlphaInterleaveNoOffset) + "_model.zip"; @@ -536,13 +503,8 @@ public void TestBackAndForthConversionWithoutAlphaInterleaveNoOffset() var pixels = new ImagePixelExtractingTransformer(env, "ImagePixels", "ImageCropped", ImagePixelExtractingEstimator.ColorBits.Rgb, true).Transform(cropped); - IDataView backToBitmaps = new VectorToImageTransform(env, new VectorToImageTransform.Arguments() - { - InterleaveArgb = true, - Columns = new VectorToImageTransform.Column[1]{ - new VectorToImageTransform.Column() { Name = "ImageRestored" , Source= "ImagePixels", ImageHeight=imageHeight, ImageWidth=imageWidth, ContainsAlpha=false} - } - }, pixels); + IDataView backToBitmaps = new VectorToImageConvertingTransformer(env, "ImageRestored", "ImagePixels", + imageHeight, imageWidth, ImagePixelExtractingEstimator.ColorBits.Rgb, true, 1, 0).Transform(pixels); var fname = nameof(TestBackAndForthConversionWithoutAlphaInterleaveNoOffset) + "_model.zip"; @@ -602,13 +564,8 @@ public void TestBackAndForthConversionWithAlphaNoInterleaveNoOffset() var pixels = new ImagePixelExtractingTransformer(env, "ImagePixels", "ImageCropped", ImagePixelExtractingEstimator.ColorBits.All).Transform(cropped); - IDataView backToBitmaps = new VectorToImageTransform(env, new VectorToImageTransform.Arguments() - { - InterleaveArgb = false, - Columns = new VectorToImageTransform.Column[1]{ - new VectorToImageTransform.Column() { Name = "ImageRestored" , Source= "ImagePixels", ImageHeight=imageHeight, ImageWidth=imageWidth, ContainsAlpha=true} - } - }, pixels); + IDataView backToBitmaps = new VectorToImageConvertingTransformer(env, "ImageRestored", "ImagePixels", + imageHeight, imageWidth, ImagePixelExtractingEstimator.ColorBits.All, false, 1, 0).Transform(pixels); var fname = nameof(TestBackAndForthConversionWithAlphaNoInterleaveNoOffset) + "_model.zip"; @@ -667,13 +624,8 @@ public void TestBackAndForthConversionWithoutAlphaNoInterleaveNoOffset() var cropped = new ImageResizingTransformer(env, "ImageCropped", imageWidth, imageHeight, "ImageReal").Transform(images); var pixels = new ImagePixelExtractingTransformer(env, "ImagePixels", "ImageCropped").Transform(cropped); - IDataView backToBitmaps = new VectorToImageTransform(env, new VectorToImageTransform.Arguments() - { - InterleaveArgb = false, - Columns = new VectorToImageTransform.Column[1]{ - new VectorToImageTransform.Column() { Name = "ImageRestored" , Source= "ImagePixels", ImageHeight=imageHeight, ImageWidth=imageWidth, ContainsAlpha=false} - } - }, pixels); + IDataView backToBitmaps = new VectorToImageConvertingTransformer(env, "ImageRestored", "ImagePixels", + imageHeight, imageWidth, ImagePixelExtractingEstimator.ColorBits.Rgb, false, 1, 0).Transform(pixels); var fname = nameof(TestBackAndForthConversionWithoutAlphaNoInterleaveNoOffset) + "_model.zip";