diff --git a/src/Microsoft.ML.Transforms/NAHandleTransform.cs b/src/Microsoft.ML.Transforms/NAHandleTransform.cs
index bf3ab5f41a..3c777e03a0 100644
--- a/src/Microsoft.ML.Transforms/NAHandleTransform.cs
+++ b/src/Microsoft.ML.Transforms/NAHandleTransform.cs
@@ -11,6 +11,7 @@
using Microsoft.ML.Runtime.Data.Conversion;
using Microsoft.ML.Runtime.EntryPoints;
using Microsoft.ML.Runtime.Internal.Utilities;
+using Microsoft.ML.Transforms;
[assembly: LoadableClass(NAHandleTransform.Summary, typeof(IDataTransform), typeof(NAHandleTransform), typeof(NAHandleTransform.Arguments), typeof(SignatureDataTransform),
NAHandleTransform.FriendlyName, "NAHandleTransform", NAHandleTransform.ShortName, "NA", DocName = "transform/NAHandle.md")]
@@ -212,7 +213,7 @@ public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataV
// Create the indicator columns.
if (naIndicatorCols.Count > 0)
- output = new NAIndicatorTransform(h, new NAIndicatorTransform.Arguments() { Column = naIndicatorCols.ToArray() }, input);
+ output = NAIndicatorTransform.Create(h, new NAIndicatorTransform.Arguments() { Column = naIndicatorCols.ToArray() }, input);
// Convert the indicator columns to the correct type so that they can be concatenated to the NAReplace outputs.
if (naConvCols.Count > 0)
diff --git a/src/Microsoft.ML.Transforms/NAHandling.cs b/src/Microsoft.ML.Transforms/NAHandling.cs
index 8f8fdaabb9..79d3b49d5a 100644
--- a/src/Microsoft.ML.Transforms/NAHandling.cs
+++ b/src/Microsoft.ML.Transforms/NAHandling.cs
@@ -4,6 +4,7 @@
using Microsoft.ML.Runtime.Data;
using Microsoft.ML.Runtime.EntryPoints;
+using Microsoft.ML.Transforms;
[assembly: EntryPointModule(typeof(NAHandling))]
@@ -71,7 +72,7 @@ public static CommonOutputs.TransformOutput Handle(IHostEnvironment env, NAHandl
public static CommonOutputs.TransformOutput Indicator(IHostEnvironment env, NAIndicatorTransform.Arguments input)
{
var h = EntryPointUtils.CheckArgsAndCreateHost(env, "NAIndicator", input);
- var xf = new NAIndicatorTransform(h, input, input.Data);
+ var xf = new NAIndicatorTransform(h, input).Transform(input.Data);
return new CommonOutputs.TransformOutput()
{
Model = new TransformModel(h, xf, input.Data),
diff --git a/src/Microsoft.ML.Transforms/NAIndicatorTransform.cs b/src/Microsoft.ML.Transforms/NAIndicatorTransform.cs
index 044751f459..cb9cc6bfbb 100644
--- a/src/Microsoft.ML.Transforms/NAIndicatorTransform.cs
+++ b/src/Microsoft.ML.Transforms/NAIndicatorTransform.cs
@@ -4,7 +4,9 @@
using System;
using System.Collections.Generic;
+using System.Linq;
using System.Text;
+using Microsoft.ML.Core.Data;
using Microsoft.ML.Runtime;
using Microsoft.ML.Runtime.CommandLine;
using Microsoft.ML.Runtime.Data;
@@ -12,17 +14,26 @@
using Microsoft.ML.Runtime.EntryPoints;
using Microsoft.ML.Runtime.Internal.Utilities;
using Microsoft.ML.Runtime.Model;
+using Microsoft.ML.StaticPipe;
+using Microsoft.ML.StaticPipe.Runtime;
+using Microsoft.ML.Transforms;
-[assembly: LoadableClass(typeof(NAIndicatorTransform), typeof(NAIndicatorTransform.Arguments), typeof(SignatureDataTransform),
- NAIndicatorTransform.FriendlyName, "NAIndicatorTransform", "NAIndicator", NAIndicatorTransform.ShortName, DocName = "transform/NAHandle.md")]
+[assembly: LoadableClass(NAIndicatorTransform.Summary, typeof(IDataTransform), typeof(NAIndicatorTransform), typeof(NAIndicatorTransform.Arguments), typeof(SignatureDataTransform),
+ NAIndicatorTransform.FriendlyName, NAIndicatorTransform.LoadName, "NAIndicator", NAIndicatorTransform.ShortName, DocName = "transform/NAHandle.md")]
-[assembly: LoadableClass(typeof(NAIndicatorTransform), null, typeof(SignatureLoadDataTransform),
- NAIndicatorTransform.FriendlyName, NAIndicatorTransform.LoaderSignature)]
+[assembly: LoadableClass(NAIndicatorTransform.Summary, typeof(IDataTransform), typeof(NAIndicatorTransform), null, typeof(SignatureLoadDataTransform),
+ NAIndicatorTransform.FriendlyName, NAIndicatorTransform.LoadName)]
-namespace Microsoft.ML.Runtime.Data
+[assembly: LoadableClass(NAIndicatorTransform.Summary, typeof(NAIndicatorTransform), null, typeof(SignatureLoadModel),
+ NAIndicatorTransform.FriendlyName, NAIndicatorTransform.LoadName)]
+
+[assembly: LoadableClass(typeof(IRowMapper), typeof(NAIndicatorTransform), null, typeof(SignatureLoadRowMapper),
+ NAIndicatorTransform.FriendlyName, NAIndicatorTransform.LoadName)]
+
+namespace Microsoft.ML.Transforms
{
/// <include file='doc.xml' path='doc/members/member[@name="NAIndicator"]'/>
- public sealed class NAIndicatorTransform : OneToOneTransformBase
+ public sealed class NAIndicatorTransform : OneToOneTransformerBase
{
public sealed class Column : OneToOneColumn
{
@@ -49,17 +60,16 @@ public sealed class Arguments : TransformInputBase
public Column[] Column;
}
- public const string LoaderSignature = "NaIndicatorTransform";
+ internal const string LoadName = "NaIndicatorTransform";
private static VersionInfo GetVersionInfo()
{
return new VersionInfo(
- // REVIEW: temporary name
modelSignature: "NAIND TF",
verWrittenCur: 0x00010001, // Initial
verReadableCur: 0x00010001,
verWeCanReadBack: 0x00010001,
- loaderSignature: LoaderSignature,
+ loaderSignature: LoadName,
loaderAssemblyName: typeof(NAIndicatorTransform).Assembly.FullName);
}
@@ -68,322 +78,538 @@ private static VersionInfo GetVersionInfo()
internal const string FriendlyName = "NA Indicator Transform";
internal const string ShortName = "NAInd";
- private static string TestType(ColumnType type)
- {
- // Item type must have an NA value. We'll get the predicate again later when we're ready to use it.
- Delegate del;
- if (Conversions.Instance.TryGetIsNAPredicate(type.ItemType, out del))
- return null;
- return string.Format("Type '{0}' is not supported by {1} since it doesn't have an NA value",
- type, LoaderSignature);
- }
-
- private const string RegistrationName = "NaIndicator";
+ private const string RegistrationName = nameof(NAIndicatorTransform);
- // The output column types, parallel to Infos.
- private readonly ColumnType[] _types;
+ public IReadOnlyList<(string input, string output)> Columns => ColumnPairs.AsReadOnly();
/// <summary>
- /// Convenience constructor for public facing API.
+ /// Initializes a new instance of <see cref="NAIndicatorTransform"/>
/// </summary>
- /// <param name="env">Host Environment.</param>
- /// <param name="input">Input <see cref="IDataView"/>. This is the output from previous transform or loader.</param>
- /// <param name="name">Name of the output column.</param>
- /// <param name="source">Name of the column to be transformed. If this is null '<paramref name="name"/>' will be used.</param>
- public NAIndicatorTransform(IHostEnvironment env, IDataView input, string name, string source = null)
- : this(env, new Arguments() { Column = new[] { new Column() { Source = source ?? name, Name = name } } }, input)
+ /// <param name="env">The environment to use.</param>
+ /// <param name="columns">The names of the input columns of the transformation and the corresponding names for the output columns.</param>
+ public NAIndicatorTransform(IHostEnvironment env, params (string input, string output)[] columns)
+ : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(NAIndicatorTransform)), columns)
{
}
- /// <summary>
- /// Public constructor corresponding to SignatureDataTransform.
- /// </summary>
- public NAIndicatorTransform(IHostEnvironment env, Arguments args, IDataView input)
- : base(env, RegistrationName, Contracts.CheckRef(args, nameof(args)).Column,
- input, TestType)
+ internal NAIndicatorTransform(IHostEnvironment env, Arguments args)
+ : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(NAIndicatorTransform)), GetColumnPairs(args.Column))
{
- Host.AssertNonEmpty(Infos);
- Host.Assert(Infos.Length == Utils.Size(args.Column));
-
- _types = GetTypesAndMetadata();
}
- private NAIndicatorTransform(IHost host, ModelLoadContext ctx, IDataView input)
- : base(host, ctx, input, TestType)
+ private NAIndicatorTransform(IHostEnvironment env, ModelLoadContext ctx)
+ : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(NAIndicatorTransform)), ctx)
{
- Host.AssertValue(ctx);
- Host.AssertNonEmpty(Infos);
-
- // *** Binary format ***
- // <base>
-
- _types = GetTypesAndMetadata();
+ Host.CheckValue(ctx, nameof(ctx));
}
- public static NAIndicatorTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
+ private static (string input, string output)[] GetColumnPairs(Column[] columns)
+ => columns.Select(c => (c.Source ?? c.Name, c.Name)).ToArray();
+
+ // Factory method for SignatureLoadModel
+ internal static NAIndicatorTransform Create(IHostEnvironment env, ModelLoadContext ctx)
{
Contracts.CheckValue(env, nameof(env));
- var h = env.Register(RegistrationName);
- h.CheckValue(ctx, nameof(ctx));
- h.CheckValue(input, nameof(input));
ctx.CheckAtModel(GetVersionInfo());
- return h.Apply("Loading Model", ch => new NAIndicatorTransform(h, ctx, input));
+
+ return new NAIndicatorTransform(env, ctx);
}
+ // Factory method for SignatureDataTransform.
+ internal static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
+ => new NAIndicatorTransform(env, args).MakeDataTransform(input);
+
+ // Factory method for SignatureLoadDataTransform.
+ internal static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
+ => Create(env, ctx).MakeDataTransform(input);
+
+ // Factory method for SignatureLoadRowMapper.
+ internal static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, ISchema inputSchema)
+ => Create(env, ctx).MakeRowMapper(inputSchema);
+
+ /// <summary>
+ /// Saves the transform.
+ /// </summary>
public override void Save(ModelSaveContext ctx)
{
Host.CheckValue(ctx, nameof(ctx));
ctx.CheckAtModel();
ctx.SetVersionInfo(GetVersionInfo());
-
- // *** Binary format ***
- // <base>
- SaveBase(ctx);
+ SaveColumns(ctx);
}
- private ColumnType[] GetTypesAndMetadata()
+ protected override IRowMapper MakeRowMapper(ISchema schema)
+ => new Mapper(this, Schema.Create(schema));
+
+ private sealed class Mapper : MapperBase
{
- var md = Metadata;
- var types = new ColumnType[Infos.Length];
- for (int iinfo = 0; iinfo < Infos.Length; iinfo++)
+ private readonly NAIndicatorTransform _parent;
+ private readonly ColInfo[] _infos;
+
+ private sealed class ColInfo
+ {
+ public readonly string Output;
+ public readonly string Input;
+ public readonly ColumnType OutputType;
+ public readonly ColumnType InputType;
+ public readonly Delegate InputIsNA;
+
+ public ColInfo(string input, string output, ColumnType inType, ColumnType outType)
+ {
+ Input = input;
+ Output = output;
+ InputType = inType;
+ OutputType = outType;
+ InputIsNA = GetIsNADelegate(InputType);
+ }
+ }
+
+ public Mapper(NAIndicatorTransform parent, Schema inputSchema)
+ : base(parent.Host.Register(nameof(Mapper)), parent, inputSchema)
+ {
+ _parent = parent;
+ _infos = CreateInfos(inputSchema);
+ }
+
+ private ColInfo[] CreateInfos(Schema inputSchema)
{
- var type = Infos[iinfo].TypeSrc;
+ Host.AssertValue(inputSchema);
+ var infos = new ColInfo[_parent.ColumnPairs.Length];
+ for (int i = 0; i < _parent.ColumnPairs.Length; i++)
+ {
+ if (!inputSchema.TryGetColumnIndex(_parent.ColumnPairs[i].input, out int colSrc))
+ throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", _parent.ColumnPairs[i].input);
+ _parent.CheckInputColumn(inputSchema, i, colSrc);
+ var inType = inputSchema.GetColumnType(colSrc);
+ ColumnType outType;
+ if (!inType.IsVector)
+ outType = BoolType.Instance;
+ else
+ outType = new VectorType(BoolType.Instance, inType.AsVector);
+ infos[i] = new ColInfo(_parent.ColumnPairs[i].input, _parent.ColumnPairs[i].output, inType, outType);
+ }
+ return infos;
+ }
- if (!type.IsVector)
- types[iinfo] = BoolType.Instance;
+ public override Schema.Column[] GetOutputColumns()
+ {
+ var result = new Schema.Column[_parent.ColumnPairs.Length];
+ for (int iinfo = 0; iinfo < _infos.Length; iinfo++)
+ {
+ InputSchema.TryGetColumnIndex(_infos[iinfo].Input, out int colIndex);
+ Host.Assert(colIndex >= 0);
+ var builder = new Schema.Metadata.Builder();
+ builder.Add(InputSchema[colIndex].Metadata, x => x == MetadataUtils.Kinds.SlotNames);
+ ValueGetter<bool> getter = (ref bool dst) =>
+ {
+ dst = true;
+ };
+ builder.Add(new Schema.Column(MetadataUtils.Kinds.IsNormalized, BoolType.Instance, null), getter);
+ result[iinfo] = new Schema.Column(_infos[iinfo].Output, _infos[iinfo].OutputType, builder.GetMetadata());
+ }
+ return result;
+ }
+
+ /// <summary>
+ /// Returns the isNA predicate for the respective type.
+ /// </summary>
+ private static Delegate GetIsNADelegate(ColumnType type)
+ {
+ Func<ColumnType, Delegate> func = GetIsNADelegate<int>;
+ return Utils.MarshalInvoke(func, type.ItemType.RawType, type);
+ }
+
+ private static Delegate GetIsNADelegate<T>(ColumnType type)
+ {
+ return Conversions.Instance.GetIsNAPredicate<T>(type.ItemType);
+ }
+
+ protected override Delegate MakeGetter(IRow input, int iinfo, out Action disposer)
+ {
+ Host.AssertValue(input);
+ Host.Assert(0 <= iinfo && iinfo < _infos.Length);
+ disposer = null;
+
+ if (!_infos[iinfo].InputType.IsVector)
+ return ComposeGetterOne(input, iinfo);
+ return ComposeGetterVec(input, iinfo);
+ }
+
+ /// <summary>
+ /// Getter generator for single valued inputs.
+ /// </summary>
+ private ValueGetter<bool> ComposeGetterOne(IRow input, int iinfo)
+ => Utils.MarshalInvoke(ComposeGetterOne<int>, _infos[iinfo].InputType.RawType, input, iinfo);
+
+ private ValueGetter<bool> ComposeGetterOne<T>(IRow input, int iinfo)
+ {
+ var getSrc = input.GetGetter<T>(ColMapNewToOld[iinfo]);
+ var src = default(T);
+ var isNA = (RefPredicate<T>)_infos[iinfo].InputIsNA;
+
+ ValueGetter<bool> getter;
+
+ return getter =
+ (ref bool dst) =>
+ {
+ getSrc(ref src);
+ dst = isNA(ref src);
+ };
+ }
+
+ /// <summary>
+ /// Getter generator for vector valued inputs.
+ /// </summary>
+ private ValueGetter<VBuffer<bool>> ComposeGetterVec(IRow input, int iinfo)
+ => Utils.MarshalInvoke(ComposeGetterVec<int>, _infos[iinfo].InputType.ItemType.RawType, input, iinfo);
+
+ private ValueGetter<VBuffer<bool>> ComposeGetterVec<T>(IRow input, int iinfo)
+ {
+ var getSrc = input.GetGetter<VBuffer<T>>(ColMapNewToOld[iinfo]);
+ var isNA = (RefPredicate<T>)_infos[iinfo].InputIsNA;
+ var val = default(T);
+ var defaultIsNA = isNA(ref val);
+ var src = default(VBuffer<T>);
+ var indices = new List<int>();
+
+ ValueGetter<VBuffer<bool>> getter;
+
+ return getter =
+ (ref VBuffer<bool> dst) =>
+ {
+ // Sense indicates if the values added to the indices list represent NAs or non-NAs.
+ bool sense;
+ getSrc(ref src);
+ FindNAs(ref src, isNA, defaultIsNA, indices, out sense);
+ FillValues(src.Length, ref dst, indices, sense);
+ };
+ }
+
+ /// <summary>
+ /// Adds all NAs (or non-NAs) to the indices List. Whether NAs or non-NAs have been added is indicated by the bool sense.
+ /// </summary>
+ private void FindNAs<T>(ref VBuffer<T> src, RefPredicate<T> isNA, bool defaultIsNA, List<int> indices, out bool sense)
+ {
+ Host.AssertValue(isNA);
+ Host.AssertValue(indices);
+
+ // Find the indices of all of the NAs.
+ indices.Clear();
+ var srcValues = src.Values;
+ var srcCount = src.Count;
+ if (src.IsDense)
+ {
+ for (int i = 0; i < srcCount; i++)
+ {
+ if (isNA(ref srcValues[i]))
+ indices.Add(i);
+ }
+ sense = true;
+ }
+ else if (!defaultIsNA)
+ {
+ var srcIndices = src.Indices;
+ for (int ii = 0; ii < srcCount; ii++)
+ {
+ if (isNA(ref srcValues[ii]))
+ indices.Add(srcIndices[ii]);
+ }
+ sense = true;
+ }
else
- types[iinfo] = new VectorType(BoolType.Instance, type.AsVector);
- // Pass through slot name metadata.
- using (var bldr = md.BuildMetadata(iinfo, Source.Schema, Infos[iinfo].Source, MetadataUtils.Kinds.SlotNames))
{
- // Output is normalized.
- bldr.AddPrimitive(MetadataUtils.Kinds.IsNormalized, BoolType.Instance, true);
+ // Note that this adds non-NAs to indices -- this is indicated by sense being false.
+ var srcIndices = src.Indices;
+ for (int ii = 0; ii < srcCount; ii++)
+ {
+ if (!isNA(ref srcValues[ii]))
+ indices.Add(srcIndices[ii]);
+ }
+ sense = false;
}
}
- md.Seal();
- return types;
- }
- protected override ColumnType GetColumnTypeCore(int iinfo)
- {
- Host.Assert(0 <= iinfo & iinfo < Infos.Length);
- return _types[iinfo];
- }
+ /// <summary>
+ /// Fills indicator values for vectors. The indices is a list that either holds all of the NAs or all
+ /// of the non-NAs, indicated by sense being true or false respectively.
+ /// </summary>
+ private void FillValues(int srcLength, ref VBuffer<bool> dst, List<int> indices, bool sense)
+ {
+ var dstValues = dst.Values;
+ var dstIndices = dst.Indices;
- protected override Delegate GetGetterCore(IChannel ch, IRow input, int iinfo, out Action disposer)
- {
- Host.AssertValueOrNull(ch);
- Host.AssertValue(input);
- Host.Assert(0 <= iinfo && iinfo < Infos.Length);
- disposer = null;
-
- if (!Infos[iinfo].TypeSrc.IsVector)
- return ComposeGetterOne(input, iinfo);
- return ComposeGetterVec(input, iinfo);
+ if (indices.Count == 0)
+ {
+ if (sense)
+ {
+ // Return empty VBuffer.
+ dst = new VBuffer<bool>(srcLength, 0, dstValues, dstIndices);
+ return;
+ }
+
+ // Return VBuffer filled with 1's.
+ Utils.EnsureSize(ref dstValues, srcLength, false);
+ for (int i = 0; i < srcLength; i++)
+ dstValues[i] = true;
+ dst = new VBuffer<bool>(srcLength, dstValues, dstIndices);
+ return;
+ }
+
+ if (sense && indices.Count < srcLength / 2)
+ {
+ // Will produce sparse output.
+ int dstCount = indices.Count;
+ Utils.EnsureSize(ref dstValues, dstCount, false);
+ Utils.EnsureSize(ref dstIndices, dstCount, false);
+
+ indices.CopyTo(dstIndices);
+ for (int ii = 0; ii < dstCount; ii++)
+ dstValues[ii] = true;
+
+ Host.Assert(dstCount <= srcLength);
+ dst = new VBuffer<bool>(srcLength, dstCount, dstValues, dstIndices);
+ }
+ else if (!sense && srcLength - indices.Count < srcLength / 2)
+ {
+ // Will produce sparse output.
+ int dstCount = srcLength - indices.Count;
+ Utils.EnsureSize(ref dstValues, dstCount, false);
+ Utils.EnsureSize(ref dstIndices, dstCount, false);
+
+ // Appends the length of the src to make the loop simpler,
+ // as the length of src will never be reached in the loop.
+ indices.Add(srcLength);
+
+ int iiDst = 0;
+ int iiSrc = 0;
+ int iNext = indices[iiSrc];
+ for (int i = 0; i < srcLength; i++)
+ {
+ Host.Assert(0 <= i && i <= iNext);
+ Host.Assert(iiSrc + iiDst == i);
+ if (i < iNext)
+ {
+ Host.Assert(iiDst < dstCount);
+ dstValues[iiDst] = true;
+ dstIndices[iiDst++] = i;
+ }
+ else
+ {
+ Host.Assert(iiSrc + 1 < indices.Count);
+ Host.Assert(iNext < indices[iiSrc + 1]);
+ iNext = indices[++iiSrc];
+ }
+ }
+ Host.Assert(srcLength == iiSrc + iiDst);
+ Host.Assert(iiDst == dstCount);
+
+ dst = new VBuffer<bool>(srcLength, dstCount, dstValues, dstIndices);
+ }
+ else
+ {
+ // Will produce dense output.
+ Utils.EnsureSize(ref dstValues, srcLength, false);
+
+ // Appends the length of the src to make the loop simpler,
+ // as the length of src will never be reached in the loop.
+ indices.Add(srcLength);
+
+ int ii = 0;
+ for (int i = 0; i < srcLength; i++)
+ {
+ Host.Assert(0 <= i && i <= indices[ii]);
+ if (i == indices[ii])
+ {
+ dstValues[i] = sense;
+ ii++;
+ Host.Assert(ii < indices.Count);
+ Host.Assert(indices[ii - 1] < indices[ii]);
+ }
+ else
+ dstValues[i] = !sense;
+ }
+
+ dst = new VBuffer<bool>(srcLength, dstValues, dstIndices);
+ }
+ }
}
+ }
+ public sealed class NAIndicatorEstimator : TrivialEstimator<NAIndicatorTransform>
+ {
/// <summary>
- /// Getter generator for single valued inputs.
+ /// Initializes a new instance of <see cref="NAIndicatorEstimator"/>
/// </summary>
- private ValueGetter<bool> ComposeGetterOne(IRow input, int iinfo)
+ /// <param name="env">The environment to use.</param>
+ /// <param name="columns">The names of the input columns of the transformation and the corresponding names for the output columns.</param>
+ public NAIndicatorEstimator(IHostEnvironment env, params (string input, string output)[] columns)
+ : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(NAIndicatorTransform)), new NAIndicatorTransform(env, columns))
{
- Func<IRow, int, ValueGetter<bool>> func = ComposeGetterOne<int>;
- return Utils.MarshalInvoke(func, Infos[iinfo].TypeSrc.RawType, input, iinfo);
+ Contracts.CheckValue(env, nameof(env));
}
/// <summary>
- /// Tests if a value is NA for scalars.
+ /// Initializes a new instance of <see cref="NAIndicatorEstimator"/>
/// </summary>
- private ValueGetter<bool> ComposeGetterOne<T>(IRow input, int iinfo)
+ /// <param name="env">The environment to use.</param>
+ /// <param name="input">The name of the input column of the transformation.</param>
+ /// <param name="output">The name of the column produced by the transformation.</param>
+ public NAIndicatorEstimator(IHostEnvironment env, string input, string output = null)
+ : this(env, (input, output ?? input))
{
- var getSrc = GetSrcGetter<T>(input, iinfo);
- var isNA = Conversions.Instance.GetIsNAPredicate<T>(input.Schema.GetColumnType(Infos[iinfo].Source));
- T src = default(T);
- return
- (ref bool dst) =>
- {
- getSrc(ref src);
- dst = isNA(ref src);
- };
}
/// <summary>
- /// Getter generator for vector valued inputs.
+ /// Returns the schema that would be produced by the transformation.
/// </summary>
- private ValueGetter<VBuffer<bool>> ComposeGetterVec(IRow input, int iinfo)
+ public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
{
- Func<IRow, int, ValueGetter<VBuffer<bool>>> func = ComposeGetterVec<int>;
- return Utils.MarshalInvoke(func, Infos[iinfo].TypeSrc.ItemType.RawType, input, iinfo);
+ Host.CheckValue(inputSchema, nameof(inputSchema));
+ var result = inputSchema.Columns.ToDictionary(x => x.Name);
+ foreach (var colPair in Transformer.Columns)
+ {
+ if (!inputSchema.TryFindColumn(colPair.input, out var col) || !Conversions.Instance.TryGetIsNAPredicate(col.ItemType, out Delegate del))
+ throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", colPair.input);
+ var metadata = new List<SchemaShape.Column>();
+ if (col.Metadata.TryFindColumn(MetadataUtils.Kinds.SlotNames, out var slotMeta))
+ metadata.Add(slotMeta);
+ metadata.Add(new SchemaShape.Column(MetadataUtils.Kinds.IsNormalized, SchemaShape.Column.VectorKind.Scalar, BoolType.Instance, false));
+ ColumnType type = !col.ItemType.IsVector ? (ColumnType) BoolType.Instance : new VectorType(BoolType.Instance, col.ItemType.AsVector);
+ result[colPair.output] = new SchemaShape.Column(colPair.output, col.Kind, type, false, new SchemaShape(metadata.ToArray()));
+ }
+ return new SchemaShape(result.Values);
}
+ }
- /// <summary>
- /// Tests if a value is NA for vectors.
- /// </summary>
- private ValueGetter<VBuffer<bool>> ComposeGetterVec<T>(IRow input, int iinfo)
+ /// <summary>
+ /// Extension methods for the static-pipeline over <see cref="PipelineColumn"/> objects.
+ /// </summary>
+ public static class NAIndicatorExtensions
+ {
+ private interface IColInput
{
- var getSrc = GetSrcGetter<VBuffer<T>>(input, iinfo);
- var isNA = Conversions.Instance.GetIsNAPredicate<T>(input.Schema.GetColumnType(Infos[iinfo].Source).ItemType);
- var val = default(T);
- bool defaultIsNA = isNA(ref val);
- var src = default(VBuffer<T>);
- var indices = new List<int>();
- return
- (ref VBuffer<bool> dst) =>
- {
- // Sense indicates if the values added to the indices list represent NAs or non-NAs.
- bool sense;
- getSrc(ref src);
- FindNAs(ref src, isNA, defaultIsNA, indices, out sense);
- FillValues(src.Length, ref dst, indices, sense);
- };
+ PipelineColumn Input { get; }
}
- /// <summary>
- /// Adds all NAs (or non-NAs) to the indices List. Whether NAs or non-NAs have been added is indicated by the bool sense.
- /// </summary>
- private void FindNAs<T>(ref VBuffer<T> src, RefPredicate<T> isNA, bool defaultIsNA, List<int> indices, out bool sense)
+ private sealed class OutScalar<TValue> : Scalar<bool>, IColInput
{
- Host.AssertValue(isNA);
- Host.AssertValue(indices);
-
- // Find the indices of all of the NAs.
- indices.Clear();
- var srcValues = src.Values;
- var srcCount = src.Count;
- if (src.IsDense)
- {
- for (int i = 0; i < srcCount; i++)
- {
- if (isNA(ref srcValues[i]))
- indices.Add(i);
- }
- sense = true;
- }
- else if (!defaultIsNA)
+ public PipelineColumn Input { get; }
+
+ public OutScalar(Scalar<TValue> input)
+ : base(Reconciler.Inst, input)
{
- var srcIndices = src.Indices;
- for (int ii = 0; ii < srcCount; ii++)
- {
- if (isNA(ref srcValues[ii]))
- indices.Add(srcIndices[ii]);
- }
- sense = true;
+ Input = input;
}
- else
+ }
+
+ private sealed class OutVectorColumn<TValue> : Vector<bool>, IColInput
+ {
+ public PipelineColumn Input { get; }
+
+ public OutVectorColumn(Vector<TValue> input)
+ : base(Reconciler.Inst, input)
{
- // Note that this adds non-NAs to indices -- this is indicated by sense being false.
- var srcIndices = src.Indices;
- for (int ii = 0; ii < srcCount; ii++)
- {
- if (!isNA(ref srcValues[ii]))
- indices.Add(srcIndices[ii]);
- }
- sense = false;
+ Input = input;
}
}
- /// <summary>
- /// Fills indicator values for vectors. The indices is a list that either holds all of the NAs or all
- /// of the non-NAs, indicated by sense being true or false respectively.
- /// </summary>
- private void FillValues(int srcLength, ref VBuffer<bool> dst, List<int> indices, bool sense)
+ private sealed class OutVarVectorColumn<TValue> : VarVector<bool>, IColInput
{
- var dstValues = dst.Values;
- var dstIndices = dst.Indices;
+ public PipelineColumn Input { get; }
- if (indices.Count == 0)
+ public OutVarVectorColumn(VarVector<TValue> input)
+ : base(Reconciler.Inst, input)
{
- if (sense)
- {
- // Return empty VBuffer.
- dst = new VBuffer<bool>(srcLength, 0, dstValues, dstIndices);
- return;
- }
-
- // Return VBuffer filled with 1's.
- Utils.EnsureSize(ref dstValues, srcLength, false);
- for (int i = 0; i < srcLength; i++)
- dstValues[i] = true;
- dst = new VBuffer<bool>(srcLength, dstValues, dstIndices);
- return;
+ Input = input;
}
+ }
- if (sense && indices.Count < srcLength / 2)
- {
- // Will produce sparse output.
- int dstCount = indices.Count;
- Utils.EnsureSize(ref dstValues, dstCount, false);
- Utils.EnsureSize(ref dstIndices, dstCount, false);
+ private sealed class Reconciler : EstimatorReconciler
+ {
+ public static Reconciler Inst = new Reconciler();
- indices.CopyTo(dstIndices);
- for (int ii = 0; ii < dstCount; ii++)
- dstValues[ii] = true;
+ private Reconciler() { }
- Host.Assert(dstCount <= srcLength);
- dst = new VBuffer<bool>(srcLength, dstCount, dstValues, dstIndices);
- }
- else if (!sense && srcLength - indices.Count < srcLength / 2)
+ public override IEstimator<ITransformer> Reconcile(IHostEnvironment env,
+ PipelineColumn[] toOutput,
+ IReadOnlyDictionary<PipelineColumn, string> inputNames,
+ IReadOnlyDictionary<PipelineColumn, string> outputNames,
+ IReadOnlyCollection<string> usedNames)
{
- // Will produce sparse output.
- int dstCount = srcLength - indices.Count;
- Utils.EnsureSize(ref dstValues, dstCount, false);
- Utils.EnsureSize(ref dstIndices, dstCount, false);
-
- // Appends the length of the src to make the loop simpler,
- // as the length of src will never be reached in the loop.
- indices.Add(srcLength);
-
- int iiDst = 0;
- int iiSrc = 0;
- int iNext = indices[iiSrc];
- for (int i = 0; i < srcLength; i++)
+ var columnPairs = new (string input, string output)[toOutput.Length];
+ for (int i = 0; i < toOutput.Length; ++i)
{
- Host.Assert(0 <= i && i <= iNext);
- Host.Assert(iiSrc + iiDst == i);
- if (i < iNext)
- {
- Host.Assert(iiDst < dstCount);
- dstValues[iiDst] = true;
- dstIndices[iiDst++] = i;
- }
- else
- {
- Host.Assert(iiSrc + 1 < indices.Count);
- Host.Assert(iNext < indices[iiSrc + 1]);
- iNext = indices[++iiSrc];
- }
+ var col = (IColInput)toOutput[i];
+ columnPairs[i] = (inputNames[col.Input], outputNames[toOutput[i]]);
}
- Host.Assert(srcLength == iiSrc + iiDst);
- Host.Assert(iiDst == dstCount);
-
- dst = new VBuffer<bool>(srcLength, dstCount, dstValues, dstIndices);
+ return new NAIndicatorEstimator(env, columnPairs);
}
- else
- {
- // Will produce dense output.
- Utils.EnsureSize(ref dstValues, srcLength, false);
+ }
- // Appends the length of the src to make the loop simpler,
- // as the length of src will never be reached in the loop.
- indices.Add(srcLength);
+ /// <summary>
+ /// Produces a column of boolean entries indicating whether input column entries were missing.
+ /// </summary>
+ /// <param name="input">The input column.</param>
+ /// <returns>A column indicating whether input column entries were missing.</returns>
+ public static Scalar<bool> IsMissingValue(this Scalar<float> input)
+ {
+ Contracts.CheckValue(input, nameof(input));
+ return new OutScalar<float>(input);
+ }
- int ii = 0;
- for (int i = 0; i < srcLength; i++)
- {
- Host.Assert(0 <= i && i <= indices[ii]);
- if (i == indices[ii])
- {
- dstValues[i] = sense;
- ii++;
- Host.Assert(ii < indices.Count);
- Host.Assert(indices[ii - 1] < indices[ii]);
- }
- else
- dstValues[i] = !sense;
- }
+ /// <summary>
+ /// Produces a column of boolean entries indicating whether input column entries were missing.
+ /// </summary>
+ /// <param name="input">The input column.</param>
+ /// <returns>A column indicating whether input column entries were missing.</returns>
+ public static Scalar<bool> IsMissingValue(this Scalar<double> input)
+ {
+ Contracts.CheckValue(input, nameof(input));
+ return new OutScalar<double>(input);
+ }
- dst = new VBuffer<bool>(srcLength, dstValues, dstIndices);
- }
+ /// <summary>
+ /// Produces a column of boolean entries indicating whether input column entries were missing.
+ /// </summary>
+ /// <param name="input">The input column.</param>
+ /// <returns>A column indicating whether input column entries were missing.</returns>
+ public static Vector<bool> IsMissingValue(this Vector<float> input)
+ {
+ Contracts.CheckValue(input, nameof(input));
+ return new OutVectorColumn<float>(input);
+ }
+
+ /// <summary>
+ /// Produces a column of boolean entries indicating whether input column entries were missing.
+ /// </summary>
+ /// <param name="input">The input column.</param>
+ /// <returns>A column indicating whether input column entries were missing.</returns>
+ public static Vector<bool> IsMissingValue(this Vector<double> input)
+ {
+ Contracts.CheckValue(input, nameof(input));
+ return new OutVectorColumn<double>(input);
+ }
+
+ /// <summary>
+ /// Produces a column of boolean entries indicating whether input column entries were missing.
+ /// </summary>
+ /// <param name="input">The input column.</param>
+ /// <returns>A column indicating whether input column entries were missing.</returns>
+ public static VarVector<bool> IsMissingValue(this VarVector<float> input)
+ {
+ Contracts.CheckValue(input, nameof(input));
+ return new OutVarVectorColumn<float>(input);
+ }
+
+ /// <summary>
+ /// Produces a column of boolean entries indicating whether input column entries were missing.
+ /// </summary>
+ /// <param name="input">The input column.</param>
+ /// <returns>A column indicating whether input column entries were missing.</returns>
+ public static VarVector<bool> IsMissingValue(this VarVector<double> input)
+ {
+ Contracts.CheckValue(input, nameof(input));
+ return new OutVarVectorColumn<double>(input);
}
}
-}
+}
\ No newline at end of file
diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv
index c4434d23b7..22f929ecb4 100644
--- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv
+++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv
@@ -112,7 +112,7 @@ Transforms.ManyHeterogeneousModelCombiner Combines a sequence of TransformModels
Transforms.MeanVarianceNormalizer Normalizes the data based on the computed mean and variance of the data. Microsoft.ML.Runtime.Data.Normalize MeanVar Microsoft.ML.Runtime.Data.NormalizeTransform+MeanVarArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput
Transforms.MinMaxNormalizer Normalizes the data based on the observed minimum and maximum values of the data. Microsoft.ML.Runtime.Data.Normalize MinMax Microsoft.ML.Runtime.Data.NormalizeTransform+MinMaxArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput
Transforms.MissingValueHandler Handle missing values by replacing them with either the default value or the mean/min/max value (for non-text columns only). An indicator column can optionally be concatenated, if theinput column type is numeric. Microsoft.ML.Runtime.Data.NAHandling Handle Microsoft.ML.Runtime.Data.NAHandleTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput
-Transforms.MissingValueIndicator Create a boolean output column with the same number of slots as the input column, where the output value is true if the value in the input column is missing. Microsoft.ML.Runtime.Data.NAHandling Indicator Microsoft.ML.Runtime.Data.NAIndicatorTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput
+Transforms.MissingValueIndicator Create a boolean output column with the same number of slots as the input column, where the output value is true if the value in the input column is missing. Microsoft.ML.Runtime.Data.NAHandling Indicator Microsoft.ML.Transforms.NAIndicatorTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput
Transforms.MissingValuesDropper Removes NAs from vector columns. Microsoft.ML.Runtime.Data.NAHandling Drop Microsoft.ML.Runtime.Data.NADropTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput
Transforms.MissingValuesRowDropper Filters out rows that contain missing values. Microsoft.ML.Runtime.Data.NAHandling Filter Microsoft.ML.Runtime.Data.NAFilter+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput
Transforms.MissingValueSubstitutor Create an output column of the same type and size of the input column, where missing values are replaced with either the default value or the mean/min/max value (for non-text columns only). Microsoft.ML.Runtime.Data.NAHandling Replace Microsoft.ML.Runtime.Data.NAReplaceTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput
diff --git a/test/BaselineOutput/SingleDebug/NAIndicator/featurized.tsv b/test/BaselineOutput/SingleDebug/NAIndicator/featurized.tsv
new file mode 100644
index 0000000000..f4dcf44107
--- /dev/null
+++ b/test/BaselineOutput/SingleDebug/NAIndicator/featurized.tsv
@@ -0,0 +1,17 @@
+#@ TextLoader{
+#@ header+
+#@ sep=tab
+#@ col=ScalarFloat:R4:0
+#@ col=ScalarDouble:R8:1
+#@ col=VectorFloat:R4:2-5
+#@ col=VectorDoulbe:R8:6-9
+#@ col=A:BL:10
+#@ col=B:BL:11
+#@ col=C:BL:12-15
+#@ col=D:BL:16-19
+#@ }
+ScalarFloat ScalarDouble 18 8:A 9:B
+5 5 5 1 1 1 5 1 1 1 10 0:0
+5 5 5 4 4 5 5 4 4 5 10 0:0
+3 3 3 1 1 1 3 1 1 1 10 0:0
+6 6 6 8 8 1 6 8 8 1 10 0:0
diff --git a/test/BaselineOutput/SingleRelease/NAIndicator/featurized.tsv b/test/BaselineOutput/SingleRelease/NAIndicator/featurized.tsv
new file mode 100644
index 0000000000..f4dcf44107
--- /dev/null
+++ b/test/BaselineOutput/SingleRelease/NAIndicator/featurized.tsv
@@ -0,0 +1,17 @@
+#@ TextLoader{
+#@ header+
+#@ sep=tab
+#@ col=ScalarFloat:R4:0
+#@ col=ScalarDouble:R8:1
+#@ col=VectorFloat:R4:2-5
+#@ col=VectorDoulbe:R8:6-9
+#@ col=A:BL:10
+#@ col=B:BL:11
+#@ col=C:BL:12-15
+#@ col=D:BL:16-19
+#@ }
+ScalarFloat ScalarDouble 18 8:A 9:B
+5 5 5 1 1 1 5 1 1 1 10 0:0
+5 5 5 4 4 5 5 4 4 5 10 0:0
+3 3 3 1 1 1 3 1 1 1 10 0:0
+6 6 6 8 8 1 6 8 8 1 10 0:0
diff --git a/test/Microsoft.ML.StaticPipelineTesting/StaticPipeTests.cs b/test/Microsoft.ML.StaticPipelineTesting/StaticPipeTests.cs
index 5166b4eb0e..6d4cae0995 100644
--- a/test/Microsoft.ML.StaticPipelineTesting/StaticPipeTests.cs
+++ b/test/Microsoft.ML.StaticPipelineTesting/StaticPipeTests.cs
@@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
+using Microsoft.ML.Data;
using Microsoft.ML.Runtime.Data;
using Microsoft.ML.Runtime.Data.IO;
using Microsoft.ML.Runtime.Internal.Utilities;
@@ -772,6 +773,50 @@ public void PrincipalComponentAnalysis()
Assert.True(type.IsVector && type.IsKnownSizeVector && type.ItemType.IsNumber);
}
+ [Fact]
+ public void NAIndicatorStatic()
+ {
+ var Env = new ConsoleEnvironment(seed: 0);
+
+ string dataPath = GetDataPath("breast-cancer.txt");
+ var reader = TextLoader.CreateReader(Env, ctx => (
+ ScalarFloat: ctx.LoadFloat(1),
+ ScalarDouble: ctx.LoadDouble(1),
+ VectorFloat: ctx.LoadFloat(1, 4),
+ VectorDoulbe: ctx.LoadDouble(1, 4)
+ ));
+
+ var data = reader.Read(new MultiFileSource(dataPath));
+
+ var est = data.MakeNewEstimator().
+ Append(row => (
+ A: row.ScalarFloat.IsMissingValue(),
+ B: row.ScalarDouble.IsMissingValue(),
+ C: row.VectorFloat.IsMissingValue(),
+ D: row.VectorDoulbe.IsMissingValue()
+ ));
+
+ IDataView newData = TakeFilter.Create(Env, est.Fit(data).Transform(data).AsDynamic, 4);
+ Assert.NotNull(newData);
+ bool[] ScalarFloat = newData.GetColumn<bool>(Env, "A").ToArray();
+ bool[] ScalarDouble = newData.GetColumn<bool>(Env, "B").ToArray();
+ bool[][] VectorFloat = newData.GetColumn<bool[]>(Env, "C").ToArray();
+ bool[][] VectorDoulbe = newData.GetColumn<bool[]>(Env, "D").ToArray();
+
+ Assert.NotNull(ScalarFloat);
+ Assert.NotNull(ScalarDouble);
+ Assert.NotNull(VectorFloat);
+ Assert.NotNull(VectorDoulbe);
+ for (int i = 0; i < 4; i++)
+ {
+ Assert.True(!ScalarFloat[i] && !ScalarDouble[i]);
+ Assert.NotNull(VectorFloat[i]);
+ Assert.NotNull(VectorDoulbe[i]);
+ for (int j = 0; j < 4; j++)
+ Assert.True(!VectorFloat[i][j] && !VectorDoulbe[i][j]);
+ }
+ }
+
[Fact]
public void TextNormalizeStatic()
{
@@ -810,7 +855,6 @@ public void TextNormalizeStatic()
Assert.True(schema.TryGetColumnIndex("norm_NoNumbers", out int numbers));
type = schema.GetColumnType(numbers);
Assert.True(!type.IsVector && type.ItemType.IsText);
-
}
}
}
\ No newline at end of file
diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/LbfgsTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/LbfgsTests.cs
index fb54ad52ce..ecbd4e4363 100644
--- a/test/Microsoft.ML.Tests/TrainerEstimators/LbfgsTests.cs
+++ b/test/Microsoft.ML.Tests/TrainerEstimators/LbfgsTests.cs
@@ -15,7 +15,7 @@ public partial class TrainerEstimators
public void TestEstimatorLogisticRegression()
{
(IEstimator<ITransformer> pipe, IDataView dataView) = GetBinaryClassificationPipeline();
- pipe.Append(new LogisticRegression(Env, "Features", "Label"));
+ pipe = pipe.Append(new LogisticRegression(Env, "Features", "Label"));
TestEstimatorCore(pipe, dataView);
Done();
}
@@ -24,7 +24,7 @@ public void TestEstimatorLogisticRegression()
public void TestEstimatorMulticlassLogisticRegression()
{
(IEstimator<ITransformer> pipe, IDataView dataView) = GetMultiClassPipeline();
- pipe.Append(new MulticlassLogisticRegression(Env, "Features", "Label"));
+ pipe = pipe.Append(new MulticlassLogisticRegression(Env, "Features", "Label"));
TestEstimatorCore(pipe, dataView);
Done();
}
diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/SymSgdClassificationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/SymSgdClassificationTests.cs
index 68e126c439..a8655b9aac 100644
--- a/test/Microsoft.ML.Tests/TrainerEstimators/SymSgdClassificationTests.cs
+++ b/test/Microsoft.ML.Tests/TrainerEstimators/SymSgdClassificationTests.cs
@@ -22,7 +22,7 @@ public partial class TrainerEstimators
public void TestEstimatorSymSgdClassificationTrainer()
{
(var pipe, var dataView) = GetBinaryClassificationPipeline();
- pipe.Append(new SymSgdClassificationTrainer(Env, "Features", "Label"));
+ pipe = pipe.Append(new SymSgdClassificationTrainer(Env, "Features", "Label"));
TestEstimatorCore(pipe, dataView);
Done();
}
diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs
index 06ffc61e26..20c2eb388b 100644
--- a/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs
+++ b/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs
@@ -84,7 +84,7 @@ public void KMeansEstimator()
public void TestEstimatorHogwildSGD()
{
(IEstimator<ITransformer> pipe, IDataView dataView) = GetBinaryClassificationPipeline();
- pipe.Append(new StochasticGradientDescentClassificationTrainer(Env, "Features", "Label"));
+ pipe = pipe.Append(new StochasticGradientDescentClassificationTrainer(Env, "Features", "Label"));
TestEstimatorCore(pipe, dataView);
Done();
}
@@ -96,7 +96,7 @@ public void TestEstimatorHogwildSGD()
public void TestEstimatorMultiClassNaiveBayesTrainer()
{
(IEstimator<ITransformer> pipe, IDataView dataView) = GetMultiClassPipeline();
- pipe.Append(new MultiClassNaiveBayesTrainer(Env, "Features", "Label"));
+ pipe = pipe.Append(new MultiClassNaiveBayesTrainer(Env, "Features", "Label"));
TestEstimatorCore(pipe, dataView);
Done();
}
diff --git a/test/Microsoft.ML.Tests/Transformers/NAIndicatorTests.cs b/test/Microsoft.ML.Tests/Transformers/NAIndicatorTests.cs
new file mode 100644
index 0000000000..505f39ebf7
--- /dev/null
+++ b/test/Microsoft.ML.Tests/Transformers/NAIndicatorTests.cs
@@ -0,0 +1,148 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Runtime.Api;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Runtime.Data.IO;
+using Microsoft.ML.Runtime.Model;
+using Microsoft.ML.Runtime.RunTests;
+using Microsoft.ML.Runtime.Tools;
+using Microsoft.ML.Transforms;
+using System;
+using System.IO;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.ML.Tests.Transformers
+{
+ public class NAIndicatorTests : TestDataPipeBase
+ {
+ private class TestClass
+ {
+ public float A;
+ public double B;
+ [VectorType(2)]
+ public float[] C;
+ [VectorType(2)]
+ public double[] D;
+ }
+
+ public NAIndicatorTests(ITestOutputHelper output) : base(output)
+ {
+ }
+
+ [Fact]
+ public void NAIndicatorWorkout()
+ {
+ var data = new[] {
+ new TestClass() { A = 1, B = 3, C = new float[2]{ 1, 2 } , D = new double[2]{ 3,4} },
+ new TestClass() { A = float.NaN, B = double.NaN, C = new float[2]{ float.NaN, float.NaN } , D = new double[2]{ double.NaN,double.NaN}},
+ new TestClass() { A = float.NegativeInfinity, B = double.NegativeInfinity, C = new float[2]{ float.NegativeInfinity, float.NegativeInfinity } , D = new double[2]{ double.NegativeInfinity, double.NegativeInfinity}},
+ new TestClass() { A = float.PositiveInfinity, B = double.PositiveInfinity, C = new float[2]{ float.PositiveInfinity, float.PositiveInfinity, } , D = new double[2]{ double.PositiveInfinity, double.PositiveInfinity}},
+ new TestClass() { A = 2, B = 1, C = new float[2]{ 3, 4 } , D = new double[2]{ 5,6}},
+ };
+
+ var dataView = ComponentCreation.CreateDataView(Env, data);
+ var pipe = new NAIndicatorEstimator(Env,
+ new (string input, string output)[] { ("A", "NAA"), ("B", "NAB"), ("C", "NAC"), ("D", "NAD") });
+ TestEstimatorCore(pipe, dataView);
+ Done();
+ }
+
+ [Fact]
+ public void TestCommandLine()
+ {
+ Assert.Equal(Maml.Main(new[] { @"showschema loader=Text{col=A:R4:0} xf=NAIndicator{col=B:A} in=f:\2.txt" }), (int)0);
+ }
+
+ [Fact]
+ public void TestOldSavingAndLoading()
+ {
+ var data = new[] {
+ new TestClass() { A = 1, B = 3, C = new float[2]{ 1, 2 } , D = new double[2]{ 3,4} },
+ new TestClass() { A = float.NaN, B = double.NaN, C = new float[2]{ float.NaN, float.NaN } , D = new double[2]{ double.NaN,double.NaN}},
+ new TestClass() { A = float.NegativeInfinity, B = double.NegativeInfinity, C = new float[2]{ float.NegativeInfinity, float.NegativeInfinity } , D = new double[2]{ double.NegativeInfinity, double.NegativeInfinity}},
+ new TestClass() { A = float.PositiveInfinity, B = double.PositiveInfinity, C = new float[2]{ float.PositiveInfinity, float.PositiveInfinity, } , D = new double[2]{ double.PositiveInfinity, double.PositiveInfinity}},
+ new TestClass() { A = 2, B = 1 , C = new float[2]{ 3, 4 } , D = new double[2]{ 5,6}},
+ };
+
+ var dataView = ComponentCreation.CreateDataView(Env, data);
+ var pipe = new NAIndicatorEstimator(Env,
+ new (string input, string output)[] { ("A", "NAA"), ("B", "NAB"), ("C", "NAC"), ("D", "NAD") });
+ var result = pipe.Fit(dataView).Transform(dataView);
+ var resultRoles = new RoleMappedData(result);
+ using (var ms = new MemoryStream())
+ {
+ TrainUtils.SaveModel(Env, Env.Start("saving"), ms, null, resultRoles);
+ ms.Position = 0;
+ var loadedView = ModelFileUtils.LoadTransforms(Env, dataView, ms);
+ }
+ }
+
+ [Fact]
+ public void NAIndicatorFileOutput()
+ {
+ string dataPath = GetDataPath("breast-cancer.txt");
+ var reader = TextLoader.CreateReader(Env, ctx => (
+ ScalarFloat: ctx.LoadFloat(1),
+ ScalarDouble: ctx.LoadDouble(1),
+ VectorFloat: ctx.LoadFloat(1, 4),
+ VectorDoulbe: ctx.LoadDouble(1, 4)
+ ));
+
+ var data = reader.Read(new MultiFileSource(dataPath)).AsDynamic;
+ var wrongCollection = new[] { new TestClass() { A = 1, B = 3, C = new float[2] { 1, 2 }, D = new double[2] { 3, 4 } } };
+ var invalidData = ComponentCreation.CreateDataView(Env, wrongCollection);
+ var est = new NAIndicatorEstimator(Env,
+ new (string input, string output)[] { ("ScalarFloat", "A"), ("ScalarDouble", "B"), ("VectorFloat", "C"), ("VectorDoulbe", "D") });
+
+ TestEstimatorCore(est, data, invalidInput: invalidData);
+ var outputPath = GetOutputPath("NAIndicator", "featurized.tsv");
+ using (var ch = Env.Start("save"))
+ {
+ var saver = new TextSaver(Env, new TextSaver.Arguments { Silent = true });
+ IDataView savedData = TakeFilter.Create(Env, est.Fit(data).Transform(data), 4);
+ using (var fs = File.Create(outputPath))
+ DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
+ }
+
+ CheckEquality("NAIndicator", "featurized.tsv");
+ Done();
+ }
+
+ [Fact]
+ public void NAIndicatorMetadataTest()
+ {
+ var data = new[] {
+ new TestClass() { A = 1, B = 3, C = new float[2]{ 1, 2 } , D = new double[2]{ 3,4} },
+ new TestClass() { A = float.NaN, B = double.NaN, C = new float[2]{ float.NaN, float.NaN } , D = new double[2]{ double.NaN,double.NaN}},
+ new TestClass() { A = float.NegativeInfinity, B = double.NegativeInfinity, C = new float[2]{ float.NegativeInfinity, float.NegativeInfinity } , D = new double[2]{ double.NegativeInfinity, double.NegativeInfinity}},
+ new TestClass() { A = float.PositiveInfinity, B = double.PositiveInfinity, C = new float[2]{ float.PositiveInfinity, float.PositiveInfinity, } , D = new double[2]{ double.PositiveInfinity, double.PositiveInfinity}},
+ new TestClass() { A = 2, B = 1, C = new float[2]{ 3, 4 } , D = new double[2]{ 5,6}},
+ };
+
+ var dataView = ComponentCreation.CreateDataView(Env, data);
+ var pipe = new CategoricalEstimator(Env, new CategoricalEstimator.ColumnInfo("A", "CatA"));
+ var newpipe = pipe.Append(new NAIndicatorEstimator(Env, new (string input, string output)[] { ("CatA", "NAA") }));
+ var result = newpipe.Fit(dataView).Transform(dataView);
+ Assert.True(result.Schema.TryGetColumnIndex("NAA", out var col));
+ // Check that the column is normalized.
+ Assert.True(result.Schema.IsNormalized(col));
+ // Check that slot names metadata was correctly created.
+ var value = new VBuffer<ReadOnlyMemory<char>>();
+ var type = result.Schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.SlotNames, col);
+ result.Schema.GetMetadata(MetadataUtils.Kinds.SlotNames, col, ref value);
+ Assert.True(value.Length == 4);
+ var mem = new ReadOnlyMemory<char>();
+ value.GetItemOrDefault(0, ref mem);
+ Assert.True(mem.ToString() == "1");
+ value.GetItemOrDefault(1, ref mem);
+ Assert.True(mem.ToString() == "-Infinity");
+ value.GetItemOrDefault(2, ref mem);
+ Assert.True(mem.ToString() == "Infinity");
+ value.GetItemOrDefault(3, ref mem);
+ Assert.True(mem.ToString() == "2");
+ }
+ }
+}