Updating DatabaseLoader to support getting column info from a given .NET type. (dotnet#4091)

* Updating DatabaseLoader to support getting column info from a given .NET type.

* Addressing PR feedback.

* Reverting DatabaseLoaderTests.IrisLightGbm to manually creating the columns.

* Fixing a naming error: source to sources

Author: tannergooding

src/Microsoft.ML.Data/Data/SchemaDefinition.cs

@@ -104,6 +104,7 @@ public sealed class ColumnNameAttribute : Attribute
         /// <summary>
         /// Column name.
         /// </summary>
+        [BestFriend]
         internal string Name { get; }
 
         /// <summary>

src/Microsoft.ML.Data/DataLoadSave/Text/LoadColumnAttribute.cs

@@ -46,6 +46,7 @@ public LoadColumnAttribute(int[] columnIndexes)
                 Sources.Add(new TextLoader.Range(col));
         }
 
+        [BestFriend]
         internal List<TextLoader.Range> Sources;
     }
 }

src/Microsoft.ML.Experimental/DataLoadSave/Database/DatabaseLoader.cs

@@ -5,7 +5,11 @@
 using System;
 using System.Collections.Generic;
 using System.Data;
+using System.Data.Common;
 using System.Linq;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Text;
 using Microsoft.ML;
 using Microsoft.ML.CommandLine;
 using Microsoft.ML.Data;
@@ -98,6 +102,71 @@ void ICanSaveModel.Save(ModelSaveContext ctx)
         /// <param name="source">The source from which to load data.</param>
         public IDataView Load(DatabaseSource source) => new BoundLoader(this, source);
 
+        internal static DatabaseLoader CreateDatabaseLoader<TInput>(IHostEnvironment host)
+        {
+            var userType = typeof(TInput);
+
+            var fieldInfos = userType.GetFields(BindingFlags.Public | BindingFlags.Instance);
+
+            var propertyInfos =
+                userType
+                .GetProperties(BindingFlags.Public | BindingFlags.Instance)
+                .Where(x => x.CanRead && x.GetGetMethod() != null && x.GetIndexParameters().Length == 0);
+
+            var memberInfos = (fieldInfos as IEnumerable<MemberInfo>).Concat(propertyInfos).ToArray();
+
+            if (memberInfos.Length == 0)
+                throw host.ExceptParam(nameof(TInput), $"Should define at least one public, readable field or property in {nameof(TInput)}.");
+
+            var columns = new List<Column>();
+
+            for (int index = 0; index < memberInfos.Length; index++)
+            {
+                var memberInfo = memberInfos[index];
+                var mappingAttrName = memberInfo.GetCustomAttribute<ColumnNameAttribute>();
+
+                var column = new Column();
+                column.Name = mappingAttrName?.Name ?? memberInfo.Name;
+
+                var mappingAttr = memberInfo.GetCustomAttribute<LoadColumnAttribute>();
+
+                if (mappingAttr is object)
+                {
+                    var sources = mappingAttr.Sources.Select((source) => Range.FromTextLoaderRange(source)).ToArray();
+                    column.Source = sources.Single().Min;
+                }
+
+                InternalDataKind dk;
+                switch (memberInfo)
+                {
+                    case FieldInfo field:
+                        if (!InternalDataKindExtensions.TryGetDataKind(field.FieldType.IsArray ? field.FieldType.GetElementType() : field.FieldType, out dk))
+                            throw Contracts.Except($"Field {memberInfo.Name} is of unsupported type.");
+
+                        break;
+
+                    case PropertyInfo property:
+                        if (!InternalDataKindExtensions.TryGetDataKind(property.PropertyType.IsArray ? property.PropertyType.GetElementType() : property.PropertyType, out dk))
+                            throw Contracts.Except($"Property {memberInfo.Name} is of unsupported type.");
+                        break;
+
+                    default:
+                        Contracts.Assert(false);
+                        throw Contracts.ExceptNotSupp("Expected a FieldInfo or a PropertyInfo");
+                }
+
+                column.Type = dk.ToDbType();
+
+                columns.Add(column);
+            }
+
+            var options = new Options
+            {
+                Columns = columns.ToArray()
+            };
+            return new DatabaseLoader(host, options);
+        }
+
         /// <summary>
         /// Describes how an input column should be mapped to an <see cref="IDataView"/> column.
         /// </summary>
@@ -128,6 +197,86 @@ public sealed class Column
             public KeyCount KeyCount;
         }
 
+        /// <summary>
+        /// Specifies the range of indices of input columns that should be mapped to an output column.
+        /// </summary>
+        public sealed class Range
+        {
+            public Range() { }
+
+            /// <summary>
+            /// A range representing a single value. Will result in a scalar column.
+            /// </summary>
+            /// <param name="index">The index of the field of the text file to read.</param>
+            public Range(int index)
+            {
+                Contracts.CheckParam(index >= 0, nameof(index), "Must be non-negative");
+                Min = index;
+                Max = index;
+            }
+
+            /// <summary>
+            /// A range representing a set of values. Will result in a vector column.
+            /// </summary>
+            /// <param name="min">The minimum inclusive index of the column.</param>
+            /// <param name="max">The maximum-inclusive index of the column. If <c>null</c>
+            /// indicates that the <see cref="TextLoader"/> should auto-detect the legnth
+            /// of the lines, and read untill the end.</param>
+            public Range(int min, int? max)
+            {
+                Contracts.CheckParam(min >= 0, nameof(min), "Must be non-negative");
+                Contracts.CheckParam(!(max < min), nameof(max), "If specified, must be greater than or equal to " + nameof(min));
+
+                Min = min;
+                Max = max;
+                // Note that without the following being set, in the case where there is a single range
+                // where Min == Max, the result will not be a vector valued but a scalar column.
+                ForceVector = true;
+                AutoEnd = max == null;
+            }
+
+            /// <summary>
+            ///  The minimum index of the column, inclusive.
+            /// </summary>
+            [Argument(ArgumentType.Required, HelpText = "First index in the range")]
+            public int Min;
+
+            /// <summary>
+            /// The maximum index of the column, inclusive. If <see langword="null"/>
+            /// indicates that the <see cref="TextLoader"/> should auto-detect the legnth
+            /// of the lines, and read untill the end.
+            /// If <see cref="Max"/> is specified, the field <see cref="AutoEnd"/> is ignored.
+            /// </summary>
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Last index in the range")]
+            public int? Max;
+
+            /// <summary>
+            /// Whether this range extends to the end of the line, but should be a fixed number of items.
+            /// If <see cref="Max"/> is specified, the field <see cref="AutoEnd"/> is ignored.
+            /// </summary>
+            [Argument(ArgumentType.AtMostOnce,
+                HelpText = "This range extends to the end of the line, but should be a fixed number of items",
+                ShortName = "auto")]
+            public bool AutoEnd;
+
+            /// <summary>
+            /// Whether this range includes only other indices not specified.
+            /// </summary>
+            [Argument(ArgumentType.AtMostOnce, HelpText = "This range includes only other indices not specified", ShortName = "other")]
+            public bool AllOther;
+
+            /// <summary>
+            /// Force scalar columns to be treated as vectors of length one.
+            /// </summary>
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Force scalar columns to be treated as vectors of length one", ShortName = "vector")]
+            public bool ForceVector;
+
+            internal static Range FromTextLoaderRange(TextLoader.Range range)
+            {
+                return new Range(range.Min, range.Max);
+            }
+        }
+
         /// <summary>
         /// The settings for <see cref="DatabaseLoader"/>
         /// </summary>

src/Microsoft.ML.Experimental/DataLoadSave/Database/DatabaseLoaderCatalog.cs

@@ -11,9 +11,7 @@ namespace Microsoft.ML
     /// </summary>
     public static class DatabaseLoaderCatalog
     {
-        /// <summary>
-        /// Create a database loader <see cref="DatabaseLoader"/>.
-        /// </summary>
+        /// <summary>Create a database loader <see cref="DatabaseLoader"/>.</summary>
         /// <param name="catalog">The <see cref="DataOperationsCatalog"/> catalog.</param>
         /// <param name="columns">Array of columns <see cref="DatabaseLoader.Column"/> defining the schema.</param>
         public static DatabaseLoader CreateDatabaseLoader(this DataOperationsCatalog catalog,
@@ -23,8 +21,22 @@ public static DatabaseLoader CreateDatabaseLoader(this DataOperationsCatalog cat
             {
                 Columns = columns,
             };
-
-            return new DatabaseLoader(CatalogUtils.GetEnvironment(catalog), options);
+            return catalog.CreateDatabaseLoader(options);
         }
+
+        /// <summary>Create a database loader <see cref="DatabaseLoader"/>.</summary>
+        /// <param name="catalog">The <see cref="DataOperationsCatalog"/> catalog.</param>
+        /// <param name="options">Defines the settings of the load operation.</param>
+        public static DatabaseLoader CreateDatabaseLoader(this DataOperationsCatalog catalog,
+            DatabaseLoader.Options options)
+            => new DatabaseLoader(CatalogUtils.GetEnvironment(catalog), options);
+
+        /// <summary>Create a database loader <see cref="DatabaseLoader"/>.</summary>
+        /// <typeparam name="TInput">Defines the schema of the data to be loaded. Use public fields or properties
+        /// decorated with <see cref="LoadColumnAttribute"/> (and possibly other attributes) to specify the column
+        /// names and their data types in the schema of the loaded data.</typeparam>
+        /// <param name="catalog">The <see cref="DataOperationsCatalog"/> catalog.</param>
+        public static DatabaseLoader CreateDatabaseLoader<TInput>(this DataOperationsCatalog catalog)
+            => DatabaseLoader.CreateDatabaseLoader<TInput>(CatalogUtils.GetEnvironment(catalog));
     }
 }

src/Microsoft.ML.Experimental/DataLoadSave/Database/DbExtensions.cs

@@ -66,5 +66,82 @@ public static Type ToType(this DbType dbType)
                     return null;
             }
         }
+
+        /// <summary>Maps a <see cref="InternalDataKind"/> to the associated <see cref="DbType"/>.</summary>
+        public static DbType ToDbType(this InternalDataKind dataKind)
+        {
+            switch (dataKind)
+            {
+                case InternalDataKind.I1:
+                {
+                    return DbType.SByte;
+                }
+
+                case InternalDataKind.U1:
+                {
+                    return DbType.Byte;
+                }
+
+                case InternalDataKind.I2:
+                {
+                    return DbType.Int16;
+                }
+
+                case InternalDataKind.U2:
+                {
+                    return DbType.UInt16;
+                }
+
+                case InternalDataKind.I4:
+                {
+                    return DbType.Int32;
+                }
+
+                case InternalDataKind.U4:
+                {
+                    return DbType.UInt32;
+                }
+
+                case InternalDataKind.I8:
+                {
+                    return DbType.Int64;
+                }
+
+                case InternalDataKind.U8:
+                {
+                    return DbType.UInt64;
+                }
+
+                case InternalDataKind.R4:
+                {
+                    return DbType.Single;
+                }
+
+                case InternalDataKind.R8:
+                {
+                    return DbType.Double;
+                }
+
+                case InternalDataKind.TX:
+                {
+                    return DbType.String;
+                }
+
+                case InternalDataKind.BL:
+                {
+                    return DbType.Boolean;
+                }
+
+                case InternalDataKind.DT:
+                {
+                    return DbType.DateTime;
+                }
+
+                default:
+                {
+                    throw new NotSupportedException();
+                }
+            }
+        }
     }
 }