Move KeyType, VectorType and VBuffer to ML.DataView (#3022)

* Move KeyType into ML.DataView assembly.
* Rename KeyType to KeyDataViewType.
* Move VBuffer to ML.DataView
* Move VectorType to ML.DataView
* Rename VectorType to VectorDataViewType.

Author: eerhardt

docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/TextClassification.cs

@@ -46,9 +46,9 @@ public static void Example()
             //      - Use it for prediction in the pipeline.
             var tensorFlowModel = mlContext.Model.LoadTensorFlowModel(modelLocation);
             var schema = tensorFlowModel.GetModelSchema();
-            var featuresType = (VectorType)schema["Features"].Type;
+            var featuresType = (VectorDataViewType)schema["Features"].Type;
             Console.WriteLine("Name: {0}, Type: {1}, Shape: (-1, {2})", "Features", featuresType.ItemType.RawType, featuresType.Dimensions[0]);
-            var predictionType = (VectorType)schema["Prediction/Softmax"].Type;
+            var predictionType = (VectorDataViewType)schema["Prediction/Softmax"].Type;
             Console.WriteLine("Name: {0}, Type: {1}, Shape: (-1, {2})", "Prediction/Softmax", predictionType.ItemType.RawType, predictionType.Dimensions[0]);
 
             // The model expects the input feature vector to be a fixed length vector.

src/Microsoft.ML.Core/Data/AnnotationBuilderExtensions.cs

@@ -16,7 +16,7 @@ internal static class AnnotationBuilderExtensions
         /// <param name="size">The size of the slot names vector.</param>
         /// <param name="getter">The getter delegate for the slot names.</param>
         public static void AddSlotNames(this DataViewSchema.Annotations.Builder builder, int size, ValueGetter<VBuffer<ReadOnlyMemory<char>>> getter)
-            => builder.Add(AnnotationUtils.Kinds.SlotNames, new VectorType(TextDataViewType.Instance, size), getter);
+            => builder.Add(AnnotationUtils.Kinds.SlotNames, new VectorDataViewType(TextDataViewType.Instance, size), getter);
 
         /// <summary>
         /// Add key values annotation.
@@ -27,6 +27,6 @@ public static void AddSlotNames(this DataViewSchema.Annotations.Builder builder,
         /// <param name="valueType">The value type of key values. Its raw type must match <typeparamref name="TValue"/>.</param>
         /// <param name="getter">The getter delegate for the key values.</param>
         public static void AddKeyValues<TValue>(this DataViewSchema.Annotations.Builder builder, int size, PrimitiveDataViewType valueType, ValueGetter<VBuffer<TValue>> getter)
-            => builder.Add(AnnotationUtils.Kinds.KeyValues, new VectorType(valueType, size), getter);
+            => builder.Add(AnnotationUtils.Kinds.KeyValues, new VectorDataViewType(valueType, size), getter);
     }
 }

src/Microsoft.ML.Core/Data/AnnotationUtils.cs

@@ -36,7 +36,7 @@ public static class Kinds
             public const string KeyValues = "KeyValues";
 
             /// <summary>
-            /// Annotation kind for sets of score columns. The value is typically a KeyType with raw type U4.
+            /// Annotation kind for sets of score columns. The value is typically a <see cref="KeyDataViewType"/> with raw type U4.
             /// </summary>
             public const string ScoreColumnSetId = "ScoreColumnSetId";
 
@@ -139,10 +139,10 @@ public static void Marshal<THave, TNeed>(this AnnotationGetter<THave> getter, in
         /// Returns a vector type with item type text and the given size. The size must be positive.
         /// This is a standard type for annotation consisting of multiple text values, eg SlotNames.
         /// </summary>
-        public static VectorType GetNamesType(int size)
+        public static VectorDataViewType GetNamesType(int size)
         {
             Contracts.CheckParam(size > 0, nameof(size), "must be known size");
-            return new VectorType(TextDataViewType.Instance, size);
+            return new VectorDataViewType(TextDataViewType.Instance, size);
         }
 
         /// <summary>
@@ -151,23 +151,23 @@ public static VectorType GetNamesType(int size)
         /// This is a standard type for annotation consisting of multiple int values that represent
         /// categorical slot ranges with in a column.
         /// </summary>
-        public static VectorType GetCategoricalType(int rangeCount)
+        public static VectorDataViewType GetCategoricalType(int rangeCount)
         {
             Contracts.CheckParam(rangeCount > 0, nameof(rangeCount), "must be known size");
-            return new VectorType(NumberDataViewType.Int32, rangeCount, 2);
+            return new VectorDataViewType(NumberDataViewType.Int32, rangeCount, 2);
         }
 
-        private static volatile KeyType _scoreColumnSetIdType;
+        private static volatile KeyDataViewType _scoreColumnSetIdType;
 
         /// <summary>
         /// The type of the ScoreColumnSetId annotation.
         /// </summary>
-        public static KeyType ScoreColumnSetIdType
+        public static KeyDataViewType ScoreColumnSetIdType
         {
             get
             {
                 return _scoreColumnSetIdType ??
-                    Interlocked.CompareExchange(ref _scoreColumnSetIdType, new KeyType(typeof(uint), int.MaxValue), null) ??
+                    Interlocked.CompareExchange(ref _scoreColumnSetIdType, new KeyDataViewType(typeof(uint), int.MaxValue), null) ??
                     _scoreColumnSetIdType;
             }
         }
@@ -211,7 +211,7 @@ public static IEnumerable<T> Prepend<T>(this IEnumerable<T> tail, params T[] hea
 
         /// <summary>
         /// Returns the max value for the specified annotation kind.
-        /// The annotation type should be a KeyType with raw type U4.
+        /// The annotation type should be a <see cref="KeyDataViewType"/> with raw type U4.
         /// colMax will be set to the first column that has the max value for the specified annotation.
         /// If no column has the specified annotation, colMax is set to -1 and the method returns zero.
         /// The filter function is called for each column, passing in the schema and the column index, and returns
@@ -224,7 +224,7 @@ public static uint GetMaxAnnotationKind(this DataViewSchema schema, out int colM
             for (int col = 0; col < schema.Count; col++)
             {
                 var columnType = schema[col].Annotations.Schema.GetColumnOrNull(annotationKind)?.Type;
-                if (!(columnType is KeyType) || columnType.RawType != typeof(uint))
+                if (!(columnType is KeyDataViewType) || columnType.RawType != typeof(uint))
                     continue;
                 if (filterFunc != null && !filterFunc(schema, col))
                     continue;
@@ -241,14 +241,14 @@ public static uint GetMaxAnnotationKind(this DataViewSchema schema, out int colM
 
         /// <summary>
         /// Returns the set of column ids which match the value of specified annotation kind.
-        /// The annotation type should be a KeyType with raw type U4.
+        /// The annotation type should be a <see cref="KeyDataViewType"/> with raw type U4.
         /// </summary>
         public static IEnumerable<int> GetColumnSet(this DataViewSchema schema, string annotationKind, uint value)
         {
             for (int col = 0; col < schema.Count; col++)
             {
                 var columnType = schema[col].Annotations.Schema.GetColumnOrNull(annotationKind)?.Type;
-                if (columnType is KeyType && columnType.RawType == typeof(uint))
+                if (columnType is KeyDataViewType && columnType.RawType == typeof(uint))
                 {
                     uint val = 0;
                     schema[col].Annotations.GetValue(annotationKind, ref val);
@@ -290,7 +290,7 @@ public static bool HasSlotNames(this DataViewSchema.Column column, int vectorSiz
             var metaColumn = column.Annotations.Schema.GetColumnOrNull(Kinds.SlotNames);
             return
                 metaColumn != null
-                && metaColumn.Value.Type is VectorType vectorType
+                && metaColumn.Value.Type is VectorDataViewType vectorType
                 && vectorType.Size == vectorSize
                 && vectorType.ItemType is TextDataViewType;
         }
@@ -382,7 +382,7 @@ public static bool TryGetCategoricalFeatureIndices(DataViewSchema schema, int co
 
             bool isValid = false;
             categoricalFeatures = null;
-            if (!(schema[colIndex].Type is VectorType vecType && vecType.Size > 0))
+            if (!(schema[colIndex].Type is VectorDataViewType vecType && vecType.Size > 0))
                 return isValid;
 
             var type = schema[colIndex].Annotations.Schema.GetColumnOrNull(Kinds.CategoricalSlotRanges)?.Type;

src/Microsoft.ML.Core/Data/ColumnTypeExtensions.cs

@@ -15,7 +15,7 @@ internal static class ColumnTypeExtensions
     {
         /// <summary>
         /// Whether this type is a standard scalar type completely determined by its <see cref="DataViewType.RawType"/>
-        /// (not a <see cref="KeyType"/> or <see cref="StructuredDataViewType"/>, etc).
+        /// (not a <see cref="KeyDataViewType"/> or <see cref="StructuredDataViewType"/>, etc).
         /// </summary>
         public static bool IsStandardScalar(this DataViewType columnType) =>
             (columnType is NumberDataViewType) || (columnType is TextDataViewType) || (columnType is BooleanDataViewType) ||
@@ -25,7 +25,7 @@ public static bool IsStandardScalar(this DataViewType columnType) =>
         /// <summary>
         /// Zero return means it's not a key type.
         /// </summary>
-        public static ulong GetKeyCount(this DataViewType columnType) => (columnType as KeyType)?.Count ?? 0;
+        public static ulong GetKeyCount(this DataViewType columnType) => (columnType as KeyDataViewType)?.Count ?? 0;
 
         /// <summary>
         /// Sometimes it is necessary to cast the Count to an int. This performs overflow check.
@@ -34,26 +34,26 @@ public static bool IsStandardScalar(this DataViewType columnType) =>
         public static int GetKeyCountAsInt32(this DataViewType columnType, IExceptionContext ectx = null)
         {
             ulong count = columnType.GetKeyCount();
-            ectx.Check(count <= int.MaxValue, nameof(KeyType) + "." + nameof(KeyType.Count) + " exceeds int.MaxValue.");
+            ectx.Check(count <= int.MaxValue, nameof(KeyDataViewType) + "." + nameof(KeyDataViewType.Count) + " exceeds int.MaxValue.");
             return (int)count;
         }
 
         /// <summary>
         /// For non-vector types, this returns the column type itself (i.e., return <paramref name="columnType"/>).
         /// For vector types, this returns the type of the items stored as values in vector.
         /// </summary>
-        public static DataViewType GetItemType(this DataViewType columnType) => (columnType as VectorType)?.ItemType ?? columnType;
+        public static DataViewType GetItemType(this DataViewType columnType) => (columnType as VectorDataViewType)?.ItemType ?? columnType;
 
         /// <summary>
         /// Zero return means either it's not a vector or the size is unknown.
         /// </summary>
-        public static int GetVectorSize(this DataViewType columnType) => (columnType as VectorType)?.Size ?? 0;
+        public static int GetVectorSize(this DataViewType columnType) => (columnType as VectorDataViewType)?.Size ?? 0;
 
         /// <summary>
         /// For non-vectors, this returns one. For unknown size vectors, it returns zero.
         /// For known sized vectors, it returns size.
         /// </summary>
-        public static int GetValueCount(this DataViewType columnType) => (columnType as VectorType)?.Size ?? 1;
+        public static int GetValueCount(this DataViewType columnType) => (columnType as VectorDataViewType)?.Size ?? 1;
 
         /// <summary>
         /// Whether this is a vector type with known size. Returns false for non-vector types.
@@ -85,7 +85,7 @@ public static bool SameSizeAndItemType(this DataViewType columnType, DataViewTyp
                 return true;
 
             // For vector types, we don't care about the factoring of the dimensions.
-            if (!(columnType is VectorType vectorType) || !(other is VectorType otherVectorType))
+            if (!(columnType is VectorDataViewType vectorType) || !(other is VectorDataViewType otherVectorType))
                 return false;
             if (!vectorType.ItemType.Equals(otherVectorType.ItemType))
                 return false;

src/Microsoft.ML.Core/Data/IEstimator.cs

@@ -63,9 +63,9 @@ internal Column(string name, VectorKind vecKind, DataViewType itemType, bool isK
             {
                 Contracts.CheckNonEmpty(name, nameof(name));
                 Contracts.CheckValueOrNull(annotations);
-                Contracts.CheckParam(!(itemType is KeyType), nameof(itemType), "Item type cannot be a key");
-                Contracts.CheckParam(!(itemType is VectorType), nameof(itemType), "Item type cannot be a vector");
-                Contracts.CheckParam(!isKey || KeyType.IsValidDataType(itemType.RawType), nameof(itemType), "The item type must be valid for a key");
+                Contracts.CheckParam(!(itemType is KeyDataViewType), nameof(itemType), "Item type cannot be a key");
+                Contracts.CheckParam(!(itemType is VectorDataViewType), nameof(itemType), "Item type cannot be a vector");
+                Contracts.CheckParam(!isKey || KeyDataViewType.IsValidDataType(itemType.RawType), nameof(itemType), "The item type must be valid for a key");
 
                 Name = name;
                 Kind = vecKind;
@@ -147,7 +147,7 @@ internal static void GetColumnTypeShape(DataViewType type,
             out DataViewType itemType,
             out bool isKey)
         {
-            if (type is VectorType vectorType)
+            if (type is VectorDataViewType vectorType)
             {
                 if (vectorType.IsKnownSize)
                 {
@@ -166,7 +166,7 @@ internal static void GetColumnTypeShape(DataViewType type,
                 itemType = type;
             }
 
-            isKey = itemType is KeyType;
+            isKey = itemType is KeyDataViewType;
             if (isKey)
                 itemType = ColumnTypeExtensions.PrimitiveTypeFromType(itemType.RawType);
         }

src/Microsoft.ML.Core/Data/KeyTypeExtensions.cs

@@ -7,17 +7,17 @@
 namespace Microsoft.ML.Data
 {
     /// <summary>
-    /// Extension methods related to the KeyType class.
+    /// Extension methods related to the <see cref="KeyDataViewType"/> class.
     /// </summary>
     [BestFriend]
     internal static class KeyTypeExtensions
     {
         /// <summary>
         /// Sometimes it is necessary to cast the Count to an int. This performs overflow check.
         /// </summary>
-        public static int GetCountAsInt32(this KeyType key, IExceptionContext ectx = null)
+        public static int GetCountAsInt32(this KeyDataViewType key, IExceptionContext ectx = null)
         {
-            ectx.Check(key.Count <= int.MaxValue, nameof(KeyType) + "." + nameof(KeyType.Count) + " exceeds int.MaxValue.");
+            ectx.Check(key.Count <= int.MaxValue, nameof(KeyDataViewType) + "." + nameof(KeyDataViewType.Count) + " exceeds int.MaxValue.");
             return (int)key.Count;
         }
     }

src/Microsoft.ML.Core/Utilities/ArrayUtils.cs

@@ -0,0 +1,104 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Diagnostics;
+
+namespace Microsoft.ML.Internal.Utilities
+{
+    internal static partial class ArrayUtils
+    {
+        // Maximum size of one-dimensional array.
+        // See: https://msdn.microsoft.com/en-us/library/hh285054(v=vs.110).aspx
+        public const int ArrayMaxSize = 0X7FEFFFFF;
+
+        public static int Size<T>(T[] x)
+        {
+            return x == null ? 0 : x.Length;
+        }
+
+        /// <summary>
+        /// Akin to <c>FindIndexSorted</c>, except stores the found index in the output
+        /// <c>index</c> parameter, and returns whether that index is a valid index
+        /// pointing to a value equal to the input parameter <c>value</c>.
+        /// </summary>
+        public static bool TryFindIndexSorted(int[] input, int min, int lim, int value, out int index)
+        {
+            index = FindIndexSorted(input, min, lim, value);
+            return index < lim && input[index] == value;
+        }
+
+        /// <summary>
+        /// Assumes input is sorted and finds value using BinarySearch.
+        /// If value is not found, returns the logical index of 'value' in the sorted list i.e index of the first element greater than value.
+        /// In case of duplicates it returns the index of the first one.
+        /// It guarantees that items before the returned index are &lt; value, while those at and after the returned index are &gt;= value.
+        /// </summary>
+        public static int FindIndexSorted(int[] input, int min, int lim, int value)
+        {
+            return FindIndexSorted(input.AsSpan(), min, lim, value);
+        }
+
+        /// <summary>
+        /// Assumes input is sorted and finds value using BinarySearch.
+        /// If value is not found, returns the logical index of 'value' in the sorted list i.e index of the first element greater than value.
+        /// In case of duplicates it returns the index of the first one.
+        /// It guarantees that items before the returned index are &lt; value, while those at and after the returned index are &gt;= value.
+        /// </summary>
+        public static int FindIndexSorted(ReadOnlySpan<int> input, int min, int lim, int value)
+        {
+            Debug.Assert(0 <= min & min <= lim & lim <= input.Length);
+
+            int minCur = min;
+            int limCur = lim;
+            while (minCur < limCur)
+            {
+                int mid = (int)(((uint)minCur + (uint)limCur) / 2);
+                Debug.Assert(minCur <= mid & mid < limCur);
+
+                if (input[mid] >= value)
+                    limCur = mid;
+                else
+                    minCur = mid + 1;
+
+                Debug.Assert(min <= minCur & minCur <= limCur & limCur <= lim);
+                Debug.Assert(minCur == min || input[minCur - 1] < value);
+                Debug.Assert(limCur == lim || input[limCur] >= value);
+            }
+            Debug.Assert(min <= minCur & minCur == limCur & limCur <= lim);
+            Debug.Assert(minCur == min || input[minCur - 1] < value);
+            Debug.Assert(limCur == lim || input[limCur] >= value);
+
+            return minCur;
+        }
+
+        public static int EnsureSize<T>(ref T[] array, int min, int max, bool keepOld, out bool resized)
+        {
+            if (min > max)
+                throw new ArgumentOutOfRangeException(nameof(max), "min must not exceed max");
+
+            // This code adapted from the private method EnsureCapacity code of List<T>.
+            int size = ArrayUtils.Size(array);
+            if (size >= min)
+            {
+                resized = false;
+                return size;
+            }
+
+            int newSize = size == 0 ? 4 : size * 2;
+            // This constant taken from the internal code of system\array.cs of mscorlib.
+            if ((uint)newSize > max)
+                newSize = max;
+            if (newSize < min)
+                newSize = min;
+            if (keepOld && size > 0)
+                Array.Resize(ref array, newSize);
+            else
+                array = new T[newSize];
+
+            resized = true;
+            return newSize;
+        }
+    }
+}