Skip to content

Commit 6ed35f0

Browse files
author
Pete Luferenko
committed
Removed GetMetadataTypes and GetMetadataTypeOrNull
1 parent 4d488dd commit 6ed35f0

38 files changed

+250
-344
lines changed

‎src/Microsoft.ML.Core/Data/IEstimator.cs

+3-3
Original file line numberDiff line numberDiff line change
@@ -177,10 +177,10 @@ internal static SchemaShape Create(Schema schema)
177177
{
178178
// First create the metadata.
179179
var mCols = new List<Column>();
180-
foreach (var metaNameType in schema.GetMetadataTypes(iCol))
180+
foreach (var metaColumn in schema[iCol].Metadata.Schema)
181181
{
182-
GetColumnTypeShape(metaNameType.Value, out var mVecKind, out var mItemType, out var mIsKey);
183-
mCols.Add(new Column(metaNameType.Key, mVecKind, mItemType, mIsKey));
182+
GetColumnTypeShape(metaColumn.Type, out var mVecKind, out var mItemType, out var mIsKey);
183+
mCols.Add(new Column(metaColumn.Name, mVecKind, mItemType, mIsKey));
184184
}
185185
var metadata = mCols.Count > 0 ? new SchemaShape(mCols) : _empty;
186186
// Next create the single column.

‎src/Microsoft.ML.Core/Data/MetadataUtils.cs

+4-4
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ public static uint GetMaxMetadataKind(this Schema schema, out int colMax, string
239239
colMax = -1;
240240
for (int col = 0; col < schema.Count; col++)
241241
{
242-
var columnType = schema.GetMetadataTypeOrNull(metadataKind, col);
242+
var columnType = schema[col].Metadata.Schema.GetColumnOrNull(metadataKind)?.Type;
243243
if (columnType == null || !columnType.IsKey || columnType.RawKind != DataKind.U4)
244244
continue;
245245
if (filterFunc != null && !filterFunc(schema, col))
@@ -264,7 +264,7 @@ internal static IEnumerable<int> GetColumnSet(this Schema schema, string metadat
264264
{
265265
for (int col = 0; col < schema.Count; col++)
266266
{
267-
var columnType = schema.GetMetadataTypeOrNull(metadataKind, col);
267+
var columnType = schema[col].Metadata.Schema.GetColumnOrNull(metadataKind)?.Type;
268268
if (columnType != null && columnType.IsKey && columnType.RawKind == DataKind.U4)
269269
{
270270
uint val = 0;
@@ -284,7 +284,7 @@ internal static IEnumerable<int> GetColumnSet(this Schema schema, string metadat
284284
{
285285
for (int col = 0; col < schema.Count; col++)
286286
{
287-
var columnType = schema.GetMetadataTypeOrNull(metadataKind, col);
287+
var columnType = schema[col].Metadata.Schema.GetColumnOrNull(metadataKind)?.Type;
288288
if (columnType != null && columnType.IsText)
289289
{
290290
ReadOnlyMemory<char> val = default;
@@ -450,7 +450,7 @@ internal static bool TryGetCategoricalFeatureIndices(Schema schema, int colIndex
450450
if (!(schema[colIndex].Type is VectorType vecType && vecType.Size > 0))
451451
return isValid;
452452

453-
var type = schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.CategoricalSlotRanges, colIndex);
453+
var type = schema[colIndex].Metadata.Schema.GetColumnOrNull(MetadataUtils.Kinds.CategoricalSlotRanges)?.Type;
454454
if (type?.RawType == typeof(VBuffer<int>))
455455
{
456456
VBuffer<int> catIndices = default(VBuffer<int>);

‎src/Microsoft.ML.Core/Data/Schema.cs

+4-16
Original file line numberDiff line numberDiff line change
@@ -310,23 +310,11 @@ private static Delegate GetMetadataGetterDelegate<TValue>(ISchema schema, int co
310310

311311
ColumnType ISchema.GetColumnType(int col) => this[col].Type;
312312

313-
public IEnumerable<KeyValuePair<string, ColumnType>> GetMetadataTypes(int col)
314-
{
315-
var meta = this[col].Metadata;
316-
if (meta == null)
317-
return Enumerable.Empty<KeyValuePair<string, ColumnType>>();
318-
return meta.Schema.Select(c => new KeyValuePair<string, ColumnType>(c.Name, c.Type));
319-
}
313+
IEnumerable<KeyValuePair<string, ColumnType>> ISchema.GetMetadataTypes(int col)
314+
=> this[col].Metadata.Schema.Select(c => new KeyValuePair<string, ColumnType>(c.Name, c.Type));
320315

321-
public ColumnType GetMetadataTypeOrNull(string kind, int col)
322-
{
323-
var meta = this[col].Metadata;
324-
if (meta == null)
325-
return null;
326-
if (meta.Schema.TryGetColumnIndex(kind, out int metaCol))
327-
return meta.Schema[metaCol].Type;
328-
return null;
329-
}
316+
ColumnType ISchema.GetMetadataTypeOrNull(string kind, int col)
317+
=> this[col].Metadata.Schema.GetColumnOrNull(kind)?.Type;
330318

331319
public void GetMetadata<TValue>(string kind, int col, ref TValue value)
332320
{

‎src/Microsoft.ML.Data/Commands/ShowSchemaCommand.cs

+6-8
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ private static void PrintSchema(TextWriter writer, Arguments args, Schema schema
153153
if (!type.IsKnownSizeVector)
154154
continue;
155155
ColumnType typeNames;
156-
if ((typeNames = schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.SlotNames, col)) == null)
156+
if ((typeNames = schema[col].Metadata.Schema.GetColumnOrNull(MetadataUtils.Kinds.SlotNames)?.Type) == null)
157157
continue;
158158
if (typeNames.VectorSize != type.VectorSize || !typeNames.ItemType.IsText)
159159
{
@@ -188,18 +188,16 @@ private static void ShowMetadata(IndentedTextWriter itw, Schema schema, int col,
188188

189189
using (itw.Nest())
190190
{
191-
foreach (var kvp in schema.GetMetadataTypes(col).OrderBy(p => p.Key))
191+
foreach (var metaColumn in schema[col].Metadata.Schema.OrderBy(mcol => mcol.Name))
192192
{
193-
Contracts.AssertNonEmpty(kvp.Key);
194-
Contracts.AssertValue(kvp.Value);
195-
var type = kvp.Value;
196-
itw.Write("Metadata '{0}': {1}", kvp.Key, type);
193+
var type = metaColumn.Type;
194+
itw.Write("Metadata '{0}': {1}", metaColumn.Name, type);
197195
if (showVals)
198196
{
199197
if (!type.IsVector)
200-
ShowMetadataValue(itw, schema, col, kvp.Key, type);
198+
ShowMetadataValue(itw, schema, col, metaColumn.Name, type);
201199
else
202-
ShowMetadataValueVec(itw, schema, col, kvp.Key, type);
200+
ShowMetadataValueVec(itw, schema, col, metaColumn.Name, type);
203201
}
204202
itw.WriteLine();
205203
}

‎src/Microsoft.ML.Data/DataLoadSave/Binary/BinarySaver.cs

+10-10
Original file line numberDiff line numberDiff line change
@@ -275,25 +275,25 @@ private long WriteMetadata(BinaryWriter writer, Schema schema, int col, IChannel
275275
// track of the location and size of each for when we write the metadata table of contents.
276276
// (To be clear, this specific layout is not required by the format.)
277277

278-
foreach (var pair in schema.GetMetadataTypes(col))
278+
foreach (var metaColumn in schema[col].Metadata.Schema)
279279
{
280-
_host.Check(!string.IsNullOrEmpty(pair.Key), "Metadata with null or empty kind detected, disallowed");
281-
_host.Check(pair.Value != null, "Metadata with null type detected, disallowed");
282-
if (!kinds.Add(pair.Key))
283-
throw _host.Except("Metadata with duplicate kind '{0}' encountered, disallowed", pair.Key, schema[col].Name);
284-
args[3] = pair.Key;
285-
args[4] = pair.Value;
286-
IValueCodec codec = (IValueCodec)methInfo.MakeGenericMethod(pair.Value.RawType).Invoke(this, args);
280+
_host.Check(!string.IsNullOrEmpty(metaColumn.Name), "Metadata with null or empty kind detected, disallowed");
281+
_host.Check(metaColumn.Type != null, "Metadata with null type detected, disallowed");
282+
if (!kinds.Add(metaColumn.Name))
283+
throw _host.Except("Metadata with duplicate kind '{0}' encountered, disallowed", metaColumn.Name, schema[col].Name);
284+
args[3] = metaColumn.Name;
285+
args[4] = metaColumn.Type;
286+
IValueCodec codec = (IValueCodec)methInfo.MakeGenericMethod(metaColumn.Type.RawType).Invoke(this, args);
287287
if (codec == null)
288288
{
289289
// Nothing was written.
290290
ch.Warning("Could not get codec for type {0}, dropping column '{1}' index {2} metadata kind '{3}'",
291-
pair.Value, schema[col].Name, col, pair.Key);
291+
metaColumn.Type, schema[col].Name, col, metaColumn.Name);
292292
continue;
293293
}
294294
offsets.Add(writer.BaseStream.Position);
295295
_host.CheckIO(offsets[offsets.Count - 1] > offsets[offsets.Count - 2], "Bad offsets detected during write");
296-
metadataInfos.Add(Tuple.Create(pair.Key, codec, (CompressionKind)args[5]));
296+
metadataInfos.Add(Tuple.Create(metaColumn.Name, codec, (CompressionKind)args[5]));
297297
count++;
298298
}
299299
if (metadataInfos.Count == 0)

‎src/Microsoft.ML.Data/DataLoadSave/Text/TextSaver.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ public VecValueWriter(RowCursor cursor, VectorType type, int source, char sep)
155155
_getSrc = cursor.GetGetter<VBuffer<T>>(source);
156156
ColumnType typeNames;
157157
if (type.IsKnownSizeVector &&
158-
(typeNames = cursor.Schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.SlotNames, source)) != null &&
158+
(typeNames = cursor.Schema[source].Metadata.Schema.GetColumnOrNull(MetadataUtils.Kinds.SlotNames)?.Type) != null &&
159159
typeNames.VectorSize == type.VectorSize && typeNames.ItemType.IsText)
160160
{
161161
cursor.Schema.GetMetadata(MetadataUtils.Kinds.SlotNames, source, ref _slotNames);
@@ -408,7 +408,7 @@ private void WriteDataCore(IChannel ch, TextWriter writer, IDataView data,
408408
}
409409
if (!type.IsKnownSizeVector)
410410
continue;
411-
var typeNames = data.Schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.SlotNames, cols[i]);
411+
var typeNames = data.Schema[cols[i]].Metadata.Schema.GetColumnOrNull(MetadataUtils.Kinds.SlotNames)?.Type;
412412
if (typeNames != null && typeNames.VectorSize == type.VectorSize && typeNames.ItemType.IsText)
413413
hasHeader = true;
414414
}

‎src/Microsoft.ML.Data/DataLoadSave/Transpose/TransposeLoader.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -640,12 +640,12 @@ public ColumnType GetColumnType(int col)
640640

641641
public ColumnType GetMetadataTypeOrNull(string kind, int col)
642642
{
643-
return Schema.GetMetadataTypeOrNull(kind, col);
643+
return Schema[col].Metadata.Schema.GetColumnOrNull(kind)?.Type;
644644
}
645645

646646
public IEnumerable<KeyValuePair<string, ColumnType>> GetMetadataTypes(int col)
647647
{
648-
return Schema.GetMetadataTypes(col);
648+
return Schema[col].Metadata.Schema.Select(c => new KeyValuePair<string, ColumnType>(c.Name, c.Type));
649649
}
650650

651651
public void GetMetadata<TValue>(string kind, int col, ref TValue value)

‎src/Microsoft.ML.Data/DataView/CompositeSchema.cs

+3-2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
using System;
66
using System.Collections.Generic;
7+
using System.Linq;
78
using Microsoft.ML.Data;
89
using Microsoft.ML.Runtime.Internal.Utilities;
910

@@ -105,13 +106,13 @@ public ColumnType GetColumnType(int col)
105106
public IEnumerable<KeyValuePair<string, ColumnType>> GetMetadataTypes(int col)
106107
{
107108
GetColumnSource(col, out int dv, out int srcCol);
108-
return _sources[dv].GetMetadataTypes(srcCol);
109+
return _sources[dv][srcCol].Metadata.Schema.Select(c => new KeyValuePair<string, ColumnType>(c.Name, c.Type));
109110
}
110111

111112
public ColumnType GetMetadataTypeOrNull(string kind, int col)
112113
{
113114
GetColumnSource(col, out int dv, out int srcCol);
114-
return _sources[dv].GetMetadataTypeOrNull(kind, srcCol);
115+
return _sources[dv][srcCol].Metadata.Schema.GetColumnOrNull(kind)?.Type;
115116
}
116117

117118
public void GetMetadata<TValue>(string kind, int col, ref TValue value)

‎src/Microsoft.ML.Data/DataView/Transposer.cs

+4-4
Original file line numberDiff line numberDiff line change
@@ -332,12 +332,12 @@ public ColumnType GetColumnType(int col)
332332

333333
public IEnumerable<KeyValuePair<string, ColumnType>> GetMetadataTypes(int col)
334334
{
335-
return InputSchema.GetMetadataTypes(col);
335+
return InputSchema[col].Metadata.Schema.Select(c => new KeyValuePair<string, ColumnType>(c.Name, c.Type));
336336
}
337337

338338
public ColumnType GetMetadataTypeOrNull(string kind, int col)
339339
{
340-
return InputSchema.GetMetadataTypeOrNull(kind, col);
340+
return InputSchema[col].Metadata.Schema.GetColumnOrNull(kind)?.Type;
341341
}
342342

343343
public void GetMetadata<TValue>(string kind, int col, ref TValue value)
@@ -1667,12 +1667,12 @@ public VectorType GetSlotType(int col)
16671667

16681668
public IEnumerable<KeyValuePair<string, ColumnType>> GetMetadataTypes(int col)
16691669
{
1670-
return _schema.GetMetadataTypes(col);
1670+
return _schema[col].Metadata.Schema.Select(c => new KeyValuePair<string, ColumnType>(c.Name, c.Type));
16711671
}
16721672

16731673
public ColumnType GetMetadataTypeOrNull(string kind, int col)
16741674
{
1675-
return _schema.GetMetadataTypeOrNull(kind, col);
1675+
return _schema[col].Metadata.Schema.GetColumnOrNull(kind)?.Type;
16761676
}
16771677

16781678
public void GetMetadata<TValue>(string kind, int col, ref TValue value)

‎src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ private ReadOnlyMemory<char>[] GetClassNames(RoleMappedSchema schema)
175175
ColumnType type;
176176
var labelNames = default(VBuffer<ReadOnlyMemory<char>>);
177177
if (schema.Label.Type.IsKey &&
178-
(type = schema.Schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.KeyValues, schema.Label.Index)) != null &&
178+
(type = schema.Schema[schema.Label.Index].Metadata.Schema.GetColumnOrNull(MetadataUtils.Kinds.KeyValues)?.Type) != null &&
179179
type.ItemType.IsKnownSizeVector && type.ItemType.IsText)
180180
{
181181
schema.Schema.GetMetadata(MetadataUtils.Kinds.KeyValues, schema.Label.Index, ref labelNames);

0 commit comments

Comments
 (0)