Skip to content

Internalize DataKind #2661

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Feb 25, 2019
Merged

Internalize DataKind #2661

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
afe3ee7
Hide the uses of DataKind in TypeConverting
wschin Feb 21, 2019
ac7f822
Hide DataKind used in TextLoader
wschin Feb 21, 2019
c7f5880
Internalize-best-friend DataKind
wschin Feb 21, 2019
93d3ffe
DataKind ---> InternalDataKind
wschin Feb 22, 2019
7f28030
ScalarType ---> DataKind (massive renaming)
wschin Feb 22, 2019
70b5936
Address comments
wschin Feb 22, 2019
c767628
Address comments
wschin Feb 22, 2019
9a803a0
Address comments
wschin Feb 23, 2019
e9fd4ae
Make R4 as default
wschin Feb 23, 2019
9d1e49c
Ok. I updated entry point...
wschin Feb 23, 2019
0057fa6
Sync with new things from master
wschin Feb 25, 2019
02e4374
Address comments
wschin Feb 25, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Calibrator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ public static void Example()
HasHeader = true,
Columns = new[]
{
new TextLoader.Column("Sentiment", DataKind.BL, 0),
new TextLoader.Column("SentimentText", DataKind.Text, 1)
new TextLoader.Column("Sentiment", DataKind.Boolean, 0),
new TextLoader.Column("SentimentText", DataKind.String, 1)
}
});

Expand Down
4 changes: 2 additions & 2 deletions docs/samples/Microsoft.ML.Samples/Dynamic/FeatureSelectionTransform.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ public static void Example()
var reader = ml.Data.CreateTextLoader(
columns: new[]
{
new TextLoader.Column("Label", DataKind.BL, 0),
new TextLoader.Column("Features", DataKind.Num, new [] { new TextLoader.Range(1, 9) })
new TextLoader.Column("Label", DataKind.Boolean, 0),
new TextLoader.Column("Features", DataKind.Single, new [] { new TextLoader.Range(1, 9) })
},
hasHeader: true
);
Expand Down
4 changes: 2 additions & 2 deletions docs/samples/Microsoft.ML.Samples/Dynamic/FieldAwareFactorizationMachine.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ public static void Example()
var reader = mlContext.Data.CreateTextLoader(
columns: new[]
{
new TextLoader.Column("Sentiment", DataKind.BL, 0),
new TextLoader.Column("SentimentText", DataKind.Text, 1)
new TextLoader.Column("Sentiment", DataKind.Boolean, 0),
new TextLoader.Column("SentimentText", DataKind.String, 1)
},
hasHeader: true
);
Expand Down
4 changes: 2 additions & 2 deletions docs/samples/Microsoft.ML.Samples/Dynamic/ImageAnalytics/ConvertToGrayScale.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ public static void Example()
{
Columns = new[]
{
new TextLoader.Column("ImagePath", DataKind.TX, 0),
new TextLoader.Column("Name", DataKind.TX, 1),
new TextLoader.Column("ImagePath", DataKind.String, 0),
new TextLoader.Column("Name", DataKind.String, 1),
}
}).Read(imagesDataFile);

Expand Down
4 changes: 2 additions & 2 deletions docs/samples/Microsoft.ML.Samples/Dynamic/ImageAnalytics/ExtractPixels.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ public static void Example()
{
Columns = new[]
{
new TextLoader.Column("ImagePath", DataKind.TX, 0),
new TextLoader.Column("Name", DataKind.TX, 1),
new TextLoader.Column("ImagePath", DataKind.String, 0),
new TextLoader.Column("Name", DataKind.String, 1),
}
}).Read(imagesDataFile);

Expand Down
4 changes: 2 additions & 2 deletions docs/samples/Microsoft.ML.Samples/Dynamic/ImageAnalytics/LoadImages.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ public static void Example()
{
Columns = new[]
{
new TextLoader.Column("ImagePath", DataKind.TX, 0),
new TextLoader.Column("Name", DataKind.TX, 1),
new TextLoader.Column("ImagePath", DataKind.String, 0),
new TextLoader.Column("Name", DataKind.String, 1),
}
}).Read(imagesDataFile);

Expand Down
4 changes: 2 additions & 2 deletions docs/samples/Microsoft.ML.Samples/Dynamic/ImageAnalytics/ResizeImages.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ public static void Example()
{
Columns = new[]
{
new TextLoader.Column("ImagePath", DataKind.TX, 0),
new TextLoader.Column("Name", DataKind.TX, 1),
new TextLoader.Column("ImagePath", DataKind.String, 0),
new TextLoader.Column("Name", DataKind.String, 1),
}
}).Read(imagesDataFile);

Expand Down
30 changes: 15 additions & 15 deletions docs/samples/Microsoft.ML.Samples/Dynamic/LogisticRegression.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,21 +37,21 @@ public static void Example()
HasHeader = true,
Columns = new[]
{
new TextLoader.Column("age", DataKind.R4, 0),
new TextLoader.Column("workclass", DataKind.Text, 1),
new TextLoader.Column("fnlwgt", DataKind.R4, 2),
new TextLoader.Column("education", DataKind.Text, 3),
new TextLoader.Column("education-num", DataKind.R4, 4),
new TextLoader.Column("marital-status", DataKind.Text, 5),
new TextLoader.Column("occupation", DataKind.Text, 6),
new TextLoader.Column("relationship", DataKind.Text, 7),
new TextLoader.Column("ethnicity", DataKind.Text, 8),
new TextLoader.Column("sex", DataKind.Text, 9),
new TextLoader.Column("capital-gain", DataKind.R4, 10),
new TextLoader.Column("capital-loss", DataKind.R4, 11),
new TextLoader.Column("hours-per-week", DataKind.R4, 12),
new TextLoader.Column("native-country", DataKind.Text, 13),
new TextLoader.Column("Label", DataKind.Bool, 14)
new TextLoader.Column("age", DataKind.Single, 0),
new TextLoader.Column("workclass", DataKind.String, 1),
new TextLoader.Column("fnlwgt", DataKind.Single, 2),
new TextLoader.Column("education", DataKind.String, 3),
new TextLoader.Column("education-num", DataKind.Single, 4),
new TextLoader.Column("marital-status", DataKind.String, 5),
new TextLoader.Column("occupation", DataKind.String, 6),
new TextLoader.Column("relationship", DataKind.String, 7),
new TextLoader.Column("ethnicity", DataKind.String, 8),
new TextLoader.Column("sex", DataKind.String, 9),
new TextLoader.Column("capital-gain", DataKind.Single, 10),
new TextLoader.Column("capital-loss", DataKind.Single, 11),
new TextLoader.Column("hours-per-week", DataKind.Single, 12),
new TextLoader.Column("native-country", DataKind.String, 13),
new TextLoader.Column("Label", DataKind.Boolean, 14)
}
});

Expand Down
4 changes: 2 additions & 2 deletions docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/TextClassification.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ public static void Example()
var lookupMap = mlContext.Data.ReadFromTextFile(Path.Combine(modelLocation, "imdb_word_index.csv"),
columns: new[]
{
new TextLoader.Column("Words", DataKind.TX, 0),
new TextLoader.Column("Ids", DataKind.I4, 1),
new TextLoader.Column("Words", DataKind.String, 0),
new TextLoader.Column("Ids", DataKind.Int32, 1),
},
separatorChar: ','
);
Expand Down
4 changes: 2 additions & 2 deletions ...ples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/SDCALogisticRegression.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ public static void Example()
var reader = mlContext.Data.CreateTextLoader(
columns: new[]
{
new TextLoader.Column("Sentiment", DataKind.BL, 0),
new TextLoader.Column("SentimentText", DataKind.Text, 1)
new TextLoader.Column("Sentiment", DataKind.Boolean, 0),
new TextLoader.Column("SentimentText", DataKind.String, 1)
},
hasHeader: true
);
Expand Down
4 changes: 2 additions & 2 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/PriorTrainerSample.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ public static void Example()
var reader = mlContext.Data.CreateTextLoader(
columns: new[]
{
new TextLoader.Column("Sentiment", DataKind.R4, 0),
new TextLoader.Column("SentimentText", DataKind.Text, 1)
new TextLoader.Column("Sentiment", DataKind.Single, 0),
new TextLoader.Column("SentimentText", DataKind.String, 1)
},
hasHeader: true
);
Expand Down
4 changes: 2 additions & 2 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/RandomTrainerSample.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ public static void Example()
var reader = mlContext.Data.CreateTextLoader(
columns: new[]
{
new TextLoader.Column("Sentiment", DataKind.R4, 0),
new TextLoader.Column("SentimentText", DataKind.Text, 1)
new TextLoader.Column("Sentiment", DataKind.Single, 0),
new TextLoader.Column("SentimentText", DataKind.String, 1)
},
hasHeader: true
);
Expand Down
4 changes: 2 additions & 2 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquares.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ public static void Example()
HasHeader = true,
Columns = new[]
{
new TextLoader.Column("Label", DataKind.R4, 0),
new TextLoader.Column("Features", DataKind.R4, 1, 6)
new TextLoader.Column("Label", DataKind.Single, 0),
new TextLoader.Column("Features", DataKind.Single, 1, 6)
}
});

Expand Down
4 changes: 2 additions & 2 deletions ...mples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/OrdinaryLeastSquaresWithOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ public static void Example()
HasHeader = true,
Columns = new[]
{
new TextLoader.Column("Label", DataKind.R4, 0),
new TextLoader.Column("Features", DataKind.R4, 1, 6)
new TextLoader.Column("Label", DataKind.Single, 0),
new TextLoader.Column("Features", DataKind.Single, 1, 6)
}
});

Expand Down
48 changes: 24 additions & 24 deletions src/Microsoft.ML.Core/Data/ColumnTypeExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,13 +61,13 @@ public static int GetKeyCountAsInt32(this DataViewType columnType, IExceptionCon
public static bool IsKnownSizeVector(this DataViewType columnType) => columnType.GetVectorSize() > 0;

/// <summary>
/// Gets the equivalent <see cref="DataKind"/> for the <paramref name="columnType"/>'s RawType.
/// This can return default(<see cref="DataKind"/>) if the RawType doesn't have a corresponding
/// <see cref="DataKind"/>.
/// Gets the equivalent <see cref="InternalDataKind"/> for the <paramref name="columnType"/>'s RawType.
/// This can return default(<see cref="InternalDataKind"/>) if the RawType doesn't have a corresponding
/// <see cref="InternalDataKind"/>.
/// </summary>
public static DataKind GetRawKind(this DataViewType columnType)
public static InternalDataKind GetRawKind(this DataViewType columnType)
{
columnType.RawType.TryGetDataKind(out DataKind result);
columnType.RawType.TryGetDataKind(out InternalDataKind result);
return result;
}

Expand Down Expand Up @@ -106,56 +106,56 @@ public static PrimitiveDataViewType PrimitiveTypeFromType(Type type)
return NumberTypeFromType(type);
}

public static PrimitiveDataViewType PrimitiveTypeFromKind(DataKind kind)
public static PrimitiveDataViewType PrimitiveTypeFromKind(InternalDataKind kind)
{
if (kind == DataKind.TX)
if (kind == InternalDataKind.TX)
return TextDataViewType.Instance;
if (kind == DataKind.BL)
if (kind == InternalDataKind.BL)
return BooleanDataViewType.Instance;
if (kind == DataKind.TS)
if (kind == InternalDataKind.TS)
return TimeSpanDataViewType.Instance;
if (kind == DataKind.DT)
if (kind == InternalDataKind.DT)
return DateTimeDataViewType.Instance;
if (kind == DataKind.DZ)
if (kind == InternalDataKind.DZ)
return DateTimeOffsetDataViewType.Instance;
return NumberTypeFromKind(kind);
}

public static NumberDataViewType NumberTypeFromType(Type type)
{
DataKind kind;
InternalDataKind kind;
if (type.TryGetDataKind(out kind))
return NumberTypeFromKind(kind);

Contracts.Assert(false);
throw new InvalidOperationException($"Bad type in {nameof(ColumnTypeExtensions)}.{nameof(NumberTypeFromType)}: {type}");
}

public static NumberDataViewType NumberTypeFromKind(DataKind kind)
public static NumberDataViewType NumberTypeFromKind(InternalDataKind kind)
{
switch (kind)
{
case DataKind.I1:
case InternalDataKind.I1:
return NumberDataViewType.SByte;
case DataKind.U1:
case InternalDataKind.U1:
return NumberDataViewType.Byte;
case DataKind.I2:
case InternalDataKind.I2:
return NumberDataViewType.Int16;
case DataKind.U2:
case InternalDataKind.U2:
return NumberDataViewType.UInt16;
case DataKind.I4:
case InternalDataKind.I4:
return NumberDataViewType.Int32;
case DataKind.U4:
case InternalDataKind.U4:
return NumberDataViewType.UInt32;
case DataKind.I8:
case InternalDataKind.I8:
return NumberDataViewType.Int64;
case DataKind.U8:
case InternalDataKind.U8:
return NumberDataViewType.UInt64;
case DataKind.R4:
case InternalDataKind.R4:
return NumberDataViewType.Single;
case DataKind.R8:
case InternalDataKind.R8:
return NumberDataViewType.Double;
case DataKind.UG:
case InternalDataKind.UG:
return NumberDataViewType.DataViewRowId;
}

Expand Down
Loading