diff --git a/ZBaselines/Common/EntryPoints/core_ep-list.tsv b/ZBaselines/Common/EntryPoints/core_ep-list.tsv index 568a6066f9..0a5c423892 100644 --- a/ZBaselines/Common/EntryPoints/core_ep-list.tsv +++ b/ZBaselines/Common/EntryPoints/core_ep-list.tsv @@ -1,6 +1,7 @@ +Data.CustomTextLoader Import a dataset from a text file Microsoft.ML.Runtime.EntryPoints.ImportTextData ImportText Microsoft.ML.Runtime.EntryPoints.ImportTextData+Input Microsoft.ML.Runtime.EntryPoints.ImportTextData+Output Data.IDataViewArrayConverter Create and array variable Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewOutput Data.PredictorModelArrayConverter Create and array variable Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelOutput -Data.TextLoader Import a dataset from a text file Microsoft.ML.Runtime.EntryPoints.ImportTextData ImportText Microsoft.ML.Runtime.EntryPoints.ImportTextData+Input Microsoft.ML.Runtime.EntryPoints.ImportTextData+Output +Data.TextLoader Import a dataset from a text file Microsoft.ML.Runtime.EntryPoints.ImportTextData TextLoader Microsoft.ML.Runtime.EntryPoints.ImportTextData+LoaderInput Microsoft.ML.Runtime.EntryPoints.ImportTextData+Output Models.AnomalyDetectionEvaluator Evaluates an anomaly detection scored dataset. Microsoft.ML.Runtime.Data.Evaluate AnomalyDetection Microsoft.ML.Runtime.Data.AnomalyDetectionMamlEvaluator+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CommonEvaluateOutput Models.BinaryClassificationEvaluator Evaluates a binary classification scored dataset. Microsoft.ML.Runtime.Data.Evaluate Binary Microsoft.ML.Runtime.Data.BinaryClassifierMamlEvaluator+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+ClassificationEvaluateOutput Models.BinaryCrossValidator Cross validation for binary classification Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro CrossValidateBinary Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+Output] diff --git a/ZBaselines/Common/EntryPoints/core_manifest.json b/ZBaselines/Common/EntryPoints/core_manifest.json index a3778a7f7f..1071227c0f 100644 --- a/ZBaselines/Common/EntryPoints/core_manifest.json +++ b/ZBaselines/Common/EntryPoints/core_manifest.json @@ -1,5 +1,43 @@ { "EntryPoints": [ + { + "Name": "Data.CustomTextLoader", + "Desc": "Import a dataset from a text file", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "InputFile", + "Type": "FileHandle", + "Desc": "Location of the input file", + "Aliases": [ + "data" + ], + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "CustomSchema", + "Type": "String", + "Desc": "Custom schema to use for parsing", + "Aliases": [ + "schema" + ], + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": null + } + ], + "Outputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "The resulting data view" + } + ] + }, { "Name": "Data.IDataViewArrayConverter", "Desc": "Create and array variable", @@ -76,16 +114,320 @@ "IsNullable": false }, { - "Name": "CustomSchema", - "Type": "String", - "Desc": "Custom schema to use for parsing", + "Name": "Arguments", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Column", + "Type": { + "Kind": "Array", + "ItemType": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Name", + "Type": "String", + "Desc": "Name of the column", + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "Type", + "Type": { + "Kind": "Enum", + "Values": [ + "I1", + "U1", + "I2", + "U2", + "I4", + "U4", + "I8", + "U8", + "R4", + "Num", + "R8", + "TX", + "Text", + "TXT", + "BL", + "Bool", + "TimeSpan", + "TS", + "DT", + "DateTime", + "DZ", + "DateTimeZone", + "UG", + "U16" + ] + }, + "Desc": "Type of the items in the column", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Source", + "Type": { + "Kind": "Array", + "ItemType": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Min", + "Type": "Int", + "Desc": "First index in the range", + "Required": true, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 0 + }, + { + "Name": "Max", + "Type": "Int", + "Desc": "Last index in the range", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "AutoEnd", + "Type": "Bool", + "Desc": "This range extends to the end of the line, but should be a fixed number of items", + "Aliases": [ + "auto" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "VariableEnd", + "Type": "Bool", + "Desc": "This range extends to the end of the line, which can vary from line to line", + "Aliases": [ + "var" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "AllOther", + "Type": "Bool", + "Desc": "This range includes only other indices not specified", + "Aliases": [ + "other" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "ForceVector", + "Type": "Bool", + "Desc": "Force scalar columns to be treated as vectors of length one", + "Aliases": [ + "vector" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + } + ] + } + }, + "Desc": "Source index range(s) of the column", + "Aliases": [ + "src" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "KeyRange", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Min", + "Type": "UInt", + "Desc": "First index in the range", + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 0 + }, + { + "Name": "Max", + "Type": "UInt", + "Desc": "Last index in the range", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Contiguous", + "Type": "Bool", + "Desc": "Whether the key is contiguous", + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + } + ] + }, + "Desc": "For a key column, this defines the range of values", + "Aliases": [ + "key" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + } + ] + } + }, + "Desc": "Column groups. Each group is specified as name:type:numeric-ranges, eg, col=Features:R4:1-17,26,35-40", + "Aliases": [ + "col" + ], + "Required": false, + "SortOrder": 1.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "UseThreads", + "Type": "Bool", + "Desc": "Use separate parsing threads?", + "Aliases": [ + "threads" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "HeaderFile", + "Type": "String", + "Desc": "File containing a header with feature names. If specified, header defined in the data file (header+) is ignored.", + "Aliases": [ + "hf" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "MaxRows", + "Type": "Int", + "Desc": "Maximum number of rows to produce", + "Aliases": [ + "rows" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "AllowQuoting", + "Type": "Bool", + "Desc": "Whether the input may include quoted values, which can contain separator characters, colons, and distinguish empty values from missing values. When true, consecutive separators denote a missing value and an empty value is denoted by \"\". When false, consecutive separators denote an empty value.", + "Aliases": [ + "quote" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "AllowSparse", + "Type": "Bool", + "Desc": "Whether the input may include sparse representations", + "Aliases": [ + "sparse" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "InputSize", + "Type": "Int", + "Desc": "Number of source columns in the text data. Default is that sparse rows contain their size information.", + "Aliases": [ + "size" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Separator", + "Type": "String", + "Desc": "Source column separator. Options: tab, space, comma, single character", + "Aliases": [ + "sep" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": "tab" + }, + { + "Name": "TrimWhitespace", + "Type": "Bool", + "Desc": "Remove trailing whitespace from lines", + "Aliases": [ + "trim" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "HasHeader", + "Type": "Bool", + "Desc": "Data file has header with feature names. Header is read only if options 'hs' and 'hf' are not specified.", + "Aliases": [ + "header" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + } + ] + }, + "Desc": "Arguments", "Aliases": [ - "schema" + "args" ], - "Required": false, - "SortOrder": 2.0, - "IsNullable": false, - "Default": null + "Required": true, + "SortOrder": 1.0, + "IsNullable": false } ], "Outputs": [ @@ -94,6 +436,9 @@ "Type": "DataView", "Desc": "The resulting data view" } + ], + "InputKind": [ + "ILearningPipelineLoader" ] }, { @@ -21578,6 +21923,10 @@ } ] }, + { + "Kind": "ILearningPipelineLoader", + "Settings": [] + }, { "Kind": "IMulticlassClassificationOutput", "Settings": [] diff --git a/src/Microsoft.ML.Core/EntryPoints/ModuleArgs.cs b/src/Microsoft.ML.Core/EntryPoints/ModuleArgs.cs index 99cfec0dd9..9b778557eb 100644 --- a/src/Microsoft.ML.Core/EntryPoints/ModuleArgs.cs +++ b/src/Microsoft.ML.Core/EntryPoints/ModuleArgs.cs @@ -527,6 +527,12 @@ public sealed class EntryPointAttribute : Attribute /// Short name of the Entry Point /// public string ShortName { get; set; } + + /// + /// Indicates if the code generated should not be sealed. + /// By default all classes are sealed. + /// + public bool NoSeal { get; set; } } /// diff --git a/src/Microsoft.ML.Core/EntryPoints/ModuleCatalog.cs b/src/Microsoft.ML.Core/EntryPoints/ModuleCatalog.cs index 3a468ad451..ac1698e520 100644 --- a/src/Microsoft.ML.Core/EntryPoints/ModuleCatalog.cs +++ b/src/Microsoft.ML.Core/EntryPoints/ModuleCatalog.cs @@ -40,6 +40,7 @@ public sealed class ModuleCatalog /// public sealed class EntryPointInfo { + public readonly bool NoSeal; public readonly string Name; public readonly string Description; public readonly string ShortName; @@ -57,6 +58,7 @@ internal EntryPointInfo(IExceptionContext ectx, MethodInfo method, TlcModule.Ent ectx.AssertValue(attribute); Name = attribute.Name ?? string.Join(".", method.DeclaringType.Name, method.Name); + NoSeal = attribute.NoSeal; Description = attribute.Desc; Method = method; ShortName = attribute.ShortName; diff --git a/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs b/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs index 3867b18f26..40519b39fd 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs @@ -317,7 +317,7 @@ public bool IsValid() } } - public sealed class Arguments : ArgumentsCore + public class Arguments : ArgumentsCore { [Argument(ArgumentType.AtMostOnce, HelpText = "Use separate parsing threads?", ShortName = "threads", Hide = true)] public bool UseThreads = true; diff --git a/src/Microsoft.ML.Data/EntryPoints/InputBase.cs b/src/Microsoft.ML.Data/EntryPoints/InputBase.cs index 57a7c9120f..a787d43cb9 100644 --- a/src/Microsoft.ML.Data/EntryPoints/InputBase.cs +++ b/src/Microsoft.ML.Data/EntryPoints/InputBase.cs @@ -191,6 +191,7 @@ public static TOut Train(IHost host, TArg input, /// public static class CommonInputs { + /// /// Interface that all API transform input classes will implement. /// diff --git a/src/Microsoft.ML/CSharpApi.cs b/src/Microsoft.ML/CSharpApi.cs index 46002c5abf..2402ecf95a 100644 --- a/src/Microsoft.ML/CSharpApi.cs +++ b/src/Microsoft.ML/CSharpApi.cs @@ -22,6 +22,18 @@ namespace Runtime { public sealed partial class Experiment { + public Microsoft.ML.Data.CustomTextLoader.Output Add(Microsoft.ML.Data.CustomTextLoader input) + { + var output = new Microsoft.ML.Data.CustomTextLoader.Output(); + Add(input, output); + return output; + } + + public void Add(Microsoft.ML.Data.CustomTextLoader input, Microsoft.ML.Data.CustomTextLoader.Output output) + { + _jsonNodes.Add(Serialize("Data.CustomTextLoader", input, output)); + } + public Microsoft.ML.Data.IDataViewArrayConverter.Output Add(Microsoft.ML.Data.IDataViewArrayConverter input) { var output = new Microsoft.ML.Data.IDataViewArrayConverter.Output(); @@ -1236,6 +1248,38 @@ public void Add(Microsoft.ML.Transforms.WordTokenizer input, Microsoft.ML.Transf } } + namespace Data + { + + /// + /// Import a dataset from a text file + /// + public sealed partial class CustomTextLoader + { + + + /// + /// Location of the input file + /// + public Var InputFile { get; set; } = new Var(); + + /// + /// Custom schema to use for parsing + /// + public string CustomSchema { get; set; } + + + public sealed class Output + { + /// + /// The resulting data view + /// + public Var Data { get; set; } = new Var(); + + } + } + } + namespace Data { @@ -1293,12 +1337,174 @@ public sealed class Output namespace Data { + public sealed class TextLoaderArguments + { + /// + /// Use separate parsing threads? + /// + public bool UseThreads { get; set; } = true; + + /// + /// File containing a header with feature names. If specified, header defined in the data file (header+) is ignored. + /// + public string HeaderFile { get; set; } + + /// + /// Maximum number of rows to produce + /// + public long? MaxRows { get; set; } + + /// + /// Whether the input may include quoted values, which can contain separator characters, colons, and distinguish empty values from missing values. When true, consecutive separators denote a missing value and an empty value is denoted by "". When false, consecutive separators denote an empty value. + /// + public bool AllowQuoting { get; set; } = true; + + /// + /// Whether the input may include sparse representations + /// + public bool AllowSparse { get; set; } = true; + + /// + /// Number of source columns in the text data. Default is that sparse rows contain their size information. + /// + public int? InputSize { get; set; } + + /// + /// Source column separator. Options: tab, space, comma, single character + /// + public string Separator { get; set; } = "tab"; + + /// + /// Column groups. Each group is specified as name:type:numeric-ranges, eg, col=Features:R4:1-17,26,35-40 + /// + public TextLoaderColumn[] Column { get; set; } + + /// + /// Remove trailing whitespace from lines + /// + public bool TrimWhitespace { get; set; } = false; + + /// + /// Data file has header with feature names. Header is read only if options 'hs' and 'hf' are not specified. + /// + public bool HasHeader { get; set; } = false; + + } + + public sealed class TextLoaderColumn + { + /// + /// Name of the column + /// + public string Name { get; set; } + + /// + /// Type of the items in the column + /// + public DataKind? Type { get; set; } + + /// + /// Source index range(s) of the column + /// + public TextLoaderRange[] Source { get; set; } + + /// + /// For a key column, this defines the range of values + /// + public KeyRange KeyRange { get; set; } + + } + + public sealed class TextLoaderRange + { + /// + /// First index in the range + /// + public int Min { get; set; } + + /// + /// Last index in the range + /// + public int? Max { get; set; } + + /// + /// This range extends to the end of the line, but should be a fixed number of items + /// + public bool AutoEnd { get; set; } = false; + + /// + /// This range extends to the end of the line, which can vary from line to line + /// + public bool VariableEnd { get; set; } = false; + + /// + /// This range includes only other indices not specified + /// + public bool AllOther { get; set; } = false; + + /// + /// Force scalar columns to be treated as vectors of length one + /// + public bool ForceVector { get; set; } = false; + + } + + public sealed class KeyRange + { + /// + /// First index in the range + /// + public ulong Min { get; set; } = 0; + + /// + /// Last index in the range + /// + public ulong? Max { get; set; } + + /// + /// Whether the key is contiguous + /// + public bool Contiguous { get; set; } = true; + + } + /// /// Import a dataset from a text file /// - public sealed partial class TextLoader + public partial class TextLoader : Microsoft.ML.ILearningPipelineLoader { + [JsonIgnore] + private string _inputFilePath = null; + public TextLoader(string filePath) + { + _inputFilePath = filePath; + } + + public void SetInput(IHostEnvironment env, Experiment experiment) + { + IFileHandle inputFile = new SimpleFileHandle(env, _inputFilePath, false, false); + experiment.SetInput(InputFile, inputFile); + } + + public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment) + { + Contracts.Assert(previousStep == null); + + return new TextLoaderPipelineStep(experiment.Add(this)); + } + + private class TextLoaderPipelineStep : ILearningPipelineDataStep + { + public TextLoaderPipelineStep(Output output) + { + Data = output.Data; + Model = null; + } + + public Var Data { get; } + public Var Model { get; } + } /// /// Location of the input file @@ -1306,9 +1512,9 @@ public sealed partial class TextLoader public Var InputFile { get; set; } = new Var(); /// - /// Custom schema to use for parsing + /// Arguments /// - public string CustomSchema { get; set; } + public Data.TextLoaderArguments Arguments { get; set; } = new Data.TextLoaderArguments(); public sealed class Output @@ -2972,13 +3178,13 @@ public sealed partial class AveragedPerceptronBinaryClassifier : Microsoft.ML.Ru /// /// Learning rate /// - [TlcModule.SweepableDiscreteParamAttribute("LearningRate", new object[]{0.01f, 0.1f, 0.5f, 1f})] + [TlcModule.SweepableDiscreteParamAttribute("LearningRate", new object[] { 0.01f, 0.1f, 0.5f, 1f })] public float LearningRate { get; set; } = 1f; /// /// Decrease learning rate /// - [TlcModule.SweepableDiscreteParamAttribute("DecreaseLearningRate", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("DecreaseLearningRate", new object[] { false, true })] public bool DecreaseLearningRate { get; set; } = false; /// @@ -3020,7 +3226,7 @@ public sealed partial class AveragedPerceptronBinaryClassifier : Microsoft.ML.Ru /// /// Number of iterations /// - [TlcModule.SweepableLongParamAttribute("NumIterations", 1, 100, stepSize:10, isLogScale:true)] + [TlcModule.SweepableLongParamAttribute("NumIterations", 1, 100, stepSize: 10, isLogScale: true)] public int NumIterations { get; set; } = 1; /// @@ -3031,13 +3237,13 @@ public sealed partial class AveragedPerceptronBinaryClassifier : Microsoft.ML.Ru /// /// Init weights diameter /// - [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0f, 1f, numSteps:5)] + [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0f, 1f, numSteps: 5)] public float InitWtsDiameter { get; set; } /// /// Whether to shuffle for each training iteration /// - [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[] { false, true })] public bool Shuffle { get; set; } = true; /// @@ -3121,25 +3327,25 @@ public sealed partial class BinaryLogisticRegressor : Microsoft.ML.Runtime.Entry /// /// L2 regularization weight /// - [TlcModule.SweepableFloatParamAttribute("L2Weight", 0f, 1f, numSteps:4)] + [TlcModule.SweepableFloatParamAttribute("L2Weight", 0f, 1f, numSteps: 4)] public float L2Weight { get; set; } = 1f; /// /// L1 regularization weight /// - [TlcModule.SweepableFloatParamAttribute("L1Weight", 0f, 1f, numSteps:4)] + [TlcModule.SweepableFloatParamAttribute("L1Weight", 0f, 1f, numSteps: 4)] public float L1Weight { get; set; } = 1f; /// /// Tolerance parameter for optimization convergence. Lower = slower, more accurate /// - [TlcModule.SweepableDiscreteParamAttribute("OptTol", new object[]{0.0001f, 1E-07f})] + [TlcModule.SweepableDiscreteParamAttribute("OptTol", new object[] { 0.0001f, 1E-07f })] public float OptTol { get; set; } = 1E-07f; /// /// Memory size for L-BFGS. Lower=faster, less accurate /// - [TlcModule.SweepableDiscreteParamAttribute("MemorySize", new object[]{5, 20, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MemorySize", new object[] { 5, 20, 50 })] public int MemorySize { get; set; } = 20; /// @@ -3161,7 +3367,7 @@ public sealed partial class BinaryLogisticRegressor : Microsoft.ML.Runtime.Entry /// /// Init weights diameter /// - [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0f, 1f, numSteps:5)] + [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0f, 1f, numSteps: 5)] public float InitWtsDiameter { get; set; } /// @@ -3177,7 +3383,7 @@ public sealed partial class BinaryLogisticRegressor : Microsoft.ML.Runtime.Entry /// /// Force densification of the internal optimization vectors /// - [TlcModule.SweepableDiscreteParamAttribute("DenseOptimizer", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("DenseOptimizer", new object[] { false, true })] public bool DenseOptimizer { get; set; } = false; /// @@ -3400,19 +3606,19 @@ public sealed partial class FastForestBinaryClassifier : Microsoft.ML.Runtime.En /// /// The max number of leaves in each regression tree /// - [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize:4, isLogScale:true)] + [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize: 4, isLogScale: true)] public int NumLeaves { get; set; } = 20; /// /// The minimal number of documents allowed in a leaf of a regression tree, out of the subsampled data /// - [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[]{1, 10, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[] { 1, 10, 50 })] public int MinDocumentsInLeafs { get; set; } = 10; /// /// Number of weak hypotheses in the ensemble /// - [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[]{20, 100, 500})] + [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[] { 20, 100, 500 })] public int NumTrees { get; set; } = 100; /// @@ -3677,19 +3883,19 @@ public sealed partial class FastForestRegressor : Microsoft.ML.Runtime.EntryPoin /// /// The max number of leaves in each regression tree /// - [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize:4, isLogScale:true)] + [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize: 4, isLogScale: true)] public int NumLeaves { get; set; } = 20; /// /// The minimal number of documents allowed in a leaf of a regression tree, out of the subsampled data /// - [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[]{1, 10, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[] { 1, 10, 50 })] public int MinDocumentsInLeafs { get; set; } = 10; /// /// Number of weak hypotheses in the ensemble /// - [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[]{20, 100, 500})] + [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[] { 20, 100, 500 })] public int NumTrees { get; set; } = 100; /// @@ -3901,19 +4107,19 @@ public sealed partial class FastTreeBinaryClassifier : Microsoft.ML.Runtime.Entr /// /// The learning rate /// - [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale: true)] public double LearningRates { get; set; } = 0.2d; /// /// Shrinkage /// - [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale: true)] public double Shrinkage { get; set; } = 1d; /// /// Dropout rate for tree regularization /// - [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[]{0f, 1E-09f, 0.05f, 0.1f, 0.2f})] + [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[] { 0f, 1E-09f, 0.05f, 0.1f, 0.2f })] public double DropoutRate { get; set; } /// @@ -4070,19 +4276,19 @@ public sealed partial class FastTreeBinaryClassifier : Microsoft.ML.Runtime.Entr /// /// The max number of leaves in each regression tree /// - [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize:4, isLogScale:true)] + [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize: 4, isLogScale: true)] public int NumLeaves { get; set; } = 20; /// /// The minimal number of documents allowed in a leaf of a regression tree, out of the subsampled data /// - [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[]{1, 10, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[] { 1, 10, 50 })] public int MinDocumentsInLeafs { get; set; } = 10; /// /// Number of weak hypotheses in the ensemble /// - [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[]{20, 100, 500})] + [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[] { 20, 100, 500 })] public int NumTrees { get; set; } = 100; /// @@ -4322,19 +4528,19 @@ public sealed partial class FastTreeRanker : Microsoft.ML.Runtime.EntryPoints.Co /// /// The learning rate /// - [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale: true)] public double LearningRates { get; set; } = 0.2d; /// /// Shrinkage /// - [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale: true)] public double Shrinkage { get; set; } = 1d; /// /// Dropout rate for tree regularization /// - [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[]{0f, 1E-09f, 0.05f, 0.1f, 0.2f})] + [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[] { 0f, 1E-09f, 0.05f, 0.1f, 0.2f })] public double DropoutRate { get; set; } /// @@ -4491,19 +4697,19 @@ public sealed partial class FastTreeRanker : Microsoft.ML.Runtime.EntryPoints.Co /// /// The max number of leaves in each regression tree /// - [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize:4, isLogScale:true)] + [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize: 4, isLogScale: true)] public int NumLeaves { get; set; } = 20; /// /// The minimal number of documents allowed in a leaf of a regression tree, out of the subsampled data /// - [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[]{1, 10, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[] { 1, 10, 50 })] public int MinDocumentsInLeafs { get; set; } = 10; /// /// Number of weak hypotheses in the ensemble /// - [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[]{20, 100, 500})] + [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[] { 20, 100, 500 })] public int NumTrees { get; set; } = 100; /// @@ -4703,19 +4909,19 @@ public sealed partial class FastTreeRegressor : Microsoft.ML.Runtime.EntryPoints /// /// The learning rate /// - [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale: true)] public double LearningRates { get; set; } = 0.2d; /// /// Shrinkage /// - [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale: true)] public double Shrinkage { get; set; } = 1d; /// /// Dropout rate for tree regularization /// - [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[]{0f, 1E-09f, 0.05f, 0.1f, 0.2f})] + [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[] { 0f, 1E-09f, 0.05f, 0.1f, 0.2f })] public double DropoutRate { get; set; } /// @@ -4872,19 +5078,19 @@ public sealed partial class FastTreeRegressor : Microsoft.ML.Runtime.EntryPoints /// /// The max number of leaves in each regression tree /// - [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize:4, isLogScale:true)] + [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize: 4, isLogScale: true)] public int NumLeaves { get; set; } = 20; /// /// The minimal number of documents allowed in a leaf of a regression tree, out of the subsampled data /// - [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[]{1, 10, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[] { 1, 10, 50 })] public int MinDocumentsInLeafs { get; set; } = 10; /// /// Number of weak hypotheses in the ensemble /// - [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[]{20, 100, 500})] + [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[] { 20, 100, 500 })] public int NumTrees { get; set; } = 100; /// @@ -5089,19 +5295,19 @@ public sealed partial class FastTreeTweedieRegressor : Microsoft.ML.Runtime.Entr /// /// The learning rate /// - [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale: true)] public double LearningRates { get; set; } = 0.2d; /// /// Shrinkage /// - [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale: true)] public double Shrinkage { get; set; } = 1d; /// /// Dropout rate for tree regularization /// - [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[]{0f, 1E-09f, 0.05f, 0.1f, 0.2f})] + [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[] { 0f, 1E-09f, 0.05f, 0.1f, 0.2f })] public double DropoutRate { get; set; } /// @@ -5258,19 +5464,19 @@ public sealed partial class FastTreeTweedieRegressor : Microsoft.ML.Runtime.Entr /// /// The max number of leaves in each regression tree /// - [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize:4, isLogScale:true)] + [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize: 4, isLogScale: true)] public int NumLeaves { get; set; } = 20; /// /// The minimal number of documents allowed in a leaf of a regression tree, out of the subsampled data /// - [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[]{1, 10, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[] { 1, 10, 50 })] public int MinDocumentsInLeafs { get; set; } = 10; /// /// Number of weak hypotheses in the ensemble /// - [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[]{20, 100, 500})] + [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[] { 20, 100, 500 })] public int NumTrees { get; set; } = 100; /// @@ -5440,7 +5646,7 @@ public sealed partial class GeneralizedAdditiveModelBinaryClassifier : Microsoft /// /// Total number of iterations over all features /// - [TlcModule.SweepableDiscreteParamAttribute("NumIterations", new object[]{200, 1500, 9500})] + [TlcModule.SweepableDiscreteParamAttribute("NumIterations", new object[] { 200, 1500, 9500 })] public int NumIterations { get; set; } = 9500; /// @@ -5451,7 +5657,7 @@ public sealed partial class GeneralizedAdditiveModelBinaryClassifier : Microsoft /// /// The learning rate /// - [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.001f, 0.1f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.001f, 0.1f, isLogScale: true)] public double LearningRates { get; set; } = 0.002d; /// @@ -5482,7 +5688,7 @@ public sealed partial class GeneralizedAdditiveModelBinaryClassifier : Microsoft /// /// Minimum number of training instances required to form a partition /// - [TlcModule.SweepableDiscreteParamAttribute("MinDocuments", new object[]{1, 10, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MinDocuments", new object[] { 1, 10, 50 })] public int MinDocuments { get; set; } = 10; /// @@ -5576,7 +5782,7 @@ public sealed partial class GeneralizedAdditiveModelRegressor : Microsoft.ML.Run /// /// Total number of iterations over all features /// - [TlcModule.SweepableDiscreteParamAttribute("NumIterations", new object[]{200, 1500, 9500})] + [TlcModule.SweepableDiscreteParamAttribute("NumIterations", new object[] { 200, 1500, 9500 })] public int NumIterations { get; set; } = 9500; /// @@ -5587,7 +5793,7 @@ public sealed partial class GeneralizedAdditiveModelRegressor : Microsoft.ML.Run /// /// The learning rate /// - [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.001f, 0.1f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.001f, 0.1f, isLogScale: true)] public double LearningRates { get; set; } = 0.002d; /// @@ -5618,7 +5824,7 @@ public sealed partial class GeneralizedAdditiveModelRegressor : Microsoft.ML.Run /// /// Minimum number of training instances required to form a partition /// - [TlcModule.SweepableDiscreteParamAttribute("MinDocuments", new object[]{1, 10, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MinDocuments", new object[] { 1, 10, 50 })] public int MinDocuments { get; set; } = 10; /// @@ -5702,7 +5908,7 @@ public sealed partial class LinearSvmBinaryClassifier : Microsoft.ML.Runtime.Ent /// /// Regularizer constant /// - [TlcModule.SweepableFloatParamAttribute("Lambda", 1E-05f, 0.1f, stepSize:10, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("Lambda", 1E-05f, 0.1f, stepSize: 10, isLogScale: true)] public float Lambda { get; set; } = 0.001f; /// @@ -5713,13 +5919,13 @@ public sealed partial class LinearSvmBinaryClassifier : Microsoft.ML.Runtime.Ent /// /// Perform projection to unit-ball? Typically used with batch size > 1. /// - [TlcModule.SweepableDiscreteParamAttribute("PerformProjection", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("PerformProjection", new object[] { false, true })] public bool PerformProjection { get; set; } = false; /// /// No bias /// - [TlcModule.SweepableDiscreteParamAttribute("NoBias", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("NoBias", new object[] { false, true })] public bool NoBias { get; set; } = false; /// @@ -5736,7 +5942,7 @@ public sealed partial class LinearSvmBinaryClassifier : Microsoft.ML.Runtime.Ent /// /// Number of iterations /// - [TlcModule.SweepableLongParamAttribute("NumIterations", 1, 100, stepSize:10, isLogScale:true)] + [TlcModule.SweepableLongParamAttribute("NumIterations", 1, 100, stepSize: 10, isLogScale: true)] public int NumIterations { get; set; } = 1; /// @@ -5747,13 +5953,13 @@ public sealed partial class LinearSvmBinaryClassifier : Microsoft.ML.Runtime.Ent /// /// Init weights diameter /// - [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0f, 1f, numSteps:5)] + [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0f, 1f, numSteps: 5)] public float InitWtsDiameter { get; set; } /// /// Whether to shuffle for each training iteration /// - [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[] { false, true })] public bool Shuffle { get; set; } = true; /// @@ -5837,25 +6043,25 @@ public sealed partial class LogisticRegressor : Microsoft.ML.Runtime.EntryPoints /// /// L2 regularization weight /// - [TlcModule.SweepableFloatParamAttribute("L2Weight", 0f, 1f, numSteps:4)] + [TlcModule.SweepableFloatParamAttribute("L2Weight", 0f, 1f, numSteps: 4)] public float L2Weight { get; set; } = 1f; /// /// L1 regularization weight /// - [TlcModule.SweepableFloatParamAttribute("L1Weight", 0f, 1f, numSteps:4)] + [TlcModule.SweepableFloatParamAttribute("L1Weight", 0f, 1f, numSteps: 4)] public float L1Weight { get; set; } = 1f; /// /// Tolerance parameter for optimization convergence. Lower = slower, more accurate /// - [TlcModule.SweepableDiscreteParamAttribute("OptTol", new object[]{0.0001f, 1E-07f})] + [TlcModule.SweepableDiscreteParamAttribute("OptTol", new object[] { 0.0001f, 1E-07f })] public float OptTol { get; set; } = 1E-07f; /// /// Memory size for L-BFGS. Lower=faster, less accurate /// - [TlcModule.SweepableDiscreteParamAttribute("MemorySize", new object[]{5, 20, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MemorySize", new object[] { 5, 20, 50 })] public int MemorySize { get; set; } = 20; /// @@ -5877,7 +6083,7 @@ public sealed partial class LogisticRegressor : Microsoft.ML.Runtime.EntryPoints /// /// Init weights diameter /// - [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0f, 1f, numSteps:5)] + [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0f, 1f, numSteps: 5)] public float InitWtsDiameter { get; set; } /// @@ -5893,7 +6099,7 @@ public sealed partial class LogisticRegressor : Microsoft.ML.Runtime.EntryPoints /// /// Force densification of the internal optimization vectors /// - [TlcModule.SweepableDiscreteParamAttribute("DenseOptimizer", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("DenseOptimizer", new object[] { false, true })] public bool DenseOptimizer { get; set; } = false; /// @@ -6051,13 +6257,13 @@ public sealed partial class OnlineGradientDescentRegressor : Microsoft.ML.Runtim /// /// Learning rate /// - [TlcModule.SweepableDiscreteParamAttribute("LearningRate", new object[]{0.01f, 0.1f, 0.5f, 1f})] + [TlcModule.SweepableDiscreteParamAttribute("LearningRate", new object[] { 0.01f, 0.1f, 0.5f, 1f })] public float LearningRate { get; set; } = 0.1f; /// /// Decrease learning rate /// - [TlcModule.SweepableDiscreteParamAttribute("DecreaseLearningRate", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("DecreaseLearningRate", new object[] { false, true })] public bool DecreaseLearningRate { get; set; } = true; /// @@ -6099,7 +6305,7 @@ public sealed partial class OnlineGradientDescentRegressor : Microsoft.ML.Runtim /// /// Number of iterations /// - [TlcModule.SweepableLongParamAttribute("NumIterations", 1, 100, stepSize:10, isLogScale:true)] + [TlcModule.SweepableLongParamAttribute("NumIterations", 1, 100, stepSize: 10, isLogScale: true)] public int NumIterations { get; set; } = 1; /// @@ -6110,13 +6316,13 @@ public sealed partial class OnlineGradientDescentRegressor : Microsoft.ML.Runtim /// /// Init weights diameter /// - [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0f, 1f, numSteps:5)] + [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0f, 1f, numSteps: 5)] public float InitWtsDiameter { get; set; } /// /// Whether to shuffle for each training iteration /// - [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[] { false, true })] public bool Shuffle { get; set; } = true; /// @@ -6195,7 +6401,7 @@ public sealed partial class OrdinaryLeastSquaresRegressor : Microsoft.ML.Runtime /// /// L2 regularization weight /// - [TlcModule.SweepableDiscreteParamAttribute("L2Weight", new object[]{1E-06f, 0.1f, 1f})] + [TlcModule.SweepableDiscreteParamAttribute("L2Weight", new object[] { 1E-06f, 0.1f, 1f })] public float L2Weight { get; set; } = 1E-06f; /// @@ -6279,25 +6485,25 @@ public sealed partial class PoissonRegressor : Microsoft.ML.Runtime.EntryPoints. /// /// L2 regularization weight /// - [TlcModule.SweepableFloatParamAttribute("L2Weight", 0f, 1f, numSteps:4)] + [TlcModule.SweepableFloatParamAttribute("L2Weight", 0f, 1f, numSteps: 4)] public float L2Weight { get; set; } = 1f; /// /// L1 regularization weight /// - [TlcModule.SweepableFloatParamAttribute("L1Weight", 0f, 1f, numSteps:4)] + [TlcModule.SweepableFloatParamAttribute("L1Weight", 0f, 1f, numSteps: 4)] public float L1Weight { get; set; } = 1f; /// /// Tolerance parameter for optimization convergence. Lower = slower, more accurate /// - [TlcModule.SweepableDiscreteParamAttribute("OptTol", new object[]{0.0001f, 1E-07f})] + [TlcModule.SweepableDiscreteParamAttribute("OptTol", new object[] { 0.0001f, 1E-07f })] public float OptTol { get; set; } = 1E-07f; /// /// Memory size for L-BFGS. Lower=faster, less accurate /// - [TlcModule.SweepableDiscreteParamAttribute("MemorySize", new object[]{5, 20, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MemorySize", new object[] { 5, 20, 50 })] public int MemorySize { get; set; } = 20; /// @@ -6319,7 +6525,7 @@ public sealed partial class PoissonRegressor : Microsoft.ML.Runtime.EntryPoints. /// /// Init weights diameter /// - [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0f, 1f, numSteps:5)] + [TlcModule.SweepableFloatParamAttribute("InitWtsDiameter", 0f, 1f, numSteps: 5)] public float InitWtsDiameter { get; set; } /// @@ -6335,7 +6541,7 @@ public sealed partial class PoissonRegressor : Microsoft.ML.Runtime.EntryPoints. /// /// Force densification of the internal optimization vectors /// - [TlcModule.SweepableDiscreteParamAttribute("DenseOptimizer", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("DenseOptimizer", new object[] { false, true })] public bool DenseOptimizer { get; set; } = false; /// @@ -6441,13 +6647,13 @@ public sealed partial class StochasticDualCoordinateAscentBinaryClassifier : Mic /// /// L2 regularizer constant. By default the l2 constant is automatically inferred based on data set. /// - [TlcModule.SweepableDiscreteParamAttribute("L2Const", new object[]{"", 1E-07f, 1E-06f, 1E-05f, 0.0001f, 0.001f, 0.01f})] + [TlcModule.SweepableDiscreteParamAttribute("L2Const", new object[] { "", 1E-07f, 1E-06f, 1E-05f, 0.0001f, 0.001f, 0.01f })] public float? L2Const { get; set; } /// /// L1 soft threshold (L1/L2). Note that it is easier to control and sweep using the threshold parameter than the raw L1-regularizer constant. By default the l1 threshold is automatically inferred based on data set. /// - [TlcModule.SweepableDiscreteParamAttribute("L1Threshold", new object[]{"", 0f, 0.25f, 0.5f, 0.75f, 1f})] + [TlcModule.SweepableDiscreteParamAttribute("L1Threshold", new object[] { "", 0f, 0.25f, 0.5f, 0.75f, 1f })] public float? L1Threshold { get; set; } /// @@ -6458,19 +6664,19 @@ public sealed partial class StochasticDualCoordinateAscentBinaryClassifier : Mic /// /// The tolerance for the ratio between duality gap and primal loss for convergence checking. /// - [TlcModule.SweepableDiscreteParamAttribute("ConvergenceTolerance", new object[]{0.001f, 0.01f, 0.1f, 0.2f})] + [TlcModule.SweepableDiscreteParamAttribute("ConvergenceTolerance", new object[] { 0.001f, 0.01f, 0.1f, 0.2f })] public float ConvergenceTolerance { get; set; } = 0.1f; /// /// Maximum number of iterations; set to 1 to simulate online learning. Defaults to automatic. /// - [TlcModule.SweepableDiscreteParamAttribute("MaxIterations", new object[]{"", 10, 20, 100})] + [TlcModule.SweepableDiscreteParamAttribute("MaxIterations", new object[] { "", 10, 20, 100 })] public int? MaxIterations { get; set; } /// /// Shuffle data every epoch? /// - [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[] { false, true })] public bool Shuffle { get; set; } = true; /// @@ -6481,7 +6687,7 @@ public sealed partial class StochasticDualCoordinateAscentBinaryClassifier : Mic /// /// The learning rate for adjusting bias from being regularized. /// - [TlcModule.SweepableDiscreteParamAttribute("BiasLearningRate", new object[]{0f, 0.01f, 0.1f, 1f})] + [TlcModule.SweepableDiscreteParamAttribute("BiasLearningRate", new object[] { 0f, 0.01f, 0.1f, 1f })] public float BiasLearningRate { get; set; } /// @@ -6561,13 +6767,13 @@ public sealed partial class StochasticDualCoordinateAscentClassifier : Microsoft /// /// L2 regularizer constant. By default the l2 constant is automatically inferred based on data set. /// - [TlcModule.SweepableDiscreteParamAttribute("L2Const", new object[]{"", 1E-07f, 1E-06f, 1E-05f, 0.0001f, 0.001f, 0.01f})] + [TlcModule.SweepableDiscreteParamAttribute("L2Const", new object[] { "", 1E-07f, 1E-06f, 1E-05f, 0.0001f, 0.001f, 0.01f })] public float? L2Const { get; set; } /// /// L1 soft threshold (L1/L2). Note that it is easier to control and sweep using the threshold parameter than the raw L1-regularizer constant. By default the l1 threshold is automatically inferred based on data set. /// - [TlcModule.SweepableDiscreteParamAttribute("L1Threshold", new object[]{"", 0f, 0.25f, 0.5f, 0.75f, 1f})] + [TlcModule.SweepableDiscreteParamAttribute("L1Threshold", new object[] { "", 0f, 0.25f, 0.5f, 0.75f, 1f })] public float? L1Threshold { get; set; } /// @@ -6578,19 +6784,19 @@ public sealed partial class StochasticDualCoordinateAscentClassifier : Microsoft /// /// The tolerance for the ratio between duality gap and primal loss for convergence checking. /// - [TlcModule.SweepableDiscreteParamAttribute("ConvergenceTolerance", new object[]{0.001f, 0.01f, 0.1f, 0.2f})] + [TlcModule.SweepableDiscreteParamAttribute("ConvergenceTolerance", new object[] { 0.001f, 0.01f, 0.1f, 0.2f })] public float ConvergenceTolerance { get; set; } = 0.1f; /// /// Maximum number of iterations; set to 1 to simulate online learning. Defaults to automatic. /// - [TlcModule.SweepableDiscreteParamAttribute("MaxIterations", new object[]{"", 10, 20, 100})] + [TlcModule.SweepableDiscreteParamAttribute("MaxIterations", new object[] { "", 10, 20, 100 })] public int? MaxIterations { get; set; } /// /// Shuffle data every epoch? /// - [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[] { false, true })] public bool Shuffle { get; set; } = true; /// @@ -6601,7 +6807,7 @@ public sealed partial class StochasticDualCoordinateAscentClassifier : Microsoft /// /// The learning rate for adjusting bias from being regularized. /// - [TlcModule.SweepableDiscreteParamAttribute("BiasLearningRate", new object[]{0f, 0.01f, 0.1f, 1f})] + [TlcModule.SweepableDiscreteParamAttribute("BiasLearningRate", new object[] { 0f, 0.01f, 0.1f, 1f })] public float BiasLearningRate { get; set; } /// @@ -6681,13 +6887,13 @@ public sealed partial class StochasticDualCoordinateAscentRegressor : Microsoft. /// /// L2 regularizer constant. By default the l2 constant is automatically inferred based on data set. /// - [TlcModule.SweepableDiscreteParamAttribute("L2Const", new object[]{"", 1E-07f, 1E-06f, 1E-05f, 0.0001f, 0.001f, 0.01f})] + [TlcModule.SweepableDiscreteParamAttribute("L2Const", new object[] { "", 1E-07f, 1E-06f, 1E-05f, 0.0001f, 0.001f, 0.01f })] public float? L2Const { get; set; } /// /// L1 soft threshold (L1/L2). Note that it is easier to control and sweep using the threshold parameter than the raw L1-regularizer constant. By default the l1 threshold is automatically inferred based on data set. /// - [TlcModule.SweepableDiscreteParamAttribute("L1Threshold", new object[]{"", 0f, 0.25f, 0.5f, 0.75f, 1f})] + [TlcModule.SweepableDiscreteParamAttribute("L1Threshold", new object[] { "", 0f, 0.25f, 0.5f, 0.75f, 1f })] public float? L1Threshold { get; set; } /// @@ -6698,19 +6904,19 @@ public sealed partial class StochasticDualCoordinateAscentRegressor : Microsoft. /// /// The tolerance for the ratio between duality gap and primal loss for convergence checking. /// - [TlcModule.SweepableDiscreteParamAttribute("ConvergenceTolerance", new object[]{0.001f, 0.01f, 0.1f, 0.2f})] + [TlcModule.SweepableDiscreteParamAttribute("ConvergenceTolerance", new object[] { 0.001f, 0.01f, 0.1f, 0.2f })] public float ConvergenceTolerance { get; set; } = 0.01f; /// /// Maximum number of iterations; set to 1 to simulate online learning. Defaults to automatic. /// - [TlcModule.SweepableDiscreteParamAttribute("MaxIterations", new object[]{"", 10, 20, 100})] + [TlcModule.SweepableDiscreteParamAttribute("MaxIterations", new object[] { "", 10, 20, 100 })] public int? MaxIterations { get; set; } /// /// Shuffle data every epoch? /// - [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[] { false, true })] public bool Shuffle { get; set; } = true; /// @@ -6721,7 +6927,7 @@ public sealed partial class StochasticDualCoordinateAscentRegressor : Microsoft. /// /// The learning rate for adjusting bias from being regularized. /// - [TlcModule.SweepableDiscreteParamAttribute("BiasLearningRate", new object[]{0f, 0.01f, 0.1f, 1f})] + [TlcModule.SweepableDiscreteParamAttribute("BiasLearningRate", new object[] { 0f, 0.01f, 0.1f, 1f })] public float BiasLearningRate { get; set; } = 1f; /// @@ -6801,7 +7007,7 @@ public sealed partial class StochasticGradientDescentBinaryClassifier : Microsof /// /// L2 regularizer constant /// - [TlcModule.SweepableDiscreteParamAttribute("L2Const", new object[]{1E-07f, 5E-07f, 1E-06f, 5E-06f, 1E-05f})] + [TlcModule.SweepableDiscreteParamAttribute("L2Const", new object[] { 1E-07f, 5E-07f, 1E-06f, 5E-06f, 1E-05f })] public float L2Const { get; set; } = 1E-06f; /// @@ -6812,13 +7018,13 @@ public sealed partial class StochasticGradientDescentBinaryClassifier : Microsof /// /// Exponential moving averaged improvement tolerance for convergence /// - [TlcModule.SweepableDiscreteParamAttribute("ConvergenceTolerance", new object[]{0.01f, 0.001f, 0.0001f, 1E-05f})] + [TlcModule.SweepableDiscreteParamAttribute("ConvergenceTolerance", new object[] { 0.01f, 0.001f, 0.0001f, 1E-05f })] public double ConvergenceTolerance { get; set; } = 0.0001d; /// /// Maximum number of iterations; set to 1 to simulate online learning. /// - [TlcModule.SweepableDiscreteParamAttribute("MaxIterations", new object[]{1, 5, 10, 20})] + [TlcModule.SweepableDiscreteParamAttribute("MaxIterations", new object[] { 1, 5, 10, 20 })] public int MaxIterations { get; set; } = 20; /// @@ -6829,7 +7035,7 @@ public sealed partial class StochasticGradientDescentBinaryClassifier : Microsof /// /// Shuffle data every epoch? /// - [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[]{false, true})] + [TlcModule.SweepableDiscreteParamAttribute("Shuffle", new object[] { false, true })] public bool Shuffle { get; set; } = true; /// @@ -7092,7 +7298,7 @@ public sealed partial class BinNormalizer : Microsoft.ML.Runtime.EntryPoints.Com public BinNormalizer() { } - + public BinNormalizer(params string[] inputColumns) { if (inputColumns != null) @@ -7103,7 +7309,7 @@ public BinNormalizer(params string[] inputColumns) } } } - + public BinNormalizer(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -7114,7 +7320,7 @@ public BinNormalizer(params ValueTuple[] inputOutputColumns) } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -7254,7 +7460,7 @@ public sealed partial class CategoricalHashOneHotVectorizer : Microsoft.ML.Runti public CategoricalHashOneHotVectorizer() { } - + public CategoricalHashOneHotVectorizer(params string[] inputColumns) { if (inputColumns != null) @@ -7265,7 +7471,7 @@ public CategoricalHashOneHotVectorizer(params string[] inputColumns) } } } - + public CategoricalHashOneHotVectorizer(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -7276,7 +7482,7 @@ public CategoricalHashOneHotVectorizer(params ValueTuple[] input } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -7424,7 +7630,7 @@ public sealed partial class CategoricalOneHotVectorizer : Microsoft.ML.Runtime.E public CategoricalOneHotVectorizer() { } - + public CategoricalOneHotVectorizer(params string[] inputColumns) { if (inputColumns != null) @@ -7435,7 +7641,7 @@ public CategoricalOneHotVectorizer(params string[] inputColumns) } } } - + public CategoricalOneHotVectorizer(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -7446,7 +7652,7 @@ public CategoricalOneHotVectorizer(params ValueTuple[] inputOutp } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -7563,7 +7769,7 @@ public sealed partial class CharacterTokenizer : Microsoft.ML.Runtime.EntryPoint public CharacterTokenizer() { } - + public CharacterTokenizer(params string[] inputColumns) { if (inputColumns != null) @@ -7574,7 +7780,7 @@ public CharacterTokenizer(params string[] inputColumns) } } } - + public CharacterTokenizer(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -7585,7 +7791,7 @@ public CharacterTokenizer(params ValueTuple[] inputOutputColumns } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -7682,12 +7888,12 @@ public sealed partial class ColumnConcatenator : Microsoft.ML.Runtime.EntryPoint public ColumnConcatenator() { } - + public ColumnConcatenator(string outputColumn, params string[] inputColumns) { AddColumn(outputColumn, inputColumns); } - + public void AddColumn(string name, params string[] source) { var list = Column == null ? new List() : new List(Column); @@ -7772,7 +7978,7 @@ public sealed partial class ColumnCopier : Microsoft.ML.Runtime.EntryPoints.Comm public ColumnCopier() { } - + public ColumnCopier(params string[] inputColumns) { if (inputColumns != null) @@ -7783,7 +7989,7 @@ public ColumnCopier(params string[] inputColumns) } } } - + public ColumnCopier(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -7794,7 +8000,7 @@ public ColumnCopier(params ValueTuple[] inputOutputColumns) } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -8044,7 +8250,7 @@ public sealed partial class ColumnTypeConverter : Microsoft.ML.Runtime.EntryPoin public ColumnTypeConverter() { } - + public ColumnTypeConverter(params string[] inputColumns) { if (inputColumns != null) @@ -8055,7 +8261,7 @@ public ColumnTypeConverter(params string[] inputColumns) } } } - + public ColumnTypeConverter(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -8066,7 +8272,7 @@ public ColumnTypeConverter(params ValueTuple[] inputOutputColumn } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -8243,7 +8449,7 @@ public sealed partial class ConditionalNormalizer : Microsoft.ML.Runtime.EntryPo public ConditionalNormalizer() { } - + public ConditionalNormalizer(params string[] inputColumns) { if (inputColumns != null) @@ -8254,7 +8460,7 @@ public ConditionalNormalizer(params string[] inputColumns) } } } - + public ConditionalNormalizer(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -8265,7 +8471,7 @@ public ConditionalNormalizer(params ValueTuple[] inputOutputColu } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -8526,7 +8732,7 @@ public sealed partial class Dictionarizer : Microsoft.ML.Runtime.EntryPoints.Com public Dictionarizer() { } - + public Dictionarizer(params string[] inputColumns) { if (inputColumns != null) @@ -8537,7 +8743,7 @@ public Dictionarizer(params string[] inputColumns) } } } - + public Dictionarizer(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -8548,7 +8754,7 @@ public Dictionarizer(params ValueTuple[] inputOutputColumns) } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -8875,7 +9081,7 @@ public sealed partial class GlobalContrastNormalizer : Microsoft.ML.Runtime.Entr public GlobalContrastNormalizer() { } - + public GlobalContrastNormalizer(params string[] inputColumns) { if (inputColumns != null) @@ -8886,7 +9092,7 @@ public GlobalContrastNormalizer(params string[] inputColumns) } } } - + public GlobalContrastNormalizer(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -8897,7 +9103,7 @@ public GlobalContrastNormalizer(params ValueTuple[] inputOutputC } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -9029,7 +9235,7 @@ public sealed partial class HashConverter : Microsoft.ML.Runtime.EntryPoints.Com public HashConverter() { } - + public HashConverter(params string[] inputColumns) { if (inputColumns != null) @@ -9040,7 +9246,7 @@ public HashConverter(params string[] inputColumns) } } } - + public HashConverter(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -9051,7 +9257,7 @@ public HashConverter(params ValueTuple[] inputOutputColumns) } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -9163,7 +9369,7 @@ public sealed partial class KeyToTextConverter : Microsoft.ML.Runtime.EntryPoint public KeyToTextConverter() { } - + public KeyToTextConverter(params string[] inputColumns) { if (inputColumns != null) @@ -9174,7 +9380,7 @@ public KeyToTextConverter(params string[] inputColumns) } } } - + public KeyToTextConverter(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -9185,7 +9391,7 @@ public KeyToTextConverter(params ValueTuple[] inputOutputColumns } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -9347,7 +9553,7 @@ public sealed partial class LabelIndicator : Microsoft.ML.Runtime.EntryPoints.Co public LabelIndicator() { } - + public LabelIndicator(params string[] inputColumns) { if (inputColumns != null) @@ -9358,7 +9564,7 @@ public LabelIndicator(params string[] inputColumns) } } } - + public LabelIndicator(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -9369,7 +9575,7 @@ public LabelIndicator(params ValueTuple[] inputOutputColumns) } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -9531,7 +9737,7 @@ public sealed partial class LogMeanVarianceNormalizer : Microsoft.ML.Runtime.Ent public LogMeanVarianceNormalizer() { } - + public LogMeanVarianceNormalizer(params string[] inputColumns) { if (inputColumns != null) @@ -9542,7 +9748,7 @@ public LogMeanVarianceNormalizer(params string[] inputColumns) } } } - + public LogMeanVarianceNormalizer(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -9553,7 +9759,7 @@ public LogMeanVarianceNormalizer(params ValueTuple[] inputOutput } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -9673,7 +9879,7 @@ public sealed partial class LpNormalizer : Microsoft.ML.Runtime.EntryPoints.Comm public LpNormalizer() { } - + public LpNormalizer(params string[] inputColumns) { if (inputColumns != null) @@ -9684,7 +9890,7 @@ public LpNormalizer(params string[] inputColumns) } } } - + public LpNormalizer(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -9695,7 +9901,7 @@ public LpNormalizer(params ValueTuple[] inputOutputColumns) } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -9815,7 +10021,7 @@ public sealed partial class MeanVarianceNormalizer : Microsoft.ML.Runtime.EntryP public MeanVarianceNormalizer() { } - + public MeanVarianceNormalizer(params string[] inputColumns) { if (inputColumns != null) @@ -9826,7 +10032,7 @@ public MeanVarianceNormalizer(params string[] inputColumns) } } } - + public MeanVarianceNormalizer(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -9837,7 +10043,7 @@ public MeanVarianceNormalizer(params ValueTuple[] inputOutputCol } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -9930,7 +10136,7 @@ public sealed partial class MinMaxNormalizer : Microsoft.ML.Runtime.EntryPoints. public MinMaxNormalizer() { } - + public MinMaxNormalizer(params string[] inputColumns) { if (inputColumns != null) @@ -9941,7 +10147,7 @@ public MinMaxNormalizer(params string[] inputColumns) } } } - + public MinMaxNormalizer(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -9952,7 +10158,7 @@ public MinMaxNormalizer(params ValueTuple[] inputOutputColumns) } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -10081,7 +10287,7 @@ public sealed partial class MissingValueHandler : Microsoft.ML.Runtime.EntryPoin public MissingValueHandler() { } - + public MissingValueHandler(params string[] inputColumns) { if (inputColumns != null) @@ -10092,7 +10298,7 @@ public MissingValueHandler(params string[] inputColumns) } } } - + public MissingValueHandler(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -10103,7 +10309,7 @@ public MissingValueHandler(params ValueTuple[] inputOutputColumn } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -10210,7 +10416,7 @@ public sealed partial class MissingValueIndicator : Microsoft.ML.Runtime.EntryPo public MissingValueIndicator() { } - + public MissingValueIndicator(params string[] inputColumns) { if (inputColumns != null) @@ -10221,7 +10427,7 @@ public MissingValueIndicator(params string[] inputColumns) } } } - + public MissingValueIndicator(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -10232,7 +10438,7 @@ public MissingValueIndicator(params ValueTuple[] inputOutputColu } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -10324,7 +10530,7 @@ public sealed partial class MissingValuesDropper : Microsoft.ML.Runtime.EntryPoi public MissingValuesDropper() { } - + public MissingValuesDropper(params string[] inputColumns) { if (inputColumns != null) @@ -10335,7 +10541,7 @@ public MissingValuesDropper(params string[] inputColumns) } } } - + public MissingValuesDropper(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -10346,7 +10552,7 @@ public MissingValuesDropper(params ValueTuple[] inputOutputColum } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -10533,7 +10739,7 @@ public sealed partial class MissingValueSubstitutor : Microsoft.ML.Runtime.Entry public MissingValueSubstitutor() { } - + public MissingValueSubstitutor(params string[] inputColumns) { if (inputColumns != null) @@ -10544,7 +10750,7 @@ public MissingValueSubstitutor(params string[] inputColumns) } } } - + public MissingValueSubstitutor(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -10555,7 +10761,7 @@ public MissingValueSubstitutor(params ValueTuple[] inputOutputCo } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -10716,7 +10922,7 @@ public sealed partial class NGramTranslator : Microsoft.ML.Runtime.EntryPoints.C public NGramTranslator() { } - + public NGramTranslator(params string[] inputColumns) { if (inputColumns != null) @@ -10727,7 +10933,7 @@ public NGramTranslator(params string[] inputColumns) } } } - + public NGramTranslator(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -10738,7 +10944,7 @@ public NGramTranslator(params ValueTuple[] inputOutputColumns) } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -11604,7 +11810,7 @@ public sealed partial class SupervisedBinNormalizer : Microsoft.ML.Runtime.Entry public SupervisedBinNormalizer() { } - + public SupervisedBinNormalizer(params string[] inputColumns) { if (inputColumns != null) @@ -11615,7 +11821,7 @@ public SupervisedBinNormalizer(params string[] inputColumns) } } } - + public SupervisedBinNormalizer(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -11626,7 +11832,7 @@ public SupervisedBinNormalizer(params ValueTuple[] inputOutputCo } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -11788,12 +11994,12 @@ public sealed partial class TextFeaturizer : Microsoft.ML.Runtime.EntryPoints.Co public TextFeaturizer() { } - + public TextFeaturizer(string outputColumn, params string[] inputColumns) { AddColumn(outputColumn, inputColumns); } - + public void AddColumn(string name, params string[] source) { Column = ManyToOneColumn.Create(name, source); @@ -11920,7 +12126,7 @@ public sealed partial class TextToKeyConverter : Microsoft.ML.Runtime.EntryPoint public TextToKeyConverter() { } - + public TextToKeyConverter(params string[] inputColumns) { if (inputColumns != null) @@ -11931,7 +12137,7 @@ public TextToKeyConverter(params string[] inputColumns) } } } - + public TextToKeyConverter(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -11942,7 +12148,7 @@ public TextToKeyConverter(params ValueTuple[] inputOutputColumns } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -12203,7 +12409,7 @@ public sealed partial class WordTokenizer : Microsoft.ML.Runtime.EntryPoints.Com public WordTokenizer() { } - + public WordTokenizer(params string[] inputColumns) { if (inputColumns != null) @@ -12214,7 +12420,7 @@ public WordTokenizer(params string[] inputColumns) } } } - + public WordTokenizer(params ValueTuple[] inputOutputColumns) { if (inputOutputColumns != null) @@ -12225,7 +12431,7 @@ public WordTokenizer(params ValueTuple[] inputOutputColumns) } } } - + public void AddColumn(string source) { var list = Column == null ? new List() : new List(Column); @@ -12298,7 +12504,7 @@ public WordTokenizerPipelineStep(Output output) namespace Runtime { - public abstract class CalibratorTrainer : ComponentKind {} + public abstract class CalibratorTrainer : ComponentKind { } /// /// @@ -12350,7 +12556,7 @@ public sealed class PlattCalibratorCalibratorTrainer : CalibratorTrainer internal override string ComponentName => "PlattCalibrator"; } - public abstract class ClassificationLossFunction : ComponentKind {} + public abstract class ClassificationLossFunction : ComponentKind { } /// /// Exponential loss. @@ -12407,7 +12613,7 @@ public sealed class SmoothedHingeLossClassificationLossFunction : Classification internal override string ComponentName => "SmoothedHingeLoss"; } - public abstract class EarlyStoppingCriterion : ComponentKind {} + public abstract class EarlyStoppingCriterion : ComponentKind { } /// /// Stop in case of loss of generality. @@ -12501,7 +12707,7 @@ public sealed class UPEarlyStoppingCriterion : EarlyStoppingCriterion internal override string ComponentName => "UP"; } - public abstract class FastTreeTrainer : ComponentKind {} + public abstract class FastTreeTrainer : ComponentKind { } /// /// Uses a logit-boost boosted tree learner to perform binary classification. @@ -12574,19 +12780,19 @@ public sealed class FastTreeBinaryClassificationFastTreeTrainer : FastTreeTraine /// /// The learning rate /// - [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale: true)] public double LearningRates { get; set; } = 0.2d; /// /// Shrinkage /// - [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale: true)] public double Shrinkage { get; set; } = 1d; /// /// Dropout rate for tree regularization /// - [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[]{0f, 1E-09f, 0.05f, 0.1f, 0.2f})] + [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[] { 0f, 1E-09f, 0.05f, 0.1f, 0.2f })] public double DropoutRate { get; set; } /// @@ -12743,19 +12949,19 @@ public sealed class FastTreeBinaryClassificationFastTreeTrainer : FastTreeTraine /// /// The max number of leaves in each regression tree /// - [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize:4, isLogScale:true)] + [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize: 4, isLogScale: true)] public int NumLeaves { get; set; } = 20; /// /// The minimal number of documents allowed in a leaf of a regression tree, out of the subsampled data /// - [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[]{1, 10, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[] { 1, 10, 50 })] public int MinDocumentsInLeafs { get; set; } = 10; /// /// Number of weak hypotheses in the ensemble /// - [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[]{20, 100, 500})] + [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[] { 20, 100, 500 })] public int NumTrees { get; set; } = 100; /// @@ -12962,19 +13168,19 @@ public sealed class FastTreeRankingFastTreeTrainer : FastTreeTrainer /// /// The learning rate /// - [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale: true)] public double LearningRates { get; set; } = 0.2d; /// /// Shrinkage /// - [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale: true)] public double Shrinkage { get; set; } = 1d; /// /// Dropout rate for tree regularization /// - [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[]{0f, 1E-09f, 0.05f, 0.1f, 0.2f})] + [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[] { 0f, 1E-09f, 0.05f, 0.1f, 0.2f })] public double DropoutRate { get; set; } /// @@ -13131,19 +13337,19 @@ public sealed class FastTreeRankingFastTreeTrainer : FastTreeTrainer /// /// The max number of leaves in each regression tree /// - [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize:4, isLogScale:true)] + [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize: 4, isLogScale: true)] public int NumLeaves { get; set; } = 20; /// /// The minimal number of documents allowed in a leaf of a regression tree, out of the subsampled data /// - [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[]{1, 10, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[] { 1, 10, 50 })] public int MinDocumentsInLeafs { get; set; } = 10; /// /// Number of weak hypotheses in the ensemble /// - [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[]{20, 100, 500})] + [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[] { 20, 100, 500 })] public int NumTrees { get; set; } = 100; /// @@ -13310,19 +13516,19 @@ public sealed class FastTreeRegressionFastTreeTrainer : FastTreeTrainer /// /// The learning rate /// - [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale: true)] public double LearningRates { get; set; } = 0.2d; /// /// Shrinkage /// - [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale: true)] public double Shrinkage { get; set; } = 1d; /// /// Dropout rate for tree regularization /// - [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[]{0f, 1E-09f, 0.05f, 0.1f, 0.2f})] + [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[] { 0f, 1E-09f, 0.05f, 0.1f, 0.2f })] public double DropoutRate { get; set; } /// @@ -13479,19 +13685,19 @@ public sealed class FastTreeRegressionFastTreeTrainer : FastTreeTrainer /// /// The max number of leaves in each regression tree /// - [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize:4, isLogScale:true)] + [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize: 4, isLogScale: true)] public int NumLeaves { get; set; } = 20; /// /// The minimal number of documents allowed in a leaf of a regression tree, out of the subsampled data /// - [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[]{1, 10, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[] { 1, 10, 50 })] public int MinDocumentsInLeafs { get; set; } = 10; /// /// Number of weak hypotheses in the ensemble /// - [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[]{20, 100, 500})] + [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[] { 20, 100, 500 })] public int NumTrees { get; set; } = 100; /// @@ -13663,19 +13869,19 @@ public sealed class FastTreeTweedieRegressionFastTreeTrainer : FastTreeTrainer /// /// The learning rate /// - [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("LearningRates", 0.025f, 0.4f, isLogScale: true)] public double LearningRates { get; set; } = 0.2d; /// /// Shrinkage /// - [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale:true)] + [TlcModule.SweepableFloatParamAttribute("Shrinkage", 0.025f, 4f, isLogScale: true)] public double Shrinkage { get; set; } = 1d; /// /// Dropout rate for tree regularization /// - [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[]{0f, 1E-09f, 0.05f, 0.1f, 0.2f})] + [TlcModule.SweepableDiscreteParamAttribute("DropoutRate", new object[] { 0f, 1E-09f, 0.05f, 0.1f, 0.2f })] public double DropoutRate { get; set; } /// @@ -13832,19 +14038,19 @@ public sealed class FastTreeTweedieRegressionFastTreeTrainer : FastTreeTrainer /// /// The max number of leaves in each regression tree /// - [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize:4, isLogScale:true)] + [TlcModule.SweepableLongParamAttribute("NumLeaves", 2, 128, stepSize: 4, isLogScale: true)] public int NumLeaves { get; set; } = 20; /// /// The minimal number of documents allowed in a leaf of a regression tree, out of the subsampled data /// - [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[]{1, 10, 50})] + [TlcModule.SweepableDiscreteParamAttribute("MinDocumentsInLeafs", new object[] { 1, 10, 50 })] public int MinDocumentsInLeafs { get; set; } = 10; /// /// Number of weak hypotheses in the ensemble /// - [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[]{20, 100, 500})] + [TlcModule.SweepableDiscreteParamAttribute("NumTrees", new object[] { 20, 100, 500 })] public int NumTrees { get; set; } = 100; /// @@ -13945,7 +14151,7 @@ public sealed class FastTreeTweedieRegressionFastTreeTrainer : FastTreeTrainer internal override string ComponentName => "FastTreeTweedieRegression"; } - public abstract class NgramExtractor : ComponentKind {} + public abstract class NgramExtractor : ComponentKind { } /// /// Extracts NGrams from text and convert them to vector using dictionary. @@ -14027,7 +14233,7 @@ public sealed class NGramHashNgramExtractor : NgramExtractor internal override string ComponentName => "NGramHash"; } - public abstract class ParallelTraining : ComponentKind {} + public abstract class ParallelTraining : ComponentKind { } /// /// Single node machine learning process. @@ -14039,7 +14245,7 @@ public sealed class SingleParallelTraining : ParallelTraining internal override string ComponentName => "Single"; } - public abstract class RegressionLossFunction : ComponentKind {} + public abstract class RegressionLossFunction : ComponentKind { } /// /// Poisson loss. @@ -14076,7 +14282,7 @@ public sealed class TweedieLossRegressionLossFunction : RegressionLossFunction internal override string ComponentName => "TweedieLoss"; } - public abstract class SDCAClassificationLossFunction : ComponentKind {} + public abstract class SDCAClassificationLossFunction : ComponentKind { } /// /// Hinge loss. @@ -14118,7 +14324,7 @@ public sealed class SmoothedHingeLossSDCAClassificationLossFunction : SDCAClassi internal override string ComponentName => "SmoothedHingeLoss"; } - public abstract class SDCARegressionLossFunction : ComponentKind {} + public abstract class SDCARegressionLossFunction : ComponentKind { } /// /// Squared loss. @@ -14130,7 +14336,7 @@ public sealed class SquaredLossSDCARegressionLossFunction : SDCARegressionLossFu internal override string ComponentName => "SquaredLoss"; } - public abstract class StopWordsRemover : ComponentKind {} + public abstract class StopWordsRemover : ComponentKind { } /// /// Remover with list of stopwords specified by the user. diff --git a/src/Microsoft.ML/Data/TextLoader.cs b/src/Microsoft.ML/Data/TextLoader.cs new file mode 100644 index 0000000000..f9ed1db012 --- /dev/null +++ b/src/Microsoft.ML/Data/TextLoader.cs @@ -0,0 +1,116 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Api; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; + +namespace Microsoft.ML.Data +{ + public sealed class TextLoader : TextLoader + { + + /// + /// Construct a TextLoader object + /// + /// Data file path + /// Does the file contains header? + /// How the columns are seperated? + /// Options: separator="tab", separator="space", separator="comma" or separator=[single character]. + /// By default separator=null means "tab" + /// Whether the input may include quoted values, + /// which can contain separator characters, colons, + /// and distinguish empty values from missing values. When true, consecutive separators + /// denote a missing value and an empty value is denoted by \"\". + /// When false, consecutive separators denote an empty value. + /// Whether the input may include sparse representations e.g. + /// if one of the row contains "5 2:6 4:3" that's mean there are 5 columns all zero + /// except for 3rd and 5th columns which have values 6 and 3 + /// Remove trailing whitespace from lines + public TextLoader(string inputFilePath, bool useHeader = false, + string separator = null, bool allowQuotedStrings = true, + bool supportSparse = true, bool trimWhitespace = false) : base(inputFilePath) + { + var fields = typeof(TInput).GetFields(); + Arguments.Column = new TextLoaderColumn[fields.Length]; + for (int index = 0; index < fields.Length; index++) + { + var field = fields[index]; + var mappingAttr = field.GetCustomAttribute(); + if (mappingAttr == null) + throw Contracts.Except($"{field.Name} is missing ColumnAttribute"); + + if (Regex.Match(mappingAttr.Ordinal, @"[^(0-9,\*\-~)]+").Success) + throw Contracts.Except($"{mappingAttr.Ordinal} contains invalid characters. " + + $"Valid characters are 0-9, *, - and ~"); + + var name = mappingAttr.Name ?? field.Name; + if (name.Any(c => !Char.IsLetterOrDigit(c))) + throw Contracts.Except($"{name} is not alphanumeric."); + + if(separator != null) + { + if(separator != "space" && separator != "tab" && separator != "comma" && separator.Length > 1) + throw Contracts.Except($"{nameof(separator)} can only be one of the following: space, tab, comma" + + $" or a single character."); + } + + var col = Runtime.Data.TextLoader.Column.Parse( + $"{name}:" + + $"{TypeToName(field.FieldType.IsArray ? field.FieldType.GetElementType() : field.FieldType)}:" + + $"{mappingAttr.Ordinal}" + ); + + TextLoaderColumn tlc = new TextLoaderColumn(); + if (col.KeyRange != null) + { + tlc.KeyRange = new KeyRange(); + tlc.KeyRange.Min = col.KeyRange.Min; + tlc.KeyRange.Max = col.KeyRange.Max; + } + + tlc.Name = col.Name; + tlc.Source = new TextLoaderRange[col.Source.Length]; + for (int indexLocal = 0; indexLocal < tlc.Source.Length; indexLocal++) + { + tlc.Source[indexLocal] = new TextLoaderRange + { + AllOther = col.Source[indexLocal].AllOther, + AutoEnd = col.Source[indexLocal].AutoEnd, + ForceVector = col.Source[indexLocal].ForceVector, + VariableEnd = col.Source[indexLocal].VariableEnd, + Max = col.Source[indexLocal].Max, + Min = col.Source[indexLocal].Min + }; + } + + tlc.Type = col.Type; + Arguments.Column[index] = tlc; + } + + Arguments.HasHeader = useHeader; + Arguments.Separator = separator; + Arguments.AllowQuoting = allowQuotedStrings; + Arguments.AllowSparse = supportSparse; + Arguments.TrimWhitespace = trimWhitespace; + } + + private string TypeToName(Type type) + { + if (type == typeof(string)) + return "TX"; + else if (type == typeof(float) || type == typeof(double)) + return "R4"; + else if (type == typeof(bool)) + return "BL"; + else + throw new Exception("Type not implemented or supported."); //Add more types. + } + } +} diff --git a/src/Microsoft.ML/LearningPipeline.cs b/src/Microsoft.ML/LearningPipeline.cs index 0ece3697a9..aa490abc09 100644 --- a/src/Microsoft.ML/LearningPipeline.cs +++ b/src/Microsoft.ML/LearningPipeline.cs @@ -66,7 +66,7 @@ public PredictionModel Train() step = currentItem.ApplyStep(step, experiment); if (step is ILearningPipelineDataStep dataStep && dataStep.Model != null) transformModels.Add(dataStep.Model); - + else if (step is ILearningPipelinePredictorStep predictorDataStep) { if (lastTransformModel != null) @@ -94,7 +94,7 @@ public PredictionModel Train() if (transformModels.Count > 0) { - transformModels.Insert(0,lastTransformModel); + transformModels.Insert(0, lastTransformModel); var modelInput = new Transforms.ModelCombiner { Models = new ArrayVar(transformModels.ToArray()) diff --git a/src/Microsoft.ML/Runtime/EntryPoints/ImportTextData.cs b/src/Microsoft.ML/Runtime/EntryPoints/ImportTextData.cs index 8038294398..77a54b2abb 100644 --- a/src/Microsoft.ML/Runtime/EntryPoints/ImportTextData.cs +++ b/src/Microsoft.ML/Runtime/EntryPoints/ImportTextData.cs @@ -27,13 +27,23 @@ public sealed class Input public string CustomSchema = null; } + [TlcModule.EntryPointKind(typeof(ILearningPipelineLoader))] + public sealed class LoaderInput + { + [Argument(ArgumentType.Required, ShortName = "data", HelpText = "Location of the input file", SortOrder = 1)] + public IFileHandle InputFile; + + [Argument(ArgumentType.Required, ShortName = "args", HelpText = "Arguments", SortOrder = 1)] + public TextLoader.Arguments Arguments = new TextLoader.Arguments(); + } + public sealed class Output { [TlcModule.Output(Desc = "The resulting data view", SortOrder = 1)] public IDataView Data; } - [TlcModule.EntryPoint(Name = "Data.TextLoader", Desc = "Import a dataset from a text file")] + [TlcModule.EntryPoint(Name = "Data.CustomTextLoader", Desc = "Import a dataset from a text file")] public static Output ImportText(IHostEnvironment env, Input input) { Contracts.CheckValue(env, nameof(env)); @@ -43,5 +53,16 @@ public static Output ImportText(IHostEnvironment env, Input input) var loader = host.CreateLoader(string.Format("Text{{{0}}}", input.CustomSchema), new FileHandleSource(input.InputFile)); return new Output { Data = loader }; } + + [TlcModule.EntryPoint(Name = "Data.TextLoader", Desc = "Import a dataset from a text file", NoSeal = true)] + public static Output TextLoader(IHostEnvironment env, LoaderInput input) + { + Contracts.CheckValue(env, nameof(env)); + var host = env.Register("ImportTextData"); + env.CheckValue(input, nameof(input)); + EntryPointUtils.CheckInputArgs(host, input); + var loader = host.CreateLoader(input.Arguments, new FileHandleSource(input.InputFile)); + return new Output { Data = loader }; + } } } diff --git a/src/Microsoft.ML/Runtime/Internal/Tools/CSharpApiGenerator.cs b/src/Microsoft.ML/Runtime/Internal/Tools/CSharpApiGenerator.cs index 17643518ca..ccd8bc47e4 100644 --- a/src/Microsoft.ML/Runtime/Internal/Tools/CSharpApiGenerator.cs +++ b/src/Microsoft.ML/Runtime/Internal/Tools/CSharpApiGenerator.cs @@ -640,6 +640,58 @@ private void GenerateStructs(IndentingTextWriter writer, } } + private void GenerateLoaderAddInputMethod(IndentingTextWriter writer, string className) + { + //Constructor. + writer.WriteLine("[JsonIgnore]"); + writer.WriteLine("private string _inputFilePath = null;"); + writer.WriteLine($"public {className}(string filePath)"); + writer.WriteLine("{"); + writer.Indent(); + writer.WriteLine("_inputFilePath = filePath;"); + writer.Outdent(); + writer.WriteLine("}"); + writer.WriteLine(""); + + //SetInput. + writer.WriteLine($"public void SetInput(IHostEnvironment env, Experiment experiment)"); + writer.WriteLine("{"); + writer.Indent(); + writer.WriteLine("IFileHandle inputFile = new SimpleFileHandle(env, _inputFilePath, false, false);"); + writer.WriteLine("experiment.SetInput(InputFile, inputFile);"); + writer.Outdent(); + writer.WriteLine("}"); + writer.WriteLine(""); + + //Apply. + writer.WriteLine($"public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment)"); + writer.WriteLine("{"); + writer.Indent(); + writer.WriteLine("Contracts.Assert(previousStep == null);"); + writer.WriteLine(""); + writer.WriteLine($"return new {className}PipelineStep(experiment.Add(this));"); + writer.Outdent(); + writer.WriteLine("}"); + writer.WriteLine(""); + + //Pipelinestep class. + writer.WriteLine($"private class {className}PipelineStep : ILearningPipelineDataStep"); + writer.WriteLine("{"); + writer.Indent(); + writer.WriteLine($"public {className}PipelineStep (Output output)"); + writer.WriteLine("{"); + writer.Indent(); + writer.WriteLine("Data = output.Data;"); + writer.WriteLine("Model = null;"); + writer.Outdent(); + writer.WriteLine("}"); + writer.WriteLine(); + writer.WriteLine("public Var Data { get; }"); + writer.WriteLine("public Var Model { get; }"); + writer.Outdent(); + writer.WriteLine("}"); + } + private void GenerateColumnAddMethods(IndentingTextWriter writer, Type inputType, ModuleCatalog catalog, @@ -798,10 +850,15 @@ private void GenerateInput(IndentingTextWriter writer, foreach (var line in entryPointInfo.Description.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) writer.WriteLine($"/// {line}"); writer.WriteLine("/// "); - writer.WriteLine($"public sealed partial class {classAndMethod.Item2}{classBase}"); + + string seal = entryPointInfo.NoSeal ? "" : "sealed "; + writer.WriteLine($"public {seal}partial class {classAndMethod.Item2}{classBase}"); writer.WriteLine("{"); writer.Indent(); writer.WriteLine(); + if (classBase.Contains("ILearningPipelineLoader")) + GenerateLoaderAddInputMethod(writer, classAndMethod.Item2); + GenerateColumnAddMethods(writer, entryPointInfo.InputType, catalog, classAndMethod.Item2, out Type transformType); writer.WriteLine(); GenerateInputFields(writer, entryPointInfo.InputType, catalog, _typesSymbolTable); diff --git a/src/Microsoft.ML/TextLoader.cs b/src/Microsoft.ML/TextLoader.cs deleted file mode 100644 index de592a4f69..0000000000 --- a/src/Microsoft.ML/TextLoader.cs +++ /dev/null @@ -1,125 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.ML.Runtime; -using Microsoft.ML.Runtime.Api; -using Microsoft.ML.Runtime.Data; -using Microsoft.ML.Runtime.EntryPoints; -using System; -using System.Linq; -using System.Reflection; -using System.Text; - -namespace Microsoft.ML -{ - public class TextLoader : ILearningPipelineLoader - { - private string _inputFilePath; - private string CustomSchema; - private Data.TextLoader ImportTextInput; - - /// - /// Construct a TextLoader object - /// - /// Data file path - /// Does the file contains header? - /// How the columns are seperated? - /// Options: separator="tab", separator="space", separator="comma" or separator=[single character]. - /// By default separator=null means "tab" - /// Whether the input may include quoted values, - /// which can contain separator characters, colons, - /// and distinguish empty values from missing values. When true, consecutive separators - /// denote a missing value and an empty value is denoted by \"\". - /// When false, consecutive separators denote an empty value. - /// Whether the input may include sparse representations e.g. - /// if one of the row contains "5 2:6 4:3" that's mean there are 5 columns all zero - /// except for 3rd and 5th columns which have values 6 and 3 - /// Remove trailing whitespace from lines - public TextLoader(string inputFilePath, bool useHeader = false, - string separator = null, bool allowQuotedStrings = true, - bool supportSparse = true, bool trimWhitespace = false) - { - _inputFilePath = inputFilePath; - SetCustomStringFromType(useHeader, separator, allowQuotedStrings, supportSparse, trimWhitespace); - } - - private IFileHandle GetTextLoaderFileHandle(IHostEnvironment env, string trainFilePath) => - new SimpleFileHandle(env, trainFilePath, false, false); - - private void SetCustomStringFromType(bool useHeader, string separator, - bool allowQuotedStrings, bool supportSparse, bool trimWhitespace) - { - StringBuilder schemaBuilder = new StringBuilder(CustomSchema); - foreach (var field in typeof(TInput).GetFields()) - { - var mappingAttr = field.GetCustomAttribute(); - if(mappingAttr == null) - throw Contracts.ExceptParam(nameof(field.Name), " is missing ColumnAttribute"); - - schemaBuilder.AppendFormat("col={0}:{1}:{2} ", - mappingAttr.Name ?? field.Name, - TypeToName(field.FieldType.IsArray ? field.FieldType.GetElementType() : field.FieldType), - mappingAttr.Ordinal); - } - - if (useHeader) - schemaBuilder.Append(nameof(TextLoader.Arguments.HasHeader)).Append("+ "); - - if (separator != null) - schemaBuilder.Append(nameof(TextLoader.Arguments.Separator)).Append("=").Append(separator).Append(" "); - - if (!allowQuotedStrings) - schemaBuilder.Append(nameof(TextLoader.Arguments.AllowQuoting)).Append("- "); - - if (!supportSparse) - schemaBuilder.Append(nameof(TextLoader.Arguments.AllowSparse)).Append("- "); - - if (trimWhitespace) - schemaBuilder.Append(nameof(TextLoader.Arguments.TrimWhitespace)).Append("+ "); - - schemaBuilder.Length--; - CustomSchema = schemaBuilder.ToString(); - } - - private string TypeToName(Type type) - { - if (type == typeof(string)) - return "TX"; - else if (type == typeof(float) || type == typeof(double)) - return "R4"; - else if (type == typeof(bool)) - return "BL"; - else - throw new Exception("Type not implemented or supported."); //Add more types. - } - - public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment) - { - Contracts.Assert(previousStep == null); - - ImportTextInput = new Data.TextLoader(); - ImportTextInput.CustomSchema = CustomSchema; - var importOutput = experiment.Add(ImportTextInput); - return new TextLoaderPipelineStep(importOutput.Data); - } - - public void SetInput(IHostEnvironment env, Experiment experiment) - { - IFileHandle inputFile = GetTextLoaderFileHandle(env, _inputFilePath); - experiment.SetInput(ImportTextInput.InputFile, inputFile); - } - - private class TextLoaderPipelineStep : ILearningPipelineDataStep - { - public TextLoaderPipelineStep(Var data) - { - Data = data; - Model = null; - } - - public Var Data { get; } - public Var Model { get; } - } - } -} diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs index b66d61ae69..8bfa4e4e78 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestCSharpApi.cs @@ -36,7 +36,7 @@ public void TestSimpleExperiment() { var experiment = env.CreateExperiment(); - var importInput = new ML.Data.TextLoader(); + var importInput = new ML.Data.CustomTextLoader(); var importOutput = experiment.Add(importInput); var normalizeInput = new ML.Transforms.MinMaxNormalizer @@ -67,7 +67,7 @@ public void TestSimpleTrainExperiment() { var experiment = env.CreateExperiment(); - var importInput = new ML.Data.TextLoader(); + var importInput = new ML.Data.CustomTextLoader(); var importOutput = experiment.Add(importInput); var catInput = new ML.Transforms.CategoricalOneHotVectorizer @@ -165,7 +165,7 @@ public void TestTrainTestMacro() var experiment = env.CreateExperiment(); - var importInput = new ML.Data.TextLoader(); + var importInput = new ML.Data.CustomTextLoader(); var importOutput = experiment.Add(importInput); var trainTestInput = new ML.Models.TrainTestBinaryEvaluator @@ -235,7 +235,7 @@ public void TestCrossValidationBinaryMacro() var experiment = env.CreateExperiment(); - var importInput = new ML.Data.TextLoader(); + var importInput = new ML.Data.CustomTextLoader(); var importOutput = experiment.Add(importInput); var crossValidateBinary = new ML.Models.BinaryCrossValidator @@ -295,7 +295,7 @@ public void TestCrossValidationMacro() var modelCombineOutput = subGraph.Add(modelCombine); var experiment = env.CreateExperiment(); - var importInput = new ML.Data.TextLoader(); + var importInput = new ML.Data.CustomTextLoader(); var importOutput = experiment.Add(importInput); var crossValidate = new ML.Models.CrossValidator diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index 4fe503b9b7..4a4032bf03 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -305,7 +305,7 @@ public void EntryPointOptionalParams() { 'Nodes': [ { - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': { 'InputFile': '$file1' }, @@ -355,7 +355,7 @@ public void EntryPointExecGraphCommand() {{ 'Nodes': [ {{ - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': {{ 'InputFile': '$file1' }}, @@ -512,7 +512,7 @@ public void EntryPointParseColumns() {{ 'Nodes': [ {{ - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': {{ 'InputFile': '$file1' }}, @@ -562,7 +562,7 @@ public void EntryPointCountFeatures() {{ 'Nodes': [ {{ - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': {{ 'InputFile': '$file1' }}, @@ -607,7 +607,7 @@ public void EntryPointMutualSelectFeatures() {{ 'Nodes': [ {{ - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': {{ 'InputFile': '$file1' }}, @@ -653,7 +653,7 @@ public void EntryPointTextToKeyToText() {{ 'Nodes': [ {{ - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': {{ 'InputFile': '$file1', 'CustomSchema': 'sep=comma col=Cat:TX:4' @@ -735,7 +735,7 @@ private void RunTrainScoreEvaluate(string learner, string evaluator, string data {{ 'Nodes': [ {{ - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': {{ 'InputFile': '$file' }}, @@ -1214,7 +1214,7 @@ internal void TestEntryPointPipelineRoutine(string dataFile, string schema, stri {{ 'Nodes': [ {{ - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': {{ 'InputFile': '$file1', 'CustomSchema': '{schema}' @@ -1287,7 +1287,7 @@ internal void TestEntryPointRoutine(string dataFile, string trainerName, string {{ 'Nodes': [ {{ - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': {{ 'InputFile': '$file1' {3} @@ -1459,7 +1459,7 @@ public void EntryPointNormalizeIfNeeded() { 'Nodes': [ { - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': { 'InputFile': '$file' }, @@ -1522,7 +1522,7 @@ public void EntryPointTrainTestBinaryMacro() { 'Nodes': [ { - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': { 'InputFile': '$file' }, @@ -1630,7 +1630,7 @@ public void EntryPointTrainTestMacroNoTransformInput() { 'Nodes': [ { - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': { 'InputFile': '$file' }, @@ -1744,7 +1744,7 @@ public void EntryPointTrainTestMacro() { 'Nodes': [ { - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': { 'InputFile': '$file' }, @@ -1843,7 +1843,7 @@ public void EntryPointChainedTrainTestMacros() { 'Nodes': [ { - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': { 'InputFile': '$file' }, @@ -2019,7 +2019,7 @@ public void EntryPointChainedCrossValMacros() { 'Nodes': [ { - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': { 'InputFile': '$file' }, @@ -2214,7 +2214,7 @@ public void EntryPointMacroEarlyExpansion() { 'Nodes': [ { - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': { 'InputFile': '$file' }, @@ -2302,7 +2302,7 @@ public void EntryPointSerialization() { 'Nodes': [ { - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': { 'InputFile': '$file' }, @@ -2368,7 +2368,7 @@ public void EntryPointNodeSchedulingFields() { 'Nodes': [ { - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'StageId': '5063dee8f19c4dd89a1fc3a9da5351a7', 'Inputs': { 'InputFile': '$file' @@ -2437,7 +2437,7 @@ public void EntryPointPrepareLabelConvertPredictedLabel() {{ 'Nodes': [ {{ - 'Name': 'Data.TextLoader', + 'Name': 'Data.CustomTextLoader', 'Inputs': {{ 'InputFile': '$file1', 'CustomSchema': 'sep=comma col=Label:TX:4 col=Features:Num:0-3' diff --git a/test/Microsoft.ML.TestFramework/ModelHelper.cs b/test/Microsoft.ML.TestFramework/ModelHelper.cs index dca360c4e3..0e8a4000ac 100644 --- a/test/Microsoft.ML.TestFramework/ModelHelper.cs +++ b/test/Microsoft.ML.TestFramework/ModelHelper.cs @@ -54,9 +54,9 @@ private static ITransformModel CreateKcHousePricePredictorModel(string dataPath) Experiment experiment = s_environment.CreateExperiment(); - var importData = new Data.TextLoader(); + var importData = new Data.CustomTextLoader(); importData.CustomSchema = dataSchema; - Data.TextLoader.Output imported = experiment.Add(importData); + Data.CustomTextLoader.Output imported = experiment.Add(importData); var numericalConcatenate = new Transforms.ColumnConcatenator(); numericalConcatenate.Data = imported.Data; diff --git a/test/Microsoft.ML.Tests/Scenarios/Scenario3_SentimentPrediction.cs b/test/Microsoft.ML.Tests/Scenarios/Scenario3_SentimentPrediction.cs index a0591d34b9..37c841e6f0 100644 --- a/test/Microsoft.ML.Tests/Scenarios/Scenario3_SentimentPrediction.cs +++ b/test/Microsoft.ML.Tests/Scenarios/Scenario3_SentimentPrediction.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.ML.Data; using Microsoft.ML.Models; using Microsoft.ML.Runtime; using Microsoft.ML.Runtime.Api; @@ -23,7 +24,32 @@ public void TrainAndPredictSentimentModelTest() { string dataPath = GetDataPath(SentimentDataPath); var pipeline = new LearningPipeline(); - pipeline.Add(new TextLoader(dataPath, useHeader: true, separator: "tab")); + + pipeline.Add(new TextLoader(dataPath) + { + Arguments = new TextLoaderArguments + { + Separator = "tab", + HasHeader = true, + Column = new[] + { + new TextLoaderColumn() + { + Name = "Label", + Source = new [] { new TextLoaderRange() { Min = 0, Max = 0} }, + Type = Runtime.Data.DataKind.Num + }, + + new TextLoaderColumn() + { + Name = "SentimentText", + Source = new [] { new TextLoaderRange() { Min = 1, Max = 1} }, + Type = Runtime.Data.DataKind.Text + } + } + } + }); + pipeline.Add(new TextFeaturizer("Features", "SentimentText") { KeepDiacritics = false, @@ -60,8 +86,30 @@ public void TrainAndPredictSentimentModelTest() Assert.True(predictions.ElementAt(1).Sentiment); string testDataPath = GetDataPath(SentimentTestPath); - var testData = new TextLoader(testDataPath, useHeader: true, separator: "tab"); - + var testData = new TextLoader(testDataPath) + { + Arguments = new TextLoaderArguments + { + Separator = "tab", + HasHeader = true, + Column = new[] + { + new TextLoaderColumn() + { + Name = "Label", + Source = new [] { new TextLoaderRange() { Min = 0, Max = 0} }, + Type = Runtime.Data.DataKind.Num + }, + + new TextLoaderColumn() + { + Name = "SentimentText", + Source = new [] { new TextLoaderRange() { Min = 1, Max = 1} }, + Type = Runtime.Data.DataKind.Text + } + } + } + }; var evaluator = new BinaryClassificationEvaluator(); BinaryClassificationMetrics metrics = evaluator.Evaluate(model, testData); diff --git a/test/Microsoft.ML.Tests/Scenarios/Scenario_TrainPredictionModel.cs b/test/Microsoft.ML.Tests/Scenarios/Scenario_TrainPredictionModel.cs index de7d2f6a00..162c6566c4 100644 --- a/test/Microsoft.ML.Tests/Scenarios/Scenario_TrainPredictionModel.cs +++ b/test/Microsoft.ML.Tests/Scenarios/Scenario_TrainPredictionModel.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.ML.Data; using Microsoft.ML.Models; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; diff --git a/test/Microsoft.ML.Tests/Scenarios/TrainAndPredictIrisModelTest.cs b/test/Microsoft.ML.Tests/Scenarios/TrainAndPredictIrisModelTest.cs index d897303e30..4fc2c99acc 100644 --- a/test/Microsoft.ML.Tests/Scenarios/TrainAndPredictIrisModelTest.cs +++ b/test/Microsoft.ML.Tests/Scenarios/TrainAndPredictIrisModelTest.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.ML.Data; using Microsoft.ML.Models; using Microsoft.ML.Runtime.Api; using Microsoft.ML.Trainers; diff --git a/test/Microsoft.ML.Tests/TextLoaderTests.cs b/test/Microsoft.ML.Tests/TextLoaderTests.cs index ad85ae47f7..3610e6e7c6 100644 --- a/test/Microsoft.ML.Tests/TextLoaderTests.cs +++ b/test/Microsoft.ML.Tests/TextLoaderTests.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using Microsoft.ML; +using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.Runtime.Api; using Microsoft.ML.Runtime.Data;