From 3ee7e64691347f764d2f9a2072d27ecbe01cc076 Mon Sep 17 00:00:00 2001 From: Zeeshan Ahmed Date: Thu, 7 Jun 2018 14:08:19 -0700 Subject: [PATCH 1/4] Enabled '_' in field name in the input type. --- src/Microsoft.ML/Data/TextLoader.cs | 4 ++-- test/Microsoft.ML.Tests/TextLoaderTests.cs | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML/Data/TextLoader.cs b/src/Microsoft.ML/Data/TextLoader.cs index 3c8550ef09..2be07ad2de 100644 --- a/src/Microsoft.ML/Data/TextLoader.cs +++ b/src/Microsoft.ML/Data/TextLoader.cs @@ -84,8 +84,8 @@ public TextLoader CreateFrom(bool useHeader = false, $"Valid characters are 0-9, *, - and ~"); var name = mappingAttr.Name ?? field.Name; - if (name.Any(c => !Char.IsLetterOrDigit(c))) - throw Contracts.Except($"{name} is not alphanumeric."); + if (name.Any(c => !Char.IsLetterOrDigit(c) && c != '_')) + throw Contracts.Except($"{name} can only have alphanumeric or '_'."); Runtime.Data.TextLoader.Range[] sources; if (!Runtime.Data.TextLoader.Column.TryParseSourceEx(mappingAttr.Ordinal, out sources)) diff --git a/test/Microsoft.ML.Tests/TextLoaderTests.cs b/test/Microsoft.ML.Tests/TextLoaderTests.cs index 40c0b6525f..61a1744dfb 100644 --- a/test/Microsoft.ML.Tests/TextLoaderTests.cs +++ b/test/Microsoft.ML.Tests/TextLoaderTests.cs @@ -32,8 +32,11 @@ public void ConstructorDoesntThrow() Assert.NotNull(new Data.TextLoader("fakeFile.txt").CreateFrom(useHeader: false, supportSparse: false, trimWhitespace: false)); Assert.NotNull(new Data.TextLoader("fakeFile.txt").CreateFrom(useHeader: false, supportSparse: false)); Assert.NotNull(new Data.TextLoader("fakeFile.txt").CreateFrom(useHeader: false, allowQuotedStrings: false)); + + Assert.NotNull(new Data.TextLoader("fakeFile.txt").CreateFrom()); } + [Fact] public void CanSuccessfullyApplyATransform() { @@ -264,6 +267,15 @@ public class Input public float Number1; } + public class InputWithUnderscore + { + [Column("0")] + public string String_1; + + [Column("1")] + public float Number_1; + } + public class ModelWithoutColumnAttribute { public string String1; From d0a4af56304277aa62583b4c3b1d87fa826d753f Mon Sep 17 00:00:00 2001 From: Zeeshan Ahmed Date: Thu, 7 Jun 2018 14:49:17 -0700 Subject: [PATCH 2/4] Using CSharpCodeProvider.IsValidIdentifier for validating column name. --- src/Microsoft.ML/Data/TextLoader.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML/Data/TextLoader.cs b/src/Microsoft.ML/Data/TextLoader.cs index 2be07ad2de..4ed35f3f86 100644 --- a/src/Microsoft.ML/Data/TextLoader.cs +++ b/src/Microsoft.ML/Data/TextLoader.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.CSharp; using Microsoft.ML.Runtime; using Microsoft.ML.Runtime.Api; using Microsoft.ML.Runtime.Data; @@ -84,7 +85,7 @@ public TextLoader CreateFrom(bool useHeader = false, $"Valid characters are 0-9, *, - and ~"); var name = mappingAttr.Name ?? field.Name; - if (name.Any(c => !Char.IsLetterOrDigit(c) && c != '_')) + if (!new CSharpCodeProvider().IsValidIdentifier(name)) throw Contracts.Except($"{name} can only have alphanumeric or '_'."); Runtime.Data.TextLoader.Range[] sources; From f30b8a8e52d5836a16827e3ca25a21933de8f30c Mon Sep 17 00:00:00 2001 From: Zeeshan Ahmed Date: Thu, 7 Jun 2018 15:09:20 -0700 Subject: [PATCH 3/4] Updated error message. --- src/Microsoft.ML/Data/TextLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML/Data/TextLoader.cs b/src/Microsoft.ML/Data/TextLoader.cs index 4ed35f3f86..54cf131b9a 100644 --- a/src/Microsoft.ML/Data/TextLoader.cs +++ b/src/Microsoft.ML/Data/TextLoader.cs @@ -86,7 +86,7 @@ public TextLoader CreateFrom(bool useHeader = false, var name = mappingAttr.Name ?? field.Name; if (!new CSharpCodeProvider().IsValidIdentifier(name)) - throw Contracts.Except($"{name} can only have alphanumeric or '_'."); + throw Contracts.Except($"{name} is not a valid identifier."); Runtime.Data.TextLoader.Range[] sources; if (!Runtime.Data.TextLoader.Column.TryParseSourceEx(mappingAttr.Ordinal, out sources)) From 637a2973ff3e440b45598788e829fc4c3077271b Mon Sep 17 00:00:00 2001 From: Zeeshan Ahmed Date: Mon, 11 Jun 2018 11:53:01 -0700 Subject: [PATCH 4/4] removed column name checking. --- src/Microsoft.ML/Data/TextLoader.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Microsoft.ML/Data/TextLoader.cs b/src/Microsoft.ML/Data/TextLoader.cs index 54cf131b9a..c9b16ad24e 100644 --- a/src/Microsoft.ML/Data/TextLoader.cs +++ b/src/Microsoft.ML/Data/TextLoader.cs @@ -85,8 +85,6 @@ public TextLoader CreateFrom(bool useHeader = false, $"Valid characters are 0-9, *, - and ~"); var name = mappingAttr.Name ?? field.Name; - if (!new CSharpCodeProvider().IsValidIdentifier(name)) - throw Contracts.Except($"{name} is not a valid identifier."); Runtime.Data.TextLoader.Range[] sources; if (!Runtime.Data.TextLoader.Column.TryParseSourceEx(mappingAttr.Ordinal, out sources))