code review fixes

Author: adamsitnik

build/Dependencies.props

@@ -39,7 +39,7 @@
   <!-- Test-only Dependencies -->
   <PropertyGroup>
     <BenchmarkDotNetVersion>0.11.2</BenchmarkDotNetVersion>
-    <TraceEventVersion>2.0.26</TraceEventVersion>
+    <MicrosoftDiagnosticsTracingTraceEventVersion>2.0.26</MicrosoftDiagnosticsTracingTraceEventVersion>
     <MicrosoftMLTestModelsPackageVersion>0.0.3-test</MicrosoftMLTestModelsPackageVersion>
   </PropertyGroup>
 

src/Microsoft.ML.Core/Utilities/LineParser.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Runtime.CompilerServices;
+
+namespace Microsoft.ML.Runtime.Internal.Utilities
+{
+    [BestFriend]
+    internal static class LineParser
+    {
+        [BestFriend] // required because the return type is a value tuple..
+        internal static (bool isSuccess, string key, float[] values) ParseKeyThenNumbers(string line)
+        {
+            if (string.IsNullOrWhiteSpace(line))
+                return (false, null, null);
+
+            ReadOnlySpan<char> trimmedLine = line.AsSpan().TrimEnd(); // TrimEnd creates a Span, no allocations
+
+            int firstSeparatorIndex = trimmedLine.IndexOfAny(' ', '\t'); // the first word is the key, we just skip it
+            ReadOnlySpan<char> valuesToParse = trimmedLine.Slice(start: firstSeparatorIndex + 1);
+
+            float[] values = AllocateFixedSizeArrayToStoreParsedValues(valuesToParse);
+
+            int toParseStartIndex = 0;
+            int valueIndex = 0;
+            for (int i = 0; i <= valuesToParse.Length; i++)
+            {
+                if (i == valuesToParse.Length || valuesToParse[i] == ' ' || valuesToParse[i] == '\t')
+                {
+                    if (DoubleParser.TryParse(valuesToParse.Slice(toParseStartIndex, i - toParseStartIndex), out float parsed))
+                        values[valueIndex++] = parsed;
+                    else
+                        return (false, null, null);
+
+                    toParseStartIndex = i + 1;
+                }
+            }
+
+            return (true, trimmedLine.Slice(0, firstSeparatorIndex).ToString(), values);
+        }
+
+        /// <summary>
+        /// we count the number of values first to allocate a single array with of proper size
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static float[] AllocateFixedSizeArrayToStoreParsedValues(ReadOnlySpan<char> valuesToParse)
+        {
+            int valuesCount = 0;
+
+            for (int i = 0; i < valuesToParse.Length; i++)
+                if (valuesToParse[i] == ' ' || valuesToParse[i] == '\t')
+                    valuesCount++;
+
+            return new float[valuesCount + 1]; // + 1 because the line is trimmed and there is no whitespace at the end
+        }
+    }
+}

src/Microsoft.ML.Transforms/Microsoft.ML.Transforms.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0;netcoreapp2.1</TargetFrameworks>
+    <TargetFramework>netstandard2.0</TargetFramework>
     <IncludeInPackage>Microsoft.ML</IncludeInPackage>
     <DefineConstants>CORECLR</DefineConstants>
   </PropertyGroup>

src/Microsoft.ML.Transforms/Text/LineParser.cs

@@ -209,7 +209,7 @@ public WordEmbeddingsTransform(IHostEnvironment env, PretrainedModelKind modelKi
 
             _modelKind = modelKind;
             _modelFileNameWithPath = EnsureModelFile(env, out _linesToSkip, (PretrainedModelKind)_modelKind);
-            _currentVocab = GetVocabularyDictionary();
+            _currentVocab = GetVocabularyDictionary(env);
         }
 
         /// <summary>
@@ -227,7 +227,7 @@ public WordEmbeddingsTransform(IHostEnvironment env, string customModelFile, par
             _modelKind = null;
             _customLookup = true;
             _modelFileNameWithPath = customModelFile;
-            _currentVocab = GetVocabularyDictionary();
+            _currentVocab = GetVocabularyDictionary(env);
         }
 
         private static (string input, string output)[] GetColumnPairs(ColumnInfo[] columns)
@@ -283,7 +283,7 @@ private WordEmbeddingsTransform(IHost host, ModelLoadContext ctx)
             }
 
             Host.CheckNonWhiteSpace(_modelFileNameWithPath, nameof(_modelFileNameWithPath));
-            _currentVocab = GetVocabularyDictionary();
+            _currentVocab = GetVocabularyDictionary(host);
         }
 
         public static WordEmbeddingsTransform Create(IHostEnvironment env, ModelLoadContext ctx)
@@ -699,7 +699,7 @@ private string EnsureModelFile(IHostEnvironment env, out int linesToSkip, Pretra
             throw Host.Except($"Can't map model kind = {kind} to specific file, please refer to https://aka.ms/MLNetIssue for assistance");
         }
 
-        private Model GetVocabularyDictionary()
+        private Model GetVocabularyDictionary(IHostEnvironment hostEnvironment)
         {
             int dimension = 0;
             if (!File.Exists(_modelFileNameWithPath))
@@ -731,7 +731,7 @@ private Model GetVocabularyDictionary()
                     var parsedData = new ConcurrentBag<(string key, float[] values, long lineNumber)>();
                     int skippedLinesCount = Math.Max(1, _linesToSkip);
 
-                    Parallel.ForEach(File.ReadLines(_modelFileNameWithPath).Skip(skippedLinesCount),
+                    Parallel.ForEach(File.ReadLines(_modelFileNameWithPath).Skip(skippedLinesCount), GetParallelOptions(hostEnvironment),
                         (line, parallelState, lineNumber) =>
                         {
                             (bool isSuccess, string key, float[] values) = LineParser.ParseKeyThenNumbers(line);
@@ -774,6 +774,15 @@ private Model GetVocabularyDictionary()
                 }
             }
         }
+
+        private static ParallelOptions GetParallelOptions(IHostEnvironment hostEnvironment)
+        {
+            //  "Less than 1 means whatever the component views as ideal." (about ConcurrencyFactor)
+            if (hostEnvironment.ConcurrencyFactor < 1)
+                return new ParallelOptions(); // we provide default options and let the Parallel decide
+            else
+                return new ParallelOptions() { MaxDegreeOfParallelism = hostEnvironment.ConcurrencyFactor };
+        }
     }
 
     /// <include file='doc.xml' path='doc/members/member[@name="WordEmbeddings"]/*' />

src/Microsoft.ML.Transforms/Text/WordEmbeddingsTransform.cs

@@ -14,7 +14,7 @@
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" Version="$(BenchmarkDotNetVersion)" />
     <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="$(BenchmarkDotNetVersion)" />
-    <PackageReference Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="$(TraceEventVersion)" />
+    <PackageReference Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="$(MicrosoftDiagnosticsTracingTraceEventVersion)" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\src\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />

test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj

@@ -11,7 +11,6 @@
 using Microsoft.ML.Trainers;
 using Microsoft.ML.Transforms.Categorical;
 using System.IO;
-using Microsoft.ML.Transforms.Text;
 
 namespace Microsoft.ML.Benchmarks
 {

test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs

@@ -1,6 +1,5 @@
-using System;
+using Microsoft.ML.Runtime.Internal.Utilities;
 using System.Collections.Generic;
-using System.Linq;
 using Xunit;
 
 namespace Microsoft.ML.Tests.Transformers
@@ -23,7 +22,7 @@ public static IEnumerable<object[]> ValidInputs()
         [MemberData(nameof(ValidInputs))]
         public void WhenProvidedAValidInputParserParsesKeyAndValues(string input, string expectedKey, float[] expectedValues)
         {
-            var result = Transforms.Text.LineParser.ParseKeyThenNumbers(input);
+            var result = LineParser.ParseKeyThenNumbers(input);
 
             Assert.True(result.isSuccess);
             Assert.Equal(expectedKey, result.key);
@@ -35,7 +34,7 @@ public void WhenProvidedAValidInputParserParsesKeyAndValues(string input, string
         [InlineData("key 0.1 NOT_A_NUMBER")] // invalid number
         public void WhenProvidedAnInvalidInputParserReturnsFailure(string input)
         {
-            Assert.False(Transforms.Text.LineParser.ParseKeyThenNumbers(input).isSuccess);
+            Assert.False(LineParser.ParseKeyThenNumbers(input).isSuccess);
         }
     }
 }