Movement and Internalization Phase 2

Microsoft.ML.sln

@@ -17,8 +17,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CpuMath", "src
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.PipelineInference", "src\Microsoft.ML.PipelineInference\Microsoft.ML.PipelineInference.csproj", "{2D7391C9-8254-4B8F-BF26-FADAF8F02F44}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.InferenceTesting", "test\Microsoft.ML.InferenceTesting\Microsoft.ML.InferenceTesting.csproj", "{E278EC99-E6EE-49FE-92E6-0A309A478D98}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Data", "src\Microsoft.ML.Data\Microsoft.ML.Data.csproj", "{AD92D96B-0E96-4F22-8DCE-892E13B1F282}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Onnx", "src\Microsoft.ML.Onnx\Microsoft.ML.Onnx.csproj", "{65D0603E-B96C-4DFC-BDD1-705891B88C18}"
@@ -175,14 +173,6 @@ Global
 		{2D7391C9-8254-4B8F-BF26-FADAF8F02F44}.Release|Any CPU.Build.0 = Release|Any CPU
 		{2D7391C9-8254-4B8F-BF26-FADAF8F02F44}.Release-Intrinsics|Any CPU.ActiveCfg = Release-Intrinsics|Any CPU
 		{2D7391C9-8254-4B8F-BF26-FADAF8F02F44}.Release-Intrinsics|Any CPU.Build.0 = Release-Intrinsics|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug-Intrinsics|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Debug-Intrinsics|Any CPU.Build.0 = Debug-Intrinsics|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Release|Any CPU.Build.0 = Release|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Release-Intrinsics|Any CPU.ActiveCfg = Release-Intrinsics|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Release-Intrinsics|Any CPU.Build.0 = Release-Intrinsics|Any CPU
 		{AD92D96B-0E96-4F22-8DCE-892E13B1F282}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{AD92D96B-0E96-4F22-8DCE-892E13B1F282}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{AD92D96B-0E96-4F22-8DCE-892E13B1F282}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug-Intrinsics|Any CPU
@@ -520,7 +510,6 @@ Global
 		{EC743D1D-7691-43B7-B9B0-5F2F7018A8F6} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 		{46F2F967-C23F-4076-858D-33F7DA9BD2DA} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{2D7391C9-8254-4B8F-BF26-FADAF8F02F44} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 		{AD92D96B-0E96-4F22-8DCE-892E13B1F282} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{65D0603E-B96C-4DFC-BDD1-705891B88C18} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{707BB22C-7E5F-497A-8C2F-74578F675705} = {09EADF06-BE25-4228-AB53-95AE3E15B530}

src/Microsoft.ML.Api/GenerateCodeCommand.cs

@@ -24,7 +24,7 @@ namespace Microsoft.ML.Runtime.Api
     ///
     /// REVIEW: Consider adding support for generating VBuffers instead of arrays, maybe for high dimensionality vectors.
     /// </summary>
-    public sealed class GenerateCodeCommand : ICommand
+    internal sealed class GenerateCodeCommand : ICommand
     {
         public const string LoadName = "GenerateSamplePredictionCode";
         private const string CodeTemplatePath = "Microsoft.ML.Api.GeneratedCodeTemplate.csresource";

src/Microsoft.ML.Api/TypedCursor.cs

@@ -528,8 +528,6 @@ public ICursor GetRootCursor()
     /// </summary>
     public static class CursoringUtils
     {
-        private const string NeedEnvObsoleteMessage = "This method is obsolete. Please use the overload that takes an additional 'env' argument. An environment can be created via new LocalEnvironment().";
-
         /// <summary>
         /// Generate a strongly-typed cursorable wrapper of the <see cref="IDataView"/>.
         /// </summary>
@@ -550,24 +548,6 @@ public static ICursorable<TRow> AsCursorable<TRow>(this IDataView data, IHostEnv
             return TypedCursorable<TRow>.Create(env, data, ignoreMissingColumns, schemaDefinition);
         }
 
-        /// <summary>
-        /// Generate a strongly-typed cursorable wrapper of the <see cref="IDataView"/>.
-        /// </summary>
-        /// <typeparam name="TRow">The user-defined row type.</typeparam>
-        /// <param name="data">The underlying data view.</param>
-        /// <param name="ignoreMissingColumns">Whether to ignore the case when a requested column is not present in the data view.</param>
-        /// <param name="schemaDefinition">Optional user-provided schema definition. If it is not present, the schema is inferred from the definition of T.</param>
-        /// <returns>The cursorable wrapper of <paramref name="data"/>.</returns>
-        [Obsolete(NeedEnvObsoleteMessage)]
-        public static ICursorable<TRow> AsCursorable<TRow>(this IDataView data, bool ignoreMissingColumns = false,
-            SchemaDefinition schemaDefinition = null)
-            where TRow : class, new()
-        {
-            // REVIEW: Take an env as a parameter.
-            var env = new ConsoleEnvironment();
-            return data.AsCursorable<TRow>(env, ignoreMissingColumns, schemaDefinition);
-        }
-
         /// <summary>
         /// Convert an <see cref="IDataView"/> into a strongly-typed <see cref="IEnumerable{TRow}"/>.
         /// </summary>
@@ -589,24 +569,5 @@ public static IEnumerable<TRow> AsEnumerable<TRow>(this IDataView data, IHostEnv
             var engine = new PipeEngine<TRow>(env, data, ignoreMissingColumns, schemaDefinition);
             return engine.RunPipe(reuseRowObject);
         }
-
-        /// <summary>
-        /// Convert an <see cref="IDataView"/> into a strongly-typed <see cref="IEnumerable{TRow}"/>.
-        /// </summary>
-        /// <typeparam name="TRow">The user-defined row type.</typeparam>
-        /// <param name="data">The underlying data view.</param>
-        /// <param name="reuseRowObject">Whether to return the same object on every row, or allocate a new one per row.</param>
-        /// <param name="ignoreMissingColumns">Whether to ignore the case when a requested column is not present in the data view.</param>
-        /// <param name="schemaDefinition">Optional user-provided schema definition. If it is not present, the schema is inferred from the definition of T.</param>
-        /// <returns>The <see cref="IEnumerable{TRow}"/> that holds the data in <paramref name="data"/>. It can be enumerated multiple times.</returns>
-        [Obsolete(NeedEnvObsoleteMessage)]
-        public static IEnumerable<TRow> AsEnumerable<TRow>(this IDataView data, bool reuseRowObject,
-            bool ignoreMissingColumns = false, SchemaDefinition schemaDefinition = null)
-            where TRow : class, new()
-        {
-            // REVIEW: Take an env as a parameter.
-            var env = new ConsoleEnvironment();
-            return data.AsEnumerable<TRow>(env, reuseRowObject, ignoreMissingColumns, schemaDefinition);
-        }
     }
 }

src/Microsoft.ML.Core/CommandLine/CmdParser.cs

@@ -40,7 +40,8 @@ internal enum SettingsFlags
     ///
     /// This allows components to be created by name, signature type, and a settings string.
     /// </summary>
-    public interface ICommandLineComponentFactory : IComponentFactory
+    [BestFriend]
+    internal interface ICommandLineComponentFactory : IComponentFactory
     {
         Type SignatureType { get; }
         string Name { get; }

src/Common/AssemblyLoadingUtils.cs → src/Microsoft.ML.Core/ComponentModel/AssemblyLoadingUtils.cs

@@ -6,11 +6,13 @@
 using System;
 using System.IO;
 using System.IO.Compression;
-using System.Linq;
 using System.Reflection;
 
 namespace Microsoft.ML.Runtime
 {
+    [Obsolete("The usage for this is intended for the internal command line utilities and is not intended for anything related to the API. " +
+        "Please consider another way of doing whatever it is you're attempting to accomplish.")]
+    [BestFriend]
     internal static class AssemblyLoadingUtils
     {
         /// <summary>

src/Microsoft.ML.Core/Data/ICommand.cs

@@ -11,9 +11,11 @@ namespace Microsoft.ML.Runtime.Command
     /// <summary>
     /// The signature for commands.
     /// </summary>
-    public delegate void SignatureCommand();
+    [BestFriend]
+    internal delegate void SignatureCommand();
 
-    public interface ICommand
+    [BestFriend]
+    internal interface ICommand
     {
         void Run();
     }

src/Microsoft.ML.Core/Data/IFileHandle.cs

@@ -61,7 +61,7 @@ public sealed class SimpleFileHandle : IFileHandle
         // handle has been disposed.
         private List<Stream> _streams;
 
-        private bool IsDisposed { get { return _streams == null; } }
+        private bool IsDisposed => _streams == null;
 
         public SimpleFileHandle(IExceptionContext ectx, string path, bool needsWrite, bool autoDelete)
         {
@@ -84,15 +84,9 @@ public SimpleFileHandle(IExceptionContext ectx, string path, bool needsWrite, bo
             _streams = new List<Stream>();
         }
 
-        public bool CanWrite
-        {
-            get { return !_wrote && !IsDisposed; }
-        }
+        public bool CanWrite => !_wrote && !IsDisposed;
 
-        public bool CanRead
-        {
-            get { return _wrote && !IsDisposed; }
-        }
+        public bool CanRead => _wrote && !IsDisposed;
 
         public void Dispose()
         {

src/Microsoft.ML.Core/Data/IHostEnvironment.cs

@@ -72,6 +72,8 @@ public interface IHostEnvironment : IChannelProvider, IProgressChannelProvider
         /// The suffix and prefix are optional. A common use for suffix is to specify an extension, eg, ".txt".
         /// The use of suffix and prefix, including whether they have any affect, is up to the host environment.
         /// </summary>
+        [Obsolete("The host environment is not disposable, so it is inappropriate to use this method. " +
+            "Please handle your own temporary files within the component yourself, including their proper disposal and deletion.")]
         IFileHandle CreateTempFile(string suffix = null, string prefix = null);
 
         /// <summary>
@@ -188,7 +190,8 @@ public readonly struct ChannelMessage
         /// </summary>
         public string Message => _args != null ? string.Format(_message, _args) : _message;
 
-        public ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, string message)
+        [BestFriend]
+        internal ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, string message)
         {
             Contracts.CheckNonEmpty(message, nameof(message));
             Kind = kind;
@@ -197,7 +200,8 @@ public ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, s
             _args = null;
         }
 
-        public ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, string fmt, params object[] args)
+        [BestFriend]
+        internal ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, string fmt, params object[] args)
         {
             Contracts.CheckNonEmpty(fmt, nameof(fmt));
             Contracts.CheckNonEmpty(args, nameof(args));

src/Microsoft.ML.Core/Data/ProgressReporter.cs

@@ -14,7 +14,8 @@ namespace Microsoft.ML.Runtime.Data
     /// <summary>
     /// The progress reporting classes used by <see cref="HostEnvironmentBase{THostEnvironmentBase}"/> descendants.
     /// </summary>
-    public static class ProgressReporting
+    [BestFriend]
+    internal static class ProgressReporting
     {
         /// <summary>
         /// The progress channel for <see cref="ConsoleEnvironment"/>.

src/Microsoft.ML.Core/Data/ServerChannel.cs

@@ -19,7 +19,8 @@ namespace Microsoft.ML.Runtime
     /// delegates will be published in some fashion, with the target scenario being
     /// that the library will publish some sort of restful API.
     /// </summary>
-    public sealed class ServerChannel : ServerChannel.IPendingBundleNotification, IDisposable
+    [BestFriend]
+    internal sealed class ServerChannel : ServerChannel.IPendingBundleNotification, IDisposable
     {
         // See ServerChannel.md for a more elaborate discussion of high level usage and design.
         private readonly IChannelProvider _chp;
@@ -250,7 +251,8 @@ public void AddDoneAction(Action onDone)
         }
     }
 
-    public static class ServerChannelUtilities
+    [BestFriend]
+    internal static class ServerChannelUtilities
     {
         /// <summary>
         /// Convenience method for <see cref="ServerChannel.Start"/> that looks more idiomatic to typical

src/Microsoft.ML.Core/EntryPoints/EntryPointModuleAttribute.cs

@@ -9,13 +9,15 @@ namespace Microsoft.ML.Runtime.EntryPoints
     /// <summary>
     /// This is a signature for classes that are 'holders' of entry points and components.
     /// </summary>
-    public delegate void SignatureEntryPointModule();
+    [BestFriend]
+    internal delegate void SignatureEntryPointModule();
 
     /// <summary>
     /// A simplified assembly attribute for marking EntryPoint modules.
     /// </summary>
     [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)]
-    public sealed class EntryPointModuleAttribute : LoadableClassAttributeBase
+    [BestFriend]
+    internal sealed class EntryPointModuleAttribute : LoadableClassAttributeBase
     {
         public EntryPointModuleAttribute(Type loaderType)
             : base(null, typeof(void), loaderType, null, new[] { typeof(SignatureEntryPointModule) }, loaderType.FullName)

src/Microsoft.ML.Core/EntryPoints/EntryPointUtils.cs

@@ -12,7 +12,8 @@
 
 namespace Microsoft.ML.Runtime.EntryPoints
 {
-    public static class EntryPointUtils
+    [BestFriend]
+    internal static class EntryPointUtils
     {
         private static bool IsValueWithinRange<T>(TlcModule.RangeAttribute range, object obj)
         {

src/Microsoft.ML.Core/EntryPoints/IMlState.cs

@@ -10,5 +10,5 @@ namespace Microsoft.ML.Runtime.EntryPoints
     /// black box to the graph. The macro itself will then case to the concrete type.
     /// </summary>
     public interface IMlState
-    {}
+    { }
 }

src/Microsoft.ML.Core/EntryPoints/ModuleArgs.cs

@@ -18,7 +18,8 @@ namespace Microsoft.ML.Runtime.EntryPoints
     /// This class defines attributes to annotate module inputs, outputs, entry points etc. when defining
     /// the module interface.
     /// </summary>
-    public static class TlcModule
+    [BestFriend]
+    internal static class TlcModule
     {
         /// <summary>
         /// An attribute used to annotate the component.

src/Microsoft.ML.Core/Environment/ConsoleEnvironment.cs

@@ -5,8 +5,6 @@
 #pragma warning disable 420 // volatile with Interlocked.CompareExchange
 
 using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Threading;
@@ -15,7 +13,12 @@ namespace Microsoft.ML.Runtime.Data
 {
     using Stopwatch = System.Diagnostics.Stopwatch;
 
-    public sealed class ConsoleEnvironment : HostEnvironmentBase<ConsoleEnvironment>
+    /// <summary>
+    /// The console environment. As its name suggests, should be limited to those applications that deliberately want
+    /// console functionality.
+    /// </summary>
+    [BestFriend]
+    internal sealed class ConsoleEnvironment : HostEnvironmentBase<ConsoleEnvironment>
     {
         public const string ComponentHistoryKey = "ComponentHistory";
 

src/Microsoft.ML.Core/Environment/HostEnvironmentBase.cs

@@ -15,7 +15,8 @@ namespace Microsoft.ML.Runtime.Data
     /// Base class for channel providers. This is a common base class for<see cref="HostEnvironmentBase{THostEnvironmentBase}"/>.
     /// The ParentFullName, ShortName, and FullName may be null or empty.
     /// </summary>
-    public abstract class ChannelProviderBase : IExceptionContext
+    [BestFriend]
+    internal abstract class ChannelProviderBase : IExceptionContext
     {
         /// <summary>
         /// Data keys that are attached to the exception thrown via the exception context.
@@ -79,42 +80,22 @@ public virtual TException Process<TException>(TException ex)
     /// <summary>
     /// Message source (a channel) that generated the message being dispatched.
     /// </summary>
-    public interface IMessageSource
+    [BestFriend]
+    internal interface IMessageSource
     {
         string ShortName { get; }
         string FullName { get; }
         bool Verbose { get; }
     }
 
-    /// <summary>
-    /// A <see cref="IHostEnvironment"/> that is also a channel listener can attach
-    /// listeners for messages, as sent through <see cref="IChannelProvider.StartPipe{TMessage}"/>.
-    /// </summary>
-    public interface IMessageDispatcher : IHostEnvironment
-    {
-        /// <summary>
-        /// Listen on this environment to messages of a particular type.
-        /// </summary>
-        /// <typeparam name="TMessage">The message type</typeparam>
-        /// <param name="listenerFunc">The action to perform when a message of the
-        /// appropriate type is received.</param>
-        void AddListener<TMessage>(Action<IMessageSource, TMessage> listenerFunc);
-
-        /// <summary>
-        /// Removes a previously added listener.
-        /// </summary>
-        /// <typeparam name="TMessage">The message type</typeparam>
-        /// <param name="listenerFunc">The previous listener function that is now being removed.</param>
-        void RemoveListener<TMessage>(Action<IMessageSource, TMessage> listenerFunc);
-    }
-
     /// <summary>
     /// A basic host environment suited for many environments.
     /// This also supports modifying the concurrency factor, provides the ability to subscribe to pipes via the
     /// AddListener/RemoveListener methods, and exposes the <see cref="ProgressReporting.ProgressTracker"/> to
     /// query progress.
     /// </summary>
-    public abstract class HostEnvironmentBase<TEnv> : ChannelProviderBase, IHostEnvironment, IDisposable, IChannelProvider, IMessageDispatcher
+    [BestFriend]
+    internal abstract class HostEnvironmentBase<TEnv> : ChannelProviderBase, IHostEnvironment, IDisposable, IChannelProvider
         where TEnv : HostEnvironmentBase<TEnv>
     {
         /// <summary>

src/Microsoft.ML.Core/Environment/TelemetryMessage.cs

@@ -13,7 +13,8 @@ namespace Microsoft.ML.Runtime
     /// <summary>
     /// A telemetry message.
     /// </summary>
-    public abstract class TelemetryMessage
+    [BestFriend]
+    internal abstract class TelemetryMessage
     {
         public static TelemetryMessage CreateCommand(string commandName, string commandText)
         {
@@ -40,7 +41,8 @@ public static TelemetryMessage CreateException(Exception exception)
     /// <summary>
     /// Message with one long text and bunch of small properties (limit on value is ~1020 chars)
     /// </summary>
-    public sealed class TelemetryTrace : TelemetryMessage
+    [BestFriend]
+    internal sealed class TelemetryTrace : TelemetryMessage
     {
         public readonly string Text;
         public readonly string Name;
@@ -57,7 +59,8 @@ public TelemetryTrace(string text, string name, string type)
     /// <summary>
     /// Message with exception
     /// </summary>
-    public sealed class TelemetryException : TelemetryMessage
+    [BestFriend]
+    internal sealed class TelemetryException : TelemetryMessage
     {
         public readonly Exception Exception;
         public TelemetryException(Exception exception)
@@ -70,7 +73,8 @@ public TelemetryException(Exception exception)
     /// <summary>
     /// Message with metric value and it properites
     /// </summary>
-    public sealed class TelemetryMetric : TelemetryMessage
+    [BestFriend]
+    internal sealed class TelemetryMetric : TelemetryMessage
     {
         public readonly string Name;
         public readonly double Value;