Skip to content

Commit d885ebc

Browse files
TomFinleyTomFinley
authored
Movement and Internalization Phase 2 (#1626)
* Internalize ITrainer * Internalize TrainContext * Internalization of environment implementations * Internalize HostEnvironmentBase/ConsoleEnvironment * Limit usage of ConsoleEnvironment * Internalize ICommandLineComponentFactory * Internalize ICommand and all implementations * Internalize much of the entry-point attribute support * Internalize much of the channel and message infrastructure * Fix two tests from failing by writing to same path from two concurrently running tests.
1 parent 31ee766 commit d885ebc

File tree

169 files changed

+4081
-4377
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

169 files changed

+4081
-4377
lines changed

‎Microsoft.ML.sln

-11
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CpuMath", "src
1717
EndProject
1818
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.PipelineInference", "src\Microsoft.ML.PipelineInference\Microsoft.ML.PipelineInference.csproj", "{2D7391C9-8254-4B8F-BF26-FADAF8F02F44}"
1919
EndProject
20-
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.InferenceTesting", "test\Microsoft.ML.InferenceTesting\Microsoft.ML.InferenceTesting.csproj", "{E278EC99-E6EE-49FE-92E6-0A309A478D98}"
21-
EndProject
2220
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Data", "src\Microsoft.ML.Data\Microsoft.ML.Data.csproj", "{AD92D96B-0E96-4F22-8DCE-892E13B1F282}"
2321
EndProject
2422
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Onnx", "src\Microsoft.ML.Onnx\Microsoft.ML.Onnx.csproj", "{65D0603E-B96C-4DFC-BDD1-705891B88C18}"
@@ -175,14 +173,6 @@ Global
175173
{2D7391C9-8254-4B8F-BF26-FADAF8F02F44}.Release|Any CPU.Build.0 = Release|Any CPU
176174
{2D7391C9-8254-4B8F-BF26-FADAF8F02F44}.Release-Intrinsics|Any CPU.ActiveCfg = Release-Intrinsics|Any CPU
177175
{2D7391C9-8254-4B8F-BF26-FADAF8F02F44}.Release-Intrinsics|Any CPU.Build.0 = Release-Intrinsics|Any CPU
178-
{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
179-
{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Debug|Any CPU.Build.0 = Debug|Any CPU
180-
{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug-Intrinsics|Any CPU
181-
{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Debug-Intrinsics|Any CPU.Build.0 = Debug-Intrinsics|Any CPU
182-
{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Release|Any CPU.ActiveCfg = Release|Any CPU
183-
{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Release|Any CPU.Build.0 = Release|Any CPU
184-
{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Release-Intrinsics|Any CPU.ActiveCfg = Release-Intrinsics|Any CPU
185-
{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Release-Intrinsics|Any CPU.Build.0 = Release-Intrinsics|Any CPU
186176
{AD92D96B-0E96-4F22-8DCE-892E13B1F282}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
187177
{AD92D96B-0E96-4F22-8DCE-892E13B1F282}.Debug|Any CPU.Build.0 = Debug|Any CPU
188178
{AD92D96B-0E96-4F22-8DCE-892E13B1F282}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug-Intrinsics|Any CPU
@@ -520,7 +510,6 @@ Global
520510
{EC743D1D-7691-43B7-B9B0-5F2F7018A8F6} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
521511
{46F2F967-C23F-4076-858D-33F7DA9BD2DA} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
522512
{2D7391C9-8254-4B8F-BF26-FADAF8F02F44} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
523-
{E278EC99-E6EE-49FE-92E6-0A309A478D98} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
524513
{AD92D96B-0E96-4F22-8DCE-892E13B1F282} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
525514
{65D0603E-B96C-4DFC-BDD1-705891B88C18} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
526515
{707BB22C-7E5F-497A-8C2F-74578F675705} = {09EADF06-BE25-4228-AB53-95AE3E15B530}

‎src/Microsoft.ML.Api/GenerateCodeCommand.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ namespace Microsoft.ML.Runtime.Api
2424
///
2525
/// REVIEW: Consider adding support for generating VBuffers instead of arrays, maybe for high dimensionality vectors.
2626
/// </summary>
27-
public sealed class GenerateCodeCommand : ICommand
27+
internal sealed class GenerateCodeCommand : ICommand
2828
{
2929
public const string LoadName = "GenerateSamplePredictionCode";
3030
private const string CodeTemplatePath = "Microsoft.ML.Api.GeneratedCodeTemplate.csresource";

‎src/Microsoft.ML.Api/TypedCursor.cs

-39
Original file line numberDiff line numberDiff line change
@@ -528,8 +528,6 @@ public ICursor GetRootCursor()
528528
/// </summary>
529529
public static class CursoringUtils
530530
{
531-
private const string NeedEnvObsoleteMessage = "This method is obsolete. Please use the overload that takes an additional 'env' argument. An environment can be created via new LocalEnvironment().";
532-
533531
/// <summary>
534532
/// Generate a strongly-typed cursorable wrapper of the <see cref="IDataView"/>.
535533
/// </summary>
@@ -550,24 +548,6 @@ public static ICursorable<TRow> AsCursorable<TRow>(this IDataView data, IHostEnv
550548
return TypedCursorable<TRow>.Create(env, data, ignoreMissingColumns, schemaDefinition);
551549
}
552550

553-
/// <summary>
554-
/// Generate a strongly-typed cursorable wrapper of the <see cref="IDataView"/>.
555-
/// </summary>
556-
/// <typeparam name="TRow">The user-defined row type.</typeparam>
557-
/// <param name="data">The underlying data view.</param>
558-
/// <param name="ignoreMissingColumns">Whether to ignore the case when a requested column is not present in the data view.</param>
559-
/// <param name="schemaDefinition">Optional user-provided schema definition. If it is not present, the schema is inferred from the definition of T.</param>
560-
/// <returns>The cursorable wrapper of <paramref name="data"/>.</returns>
561-
[Obsolete(NeedEnvObsoleteMessage)]
562-
public static ICursorable<TRow> AsCursorable<TRow>(this IDataView data, bool ignoreMissingColumns = false,
563-
SchemaDefinition schemaDefinition = null)
564-
where TRow : class, new()
565-
{
566-
// REVIEW: Take an env as a parameter.
567-
var env = new ConsoleEnvironment();
568-
return data.AsCursorable<TRow>(env, ignoreMissingColumns, schemaDefinition);
569-
}
570-
571551
/// <summary>
572552
/// Convert an <see cref="IDataView"/> into a strongly-typed <see cref="IEnumerable{TRow}"/>.
573553
/// </summary>
@@ -589,24 +569,5 @@ public static IEnumerable<TRow> AsEnumerable<TRow>(this IDataView data, IHostEnv
589569
var engine = new PipeEngine<TRow>(env, data, ignoreMissingColumns, schemaDefinition);
590570
return engine.RunPipe(reuseRowObject);
591571
}
592-
593-
/// <summary>
594-
/// Convert an <see cref="IDataView"/> into a strongly-typed <see cref="IEnumerable{TRow}"/>.
595-
/// </summary>
596-
/// <typeparam name="TRow">The user-defined row type.</typeparam>
597-
/// <param name="data">The underlying data view.</param>
598-
/// <param name="reuseRowObject">Whether to return the same object on every row, or allocate a new one per row.</param>
599-
/// <param name="ignoreMissingColumns">Whether to ignore the case when a requested column is not present in the data view.</param>
600-
/// <param name="schemaDefinition">Optional user-provided schema definition. If it is not present, the schema is inferred from the definition of T.</param>
601-
/// <returns>The <see cref="IEnumerable{TRow}"/> that holds the data in <paramref name="data"/>. It can be enumerated multiple times.</returns>
602-
[Obsolete(NeedEnvObsoleteMessage)]
603-
public static IEnumerable<TRow> AsEnumerable<TRow>(this IDataView data, bool reuseRowObject,
604-
bool ignoreMissingColumns = false, SchemaDefinition schemaDefinition = null)
605-
where TRow : class, new()
606-
{
607-
// REVIEW: Take an env as a parameter.
608-
var env = new ConsoleEnvironment();
609-
return data.AsEnumerable<TRow>(env, reuseRowObject, ignoreMissingColumns, schemaDefinition);
610-
}
611572
}
612573
}

‎src/Microsoft.ML.Core/CommandLine/CmdParser.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ internal enum SettingsFlags
4040
///
4141
/// This allows components to be created by name, signature type, and a settings string.
4242
/// </summary>
43-
public interface ICommandLineComponentFactory : IComponentFactory
43+
[BestFriend]
44+
internal interface ICommandLineComponentFactory : IComponentFactory
4445
{
4546
Type SignatureType { get; }
4647
string Name { get; }

‎src/Common/AssemblyLoadingUtils.cs renamed to ‎src/Microsoft.ML.Core/ComponentModel/AssemblyLoadingUtils.cs

+3-1
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,13 @@
66
using System;
77
using System.IO;
88
using System.IO.Compression;
9-
using System.Linq;
109
using System.Reflection;
1110

1211
namespace Microsoft.ML.Runtime
1312
{
13+
[Obsolete("The usage for this is intended for the internal command line utilities and is not intended for anything related to the API. " +
14+
"Please consider another way of doing whatever it is you're attempting to accomplish.")]
15+
[BestFriend]
1416
internal static class AssemblyLoadingUtils
1517
{
1618
/// <summary>

‎src/Microsoft.ML.Core/Data/ICommand.cs

+4-2
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,11 @@ namespace Microsoft.ML.Runtime.Command
1111
/// <summary>
1212
/// The signature for commands.
1313
/// </summary>
14-
public delegate void SignatureCommand();
14+
[BestFriend]
15+
internal delegate void SignatureCommand();
1516

16-
public interface ICommand
17+
[BestFriend]
18+
internal interface ICommand
1719
{
1820
void Run();
1921
}

‎src/Microsoft.ML.Core/Data/IFileHandle.cs

+3-9
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public sealed class SimpleFileHandle : IFileHandle
6161
// handle has been disposed.
6262
private List<Stream> _streams;
6363

64-
private bool IsDisposed { get { return _streams == null; } }
64+
private bool IsDisposed => _streams == null;
6565

6666
public SimpleFileHandle(IExceptionContext ectx, string path, bool needsWrite, bool autoDelete)
6767
{
@@ -84,15 +84,9 @@ public SimpleFileHandle(IExceptionContext ectx, string path, bool needsWrite, bo
8484
_streams = new List<Stream>();
8585
}
8686

87-
public bool CanWrite
88-
{
89-
get { return !_wrote && !IsDisposed; }
90-
}
87+
public bool CanWrite => !_wrote && !IsDisposed;
9188

92-
public bool CanRead
93-
{
94-
get { return _wrote && !IsDisposed; }
95-
}
89+
public bool CanRead => _wrote && !IsDisposed;
9690

9791
public void Dispose()
9892
{

‎src/Microsoft.ML.Core/Data/IHostEnvironment.cs

+6-2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ public interface IHostEnvironment : IChannelProvider, IProgressChannelProvider
7272
/// The suffix and prefix are optional. A common use for suffix is to specify an extension, eg, ".txt".
7373
/// The use of suffix and prefix, including whether they have any affect, is up to the host environment.
7474
/// </summary>
75+
[Obsolete("The host environment is not disposable, so it is inappropriate to use this method. " +
76+
"Please handle your own temporary files within the component yourself, including their proper disposal and deletion.")]
7577
IFileHandle CreateTempFile(string suffix = null, string prefix = null);
7678

7779
/// <summary>
@@ -188,7 +190,8 @@ public readonly struct ChannelMessage
188190
/// </summary>
189191
public string Message => _args != null ? string.Format(_message, _args) : _message;
190192

191-
public ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, string message)
193+
[BestFriend]
194+
internal ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, string message)
192195
{
193196
Contracts.CheckNonEmpty(message, nameof(message));
194197
Kind = kind;
@@ -197,7 +200,8 @@ public ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, s
197200
_args = null;
198201
}
199202

200-
public ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, string fmt, params object[] args)
203+
[BestFriend]
204+
internal ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, string fmt, params object[] args)
201205
{
202206
Contracts.CheckNonEmpty(fmt, nameof(fmt));
203207
Contracts.CheckNonEmpty(args, nameof(args));

‎src/Microsoft.ML.Core/Data/ProgressReporter.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ namespace Microsoft.ML.Runtime.Data
1414
/// <summary>
1515
/// The progress reporting classes used by <see cref="HostEnvironmentBase{THostEnvironmentBase}"/> descendants.
1616
/// </summary>
17-
public static class ProgressReporting
17+
[BestFriend]
18+
internal static class ProgressReporting
1819
{
1920
/// <summary>
2021
/// The progress channel for <see cref="ConsoleEnvironment"/>.

‎src/Microsoft.ML.Core/Data/ServerChannel.cs

+4-2
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ namespace Microsoft.ML.Runtime
1919
/// delegates will be published in some fashion, with the target scenario being
2020
/// that the library will publish some sort of restful API.
2121
/// </summary>
22-
public sealed class ServerChannel : ServerChannel.IPendingBundleNotification, IDisposable
22+
[BestFriend]
23+
internal sealed class ServerChannel : ServerChannel.IPendingBundleNotification, IDisposable
2324
{
2425
// See ServerChannel.md for a more elaborate discussion of high level usage and design.
2526
private readonly IChannelProvider _chp;
@@ -250,7 +251,8 @@ public void AddDoneAction(Action onDone)
250251
}
251252
}
252253

253-
public static class ServerChannelUtilities
254+
[BestFriend]
255+
internal static class ServerChannelUtilities
254256
{
255257
/// <summary>
256258
/// Convenience method for <see cref="ServerChannel.Start"/> that looks more idiomatic to typical

‎src/Microsoft.ML.Core/EntryPoints/EntryPointModuleAttribute.cs

+4-2
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,15 @@ namespace Microsoft.ML.Runtime.EntryPoints
99
/// <summary>
1010
/// This is a signature for classes that are 'holders' of entry points and components.
1111
/// </summary>
12-
public delegate void SignatureEntryPointModule();
12+
[BestFriend]
13+
internal delegate void SignatureEntryPointModule();
1314

1415
/// <summary>
1516
/// A simplified assembly attribute for marking EntryPoint modules.
1617
/// </summary>
1718
[AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)]
18-
public sealed class EntryPointModuleAttribute : LoadableClassAttributeBase
19+
[BestFriend]
20+
internal sealed class EntryPointModuleAttribute : LoadableClassAttributeBase
1921
{
2022
public EntryPointModuleAttribute(Type loaderType)
2123
: base(null, typeof(void), loaderType, null, new[] { typeof(SignatureEntryPointModule) }, loaderType.FullName)

‎src/Microsoft.ML.Core/EntryPoints/EntryPointUtils.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@
1212

1313
namespace Microsoft.ML.Runtime.EntryPoints
1414
{
15-
public static class EntryPointUtils
15+
[BestFriend]
16+
internal static class EntryPointUtils
1617
{
1718
private static bool IsValueWithinRange<T>(TlcModule.RangeAttribute range, object obj)
1819
{

‎src/Microsoft.ML.Core/EntryPoints/IMlState.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@ namespace Microsoft.ML.Runtime.EntryPoints
1010
/// black box to the graph. The macro itself will then case to the concrete type.
1111
/// </summary>
1212
public interface IMlState
13-
{}
13+
{ }
1414
}

‎src/Microsoft.ML.Core/EntryPoints/ModuleArgs.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ namespace Microsoft.ML.Runtime.EntryPoints
1818
/// This class defines attributes to annotate module inputs, outputs, entry points etc. when defining
1919
/// the module interface.
2020
/// </summary>
21-
public static class TlcModule
21+
[BestFriend]
22+
internal static class TlcModule
2223
{
2324
/// <summary>
2425
/// An attribute used to annotate the component.

‎src/Microsoft.ML.Core/Environment/ConsoleEnvironment.cs

+6-3
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@
55
#pragma warning disable 420 // volatile with Interlocked.CompareExchange
66

77
using System;
8-
using System.Collections.Concurrent;
9-
using System.Collections.Generic;
108
using System.IO;
119
using System.Linq;
1210
using System.Threading;
@@ -15,7 +13,12 @@ namespace Microsoft.ML.Runtime.Data
1513
{
1614
using Stopwatch = System.Diagnostics.Stopwatch;
1715

18-
public sealed class ConsoleEnvironment : HostEnvironmentBase<ConsoleEnvironment>
16+
/// <summary>
17+
/// The console environment. As its name suggests, should be limited to those applications that deliberately want
18+
/// console functionality.
19+
/// </summary>
20+
[BestFriend]
21+
internal sealed class ConsoleEnvironment : HostEnvironmentBase<ConsoleEnvironment>
1922
{
2023
public const string ComponentHistoryKey = "ComponentHistory";
2124

‎src/Microsoft.ML.Core/Environment/HostEnvironmentBase.cs

+6-25
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ namespace Microsoft.ML.Runtime.Data
1515
/// Base class for channel providers. This is a common base class for<see cref="HostEnvironmentBase{THostEnvironmentBase}"/>.
1616
/// The ParentFullName, ShortName, and FullName may be null or empty.
1717
/// </summary>
18-
public abstract class ChannelProviderBase : IExceptionContext
18+
[BestFriend]
19+
internal abstract class ChannelProviderBase : IExceptionContext
1920
{
2021
/// <summary>
2122
/// Data keys that are attached to the exception thrown via the exception context.
@@ -79,42 +80,22 @@ public virtual TException Process<TException>(TException ex)
7980
/// <summary>
8081
/// Message source (a channel) that generated the message being dispatched.
8182
/// </summary>
82-
public interface IMessageSource
83+
[BestFriend]
84+
internal interface IMessageSource
8385
{
8486
string ShortName { get; }
8587
string FullName { get; }
8688
bool Verbose { get; }
8789
}
8890

89-
/// <summary>
90-
/// A <see cref="IHostEnvironment"/> that is also a channel listener can attach
91-
/// listeners for messages, as sent through <see cref="IChannelProvider.StartPipe{TMessage}"/>.
92-
/// </summary>
93-
public interface IMessageDispatcher : IHostEnvironment
94-
{
95-
/// <summary>
96-
/// Listen on this environment to messages of a particular type.
97-
/// </summary>
98-
/// <typeparam name="TMessage">The message type</typeparam>
99-
/// <param name="listenerFunc">The action to perform when a message of the
100-
/// appropriate type is received.</param>
101-
void AddListener<TMessage>(Action<IMessageSource, TMessage> listenerFunc);
102-
103-
/// <summary>
104-
/// Removes a previously added listener.
105-
/// </summary>
106-
/// <typeparam name="TMessage">The message type</typeparam>
107-
/// <param name="listenerFunc">The previous listener function that is now being removed.</param>
108-
void RemoveListener<TMessage>(Action<IMessageSource, TMessage> listenerFunc);
109-
}
110-
11191
/// <summary>
11292
/// A basic host environment suited for many environments.
11393
/// This also supports modifying the concurrency factor, provides the ability to subscribe to pipes via the
11494
/// AddListener/RemoveListener methods, and exposes the <see cref="ProgressReporting.ProgressTracker"/> to
11595
/// query progress.
11696
/// </summary>
117-
public abstract class HostEnvironmentBase<TEnv> : ChannelProviderBase, IHostEnvironment, IDisposable, IChannelProvider, IMessageDispatcher
97+
[BestFriend]
98+
internal abstract class HostEnvironmentBase<TEnv> : ChannelProviderBase, IHostEnvironment, IDisposable, IChannelProvider
11899
where TEnv : HostEnvironmentBase<TEnv>
119100
{
120101
/// <summary>

‎src/Microsoft.ML.Core/Environment/TelemetryMessage.cs

+8-4
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ namespace Microsoft.ML.Runtime
1313
/// <summary>
1414
/// A telemetry message.
1515
/// </summary>
16-
public abstract class TelemetryMessage
16+
[BestFriend]
17+
internal abstract class TelemetryMessage
1718
{
1819
public static TelemetryMessage CreateCommand(string commandName, string commandText)
1920
{
@@ -40,7 +41,8 @@ public static TelemetryMessage CreateException(Exception exception)
4041
/// <summary>
4142
/// Message with one long text and bunch of small properties (limit on value is ~1020 chars)
4243
/// </summary>
43-
public sealed class TelemetryTrace : TelemetryMessage
44+
[BestFriend]
45+
internal sealed class TelemetryTrace : TelemetryMessage
4446
{
4547
public readonly string Text;
4648
public readonly string Name;
@@ -57,7 +59,8 @@ public TelemetryTrace(string text, string name, string type)
5759
/// <summary>
5860
/// Message with exception
5961
/// </summary>
60-
public sealed class TelemetryException : TelemetryMessage
62+
[BestFriend]
63+
internal sealed class TelemetryException : TelemetryMessage
6164
{
6265
public readonly Exception Exception;
6366
public TelemetryException(Exception exception)
@@ -70,7 +73,8 @@ public TelemetryException(Exception exception)
7073
/// <summary>
7174
/// Message with metric value and it properites
7275
/// </summary>
73-
public sealed class TelemetryMetric : TelemetryMessage
76+
[BestFriend]
77+
internal sealed class TelemetryMetric : TelemetryMessage
7478
{
7579
public readonly string Name;
7680
public readonly double Value;

0 commit comments

Comments
 (0)