Skip to content

Rename Save API for DataLoader to SaveDataLoader. #3018

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
+80 −54
Closed

Rename Save API for DataLoader to SaveDataLoader. #3018

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 44 additions & 7 deletions src/Microsoft.ML.Data/Model/ModelOperationsCatalog.cs
View file
Original file line number Diff line number Diff line change
@@ -33,7 +33,7 @@ internal ModelOperationsCatalog(IHostEnvironment env)
/// </summary>
/// <param name="model">The trained model to be saved.</param>
/// <param name="stream">A writeable, seekable stream to save to.</param>
public void Save<TSource>(IDataLoader<TSource> model, Stream stream)
public void SaveDataLoader<TSource>(IDataLoader<TSource> model, Stream stream)
{
_env.CheckValue(model, nameof(model));
_env.CheckValue(stream, nameof(stream));
@@ -50,10 +50,10 @@ public void Save<TSource>(IDataLoader<TSource> model, Stream stream)
/// </summary>
/// <param name="model">The trained model to be saved.</param>
/// <param name="filePath">Path where model should be saved.</param>
public void Save<TSource>(IDataLoader<TSource> model, string filePath)
public void SaveDataLoader<TSource>(IDataLoader<TSource> model, string filePath)
{
using (var stream = File.Create(filePath))
Save(model, stream);
SaveDataLoader(model, stream);
}

/// <summary>
@@ -63,7 +63,7 @@ public void Save<TSource>(IDataLoader<TSource> model, string filePath)
/// <param name="model">The trained model to be saved</param>
/// <param name="stream">A writeable, seekable stream to save to.</param>
public void Save<TSource>(IDataLoader<TSource> loader, ITransformer model, Stream stream) =>
Save(new CompositeDataLoader<TSource, ITransformer>(loader, new TransformerChain<ITransformer>(model)), stream);
SaveDataLoader(new CompositeDataLoader<TSource, ITransformer>(loader, new TransformerChain<ITransformer>(model)), stream);

/// <summary>
/// Save a transformer model and the loader used to create its input data to the file.
@@ -182,12 +182,25 @@ public ITransformer Load(Stream stream, out DataViewSchema inputSchema)
}

/// <summary>
/// Load the model and its input schema from the stream.
/// Load the model and its input schema from the file.
/// </summary>
/// <param name="modelPath">Path to model.</param>
/// <param name="inputSchema">Will contain the input schema for the model. If the model was saved using older APIs
/// it may not contain an input schema, in this case <paramref name="inputSchema"/> will be null.</param>
/// <returns>The loaded model.</returns>
public ITransformer Load(string modelPath, out DataViewSchema inputSchema)
{
using (var stream = File.OpenRead(modelPath))
return Load(stream, out inputSchema);
}

/// <summary>
/// Load the model from the stream.
/// </summary>
/// <param name="stream">A readable, seekable stream to load from.</param>
/// <returns>A model of type <see cref="CompositeDataLoader{IMultiStreamSource, ITransformer}"/> containing the loader
/// and the transformer chain.</returns>
public IDataLoader<IMultiStreamSource> Load(Stream stream)
public IDataLoader<IMultiStreamSource> LoadDataLoader(Stream stream)
{
_env.CheckValue(stream, nameof(stream));

@@ -205,6 +218,18 @@ public IDataLoader<IMultiStreamSource> Load(Stream stream)
}
}

/// <summary>
/// Load the model from the file.
/// </summary>
/// <param name="modelPath">Path to model.</param>
/// <returns>A model of type <see cref="CompositeDataLoader{IMultiStreamSource, ITransformer}"/> containing the loader
/// and the transformer chain.</returns>
public IDataLoader<IMultiStreamSource> LoadDataLoader(string modelPath)
{
using (var stream = File.OpenRead(modelPath))
return LoadDataLoader(stream);
}

/// <summary>
/// Load a transformer model and a data loader model from the stream.
/// </summary>
@@ -215,7 +240,7 @@ public ITransformer LoadWithDataLoader(Stream stream, out IDataLoader<IMultiStre
{
_env.CheckValue(stream, nameof(stream));

loader = Load(stream);
loader = LoadDataLoader(stream);
if (loader is CompositeDataLoader<IMultiStreamSource, ITransformer> composite)
{
loader = composite.Loader;
@@ -224,6 +249,18 @@ public ITransformer LoadWithDataLoader(Stream stream, out IDataLoader<IMultiStre
return new TransformerChain<ITransformer>();
}

/// <summary>
/// Load a transformer model and a data loader model from the file.
/// </summary>
/// <param name="modelPath">Path to model.</param>
/// <param name="loader">The data loader from the model stream.</param>
/// <returns>The transformer model from the model stream.</returns>
public ITransformer LoadWithDataLoader(string modelPath, out IDataLoader<IMultiStreamSource> loader)
{
using (var stream = File.OpenRead(modelPath))
return LoadWithDataLoader(stream, out loader);
}

/// <summary>
/// Create a prediction engine for one-time prediction.
/// </summary>
83 changes: 36 additions & 47 deletions test/Microsoft.ML.Functional.Tests/ModelLoading.cs
View file
Original file line number Diff line number Diff line change
@@ -61,31 +61,26 @@ public void LoadModelAndExtractPredictor()
string modelAndSchemaPath = GetOutputPath(FullTestName + "-model-schema.zip");
_ml.Model.Save(transformerModel, data.Schema, modelAndSchemaPath);
string compositeLoaderModelPath = GetOutputPath(FullTestName + "-composite-model.zip");
_ml.Model.Save(compositeLoaderModel, compositeLoaderModelPath);
_ml.Model.SaveDataLoader(compositeLoaderModel, compositeLoaderModelPath);
string loaderAndTransformerModelPath = GetOutputPath(FullTestName + "-loader-transformer.zip");
_ml.Model.Save(loader, transformerModel, loaderAndTransformerModelPath);

ITransformer loadedTransformerModel;
IDataLoader<IMultiStreamSource> loadedCompositeLoader;
ITransformer loadedTransformerModel1;
using (var fs = File.OpenRead(modelAndSchemaPath))
loadedTransformerModel = _ml.Model.Load(fs, out var loadedSchema);
using (var fs = File.OpenRead(compositeLoaderModelPath))
{
// This model can be loaded either as a composite data loader,
// a transformer model + an input schema, or a transformer model + a data loader.
var t = _ml.Model.LoadWithDataLoader(fs, out IDataLoader<IMultiStreamSource> l);
var t1 = _ml.Model.Load(fs, out var s);
loadedCompositeLoader = _ml.Model.Load(fs);
}
using (var fs = File.OpenRead(loaderAndTransformerModelPath))
{
// This model can be loaded either as a composite data loader,
// a transformer model + an input schema, or a transformer model + a data loader.
var t = _ml.Model.Load(fs, out var s);
var c = _ml.Model.Load(fs);
loadedTransformerModel1 = _ml.Model.LoadWithDataLoader(fs, out IDataLoader<IMultiStreamSource> l);
}
loadedTransformerModel = _ml.Model.Load(modelAndSchemaPath, out var loadedSchema);

// This model can be loaded either as a composite data loader,
// a transformer model + an input schema, or a transformer model + a data loader.
var t = _ml.Model.LoadWithDataLoader(compositeLoaderModelPath, out IDataLoader<IMultiStreamSource> l);
var t1 = _ml.Model.Load(compositeLoaderModelPath, out var s);
loadedCompositeLoader = _ml.Model.LoadDataLoader(compositeLoaderModelPath);

// This model can be loaded either as a composite data loader,
// a transformer model + an input schema, or a transformer model + a data loader.
var tt = _ml.Model.Load(loaderAndTransformerModelPath, out var ss);
var c = _ml.Model.LoadDataLoader(loaderAndTransformerModelPath);
loadedTransformerModel1 = _ml.Model.LoadWithDataLoader(loaderAndTransformerModelPath, out IDataLoader<IMultiStreamSource> ll);

var gam = ((loadedTransformerModel as ISingleFeaturePredictionTransformer<object>).Model
as CalibratedModelParametersBase).SubModel
@@ -125,11 +120,8 @@ public void SaveAndLoadModelWithLoader()
IDataLoader<IMultiStreamSource> loadedModel;
ITransformer loadedModelWithoutLoader;
DataViewSchema loadedSchema;
using (var fs = File.OpenRead(modelPath))
{
loadedModel = _ml.Model.Load(fs);
loadedModelWithoutLoader = _ml.Model.Load(fs, out loadedSchema);
}
loadedModel = _ml.Model.LoadDataLoader(modelPath);
loadedModelWithoutLoader = _ml.Model.Load(modelPath, out loadedSchema);

// Without deserializing the loader from the model we lose the slot names.
data = _ml.Data.LoadFromEnumerable(new[] { new InputData() });
@@ -171,8 +163,7 @@ public void LoadSchemaAndCreateNewData()

ITransformer loadedModel;
DataViewSchema loadedSchema;
using (var fs = File.OpenRead(modelPath))
loadedModel = _ml.Model.Load(fs, out loadedSchema);
loadedModel = _ml.Model.Load(modelPath, out loadedSchema);

// Without using the schema from the model we lose the slot names.
data = _ml.Data.LoadFromEnumerable(new[] { new InputData() });
@@ -190,7 +181,7 @@ public void SaveTextLoaderAndLoad()
var loader = _ml.Data.CreateTextLoader<InputData>(hasHeader: true, dataSample: file);

string modelPath = GetOutputPath(FullTestName + "-model.zip");
_ml.Model.Save(loader, modelPath);
_ml.Model.SaveDataLoader(loader, modelPath);

Load(modelPath, out var loadedWithSchema, out var loadedSchema, out var loadedLoader,
out var loadedWithLoader, out var loadedLoaderWithTransformer);
@@ -220,7 +211,7 @@ public void SaveCompositeLoaderAndLoad()
var model = composite.Fit(file);

string modelPath = GetOutputPath(FullTestName + "-model.zip");
_ml.Model.Save(model, modelPath);
_ml.Model.SaveDataLoader(model, modelPath);

Load(modelPath, out var loadedWithSchema, out var loadedSchema, out var loadedLoader,
out var loadedWithLoader, out var loadedLoaderWithTransformer);
@@ -298,26 +289,24 @@ private void Load(string filename, out ITransformer loadedWithSchema, out DataVi
out IDataLoader<IMultiStreamSource> loadedLoader, out ITransformer loadedWithLoader,
out IDataLoader<IMultiStreamSource> loadedLoaderWithTransformer)
{
using (var fs = File.OpenRead(filename))

try
{
loadedLoader = _ml.Model.LoadDataLoader(filename);
}
catch (Exception)
{
loadedLoader = null;
}
loadedWithSchema = _ml.Model.Load(filename, out loadedSchema);
try
{
loadedWithLoader = _ml.Model.LoadWithDataLoader(filename, out loadedLoaderWithTransformer);
}
catch (Exception)
{
try
{
loadedLoader = _ml.Model.Load(fs);
}
catch (Exception)
{
loadedLoader = null;
}
loadedWithSchema = _ml.Model.Load(fs, out loadedSchema);
try
{
loadedWithLoader = _ml.Model.LoadWithDataLoader(fs, out loadedLoaderWithTransformer);
}
catch (Exception)
{
loadedWithLoader = null;
loadedLoaderWithTransformer = null;
}
loadedWithLoader = null;
loadedLoaderWithTransformer = null;
}
}