Skip to content

Commit b4eebc5

Browse files
authored
Adding binary saving and loading to MLContext.Data (#1678)
* Adding binary saving and loading to MLContext.Data * Addressing PR comments * Adding ReadFromBinary extensions to read from both a stream and from a file
1 parent df1c2af commit b4eebc5

File tree

1 file changed

+65
-0
lines changed

1 file changed

+65
-0
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System.IO;
6+
using Microsoft.ML.Runtime;
7+
using Microsoft.ML.Runtime.Data;
8+
using Microsoft.ML.Runtime.Data.IO;
9+
10+
namespace Microsoft.ML
11+
{
12+
public static class BinaryLoaderSaverCatalog
13+
{
14+
/// <summary>
15+
/// Read a data view from a Stream on a binary file using <see cref="BinaryLoader"/>.
16+
/// </summary>
17+
/// <param name="catalog">The catalog.</param>
18+
/// <param name="stream">The stream to read from.</param>
19+
public static IDataView ReadFromBinary(this DataOperations catalog, Stream stream)
20+
{
21+
Contracts.CheckValue(stream, nameof(stream));
22+
23+
var env = catalog.GetEnvironment();
24+
25+
var reader = new BinaryLoader(env, new BinaryLoader.Arguments(), stream);
26+
return reader;
27+
}
28+
29+
/// <summary>
30+
/// Read a data view from a binary file using <see cref="BinaryLoader"/>.
31+
/// </summary>
32+
/// <param name="catalog">The catalog.</param>
33+
/// <param name="path">The path to the file to read from.</param>
34+
public static IDataView ReadFromBinary(this DataOperations catalog, string path)
35+
{
36+
Contracts.CheckNonEmpty(path, nameof(path));
37+
38+
var env = catalog.GetEnvironment();
39+
40+
var reader = new BinaryLoader(env, new BinaryLoader.Arguments(), path);
41+
return reader;
42+
}
43+
44+
/// <summary>
45+
/// Save the data view into a binary stream.
46+
/// </summary>
47+
/// <param name="catalog">The catalog.</param>
48+
/// <param name="data">The data view to save.</param>
49+
/// <param name="stream">The stream to write to.</param>
50+
/// <param name="keepHidden">Whether to keep hidden columns in the dataset.</param>
51+
public static void SaveAsBinary(this DataOperations catalog, IDataView data, Stream stream,
52+
bool keepHidden = false)
53+
{
54+
Contracts.CheckValue(catalog, nameof(catalog));
55+
Contracts.CheckValue(data, nameof(data));
56+
Contracts.CheckValue(stream, nameof(stream));
57+
58+
var env = catalog.GetEnvironment();
59+
var saver = new BinarySaver(env, new BinarySaver.Arguments());
60+
61+
using (var ch = env.Start("Saving data"))
62+
DataSaverUtils.SaveDataView(ch, saver, data, stream, keepHidden);
63+
}
64+
}
65+
}

0 commit comments

Comments
 (0)