diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ImageClassificationDefault.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ImageClassificationDefault.cs new file mode 100644 index 0000000000..e8edfed9c3 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ImageClassificationDefault.cs @@ -0,0 +1,329 @@ + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Transforms; +using static Microsoft.ML.DataOperationsCatalog; + +namespace Samples.Dynamic +{ + public class ImageClassificationDefault + { + public static void Example() + { + string assetsRelativePath = @"../../../assets"; + string assetsPath = GetAbsolutePath(assetsRelativePath); + + var outputMlNetModelFilePath = Path.Combine(assetsPath, "outputs", + "imageClassifier.zip"); + + string imagesDownloadFolderPath = Path.Combine(assetsPath, "inputs", + "images"); + + //Download the image set and unzip + string finalImagesFolderName = DownloadImageSet( + imagesDownloadFolderPath); + string fullImagesetFolderPath = Path.Combine( + imagesDownloadFolderPath, finalImagesFolderName); + + try + { + + MLContext mlContext = new MLContext(seed: 1); + + //Load all the original images info + IEnumerable images = LoadImagesFromDirectory( + folder: fullImagesetFolderPath, useFolderNameAsLabel: true); + + IDataView shuffledFullImagesDataset = mlContext.Data.ShuffleRows( + mlContext.Data.LoadFromEnumerable(images)); + + shuffledFullImagesDataset = mlContext.Transforms.Conversion + .MapValueToKey("Label") + .Append(mlContext.Transforms.LoadImages("Image", + fullImagesetFolderPath, false, "ImagePath")) + .Fit(shuffledFullImagesDataset) + .Transform(shuffledFullImagesDataset); + + // Split the data 90:10 into train and test sets, train and + // evaluate. + TrainTestData trainTestData = mlContext.Data.TrainTestSplit( + shuffledFullImagesDataset, testFraction: 0.1, seed: 1); + + IDataView trainDataset = trainTestData.TrainSet; + IDataView testDataset = trainTestData.TestSet; + + var pipeline = mlContext.Model.ImageClassification("Image", "Label", validationSet: testDataset) + .Append(mlContext.Transforms.Conversion.MapKeyToValue( + outputColumnName: "PredictedLabel", + inputColumnName: "PredictedLabel")); + + + Console.WriteLine("*** Training the image classification model " + + "with DNN Transfer Learning on top of the selected " + + "pre-trained model/architecture ***"); + + // Measuring training time + var watch = System.Diagnostics.Stopwatch.StartNew(); + + var trainedModel = pipeline.Fit(trainDataset); + + watch.Stop(); + long elapsedMs = watch.ElapsedMilliseconds; + + Console.WriteLine("Training with transfer learning took: " + + (elapsedMs / 1000).ToString() + " seconds"); + + mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, + "model.zip"); + + ITransformer loadedModel; + DataViewSchema schema; + using (var file = File.OpenRead("model.zip")) + loadedModel = mlContext.Model.Load(file, out schema); + + EvaluateModel(mlContext, testDataset, loadedModel); + + watch = System.Diagnostics.Stopwatch.StartNew(); + + // Predict image class using an in-memory image. + TrySinglePrediction(fullImagesetFolderPath, mlContext, loadedModel); + + watch.Stop(); + elapsedMs = watch.ElapsedMilliseconds; + + Console.WriteLine("Prediction engine took: " + + (elapsedMs / 1000).ToString() + " seconds"); + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + } + + Console.WriteLine("Press any key to finish"); + Console.ReadKey(); + } + + private static void TrySinglePrediction(string imagesForPredictions, + MLContext mlContext, ITransformer trainedModel) + { + // Create prediction function to try one prediction + var predictionEngine = mlContext.Model + .CreatePredictionEngine(trainedModel); + + IEnumerable testImages = LoadInMemoryImagesFromDirectory( + imagesForPredictions, false); + + InMemoryImageData imageToPredict = new InMemoryImageData + { + Image = testImages.First().Image + }; + + var prediction = predictionEngine.Predict(imageToPredict); + + Console.WriteLine($"Scores : [{string.Join(",", prediction.Score)}], " + + $"Predicted Label : {prediction.PredictedLabel}"); + } + + + private static void EvaluateModel(MLContext mlContext, + IDataView testDataset, ITransformer trainedModel) + { + Console.WriteLine("Making bulk predictions and evaluating model's " + + "quality..."); + + // Measuring time + var watch2 = System.Diagnostics.Stopwatch.StartNew(); + + IDataView predictions = trainedModel.Transform(testDataset); + var metrics = mlContext.MulticlassClassification.Evaluate(predictions); + + Console.WriteLine($"Micro-accuracy: {metrics.MicroAccuracy}," + + $"macro-accuracy = {metrics.MacroAccuracy}"); + + watch2.Stop(); + long elapsed2Ms = watch2.ElapsedMilliseconds; + + Console.WriteLine("Predicting and Evaluation took: " + + (elapsed2Ms / 1000).ToString() + " seconds"); + } + + public static IEnumerable LoadImagesFromDirectory(string folder, + bool useFolderNameAsLabel = true) + { + var files = Directory.GetFiles(folder, "*", + searchOption: SearchOption.AllDirectories); + foreach (var file in files) + { + if (Path.GetExtension(file) != ".jpg") + continue; + + var label = Path.GetFileName(file); + if (useFolderNameAsLabel) + label = Directory.GetParent(file).Name; + else + { + for (int index = 0; index < label.Length; index++) + { + if (!char.IsLetter(label[index])) + { + label = label.Substring(0, index); + break; + } + } + } + + yield return new ImageData() + { + ImagePath = file, + Label = label + }; + + } + } + + public static IEnumerable + LoadInMemoryImagesFromDirectory(string folder, + bool useFolderNameAsLabel = true) + { + var files = Directory.GetFiles(folder, "*", + searchOption: SearchOption.AllDirectories); + foreach (var file in files) + { + if (Path.GetExtension(file) != ".jpg") + continue; + + var label = Path.GetFileName(file); + if (useFolderNameAsLabel) + label = Directory.GetParent(file).Name; + else + { + for (int index = 0; index < label.Length; index++) + { + if (!char.IsLetter(label[index])) + { + label = label.Substring(0, index); + break; + } + } + } + + yield return new InMemoryImageData() + { + Image = File.ReadAllBytes(file), + Label = label + }; + + } + } + + public static string DownloadImageSet(string imagesDownloadFolder) + { + // get a set of images to teach the network about the new classes + + //SINGLE SMALL FLOWERS IMAGESET (200 files) + string fileName = "flower_photos_small_set.zip"; + string url = $"https://mlnetfilestorage.file.core.windows.net/" + + $"imagesets/flower_images/flower_photos_small_set.zip?st=2019-08-" + + $"07T21%3A27%3A44Z&se=2030-08-08T21%3A27%3A00Z&sp=rl&sv=2018-03-" + + $"28&sr=f&sig=SZ0UBX47pXD0F1rmrOM%2BfcwbPVob8hlgFtIlN89micM%3D"; + + Download(url, imagesDownloadFolder, fileName); + UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder); + + return Path.GetFileNameWithoutExtension(fileName); + } + + public static bool Download(string url, string destDir, string destFileName) + { + if (destFileName == null) + destFileName = url.Split(Path.DirectorySeparatorChar).Last(); + + Directory.CreateDirectory(destDir); + + string relativeFilePath = Path.Combine(destDir, destFileName); + + if (File.Exists(relativeFilePath)) + { + Console.WriteLine($"{relativeFilePath} already exists."); + return false; + } + + var wc = new WebClient(); + Console.WriteLine($"Downloading {relativeFilePath}"); + var download = Task.Run(() => wc.DownloadFile(url, relativeFilePath)); + while (!download.IsCompleted) + { + Thread.Sleep(1000); + Console.Write("."); + } + Console.WriteLine(""); + Console.WriteLine($"Downloaded {relativeFilePath}"); + + return true; + } + + public static void UnZip(String gzArchiveName, String destFolder) + { + var flag = gzArchiveName.Split(Path.DirectorySeparatorChar) + .Last() + .Split('.') + .First() + ".bin"; + + if (File.Exists(Path.Combine(destFolder, flag))) return; + + Console.WriteLine($"Extracting."); + ZipFile.ExtractToDirectory(gzArchiveName, destFolder); + + File.Create(Path.Combine(destFolder, flag)); + Console.WriteLine(""); + Console.WriteLine("Extracting is completed."); + } + + public static string GetAbsolutePath(string relativePath) + { + FileInfo _dataRoot = new FileInfo(typeof( + ResnetV2101TransferLearningTrainTestSplit).Assembly.Location); + + string assemblyFolderPath = _dataRoot.Directory.FullName; + + string fullPath = Path.Combine(assemblyFolderPath, relativePath); + + return fullPath; + } + + public class InMemoryImageData + { + [LoadColumn(0)] + public byte[] Image; + + [LoadColumn(1)] + public string Label; + } + + public class ImageData + { + [LoadColumn(0)] + public string ImagePath; + + [LoadColumn(1)] + public string Label; + } + + public class ImagePrediction + { + [ColumnName("Score")] + public float[] Score; + + [ColumnName("PredictedLabel")] + public string PredictedLabel; + } + } +} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/MNISTFullModelRetrain.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/MNISTFullModelRetrain.cs deleted file mode 100644 index 340109eaf8..0000000000 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/MNISTFullModelRetrain.cs +++ /dev/null @@ -1,236 +0,0 @@ -using System; -using System.IO; -using System.Linq; -using Microsoft.ML; -using Microsoft.ML.Data; - -namespace Samples.Dynamic -{ - public static class MNISTFullModelRetrain - { - /// - /// Example full model retrain using the MNIST model in a ML.NET pipeline. - /// - - private static string sourceDir = Directory.GetCurrentDirectory(); - - // Represents the path to the machinelearning directory - private static string mlDir = @"..\..\..\..\"; - - public static void Example() - { - var mlContext = new MLContext(seed: 1); - - // Download training data into current directory and load into IDataView - var trainData = DataDownload("Train-Tiny-28x28.txt", mlContext); - - // Download testing data into current directory and load into IDataView - var testData = DataDownload("MNIST.Test.tiny.txt", mlContext); - - // Download the MNIST model and its variables into current directory - ModelDownload(); - - // Full model retrain pipeline - var pipe = mlContext.Transforms.CopyColumns("Features", "Placeholder") - .Append(mlContext.Model.RetrainDnnModel( - inputColumnNames: new[] { "Features" }, - outputColumnNames: new[] { "Prediction" }, - labelColumnName: "TfLabel", - dnnLabel: "Label", - optimizationOperation: "MomentumOp", - lossOperation: "Loss", - modelPath: "mnist_conv_model", - metricOperation: "Accuracy", - epoch: 10, - learningRateOperation: "learning_rate", - learningRate: 0.01f, - batchSize: 20)) - .Append(mlContext.Transforms.Concatenate("Features", - "Prediction")) - .AppendCacheCheckpoint(mlContext) - .Append(mlContext.MulticlassClassification.Trainers.LightGbm( - new Microsoft.ML.Trainers.LightGbm - .LightGbmMulticlassTrainer.Options() - { - LabelColumnName = "Label", - FeatureColumnName = "Features", - Seed = 1, - NumberOfThreads = 1, - NumberOfIterations = 1 - })); - - var trainedModel = pipe.Fit(trainData); - var predicted = trainedModel.Transform(testData); - var metrics = mlContext.MulticlassClassification.Evaluate(predicted); - - // Print out metrics - Console.WriteLine(); - Console.WriteLine($"Micro-accuracy: {metrics.MicroAccuracy}, " + - $"macro-accuracy = {metrics.MacroAccuracy}"); - - // Get one sample for the fully retrained model to predict on - var sample = GetOneMNISTExample(); - - // Create a prediction engine to predict on one sample - var predictionEngine = mlContext.Model.CreatePredictionEngine< - MNISTData, MNISTPrediction>(trainedModel); - - var prediction = predictionEngine.Predict(sample); - - // Print predicted labels - Console.WriteLine("Predicted Labels: "); - foreach (var pLabel in prediction.PredictedLabels) - { - Console.Write(pLabel + " "); - } - - // Clean up folder by deleting extra files made during retrain - CleanUp("mnist_conv_model"); - } - - // Copies data from another location into current directory - // and loads it into IDataView using a TextLoader - private static IDataView DataDownload(string fileName, MLContext mlContext) - { - string dataPath = Path.Combine(mlDir, "test", "data", fileName); - if (!File.Exists(fileName)) - { - System.IO.File.Copy(dataPath, Path.Combine(sourceDir, fileName)); - } - - return mlContext.Data.CreateTextLoader( - new[] - { - new TextLoader.Column("Label", DataKind.UInt32, - new[] { new TextLoader.Range(0) }, new KeyCount(10)), - new TextLoader.Column("TfLabel", DataKind.Int64, 0), - new TextLoader.Column("Placeholder", DataKind.Single, - new[] { new TextLoader.Range(1, 784) }) - }, - allowSparse: true - ).Load(fileName); - } - - // Copies MNIST model folder from another location into current directory - private static void ModelDownload() - { - if (!Directory.Exists(Path.Combine(sourceDir, "mnist_conv_model"))) - { - // The original path to the MNIST model - var oldModel = Path.Combine(new[] { mlDir, "packages", - "microsoft.ml.tensorflow.testmodels", "0.0.11-test", - "contentfiles", "any", "any", "mnist_conv_model" }); - - // Create a new folder in the current directory for the MNIST model - string newModel = Directory.CreateDirectory(Path.Combine(sourceDir, - "mnist_conv_model")).FullName; - - // Copy the model into the new mnist_conv_model folder - System.IO.File.Copy(Path.Combine(oldModel, "saved_model.pb"), - Path.Combine(newModel, "saved_model.pb")); - - // The original folder that the model variables are in. - // Because the folder already exists, the "CreateDirectory" method - // call creates a DirectoryInfo object for the existing folder - // rather than making a new directory. - var oldVariables = Directory.CreateDirectory(Path.Combine(oldModel, - "variables")); - - // Create a new folder in the new mnist_conv_model directory to - // store the model variables - var newVariables = Directory.CreateDirectory(Path.Combine(newModel, - "variables")); - - // Get the files in the original variables folder - var variableNames = oldVariables.GetFiles(); - - foreach (var vName in variableNames) - { - // Copy each file from the original variables folder into the - // new variables folder - System.IO.File.Copy(vName.FullName, Path.Combine( - newVariables.FullName, vName.Name)); - } - - } - } - public class MNISTData - { - public long Label; - - [VectorType(784)] - public float[] Placeholder; - } - - public class MNISTPrediction - { - [ColumnName("Score")] - public float[] PredictedLabels; - } - - // Returns one sample - private static MNISTData GetOneMNISTExample() - { - return new MNISTData() - { - Placeholder = new float[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 18, 18, 18, 126, - 136, 175, 26, 166, 255, 247, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 30, 36, 94, 154, 170, 253, 253, 253, 253, 253, 225, 172, - 253, 242, 195, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 238, - 253, 253, 253, 253, 253, 253, 253, 253, 251, 93, 82, 82, 56, - 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 219, 253, 253, 253, - 253, 253, 198, 182, 247, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 80, 156, 107, 253, 253, 205, 11, 0, 43, - 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 14, 1, 154, 253, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 253, 190, 2, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, - 190, 253, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 241, 225, 160, 108, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, - 240, 253, 253, 119, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 186, 253, 253, 150, 27, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 16, 93, 252, 253, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 253, 249, 64, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 130, - 183, 253, 253, 207, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 39, 148, 229, 253, 253, 253, 250, 182, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 114, 221, - 253, 253, 253, 253, 201, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 23, 66, 213, 253, 253, 253, 253, 198, 81, 2, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 171, 219, - 253, 253, 253, 253, 195, 80, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 55, 172, 226, 253, 253, 253, 253, 244, 133, - 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, - 253, 253, 253, 212, 135, 132, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0 } - }; - } - - // Deletes extra variable folders produced during retrain - private static void CleanUp(string model_location) - { - var directories = Directory.GetDirectories(model_location, - "variables-*"); - if (directories != null && directories.Length > 0) - { - var varDir = Path.Combine(model_location, "variables"); - if (Directory.Exists(varDir)) - Directory.Delete(varDir, true); - Directory.Move(directories[0], varDir); - } - } - } -} \ No newline at end of file diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs index 43638a91e2..1224975cd7 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs @@ -62,19 +62,23 @@ public static void Example() .Fit(testDataset) .Transform(testDataset); - var pipeline = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath") // false indicates we want the image as a VBuffer - .Append(mlContext.Model.ImageClassification( - "Image", "Label", - // Just by changing/selecting InceptionV3/MobilenetV2 here instead of - // ResnetV2101 you can try a different architecture/pre-trained - // model. - arch: ImageClassificationEstimator.Architecture.ResnetV2101, - batchSize: 10, - learningRate: 0.01f, - earlyStopping: new ImageClassificationEstimator.EarlyStopping(minDelta: 0.001f, patience: 20, metric: ImageClassificationEstimator.EarlyStoppingMetric.Loss), - metricsCallback: (metrics) => Console.WriteLine(metrics), - validationSet: validationSet)); + var options = new ImageClassificationEstimator.Options() + { + FeaturesColumnName = "Image", + LabelColumnName = "Label", + // Just by changing/selecting InceptionV3/MobilenetV2 here instead of + // ResnetV2101 you can try a different architecture/ + // pre-trained model. + Arch = ImageClassificationEstimator.Architecture.ResnetV2101, + BatchSize = 10, + LearningRate = 0.01f, + EarlyStoppingCriteria = new ImageClassificationEstimator.EarlyStopping(minDelta: 0.001f, patience: 20, metric: ImageClassificationEstimator.EarlyStoppingMetric.Loss), + MetricsCallback = (metrics) => Console.WriteLine(metrics), + ValidationSet = validationSet + }; + var pipeline = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath") // false indicates we want the image as a VBuffer + .Append(mlContext.Model.ImageClassification(options)); Console.WriteLine("*** Training the image classification model with " + "DNN Transfer Learning on top of the selected pre-trained " + diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs index 2b20ec0695..6d9ea650ae 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs @@ -47,7 +47,7 @@ public static void Example() shuffledFullImagesDataset = mlContext.Transforms.Conversion .MapValueToKey("Label") - .Append(mlContext.Transforms.LoadImages("Image", + .Append(mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath")) .Fit(shuffledFullImagesDataset) .Transform(shuffledFullImagesDataset); @@ -60,18 +60,23 @@ public static void Example() IDataView trainDataset = trainTestData.TrainSet; IDataView testDataset = trainTestData.TestSet; - var pipeline = mlContext.Model.ImageClassification( - "Image", "Label", - // Just by changing/selecting InceptionV3/MobilenetV2 here instead of - // ResnetV2101 you can try a different architecture/ - // pre-trained model. - arch: ImageClassificationEstimator.Architecture.ResnetV2101, - epoch: 50, - batchSize: 10, - learningRate: 0.01f, - metricsCallback: (metrics) => Console.WriteLine(metrics), - validationSet: testDataset, - disableEarlyStopping: true) + var options = new ImageClassificationEstimator.Options() + { + FeaturesColumnName = "Image", + LabelColumnName = "Label", + // Just by changing/selecting InceptionV3/MobilenetV2 here instead of + // ResnetV2101 you can try a different architecture/ + // pre-trained model. + Arch = ImageClassificationEstimator.Architecture.ResnetV2101, + Epoch = 50, + BatchSize = 10, + LearningRate = 0.01f, + MetricsCallback = (metrics) => Console.WriteLine(metrics), + ValidationSet = testDataset, + DisableEarlyStopping = true + }; + + var pipeline = mlContext.Model.ImageClassification(options) .Append(mlContext.Transforms.Conversion.MapKeyToValue( outputColumnName: "PredictedLabel", inputColumnName: "PredictedLabel")); diff --git a/src/Microsoft.ML.Dnn/DnnCatalog.cs b/src/Microsoft.ML.Dnn/DnnCatalog.cs index 55b66c79dd..a674ebb855 100644 --- a/src/Microsoft.ML.Dnn/DnnCatalog.cs +++ b/src/Microsoft.ML.Dnn/DnnCatalog.cs @@ -2,15 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; -using System.IO; -using System.IO.Compression; -using System.Net; using Microsoft.ML.Data; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.Dnn; using static Microsoft.ML.Transforms.ImageClassificationEstimator; -using Options = Microsoft.ML.Transforms.DnnRetrainEstimator.Options; namespace Microsoft.ML { @@ -46,7 +41,7 @@ public static class DnnCatalog /// /// The support for retraining is under preview. /// - public static DnnRetrainEstimator RetrainDnnModel( + internal static DnnRetrainEstimator RetrainDnnModel( this ModelOperationsCatalog catalog, string[] outputColumnNames, string[] inputColumnNames, @@ -62,7 +57,7 @@ public static DnnRetrainEstimator RetrainDnnModel( float learningRate = 0.01f, bool addBatchDimensionInput = false) { - var options = new Options() + var options = new DnnRetrainEstimator.Options() { ModelLocation = modelPath, InputColumns = inputColumnNames, @@ -97,115 +92,46 @@ public static DnnRetrainEstimator RetrainDnnModel( /// The name of the labels column. /// The name of the output score column. /// The name of the output predicted label columns. - /// The architecture of the image recognition DNN model. - /// Number of training iterations. Each iteration/epoch refers to one pass over the dataset. - /// The batch size for training. - /// The learning rate for training. - /// Whether to disable use of early stopping technique. Training will go on for the full epoch count. - /// Early stopping technique parameters to be used to terminate training when training metric stops improving. - /// Callback for reporting model statistics during training phase. - /// Indicates the frequency of epochs at which to report model statistics during training phase. - /// Indicates the choice of DNN training framework. Currently only tensorflow is supported. - /// Optional name of the path where a copy new graph should be saved. The graph will be saved as part of model. - /// The name of the prefix for the final mode and checkpoint files. /// Validation set. - /// Indicates to evaluate the model on train set after every epoch. - /// Indicates to not re-compute cached trainset bottleneck values if already available in the bin folder. - /// Indicates to not re-compute validataionset cached bottleneck validationset values if already available in the bin folder. - /// Indicates the file path to store trainset bottleneck values for caching. - /// Indicates the file path to store validationset bottleneck values for caching. - /// - /// The support for image classification is under preview. - /// public static ImageClassificationEstimator ImageClassification( this ModelOperationsCatalog catalog, string featuresColumnName, string labelColumnName, string scoreColumnName = "Score", string predictedLabelColumnName = "PredictedLabel", - Architecture arch = Architecture.InceptionV3, - int epoch = 100, - int batchSize = 10, - float learningRate = 0.01f, - bool disableEarlyStopping = false, - EarlyStopping earlyStopping = null, - ImageClassificationMetricsCallback metricsCallback = null, - int statisticFrequency = 1, - DnnFramework framework = DnnFramework.Tensorflow, - string modelSavePath = null, - string finalModelPrefix = "custom_retrained_model_based_on_", - IDataView validationSet = null, - bool testOnTrainSet = true, - bool reuseTrainSetBottleneckCachedValues = false, - bool reuseValidationSetBottleneckCachedValues = false, - string trainSetBottleneckCachedValuesFilePath = "trainSetBottleneckFile.csv", - string validationSetBottleneckCachedValuesFilePath = "validationSetBottleneckFile.csv" + IDataView validationSet = null ) { var options = new ImageClassificationEstimator.Options() { - ModelLocation = ModelLocation[arch], - InputColumns = new[] { featuresColumnName }, - OutputColumns = new[] { scoreColumnName, predictedLabelColumnName }, - LabelColumn = labelColumnName, - TensorFlowLabel = labelColumnName, - Epoch = epoch, - LearningRate = learningRate, - BatchSize = batchSize, - EarlyStoppingCriteria = disableEarlyStopping ? null : earlyStopping == null ? new EarlyStopping() : earlyStopping, + FeaturesColumnName = featuresColumnName, + LabelColumnName = labelColumnName, ScoreColumnName = scoreColumnName, PredictedLabelColumnName = predictedLabelColumnName, - FinalModelPrefix = finalModelPrefix, - Arch = arch, - MetricsCallback = metricsCallback, - StatisticsFrequency = statisticFrequency, - Framework = framework, - ModelSavePath = modelSavePath, ValidationSet = validationSet, - TestOnTrainSet = testOnTrainSet, - TrainSetBottleneckCachedValuesFilePath = trainSetBottleneckCachedValuesFilePath, - ValidationSetBottleneckCachedValuesFilePath = validationSetBottleneckCachedValuesFilePath, - ReuseTrainSetBottleneckCachedValues = reuseTrainSetBottleneckCachedValues, - ReuseValidationSetBottleneckCachedValues = reuseValidationSetBottleneckCachedValues }; - if (!File.Exists(options.ModelLocation)) - { - if (options.Arch == Architecture.InceptionV3) - { - var baseGitPath = @"https://raw.githubusercontent.com/SciSharp/TensorFlow.NET/master/graph/InceptionV3.meta"; - using (WebClient client = new WebClient()) - { - client.DownloadFile(new Uri($"{baseGitPath}"), @"InceptionV3.meta"); - } + return ImageClassification(catalog, options); + } - baseGitPath = @"https://github.com/SciSharp/TensorFlow.NET/raw/master/data/tfhub_modules.zip"; - using (WebClient client = new WebClient()) - { - client.DownloadFile(new Uri($"{baseGitPath}"), @"tfhub_modules.zip"); - ZipFile.ExtractToDirectory(Path.Combine(Directory.GetCurrentDirectory(), @"tfhub_modules.zip"), @"tfhub_modules"); - } - } - else if(options.Arch == Architecture.ResnetV2101) - { - var baseGitPath = @"https://aka.ms/mlnet-resources/image/ResNet101Tensorflow/resnet_v2_101_299.meta"; - using (WebClient client = new WebClient()) - { - client.DownloadFile(new Uri($"{baseGitPath}"), @"resnet_v2_101_299.meta"); - } - } - else if(options.Arch == Architecture.MobilenetV2) - { - var baseGitPath = @"https://tlcresources.blob.core.windows.net/image/MobileNetV2TensorFlow/mobilenet_v2.meta"; - using (WebClient client = new WebClient()) - { - client.DownloadFile(new Uri($"{baseGitPath}"), @"mobilenet_v2.meta"); - } - } - } + /// + /// Performs image classification using transfer learning. + /// usage of this API requires additional NuGet dependencies on TensorFlow redist, see linked document for more information. + /// + /// + /// + /// + /// + /// An object specifying advanced options for . + public static ImageClassificationEstimator ImageClassification( + this ModelOperationsCatalog catalog, Options options) + { + options.EarlyStoppingCriteria = options.DisableEarlyStopping ? null : options.EarlyStoppingCriteria ?? new EarlyStopping(); var env = CatalogUtils.GetEnvironment(catalog); - return new ImageClassificationEstimator(env, options, DnnUtils.LoadDnnModel(env, options.ModelLocation, true)); + return new ImageClassificationEstimator(env, options, DnnUtils.LoadDnnModel(env, options.Arch, true)); } } } diff --git a/src/Microsoft.ML.Dnn/DnnRetrainTransform.cs b/src/Microsoft.ML.Dnn/DnnRetrainTransform.cs index 27e55b0cdf..5ca961063c 100644 --- a/src/Microsoft.ML.Dnn/DnnRetrainTransform.cs +++ b/src/Microsoft.ML.Dnn/DnnRetrainTransform.cs @@ -37,7 +37,7 @@ namespace Microsoft.ML.Transforms /// /// for the . /// - public sealed class DnnRetrainTransformer : RowToRowTransformerBase + internal sealed class DnnRetrainTransformer : RowToRowTransformerBase { private readonly IHostEnvironment _env; private readonly string _modelLocation; @@ -1184,7 +1184,7 @@ public Tensor GetBufferedBatchTensor() } /// - public sealed class DnnRetrainEstimator : IEstimator + internal sealed class DnnRetrainEstimator : IEstimator { /// /// The options for the . diff --git a/src/Microsoft.ML.Dnn/DnnUtils.cs b/src/Microsoft.ML.Dnn/DnnUtils.cs index 87ebeb5c93..f4fb5f704c 100644 --- a/src/Microsoft.ML.Dnn/DnnUtils.cs +++ b/src/Microsoft.ML.Dnn/DnnUtils.cs @@ -5,7 +5,9 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; +using System.Net; using System.Security.AccessControl; using System.Security.Principal; using Microsoft.ML.Data; @@ -264,6 +266,56 @@ internal static DnnModel LoadDnnModel(IHostEnvironment env, string modelPath, bo return new DnnModel(env, session, modelPath); } + /// + /// Load TensorFlow model into memory. + /// + /// The environment to use. + /// The architecture of the model to load. + /// + /// + internal static DnnModel LoadDnnModel(IHostEnvironment env, ImageClassificationEstimator.Architecture arch, bool metaGraph = false) + { + var modelPath = ImageClassificationEstimator.ModelLocation[arch]; + if (!File.Exists(modelPath)) + { + if (arch == ImageClassificationEstimator.Architecture.InceptionV3) + { + var baseGitPath = @"https://raw.githubusercontent.com/SciSharp/TensorFlow.NET/master/graph/InceptionV3.meta"; + using (WebClient client = new WebClient()) + { + client.DownloadFile(new Uri($"{baseGitPath}"), @"InceptionV3.meta"); + } + + baseGitPath = @"https://github.com/SciSharp/TensorFlow.NET/raw/master/data/tfhub_modules.zip"; + using (WebClient client = new WebClient()) + { + client.DownloadFile(new Uri($"{baseGitPath}"), @"tfhub_modules.zip"); + ZipFile.ExtractToDirectory(Path.Combine(Directory.GetCurrentDirectory(), @"tfhub_modules.zip"), @"tfhub_modules"); + } + } + else if (arch == ImageClassificationEstimator.Architecture.ResnetV2101) + { + var baseGitPath = @"https://aka.ms/mlnet-resources/image/ResNet101Tensorflow/resnet_v2_101_299.meta"; + using (WebClient client = new WebClient()) + { + client.DownloadFile(new Uri($"{baseGitPath}"), @"resnet_v2_101_299.meta"); + } + } + else if (arch == ImageClassificationEstimator.Architecture.MobilenetV2) + { + var baseGitPath = @"https://tlcresources.blob.core.windows.net/image/MobileNetV2TensorFlow/mobilenet_v2.meta"; + using (WebClient client = new WebClient()) + { + client.DownloadFile(new Uri($"{baseGitPath}"), @"mobilenet_v2.meta"); + } + } + + } + + var session = GetSession(env, modelPath, metaGraph); + return new DnnModel(env, session, modelPath); + } + internal static Session GetSession(IHostEnvironment env, string modelPath, bool metaGraph = false) { Contracts.Check(env != null, nameof(env)); diff --git a/src/Microsoft.ML.Dnn/ImageClassificationTransform.cs b/src/Microsoft.ML.Dnn/ImageClassificationTransform.cs index 5688e99f60..19227bc4b2 100644 --- a/src/Microsoft.ML.Dnn/ImageClassificationTransform.cs +++ b/src/Microsoft.ML.Dnn/ImageClassificationTransform.cs @@ -131,7 +131,7 @@ private static ImageClassificationTransformer Create(IHostEnvironment env, Model throw env.ExceptDecode(); return new ImageClassificationTransformer(env, DnnUtils.LoadTFSession(env, modelBytes), outputs, inputs, - null, addBatchDimensionInput, 1, labelColumn, checkpointName, arch, + addBatchDimensionInput, 1, labelColumn, checkpointName, arch, scoreColumnName, predictedColumnName, learningRate, null, classCount, true, predictionTensorName, softMaxTensorName, jpegDataTensorName, resizeTensorName, keyValueAnnotations); @@ -150,14 +150,13 @@ internal static IDataTransform Create(IHostEnvironment env, ImageClassificationE } internal ImageClassificationTransformer(IHostEnvironment env, ImageClassificationEstimator.Options options, IDataView input) - : this(env, options, DnnUtils.LoadDnnModel(env, options.ModelLocation), input) + : this(env, options, DnnUtils.LoadDnnModel(env, options.Arch), input) { } internal ImageClassificationTransformer(IHostEnvironment env, ImageClassificationEstimator.Options options, DnnModel tensorFlowModel, IDataView input) - : this(env, tensorFlowModel.Session, options.OutputColumns, options.InputColumns, - options.ModelLocation, null, options.BatchSize, - options.LabelColumn, options.FinalModelPrefix, options.Arch, options.ScoreColumnName, + : this(env, tensorFlowModel.Session, options.OutputColumns, options.InputColumns, null, options.BatchSize, + options.LabelColumnName, options.FinalModelPrefix, options.Arch, options.ScoreColumnName, options.PredictedLabelColumnName, options.LearningRate, input.Schema) { Contracts.CheckValue(env, nameof(env)); @@ -166,13 +165,13 @@ internal ImageClassificationTransformer(IHostEnvironment env, ImageClassificatio CheckTrainingParameters(options); var imageProcessor = new ImageProcessor(this); if (!options.ReuseTrainSetBottleneckCachedValues || !File.Exists(options.TrainSetBottleneckCachedValuesFilePath)) - CacheFeaturizedImagesToDisk(input, options.LabelColumn, options.InputColumns[0], imageProcessor, + CacheFeaturizedImagesToDisk(input, options.LabelColumnName, options.InputColumns[0], imageProcessor, _inputTensorName, _bottleneckTensor.name, options.TrainSetBottleneckCachedValuesFilePath, ImageClassificationMetrics.Dataset.Train, options.MetricsCallback); if (options.ValidationSet != null && (!options.ReuseTrainSetBottleneckCachedValues || !File.Exists(options.ValidationSetBottleneckCachedValuesFilePath))) - CacheFeaturizedImagesToDisk(options.ValidationSet, options.LabelColumn, options.InputColumns[0], + CacheFeaturizedImagesToDisk(options.ValidationSet, options.LabelColumnName, options.InputColumns[0], imageProcessor, _inputTensorName, _bottleneckTensor.name, options.ValidationSetBottleneckCachedValuesFilePath, ImageClassificationMetrics.Dataset.Validation, options.MetricsCallback); @@ -181,11 +180,10 @@ internal ImageClassificationTransformer(IHostEnvironment env, ImageClassificatio private void CheckTrainingParameters(ImageClassificationEstimator.Options options) { - Host.CheckNonWhiteSpace(options.LabelColumn, nameof(options.LabelColumn)); - Host.CheckNonWhiteSpace(options.TensorFlowLabel, nameof(options.TensorFlowLabel)); + Host.CheckNonWhiteSpace(options.LabelColumnName, nameof(options.LabelColumnName)); if (_session.graph.OperationByName(_labelTensor.name.Split(':')[0]) == null) - throw Host.ExceptParam(nameof(options.TensorFlowLabel), $"'{options.TensorFlowLabel}' does not exist in the model"); + throw Host.ExceptParam(nameof(_labelTensor.name), $"'{_labelTensor.name}' does not exist in the model"); if (options.EarlyStoppingCriteria != null && options.ValidationSet == null && options.TestOnTrainSet == false) throw Host.ExceptParam(nameof(options.EarlyStoppingCriteria), $"Early stopping enabled but unable to find a validation" + $" set and/or train set testing disabled. Please disable early stopping or either provide a validation set or enable train set training."); @@ -577,13 +575,13 @@ private void TrainAndEvaluateClassificationLayer(string trainBottleneckFilePath, private (Session, Tensor, Tensor, Tensor) BuildEvaluationSession(ImageClassificationEstimator.Options options, int classCount) { - var evalGraph = DnnUtils.LoadMetaGraph(options.ModelLocation); + var evalGraph = DnnUtils.LoadMetaGraph(ModelLocation[options.Arch]); var evalSess = tf.Session(graph: evalGraph); Tensor evaluationStep = null; Tensor prediction = null; Tensor bottleneckTensor = evalGraph.OperationByName(_bottleneckOperationName); evalGraph.as_default(); - var (_, _, groundTruthInput, finalTensor) = AddFinalRetrainOps(classCount, options.LabelColumn, + var (_, _, groundTruthInput, finalTensor) = AddFinalRetrainOps(classCount, options.LabelColumnName, options.ScoreColumnName, options.LearningRate, bottleneckTensor, false); tf.train.Saver().restore(evalSess, _checkpointPath); @@ -775,7 +773,7 @@ private static void GetModelInfo(IHostEnvironment env, ModelLoadContext ctx, out } internal ImageClassificationTransformer(IHostEnvironment env, Session session, string[] outputColumnNames, - string[] inputColumnNames, string modelLocation, + string[] inputColumnNames, bool? addBatchDimensionInput, int batchSize, string labelColumnName, string finalModelPrefix, Architecture arch, string scoreColumnName, string predictedLabelColumnName, float learningRate, DataViewSchema inputSchema, int? classCount = null, bool loadModel = false, string predictionTensorName = null, string softMaxTensorName = null, string jpegDataTensorName = null, string resizeTensorName = null, string[] labelAnnotations = null) @@ -815,7 +813,7 @@ internal ImageClassificationTransformer(IHostEnvironment env, Session session, s else _classCount = classCount.Value; - _checkpointPath = Path.Combine(Directory.GetCurrentDirectory(), finalModelPrefix + modelLocation); + _checkpointPath = Path.Combine(Directory.GetCurrentDirectory(), finalModelPrefix + ModelLocation[arch]); // Configure bottleneck tensor based on the model. if (arch == ImageClassificationEstimator.Architecture.ResnetV2101) @@ -1341,37 +1339,31 @@ public enum Dataset /// /// The options for the . /// - internal sealed class Options : TransformInputBase + public sealed class Options { - /// - /// Location of the TensorFlow model. - /// - [Argument(ArgumentType.Required, HelpText = "TensorFlow model used by the transform. Please see https://www.tensorflow.org/mobile/prepare_models for more details.", SortOrder = 0)] - public string ModelLocation; - /// /// The names of the model inputs. /// - [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "The names of the model inputs", ShortName = "inputs", SortOrder = 1)] - public string[] InputColumns; + [Argument(ArgumentType.Multiple , HelpText = "The names of the model inputs", ShortName = "inputs", SortOrder = 1)] + internal string[] InputColumns; /// /// The names of the requested model outputs. /// - [Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "The name of the outputs", ShortName = "outputs", SortOrder = 2)] - public string[] OutputColumns; + [Argument(ArgumentType.Multiple , HelpText = "The name of the outputs", ShortName = "outputs", SortOrder = 2)] + internal string[] OutputColumns; /// - /// The name of the label column in that will be mapped to label node in TensorFlow model. + /// The names of the model input features. /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Training labels.", ShortName = "label", SortOrder = 4)] - public string LabelColumn; + [Argument(ArgumentType.AtMostOnce | ArgumentType.Required, HelpText = "The names of the model inputs", ShortName = "features", SortOrder = 1)] + public string FeaturesColumnName; /// - /// The name of the label in TensorFlow model. + /// The name of the label column in that will be mapped to label node in TensorFlow model. /// - [Argument(ArgumentType.AtMostOnce, HelpText = "TensorFlow label node.", ShortName = "TFLabel", SortOrder = 5)] - public string TensorFlowLabel; + [Argument(ArgumentType.AtMostOnce | ArgumentType.Required, HelpText = "Training labels.", ShortName = "label", SortOrder = 4)] + public string LabelColumnName; /// /// Number of samples to use for mini-batch training. @@ -1383,7 +1375,7 @@ internal sealed class Options : TransformInputBase /// Number of training iterations. /// [Argument(ArgumentType.AtMostOnce, HelpText = "Number of training iterations.", SortOrder = 10)] - public int Epoch = 5; + public int Epoch = 100; /// /// Learning rate to use during optimization. @@ -1392,9 +1384,15 @@ internal sealed class Options : TransformInputBase public float LearningRate = 0.01f; /// - /// Early Stopping technique to stop training when accuracy stops improving. + /// Whether to disable use of early stopping technique. Training will go on for the full epoch count. /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Early Stopping technique to stop training when accuracy stops improving.", SortOrder = 15)] + [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to disable use of early stopping technique. Training will go on for the full epoch count.", SortOrder = 15)] + public bool DisableEarlyStopping = false; + + /// + /// Early stopping technique parameters to be used to terminate training when training metric stops improving. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Early stopping technique parameters to be used to terminate training when training metric stops improving.", SortOrder = 15)] public EarlyStopping EarlyStoppingCriteria; /// @@ -1407,7 +1405,7 @@ internal sealed class Options : TransformInputBase /// Name of the tensor that will contain the output scores of the last layer when transfer learning is done. /// [Argument(ArgumentType.AtMostOnce, HelpText = "Softmax tensor of the last layer in transfer learning.", SortOrder = 15)] - public string ScoreColumnName = "Scores"; + public string ScoreColumnName = "Score"; /// /// Name of the tensor that will contain the predicted label from output scores of the last layer when transfer learning is done. @@ -1427,41 +1425,23 @@ internal sealed class Options : TransformInputBase [Argument(ArgumentType.AtMostOnce, HelpText = "Callback to report metrics during training and validation phase.", SortOrder = 15)] public ImageClassificationMetricsCallback MetricsCallback = null; - /// - /// Frequency of epochs at which statistics on training phase should be reported. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Frequency of epochs at which statistics on training/validation phase should be reported.", SortOrder = 15)] - public int StatisticsFrequency = 1; - - /// - /// Indicates the choice DNN training framework. Currently only TensorFlow is supported. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Indicates the choice DNN training framework. Currently only TensorFlow is supported.", SortOrder = 15)] - public DnnFramework Framework = DnnFramework.Tensorflow; - - /// - /// Indicates the path where the newly retrained model should be saved. - /// - [Argument(ArgumentType.AtMostOnce, HelpText = "Indicates the path where the newly retrained model should be saved.", SortOrder = 15)] - public string ModelSavePath = null; - /// /// Indicates to evaluate the model on train set after every epoch. /// [Argument(ArgumentType.AtMostOnce, HelpText = "Indicates to evaluate the model on train set after every epoch.", SortOrder = 15)] - public bool TestOnTrainSet; + public bool TestOnTrainSet = true; /// /// Indicates to not re-compute cached bottleneck trainset values if already available in the bin folder. /// [Argument(ArgumentType.AtMostOnce, HelpText = "Indicates to not re-compute trained cached bottleneck values if already available in the bin folder.", SortOrder = 15)] - public bool ReuseTrainSetBottleneckCachedValues; + public bool ReuseTrainSetBottleneckCachedValues = false; /// /// Indicates to not re-compute cached bottleneck validationset values if already available in the bin folder. /// [Argument(ArgumentType.AtMostOnce, HelpText = "Indicates to not re-compute validataionset cached bottleneck validationset values if already available in the bin folder.", SortOrder = 15)] - public bool ReuseValidationSetBottleneckCachedValues; + public bool ReuseValidationSetBottleneckCachedValues = false; /// /// Validation set. @@ -1473,13 +1453,13 @@ internal sealed class Options : TransformInputBase /// Indicates the file path to store trainset bottleneck values for caching. /// [Argument(ArgumentType.AtMostOnce, HelpText = "Indicates the file path to store trainset bottleneck values for caching.", SortOrder = 15)] - public string TrainSetBottleneckCachedValuesFilePath; + public string TrainSetBottleneckCachedValuesFilePath = "trainSetBottleneckFile.csv"; /// /// Indicates the file path to store validationset bottleneck values for caching. /// [Argument(ArgumentType.AtMostOnce, HelpText = "Indicates the file path to store validationset bottleneck values for caching.", SortOrder = 15)] - public string ValidationSetBottleneckCachedValuesFilePath; + public string ValidationSetBottleneckCachedValuesFilePath = "validationSetBottleneckFile.csv"; } private readonly IHost _host; @@ -1494,12 +1474,13 @@ internal ImageClassificationEstimator(IHostEnvironment env, Options options, Dnn _options = options; _dnnModel = dnnModel; _inputTypes = new[] { new VectorDataViewType(NumberDataViewType.Byte) }; + options.InputColumns = new[] { options.FeaturesColumnName }; + options.OutputColumns = new[] { options.ScoreColumnName, options.PredictedLabelColumnName }; } private static Options CreateArguments(DnnModel tensorFlowModel, string[] outputColumnNames, string[] inputColumnName, bool addBatchDimensionInput) { var options = new Options(); - options.ModelLocation = tensorFlowModel.ModelPath; options.InputColumns = inputColumnName; options.OutputColumns = outputColumnNames; return options; diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index 3a0efe6eaa..4e1d9c4cc7 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -1207,6 +1207,82 @@ public void TensorFlowStringTest() Assert.Equal(string.Join(" ", input.B).Replace("/", " "), textOutput.BOut[0]); } + [TensorFlowFact] + public void TensorFlowImageClassificationDefault() + { + string assetsRelativePath = @"assets"; + string assetsPath = GetAbsolutePath(assetsRelativePath); + string imagesDownloadFolderPath = Path.Combine(assetsPath, "inputs", + "images"); + + //Download the image set and unzip + string finalImagesFolderName = DownloadImageSet( + imagesDownloadFolderPath); + + string fullImagesetFolderPath = Path.Combine( + imagesDownloadFolderPath, finalImagesFolderName); + + MLContext mlContext = new MLContext(seed: 1); + + //Load all the original images info + IEnumerable images = LoadImagesFromDirectory( + folder: fullImagesetFolderPath, useFolderNameAsLabel: true); + + IDataView shuffledFullImagesDataset = mlContext.Data.ShuffleRows( + mlContext.Data.LoadFromEnumerable(images), seed: 1); + + shuffledFullImagesDataset = mlContext.Transforms.Conversion + .MapValueToKey("Label") + .Fit(shuffledFullImagesDataset) + .Transform(shuffledFullImagesDataset); + + // Split the data 80:10 into train and test sets, train and evaluate. + TrainTestData trainTestData = mlContext.Data.TrainTestSplit( + shuffledFullImagesDataset, testFraction: 0.2, seed: 1); + + IDataView trainDataset = trainTestData.TrainSet; + IDataView testDataset = trainTestData.TestSet; + var validationSet = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath") // false indicates we want the image as a VBuffer + .Fit(testDataset) + .Transform(testDataset); + + var pipeline = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath") // false indicates we want the image as a VBuffer + .Append(mlContext.Model.ImageClassification("Image", "Label", validationSet: validationSet) + .Append(mlContext.Transforms.Conversion.MapKeyToValue(outputColumnName: "PredictedLabel", inputColumnName: "PredictedLabel"))); ; + + var trainedModel = pipeline.Fit(trainDataset); + + mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, + "model.zip"); + + ITransformer loadedModel; + DataViewSchema schema; + using (var file = File.OpenRead("model.zip")) + loadedModel = mlContext.Model.Load(file, out schema); + + // Testing EvaluateModel: group testing on test dataset + IDataView predictions = trainedModel.Transform(testDataset); + var metrics = mlContext.MulticlassClassification.Evaluate(predictions); + + // On Ubuntu the results seem to vary quite a bit but they can probably be + // controlled by training more epochs, however that will slow the + // build down. Accuracy values seen were 0.33, 0.66, 0.70+. The model + // seems to be unstable, there could be many reasons, will need to + // investigate this further. + if (!(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || + (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)))) + { + Assert.InRange(metrics.MicroAccuracy, 0.3, 1); + Assert.InRange(metrics.MacroAccuracy, 0.3, 1); + } + else + { + Assert.InRange(metrics.MicroAccuracy, 0.8, 1); + Assert.InRange(metrics.MacroAccuracy, 0.8, 1); + } + + } + [TensorFlowTheory] [InlineData(ImageClassificationEstimator.Architecture.ResnetV2101)] [InlineData(ImageClassificationEstimator.Architecture.MobilenetV2)] @@ -1248,20 +1324,25 @@ public void TensorFlowImageClassification(ImageClassificationEstimator.Architect .Fit(testDataset) .Transform(testDataset); + var options = new ImageClassificationEstimator.Options() + { + FeaturesColumnName = "Image", + LabelColumnName = "Label", + // Just by changing/selecting InceptionV3/MobilenetV2 here instead of + // ResnetV2101 you can try a different architecture/ + // pre-trained model. + Arch = arch, + Epoch = 50, + BatchSize = 10, + LearningRate = 0.01f, + MetricsCallback = (metrics) => Console.WriteLine(metrics), + TestOnTrainSet = false, + ValidationSet = validationSet, + DisableEarlyStopping = true + }; + var pipeline = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath") // false indicates we want the image as a VBuffer - .Append(mlContext.Model.ImageClassification( - "Image", "Label", - // Just by changing/selecting InceptionV3/MobilenetV2 here instead of - // ResnetV2101 you can try a different architecture/pre-trained - // model. - arch: arch, - epoch: 50, - batchSize: 10, - learningRate: 0.01f, - metricsCallback: (metrics) => Console.WriteLine(metrics), - testOnTrainSet: false, - validationSet: validationSet, - disableEarlyStopping: true) + .Append(mlContext.Model.ImageClassification(options) .Append(mlContext.Transforms.Conversion.MapKeyToValue(outputColumnName: "PredictedLabel", inputColumnName: "PredictedLabel"))); var trainedModel = pipeline.Fit(trainDataset); @@ -1383,20 +1464,25 @@ public void TensorFlowImageClassificationEarlyStoppingIncreasing() .Fit(testDataset) .Transform(testDataset); + var options = new ImageClassificationEstimator.Options() + { + FeaturesColumnName = "Image", + LabelColumnName = "Label", + // Just by changing/selecting InceptionV3/MobilenetV2 here instead of + // ResnetV2101 you can try a different architecture/ + // pre-trained model. + Arch = ImageClassificationEstimator.Architecture.ResnetV2101, + Epoch = 100, + BatchSize = 5, + LearningRate = 0.01f, + EarlyStoppingCriteria = new ImageClassificationEstimator.EarlyStopping(), + MetricsCallback = (metrics) => { Console.WriteLine(metrics); lastEpoch = metrics.Train != null ? metrics.Train.Epoch : 0; }, + TestOnTrainSet = false, + ValidationSet = validationSet, + }; + var pipeline = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath") // false indicates we want the image as a VBuffer - .Append(mlContext.Model.ImageClassification( - "Image", "Label", - // Just by changing/selecting InceptionV3/MobilenetV2 here instead of - // ResnetV2101 you can try a different architecture/pre-trained - // model. - arch: ImageClassificationEstimator.Architecture.ResnetV2101, - epoch: 100, - batchSize: 5, - learningRate: 0.01f, - earlyStopping: new ImageClassificationEstimator.EarlyStopping(), - metricsCallback: (metrics) => { Console.WriteLine(metrics); lastEpoch = metrics.Train != null ? metrics.Train.Epoch : 0; }, - testOnTrainSet: false, - validationSet: validationSet)); + .Append(mlContext.Model.ImageClassification(options)); var trainedModel = pipeline.Fit(trainDataset); mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, @@ -1472,20 +1558,25 @@ public void TensorFlowImageClassificationEarlyStoppingDecreasing() .Fit(testDataset) .Transform(testDataset); + var options = new ImageClassificationEstimator.Options() + { + FeaturesColumnName = "Image", + LabelColumnName = "Label", + // Just by changing/selecting InceptionV3/MobilenetV2 here instead of + // ResnetV2101 you can try a different architecture/ + // pre-trained model. + Arch = ImageClassificationEstimator.Architecture.ResnetV2101, + Epoch = 100, + BatchSize = 5, + LearningRate = 0.01f, + EarlyStoppingCriteria = new ImageClassificationEstimator.EarlyStopping(metric: ImageClassificationEstimator.EarlyStoppingMetric.Loss), + MetricsCallback = (metrics) => { Console.WriteLine(metrics); lastEpoch = metrics.Train != null ? metrics.Train.Epoch : 0; }, + TestOnTrainSet = false, + ValidationSet = validationSet, + }; + var pipeline = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath") // false indicates we want the image as a VBuffer - .Append(mlContext.Model.ImageClassification( - "Image", "Label", - // Just by changing/selecting InceptionV3/MobilenetV2 here instead of - // ResnetV2101 you can try a different architecture/pre-trained - // model. - arch: ImageClassificationEstimator.Architecture.ResnetV2101, - epoch: 100, - batchSize: 5, - learningRate: 0.01f, - earlyStopping: new ImageClassificationEstimator.EarlyStopping(metric: ImageClassificationEstimator.EarlyStoppingMetric.Loss), - metricsCallback: (metrics) => {Console.WriteLine(metrics); lastEpoch = metrics.Train != null ? metrics.Train.Epoch : 0;}, - testOnTrainSet: false, - validationSet: validationSet)); + .Append(mlContext.Model.ImageClassification(options)); var trainedModel = pipeline.Fit(trainDataset); mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema,