From a62e956ce6a90fca25d13e43ba3f8b549f25d4f0 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandare Date: Thu, 17 Oct 2019 18:02:45 -0700 Subject: [PATCH 1/4] add Mobilenet_V2 architecture support and sample --- ...bilenetV2TransferLearningTrainTestSplit.cs | 350 ++++++++++++++++++ src/Microsoft.ML.Dnn/DnnCatalog.cs | 7 +- .../ImageClassificationTransform.cs | 86 +++-- 3 files changed, 420 insertions(+), 23 deletions(-) create mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/MobilenetV2TransferLearningTrainTestSplit.cs diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/MobilenetV2TransferLearningTrainTestSplit.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/MobilenetV2TransferLearningTrainTestSplit.cs new file mode 100644 index 0000000000..721a814f9b --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/MobilenetV2TransferLearningTrainTestSplit.cs @@ -0,0 +1,350 @@ + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Transforms; +using static Microsoft.ML.DataOperationsCatalog; + +namespace Samples.Dynamic +{ + public class MobilenetV2TransferLearningTrainTestSplit + { + public static void Example() + { + string assetsRelativePath = @"../../../assets"; + string assetsPath = GetAbsolutePath(assetsRelativePath); + + var outputMlNetModelFilePath = Path.Combine(assetsPath, "outputs", + "imageClassifier.zip"); + + string imagesDownloadFolderPath = Path.Combine(assetsPath, "inputs", + "images"); + + //Download the image set and unzip + string finalImagesFolderName = DownloadImageSet( + imagesDownloadFolderPath); + string fullImagesetFolderPath = Path.Combine( + imagesDownloadFolderPath, finalImagesFolderName); + + try + { + + MLContext mlContext = new MLContext(seed: 1); + + //Load all the original images info + IEnumerable images = LoadImagesFromDirectory( + folder: fullImagesetFolderPath, useFolderNameAsLabel: true); + + IDataView shuffledFullImagesDataset = mlContext.Data.ShuffleRows( + mlContext.Data.LoadFromEnumerable(images)); + + shuffledFullImagesDataset = mlContext.Transforms.Conversion + .MapValueToKey("Label") + .Append(mlContext.Transforms.LoadImages("Image", + fullImagesetFolderPath, false, "ImagePath")) + .Fit(shuffledFullImagesDataset) + .Transform(shuffledFullImagesDataset); + + // Split the data 90:10 into train and test sets, train and + // evaluate. + TrainTestData trainTestData = mlContext.Data.TrainTestSplit( + shuffledFullImagesDataset, testFraction: 0.1, seed: 1); + + IDataView trainDataset = trainTestData.TrainSet; + IDataView testDataset = trainTestData.TestSet; + + var pipeline = mlContext.Model.ImageClassification( + "Image", "Label", + // Just by changing/selecting InceptionV3 here instead of + // ResnetV2101 you can try a different architecture/ + // pre-trained model. + arch: ImageClassificationEstimator.Architecture.MobilenetV2, + epoch: 10, + batchSize: 32, + learningRate: 0.0001f, + metricsCallback: (metrics) => Console.WriteLine(metrics), + validationSet: testDataset, + disableEarlyStopping: true) + .Append(mlContext.Transforms.Conversion.MapKeyToValue( + outputColumnName: "PredictedLabel", + inputColumnName: "PredictedLabel")); + + + Console.WriteLine("*** Training the image classification model " + + "with DNN Transfer Learning on top of the selected " + + "pre-trained model/architecture ***"); + + // Measuring training time + var watch = System.Diagnostics.Stopwatch.StartNew(); + + var trainedModel = pipeline.Fit(trainDataset); + + watch.Stop(); + long elapsedMs = watch.ElapsedMilliseconds; + + Console.WriteLine("Training with transfer learning took: " + + (elapsedMs / 1000).ToString() + " seconds"); + + mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, + "model.zip"); + + ITransformer loadedModel; + DataViewSchema schema; + using (var file = File.OpenRead("model.zip")) + loadedModel = mlContext.Model.Load(file, out schema); + + EvaluateModel(mlContext, testDataset, loadedModel); + + watch = System.Diagnostics.Stopwatch.StartNew(); + + // Predict image class using an in-memory image. + TrySinglePrediction(fullImagesetFolderPath, mlContext, loadedModel); + + watch.Stop(); + elapsedMs = watch.ElapsedMilliseconds; + + Console.WriteLine("Prediction engine took: " + + (elapsedMs / 1000).ToString() + " seconds"); + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + } + + Console.WriteLine("Press any key to finish"); + Console.ReadKey(); + } + + private static void TrySinglePrediction(string imagesForPredictions, + MLContext mlContext, ITransformer trainedModel) + { + // Create prediction function to try one prediction + var predictionEngine = mlContext.Model + .CreatePredictionEngine(trainedModel); + + IEnumerable testImages = LoadInMemoryImagesFromDirectory( + imagesForPredictions, false); + + InMemoryImageData imageToPredict = new InMemoryImageData + { + Image = testImages.First().Image + }; + + var prediction = predictionEngine.Predict(imageToPredict); + + Console.WriteLine($"Scores : [{string.Join(",", prediction.Score)}], " + + $"Predicted Label : {prediction.PredictedLabel}"); + } + + + private static void EvaluateModel(MLContext mlContext, + IDataView testDataset, ITransformer trainedModel) + { + Console.WriteLine("Making bulk predictions and evaluating model's " + + "quality..."); + + // Measuring time + var watch2 = System.Diagnostics.Stopwatch.StartNew(); + + IDataView predictions = trainedModel.Transform(testDataset); + var metrics = mlContext.MulticlassClassification.Evaluate(predictions); + + Console.WriteLine($"Micro-accuracy: {metrics.MicroAccuracy}," + + $"macro-accuracy = {metrics.MacroAccuracy}"); + + watch2.Stop(); + long elapsed2Ms = watch2.ElapsedMilliseconds; + + Console.WriteLine("Predicting and Evaluation took: " + + (elapsed2Ms / 1000).ToString() + " seconds"); + } + + public static IEnumerable LoadImagesFromDirectory(string folder, + bool useFolderNameAsLabel = true) + { + var files = Directory.GetFiles(folder, "*", + searchOption: SearchOption.AllDirectories); + foreach (var file in files) + { + if (Path.GetExtension(file) != ".jpg") + continue; + + var label = Path.GetFileName(file); + if (useFolderNameAsLabel) + label = Directory.GetParent(file).Name; + else + { + for (int index = 0; index < label.Length; index++) + { + if (!char.IsLetter(label[index])) + { + label = label.Substring(0, index); + break; + } + } + } + + yield return new ImageData() + { + ImagePath = file, + Label = label + }; + + } + } + + public static IEnumerable + LoadInMemoryImagesFromDirectory(string folder, + bool useFolderNameAsLabel = true) + { + var files = Directory.GetFiles(folder, "*", + searchOption: SearchOption.AllDirectories); + foreach (var file in files) + { + if (Path.GetExtension(file) != ".jpg") + continue; + + var label = Path.GetFileName(file); + if (useFolderNameAsLabel) + label = Directory.GetParent(file).Name; + else + { + for (int index = 0; index < label.Length; index++) + { + if (!char.IsLetter(label[index])) + { + label = label.Substring(0, index); + break; + } + } + } + + yield return new InMemoryImageData() + { + Image = File.ReadAllBytes(file), + Label = label + }; + + } + } + + public static string DownloadImageSet(string imagesDownloadFolder) + { + // get a set of images to teach the network about the new classes + + //SINGLE SMALL FLOWERS IMAGESET (200 files) + string fileName = "CatsVDogs.zip"; + string url = $"https://mlnetfilestorage.file.core.windows.net/" + + $"imagesets/flower_images/flower_photos_small_set.zip?st=2019-08-" + + $"07T21%3A27%3A44Z&se=2030-08-08T21%3A27%3A00Z&sp=rl&sv=2018-03-" + + $"28&sr=f&sig=SZ0UBX47pXD0F1rmrOM%2BfcwbPVob8hlgFtIlN89micM%3D"; + + //Download(url, imagesDownloadFolder, fileName); + //UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder); + + return Path.GetFileNameWithoutExtension(fileName); + //return "CatsVsDogs/PetImages"; + } + + public static bool Download(string url, string destDir, string destFileName) + { + if (destFileName == null) + destFileName = url.Split(Path.DirectorySeparatorChar).Last(); + + Directory.CreateDirectory(destDir); + + string relativeFilePath = Path.Combine(destDir, destFileName); + + if (File.Exists(relativeFilePath)) + { + Console.WriteLine($"{relativeFilePath} already exists."); + return false; + } + + var wc = new WebClient(); + Console.WriteLine($"Downloading {relativeFilePath}"); + var download = Task.Run(() => wc.DownloadFile(url, relativeFilePath)); + while (!download.IsCompleted) + { + Thread.Sleep(1000); + Console.Write("."); + } + Console.WriteLine(""); + Console.WriteLine($"Downloaded {relativeFilePath}"); + + return true; + } + + public static void UnZip(String gzArchiveName, String destFolder) + { + var flag = gzArchiveName.Split(Path.DirectorySeparatorChar) + .Last() + .Split('.') + .First() + ".bin"; + + if (File.Exists(Path.Combine(destFolder, flag))) return; + + Console.WriteLine($"Extracting."); + var task = Task.Run(() => + { + ZipFile.ExtractToDirectory(gzArchiveName, destFolder); + }); + + while (!task.IsCompleted) + { + Thread.Sleep(200); + Console.Write("."); + } + + File.Create(Path.Combine(destFolder, flag)); + Console.WriteLine(""); + Console.WriteLine("Extracting is completed."); + } + + public static string GetAbsolutePath(string relativePath) + { + FileInfo _dataRoot = new FileInfo(typeof( + ResnetV2101TransferLearningTrainTestSplit).Assembly.Location); + + string assemblyFolderPath = _dataRoot.Directory.FullName; + + string fullPath = Path.Combine(assemblyFolderPath, relativePath); + + return fullPath; + } + + public class InMemoryImageData + { + [LoadColumn(0)] + public byte[] Image; + + [LoadColumn(1)] + public string Label; + } + + public class ImageData + { + [LoadColumn(0)] + public string ImagePath; + + [LoadColumn(1)] + public string Label; + } + + public class ImagePrediction + { + [ColumnName("Score")] + public float[] Score; + + [ColumnName("PredictedLabel")] + public string PredictedLabel; + } + } +} diff --git a/src/Microsoft.ML.Dnn/DnnCatalog.cs b/src/Microsoft.ML.Dnn/DnnCatalog.cs index b7c5fa5886..40158a86fd 100644 --- a/src/Microsoft.ML.Dnn/DnnCatalog.cs +++ b/src/Microsoft.ML.Dnn/DnnCatalog.cs @@ -144,7 +144,7 @@ public static ImageClassificationEstimator ImageClassification( { var options = new ImageClassificationEstimator.Options() { - ModelLocation = arch == Architecture.ResnetV2101 ? @"resnet_v2_101_299.meta" : @"InceptionV3.meta", + ModelLocation = ModelLocation[arch], InputColumns = new[] { featuresColumnName }, OutputColumns = new[] { scoreColumnName, predictedLabelColumnName }, LabelColumn = labelColumnName, @@ -194,6 +194,11 @@ public static ImageClassificationEstimator ImageClassification( client.DownloadFile(new Uri($"{baseGitPath}"), @"resnet_v2_101_299.meta"); } } + else if(options.Arch == Architecture.MobilenetV2) + { + if (!File.Exists(@"mobilenet_v2.pb")) + throw new ArgumentException("mobilenet_v2.pb model file not found"); + } } var env = CatalogUtils.GetEnvironment(catalog); diff --git a/src/Microsoft.ML.Dnn/ImageClassificationTransform.cs b/src/Microsoft.ML.Dnn/ImageClassificationTransform.cs index 9332195b72..556f7f2bc7 100644 --- a/src/Microsoft.ML.Dnn/ImageClassificationTransform.cs +++ b/src/Microsoft.ML.Dnn/ImageClassificationTransform.cs @@ -241,9 +241,17 @@ public ImageProcessor(ImageClassificationTransformer transformer) public Tensor ProcessImage(in VBuffer imageBuffer) { var imageTensor = EncodeByteAsString(imageBuffer); - var processedTensor = _imagePreprocessingRunner.AddInput(imageTensor, 0).Run()[0]; - imageTensor.Dispose(); - return processedTensor; + try + { + var processedTensor = _imagePreprocessingRunner.AddInput(imageTensor, 0).Run()[0]; + imageTensor.Dispose(); + return processedTensor; + } + catch + { + imageTensor.Dispose(); + return null; + } } } @@ -282,15 +290,18 @@ private void CacheFeaturizedImagesToDisk(IDataView input, string labelColumnName continue; //Empty Image var imageTensor = imageProcessor.ProcessImage(image); - runner.AddInput(imageTensor, 0); - var featurizedImage = runner.Run()[0]; // Reuse memory - featurizedImage.ToArray(ref imageArray); - Host.Assert((int)featurizedImage.size == imageArray.Length); - writer.WriteLine(label - 1 + "," + string.Join(",", imageArray)); - featurizedImage.Dispose(); - imageTensor.Dispose(); - metrics.Bottleneck.Index++; - metricsCallback?.Invoke(metrics); + if (imageTensor != null) + { + runner.AddInput(imageTensor, 0); + var featurizedImage = runner.Run()[0]; // Reuse memory + featurizedImage.ToArray(ref imageArray); + Host.Assert((int)featurizedImage.size == imageArray.Length); + writer.WriteLine(label - 1 + "," + string.Join(",", imageArray)); + featurizedImage.Dispose(); + imageTensor.Dispose(); + metrics.Bottleneck.Index++; + metricsCallback?.Invoke(metrics); + } } } } @@ -588,7 +599,8 @@ private void TrainAndEvaluateClassificationLayer(string trainBottleneckFilePath, tf.train.Saver().restore(evalSess, _checkpointPath); (evaluationStep, prediction) = AddEvaluationStep(finalTensor, groundTruthInput); - (_jpegData, _resizedImage) = AddJpegDecoding(299, 299, 3); + var imageSize = ImageClassificationEstimator.ImagePreprocessingSize[options.Arch]; + (_jpegData, _resizedImage) = AddJpegDecoding(imageSize.Item1, imageSize.Item2, 3); return (evalSess, _labelTensor, evaluationStep, prediction); } @@ -827,12 +839,18 @@ internal ImageClassificationTransformer(IHostEnvironment env, Session session, s _bottleneckOperationName = "module_apply_default/hub_output/feature_vector/SpatialSqueeze"; _inputTensorName = "Placeholder"; } + else if(arch == ImageClassificationEstimator.Architecture.MobilenetV2) + { + _bottleneckOperationName = "import/MobilenetV2/Logits/Squeeze"; + _inputTensorName = "import/input"; + } _outputs = new[] { scoreColumnName, predictedLabelColumnName }; if (loadModel == false) { - (_jpegData, _resizedImage) = AddJpegDecoding(299, 299, 3); + var imageSize = ImageClassificationEstimator.ImagePreprocessingSize[arch]; + (_jpegData, _resizedImage) = AddJpegDecoding(imageSize.Item1, imageSize.Item2, 3); _jpegDataTensorName = _jpegData.name; _resizedImageTensorName = _resizedImage.name; @@ -1005,13 +1023,16 @@ public void UpdateCacheIfNeeded() Position = _inputRow.Position; _imageGetter(ref _image); var processedTensor = _imageProcessor.ProcessImage(_image); - var outputTensor = _runner.AddInput(processedTensor, 0).Run(); - outputTensor[0].ToArray(ref _classProbability); - outputTensor[1].ToScalar(ref _predictedLabel); - _predictedLabel += 1; - outputTensor[0].Dispose(); - outputTensor[1].Dispose(); - processedTensor.Dispose(); + if (processedTensor != null) + { + var outputTensor = _runner.AddInput(processedTensor, 0).Run(); + outputTensor[0].ToArray(ref _classProbability); + outputTensor[1].ToScalar(ref _predictedLabel); + _predictedLabel += 1; + outputTensor[0].Dispose(); + outputTensor[1].Dispose(); + processedTensor.Dispose(); + } } } } @@ -1080,7 +1101,28 @@ public sealed class ImageClassificationEstimator : IEstimator + /// Dictionary mapping model architecture to model location. + /// + internal static IReadOnlyDictionary ModelLocation = new Dictionary + { + { Architecture.ResnetV2101, @"resnet_v2_101_299.meta" }, + { Architecture.InceptionV3, @"InceptionV3.meta" }, + { Architecture.MobilenetV2, @"mobilenet_v2.meta" } + }; + + /// + /// Dictionary mapping model architecture to image input size supported. + /// + internal static IReadOnlyDictionary> ImagePreprocessingSize = new Dictionary> + { + { Architecture.ResnetV2101, new Tuple(299,299) }, + { Architecture.InceptionV3, new Tuple(299,299) }, + { Architecture.MobilenetV2, new Tuple(224,224) } }; /// From 7afce9bef89cf005c8f6d575fd4428562c325121 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandare Date: Fri, 18 Oct 2019 13:19:17 -0700 Subject: [PATCH 2/4] Separated try catch fix in different issue, added model download location --- src/Microsoft.ML.Dnn/DnnCatalog.cs | 7 ++- .../ImageClassificationTransform.cs | 52 +++++++------------ 2 files changed, 24 insertions(+), 35 deletions(-) diff --git a/src/Microsoft.ML.Dnn/DnnCatalog.cs b/src/Microsoft.ML.Dnn/DnnCatalog.cs index 40158a86fd..55b66c79dd 100644 --- a/src/Microsoft.ML.Dnn/DnnCatalog.cs +++ b/src/Microsoft.ML.Dnn/DnnCatalog.cs @@ -196,8 +196,11 @@ public static ImageClassificationEstimator ImageClassification( } else if(options.Arch == Architecture.MobilenetV2) { - if (!File.Exists(@"mobilenet_v2.pb")) - throw new ArgumentException("mobilenet_v2.pb model file not found"); + var baseGitPath = @"https://tlcresources.blob.core.windows.net/image/MobileNetV2TensorFlow/mobilenet_v2.meta"; + using (WebClient client = new WebClient()) + { + client.DownloadFile(new Uri($"{baseGitPath}"), @"mobilenet_v2.meta"); + } } } diff --git a/src/Microsoft.ML.Dnn/ImageClassificationTransform.cs b/src/Microsoft.ML.Dnn/ImageClassificationTransform.cs index 556f7f2bc7..5688e99f60 100644 --- a/src/Microsoft.ML.Dnn/ImageClassificationTransform.cs +++ b/src/Microsoft.ML.Dnn/ImageClassificationTransform.cs @@ -241,17 +241,9 @@ public ImageProcessor(ImageClassificationTransformer transformer) public Tensor ProcessImage(in VBuffer imageBuffer) { var imageTensor = EncodeByteAsString(imageBuffer); - try - { - var processedTensor = _imagePreprocessingRunner.AddInput(imageTensor, 0).Run()[0]; - imageTensor.Dispose(); - return processedTensor; - } - catch - { - imageTensor.Dispose(); - return null; - } + var processedTensor = _imagePreprocessingRunner.AddInput(imageTensor, 0).Run()[0]; + imageTensor.Dispose(); + return processedTensor; } } @@ -290,18 +282,15 @@ private void CacheFeaturizedImagesToDisk(IDataView input, string labelColumnName continue; //Empty Image var imageTensor = imageProcessor.ProcessImage(image); - if (imageTensor != null) - { - runner.AddInput(imageTensor, 0); - var featurizedImage = runner.Run()[0]; // Reuse memory - featurizedImage.ToArray(ref imageArray); - Host.Assert((int)featurizedImage.size == imageArray.Length); - writer.WriteLine(label - 1 + "," + string.Join(",", imageArray)); - featurizedImage.Dispose(); - imageTensor.Dispose(); - metrics.Bottleneck.Index++; - metricsCallback?.Invoke(metrics); - } + runner.AddInput(imageTensor, 0); + var featurizedImage = runner.Run()[0]; // Reuse memory + featurizedImage.ToArray(ref imageArray); + Host.Assert((int)featurizedImage.size == imageArray.Length); + writer.WriteLine(label - 1 + "," + string.Join(",", imageArray)); + featurizedImage.Dispose(); + imageTensor.Dispose(); + metrics.Bottleneck.Index++; + metricsCallback?.Invoke(metrics); } } } @@ -1023,16 +1012,13 @@ public void UpdateCacheIfNeeded() Position = _inputRow.Position; _imageGetter(ref _image); var processedTensor = _imageProcessor.ProcessImage(_image); - if (processedTensor != null) - { - var outputTensor = _runner.AddInput(processedTensor, 0).Run(); - outputTensor[0].ToArray(ref _classProbability); - outputTensor[1].ToScalar(ref _predictedLabel); - _predictedLabel += 1; - outputTensor[0].Dispose(); - outputTensor[1].Dispose(); - processedTensor.Dispose(); - } + var outputTensor = _runner.AddInput(processedTensor, 0).Run(); + outputTensor[0].ToArray(ref _classProbability); + outputTensor[1].ToScalar(ref _predictedLabel); + _predictedLabel += 1; + outputTensor[0].Dispose(); + outputTensor[1].Dispose(); + processedTensor.Dispose(); } } } From be599451eb18f15d106ecb61f84b2007ccacd35f Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandare Date: Fri, 18 Oct 2019 14:12:14 -0700 Subject: [PATCH 3/4] removed redundant sample, changed comment message to add MobilenetV2 --- ...bilenetV2TransferLearningTrainTestSplit.cs | 350 ------------------ ...esnetV2101TransferLearningEarlyStopping.cs | 2 +- ...snetV2101TransferLearningTrainTestSplit.cs | 2 +- .../TensorflowTests.cs | 6 +- 4 files changed, 5 insertions(+), 355 deletions(-) delete mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/MobilenetV2TransferLearningTrainTestSplit.cs diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/MobilenetV2TransferLearningTrainTestSplit.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/MobilenetV2TransferLearningTrainTestSplit.cs deleted file mode 100644 index 721a814f9b..0000000000 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/MobilenetV2TransferLearningTrainTestSplit.cs +++ /dev/null @@ -1,350 +0,0 @@ - -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Net; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.ML; -using Microsoft.ML.Data; -using Microsoft.ML.Transforms; -using static Microsoft.ML.DataOperationsCatalog; - -namespace Samples.Dynamic -{ - public class MobilenetV2TransferLearningTrainTestSplit - { - public static void Example() - { - string assetsRelativePath = @"../../../assets"; - string assetsPath = GetAbsolutePath(assetsRelativePath); - - var outputMlNetModelFilePath = Path.Combine(assetsPath, "outputs", - "imageClassifier.zip"); - - string imagesDownloadFolderPath = Path.Combine(assetsPath, "inputs", - "images"); - - //Download the image set and unzip - string finalImagesFolderName = DownloadImageSet( - imagesDownloadFolderPath); - string fullImagesetFolderPath = Path.Combine( - imagesDownloadFolderPath, finalImagesFolderName); - - try - { - - MLContext mlContext = new MLContext(seed: 1); - - //Load all the original images info - IEnumerable images = LoadImagesFromDirectory( - folder: fullImagesetFolderPath, useFolderNameAsLabel: true); - - IDataView shuffledFullImagesDataset = mlContext.Data.ShuffleRows( - mlContext.Data.LoadFromEnumerable(images)); - - shuffledFullImagesDataset = mlContext.Transforms.Conversion - .MapValueToKey("Label") - .Append(mlContext.Transforms.LoadImages("Image", - fullImagesetFolderPath, false, "ImagePath")) - .Fit(shuffledFullImagesDataset) - .Transform(shuffledFullImagesDataset); - - // Split the data 90:10 into train and test sets, train and - // evaluate. - TrainTestData trainTestData = mlContext.Data.TrainTestSplit( - shuffledFullImagesDataset, testFraction: 0.1, seed: 1); - - IDataView trainDataset = trainTestData.TrainSet; - IDataView testDataset = trainTestData.TestSet; - - var pipeline = mlContext.Model.ImageClassification( - "Image", "Label", - // Just by changing/selecting InceptionV3 here instead of - // ResnetV2101 you can try a different architecture/ - // pre-trained model. - arch: ImageClassificationEstimator.Architecture.MobilenetV2, - epoch: 10, - batchSize: 32, - learningRate: 0.0001f, - metricsCallback: (metrics) => Console.WriteLine(metrics), - validationSet: testDataset, - disableEarlyStopping: true) - .Append(mlContext.Transforms.Conversion.MapKeyToValue( - outputColumnName: "PredictedLabel", - inputColumnName: "PredictedLabel")); - - - Console.WriteLine("*** Training the image classification model " + - "with DNN Transfer Learning on top of the selected " + - "pre-trained model/architecture ***"); - - // Measuring training time - var watch = System.Diagnostics.Stopwatch.StartNew(); - - var trainedModel = pipeline.Fit(trainDataset); - - watch.Stop(); - long elapsedMs = watch.ElapsedMilliseconds; - - Console.WriteLine("Training with transfer learning took: " + - (elapsedMs / 1000).ToString() + " seconds"); - - mlContext.Model.Save(trainedModel, shuffledFullImagesDataset.Schema, - "model.zip"); - - ITransformer loadedModel; - DataViewSchema schema; - using (var file = File.OpenRead("model.zip")) - loadedModel = mlContext.Model.Load(file, out schema); - - EvaluateModel(mlContext, testDataset, loadedModel); - - watch = System.Diagnostics.Stopwatch.StartNew(); - - // Predict image class using an in-memory image. - TrySinglePrediction(fullImagesetFolderPath, mlContext, loadedModel); - - watch.Stop(); - elapsedMs = watch.ElapsedMilliseconds; - - Console.WriteLine("Prediction engine took: " + - (elapsedMs / 1000).ToString() + " seconds"); - } - catch (Exception ex) - { - Console.WriteLine(ex.ToString()); - } - - Console.WriteLine("Press any key to finish"); - Console.ReadKey(); - } - - private static void TrySinglePrediction(string imagesForPredictions, - MLContext mlContext, ITransformer trainedModel) - { - // Create prediction function to try one prediction - var predictionEngine = mlContext.Model - .CreatePredictionEngine(trainedModel); - - IEnumerable testImages = LoadInMemoryImagesFromDirectory( - imagesForPredictions, false); - - InMemoryImageData imageToPredict = new InMemoryImageData - { - Image = testImages.First().Image - }; - - var prediction = predictionEngine.Predict(imageToPredict); - - Console.WriteLine($"Scores : [{string.Join(",", prediction.Score)}], " + - $"Predicted Label : {prediction.PredictedLabel}"); - } - - - private static void EvaluateModel(MLContext mlContext, - IDataView testDataset, ITransformer trainedModel) - { - Console.WriteLine("Making bulk predictions and evaluating model's " + - "quality..."); - - // Measuring time - var watch2 = System.Diagnostics.Stopwatch.StartNew(); - - IDataView predictions = trainedModel.Transform(testDataset); - var metrics = mlContext.MulticlassClassification.Evaluate(predictions); - - Console.WriteLine($"Micro-accuracy: {metrics.MicroAccuracy}," + - $"macro-accuracy = {metrics.MacroAccuracy}"); - - watch2.Stop(); - long elapsed2Ms = watch2.ElapsedMilliseconds; - - Console.WriteLine("Predicting and Evaluation took: " + - (elapsed2Ms / 1000).ToString() + " seconds"); - } - - public static IEnumerable LoadImagesFromDirectory(string folder, - bool useFolderNameAsLabel = true) - { - var files = Directory.GetFiles(folder, "*", - searchOption: SearchOption.AllDirectories); - foreach (var file in files) - { - if (Path.GetExtension(file) != ".jpg") - continue; - - var label = Path.GetFileName(file); - if (useFolderNameAsLabel) - label = Directory.GetParent(file).Name; - else - { - for (int index = 0; index < label.Length; index++) - { - if (!char.IsLetter(label[index])) - { - label = label.Substring(0, index); - break; - } - } - } - - yield return new ImageData() - { - ImagePath = file, - Label = label - }; - - } - } - - public static IEnumerable - LoadInMemoryImagesFromDirectory(string folder, - bool useFolderNameAsLabel = true) - { - var files = Directory.GetFiles(folder, "*", - searchOption: SearchOption.AllDirectories); - foreach (var file in files) - { - if (Path.GetExtension(file) != ".jpg") - continue; - - var label = Path.GetFileName(file); - if (useFolderNameAsLabel) - label = Directory.GetParent(file).Name; - else - { - for (int index = 0; index < label.Length; index++) - { - if (!char.IsLetter(label[index])) - { - label = label.Substring(0, index); - break; - } - } - } - - yield return new InMemoryImageData() - { - Image = File.ReadAllBytes(file), - Label = label - }; - - } - } - - public static string DownloadImageSet(string imagesDownloadFolder) - { - // get a set of images to teach the network about the new classes - - //SINGLE SMALL FLOWERS IMAGESET (200 files) - string fileName = "CatsVDogs.zip"; - string url = $"https://mlnetfilestorage.file.core.windows.net/" + - $"imagesets/flower_images/flower_photos_small_set.zip?st=2019-08-" + - $"07T21%3A27%3A44Z&se=2030-08-08T21%3A27%3A00Z&sp=rl&sv=2018-03-" + - $"28&sr=f&sig=SZ0UBX47pXD0F1rmrOM%2BfcwbPVob8hlgFtIlN89micM%3D"; - - //Download(url, imagesDownloadFolder, fileName); - //UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder); - - return Path.GetFileNameWithoutExtension(fileName); - //return "CatsVsDogs/PetImages"; - } - - public static bool Download(string url, string destDir, string destFileName) - { - if (destFileName == null) - destFileName = url.Split(Path.DirectorySeparatorChar).Last(); - - Directory.CreateDirectory(destDir); - - string relativeFilePath = Path.Combine(destDir, destFileName); - - if (File.Exists(relativeFilePath)) - { - Console.WriteLine($"{relativeFilePath} already exists."); - return false; - } - - var wc = new WebClient(); - Console.WriteLine($"Downloading {relativeFilePath}"); - var download = Task.Run(() => wc.DownloadFile(url, relativeFilePath)); - while (!download.IsCompleted) - { - Thread.Sleep(1000); - Console.Write("."); - } - Console.WriteLine(""); - Console.WriteLine($"Downloaded {relativeFilePath}"); - - return true; - } - - public static void UnZip(String gzArchiveName, String destFolder) - { - var flag = gzArchiveName.Split(Path.DirectorySeparatorChar) - .Last() - .Split('.') - .First() + ".bin"; - - if (File.Exists(Path.Combine(destFolder, flag))) return; - - Console.WriteLine($"Extracting."); - var task = Task.Run(() => - { - ZipFile.ExtractToDirectory(gzArchiveName, destFolder); - }); - - while (!task.IsCompleted) - { - Thread.Sleep(200); - Console.Write("."); - } - - File.Create(Path.Combine(destFolder, flag)); - Console.WriteLine(""); - Console.WriteLine("Extracting is completed."); - } - - public static string GetAbsolutePath(string relativePath) - { - FileInfo _dataRoot = new FileInfo(typeof( - ResnetV2101TransferLearningTrainTestSplit).Assembly.Location); - - string assemblyFolderPath = _dataRoot.Directory.FullName; - - string fullPath = Path.Combine(assemblyFolderPath, relativePath); - - return fullPath; - } - - public class InMemoryImageData - { - [LoadColumn(0)] - public byte[] Image; - - [LoadColumn(1)] - public string Label; - } - - public class ImageData - { - [LoadColumn(0)] - public string ImagePath; - - [LoadColumn(1)] - public string Label; - } - - public class ImagePrediction - { - [ColumnName("Score")] - public float[] Score; - - [ColumnName("PredictedLabel")] - public string PredictedLabel; - } - } -} diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs index a7192a3157..43638a91e2 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs @@ -65,7 +65,7 @@ public static void Example() var pipeline = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath") // false indicates we want the image as a VBuffer .Append(mlContext.Model.ImageClassification( "Image", "Label", - // Just by changing/selecting InceptionV3 here instead of + // Just by changing/selecting InceptionV3/MobilenetV2 here instead of // ResnetV2101 you can try a different architecture/pre-trained // model. arch: ImageClassificationEstimator.Architecture.ResnetV2101, diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs index 2d15316c0a..2b20ec0695 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs @@ -62,7 +62,7 @@ public static void Example() var pipeline = mlContext.Model.ImageClassification( "Image", "Label", - // Just by changing/selecting InceptionV3 here instead of + // Just by changing/selecting InceptionV3/MobilenetV2 here instead of // ResnetV2101 you can try a different architecture/ // pre-trained model. arch: ImageClassificationEstimator.Architecture.ResnetV2101, diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index bb70334956..ed4dce2fd8 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -1249,7 +1249,7 @@ public void TensorFlowImageClassification() var pipeline = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath") // false indicates we want the image as a VBuffer .Append(mlContext.Model.ImageClassification( "Image", "Label", - // Just by changing/selecting InceptionV3 here instead of + // Just by changing/selecting InceptionV3/MobilenetV2 here instead of // ResnetV2101 you can try a different architecture/pre-trained // model. arch: ImageClassificationEstimator.Architecture.ResnetV2101, @@ -1384,7 +1384,7 @@ public void TensorFlowImageClassificationEarlyStoppingIncreasing() var pipeline = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath") // false indicates we want the image as a VBuffer .Append(mlContext.Model.ImageClassification( "Image", "Label", - // Just by changing/selecting InceptionV3 here instead of + // Just by changing/selecting InceptionV3/MobilenetV2 here instead of // ResnetV2101 you can try a different architecture/pre-trained // model. arch: ImageClassificationEstimator.Architecture.ResnetV2101, @@ -1473,7 +1473,7 @@ public void TensorFlowImageClassificationEarlyStoppingDecreasing() var pipeline = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPath, false, "ImagePath") // false indicates we want the image as a VBuffer .Append(mlContext.Model.ImageClassification( "Image", "Label", - // Just by changing/selecting InceptionV3 here instead of + // Just by changing/selecting InceptionV3/MobilenetV2 here instead of // ResnetV2101 you can try a different architecture/pre-trained // model. arch: ImageClassificationEstimator.Architecture.ResnetV2101, From 251312fd49e353a2e40787fed707df1cba048bd6 Mon Sep 17 00:00:00 2001 From: Aishwarya Bhandare Date: Fri, 18 Oct 2019 15:18:13 -0700 Subject: [PATCH 4/4] added test for MobileNetV2, reusing test with Theory --- .../Attributes/TensorflowTheoryAttribute.cs | 23 +++++++++++++++++++ .../TensorflowTests.cs | 8 ++++--- 2 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 test/Microsoft.ML.TestFramework/Attributes/TensorflowTheoryAttribute.cs diff --git a/test/Microsoft.ML.TestFramework/Attributes/TensorflowTheoryAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/TensorflowTheoryAttribute.cs new file mode 100644 index 0000000000..24b73a56f5 --- /dev/null +++ b/test/Microsoft.ML.TestFramework/Attributes/TensorflowTheoryAttribute.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +namespace Microsoft.ML.TestFramework.Attributes +{ + /// + /// A theory for tests requiring TensorFlow. + /// + public sealed class TensorFlowTheoryAttribute : EnvironmentSpecificTheoryAttribute + { + public TensorFlowTheoryAttribute() : base("TensorFlow is 64-bit only") + { + } + + /// + protected override bool IsEnvironmentSupported() + { + return Environment.Is64BitProcess; + } + } +} \ No newline at end of file diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index ed4dce2fd8..3a0efe6eaa 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -1207,8 +1207,10 @@ public void TensorFlowStringTest() Assert.Equal(string.Join(" ", input.B).Replace("/", " "), textOutput.BOut[0]); } - [TensorFlowFact] - public void TensorFlowImageClassification() + [TensorFlowTheory] + [InlineData(ImageClassificationEstimator.Architecture.ResnetV2101)] + [InlineData(ImageClassificationEstimator.Architecture.MobilenetV2)] + public void TensorFlowImageClassification(ImageClassificationEstimator.Architecture arch) { string assetsRelativePath = @"assets"; string assetsPath = GetAbsolutePath(assetsRelativePath); @@ -1252,7 +1254,7 @@ public void TensorFlowImageClassification() // Just by changing/selecting InceptionV3/MobilenetV2 here instead of // ResnetV2101 you can try a different architecture/pre-trained // model. - arch: ImageClassificationEstimator.Architecture.ResnetV2101, + arch: arch, epoch: 50, batchSize: 10, learningRate: 0.01f,