From 6ec24f32005606f3fffe710a078f659d5daed707 Mon Sep 17 00:00:00 2001 From: Yael Dekel Date: Fri, 22 Jun 2018 13:54:48 -0700 Subject: [PATCH 1/4] Create a shorter temp file name for model loading. --- src/Microsoft.ML.Data/Model/Repository.cs | 24 ++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Data/Model/Repository.cs b/src/Microsoft.ML.Data/Model/Repository.cs index 7556cc970e..f0c8283ab3 100644 --- a/src/Microsoft.ML.Data/Model/Repository.cs +++ b/src/Microsoft.ML.Data/Model/Repository.cs @@ -109,13 +109,35 @@ internal Repository(bool needDir, IExceptionContext ectx) _open = new List(); if (needDir) { - DirTemp = GetTempPath(); + DirTemp = GetShortTempPath(); Directory.CreateDirectory(DirTemp); } else GC.SuppressFinalize(this); } + private static string GetShortTempPath() + { + Guid guid = Guid.NewGuid(); + var guidName = guid.ToString(); + int index = guidName.IndexOf('-'); + if (index < 0) + index = Math.Min(8, guidName.Length); + guidName = guidName.Substring(0, index); + var path = Path.Combine(Path.GetTempPath(), "TLC_" + guidName); + while (Directory.Exists(path)) + { + guid = Guid.NewGuid(); + guidName = guid.ToString(); + index = guidName.IndexOf('-'); + if (index < 0) + index = Math.Min(8, guidName.Length); + guidName = guidName.Substring(0, index); + path = Path.Combine(Path.GetTempPath(), "TLC_" + guidName); + } + return Path.GetFullPath(path); + } + // REVIEW: This should use host environment functionality. private static string GetTempPath() { From 35e7a6c03d0ef1a6c7a234d7f8692b62afacd2e1 Mon Sep 17 00:00:00 2001 From: Yael Dekel Date: Mon, 25 Jun 2018 13:43:36 -0700 Subject: [PATCH 2/4] Address PR comments. --- src/Microsoft.ML.Data/Model/Repository.cs | 26 +++++++---------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/Microsoft.ML.Data/Model/Repository.cs b/src/Microsoft.ML.Data/Model/Repository.cs index f0c8283ab3..e50df9b9ab 100644 --- a/src/Microsoft.ML.Data/Model/Repository.cs +++ b/src/Microsoft.ML.Data/Model/Repository.cs @@ -109,32 +109,22 @@ internal Repository(bool needDir, IExceptionContext ectx) _open = new List(); if (needDir) { - DirTemp = GetShortTempPath(); + DirTemp = GetShortTempDir(); Directory.CreateDirectory(DirTemp); } else GC.SuppressFinalize(this); } - private static string GetShortTempPath() + private static string GetShortTempDir() { - Guid guid = Guid.NewGuid(); - var guidName = guid.ToString(); - int index = guidName.IndexOf('-'); - if (index < 0) - index = Math.Min(8, guidName.Length); - guidName = guidName.Substring(0, index); - var path = Path.Combine(Path.GetTempPath(), "TLC_" + guidName); - while (Directory.Exists(path)) + var rnd = RandomUtils.Create(); + string path; + do { - guid = Guid.NewGuid(); - guidName = guid.ToString(); - index = guidName.IndexOf('-'); - if (index < 0) - index = Math.Min(8, guidName.Length); - guidName = guidName.Substring(0, index); - path = Path.Combine(Path.GetTempPath(), "TLC_" + guidName); + path = Path.Combine(Path.GetTempPath(), "TLC_" + rnd.Next().ToString("X")); } + while (Directory.Exists(path)); return Path.GetFullPath(path); } @@ -254,7 +244,7 @@ protected void GetPath(out string pathEnt, out string pathTemp, string dir, stri _ectx.CheckParam(!name.Contains(".."), nameof(name)); // The gymnastics below are meant to deal with bad invocations including absolute paths, etc. - // That's why we go through it even if _dirTemp is null. + // That's why we go through it even if DirTemp is null. string root = Path.GetFullPath(DirTemp ?? @"x:\dummy"); string entityPath = Path.Combine(root, dir ?? "", name); entityPath = Path.GetFullPath(entityPath); From 35864ad60b057d691a42d9c4b6ff682fdc6c2ad5 Mon Sep 17 00:00:00 2001 From: Yael Dekel Date: Tue, 26 Jun 2018 11:54:58 -0700 Subject: [PATCH 3/4] Address PR comments. --- src/Microsoft.ML.Data/Model/Repository.cs | 27 ++++++++++++++--------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/Microsoft.ML.Data/Model/Repository.cs b/src/Microsoft.ML.Data/Model/Repository.cs index e50df9b9ab..3481193a0e 100644 --- a/src/Microsoft.ML.Data/Model/Repository.cs +++ b/src/Microsoft.ML.Data/Model/Repository.cs @@ -6,7 +6,6 @@ using System.Collections.Generic; using System.IO; using System.IO.Compression; -using Microsoft.ML.Runtime.Data; using Microsoft.ML.Runtime.Internal.Utilities; namespace Microsoft.ML.Runtime.Model @@ -73,7 +72,7 @@ public void Dispose() } } - // These are the open entries that may contain streams into our _dirTemp. + // These are the open entries that may contain streams into our DirTemp. private List _open; private bool _disposed; @@ -108,10 +107,7 @@ internal Repository(bool needDir, IExceptionContext ectx) PathMap = new Dictionary(); _open = new List(); if (needDir) - { DirTemp = GetShortTempDir(); - Directory.CreateDirectory(DirTemp); - } else GC.SuppressFinalize(this); } @@ -123,16 +119,25 @@ private static string GetShortTempDir() do { path = Path.Combine(Path.GetTempPath(), "TLC_" + rnd.Next().ToString("X")); + path = Path.GetFullPath(path); + Directory.CreateDirectory(path); } - while (Directory.Exists(path)); - return Path.GetFullPath(path); + while (!EnsureDirectory(path)); + return path; } - // REVIEW: This should use host environment functionality. - private static string GetTempPath() + private static bool EnsureDirectory(string path) { - Guid guid = Guid.NewGuid(); - return Path.GetFullPath(Path.Combine(Path.GetTempPath(), "TLC_" + guid.ToString())); + var fullPath = Path.GetFullPath(Path.Combine(path, ".lock")); + try + { + using (var stream = new FileStream(fullPath, FileMode.CreateNew)) + return true; + } + catch + { + return false; + } } ~Repository() From cf27789909cc00a57145af830becd0477ea65677 Mon Sep 17 00:00:00 2001 From: Yael Dekel Date: Tue, 26 Jun 2018 15:39:05 -0700 Subject: [PATCH 4/4] Trigger build. --- src/Microsoft.ML.Data/Model/Repository.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.Data/Model/Repository.cs b/src/Microsoft.ML.Data/Model/Repository.cs index 3481193a0e..b19fbc8eba 100644 --- a/src/Microsoft.ML.Data/Model/Repository.cs +++ b/src/Microsoft.ML.Data/Model/Repository.cs @@ -128,10 +128,10 @@ private static string GetShortTempDir() private static bool EnsureDirectory(string path) { - var fullPath = Path.GetFullPath(Path.Combine(path, ".lock")); + path = Path.GetFullPath(Path.Combine(path, ".lock")); try { - using (var stream = new FileStream(fullPath, FileMode.CreateNew)) + using (var stream = new FileStream(path, FileMode.CreateNew)) return true; } catch