From 8886bd860763d4cf8592770464d9fbb9b7e0180d Mon Sep 17 00:00:00 2001 From: Daniel Drews Date: Mon, 9 Jul 2018 22:08:02 -0500 Subject: [PATCH 1/6] First attempt at removing extra code comments --- .../Utilities/DoubleParser.cs | 6 - src/Microsoft.ML.Core/Utilities/MathUtils.cs | 6 +- .../InternalStreams.cs | 273 +----------------- .../LowFragmentationStream.cs | 28 -- src/Microsoft.ML.InternalStreams/TableIO.cs | 27 -- .../UnbufferedStream.cs | 172 +---------- .../Optimizer/OptimizationMonitor.cs | 4 - .../UnitTests/TestEntryPoints.cs | 106 ------- 8 files changed, 15 insertions(+), 607 deletions(-) diff --git a/src/Microsoft.ML.Core/Utilities/DoubleParser.cs b/src/Microsoft.ML.Core/Utilities/DoubleParser.cs index babc135eb0..f2d5573211 100644 --- a/src/Microsoft.ML.Core/Utilities/DoubleParser.cs +++ b/src/Microsoft.ML.Core/Utilities/DoubleParser.cs @@ -433,12 +433,6 @@ public static bool TryParse(out Double value, string s, int ichMin, int ichLim, if (FloatUtils.GetBits(x) != 0 || FloatUtils.GetBits(value) != TopBit || !neg) { System.Diagnostics.Debug.WriteLine("*** FloatParser disagrees with Double.TryParse on: {0} ({1} vs {2})", str, FloatUtils.GetBits(x), FloatUtils.GetBits(value)); - //if (!_failed) - //{ - // // REVIEW: Double.Parse gets several things wrong, like mapping 148e-325 to 0x2 instead of 0x3. - // _failed = true; - // Contracts.Assert(false, string.Format("FloatParser disagrees with Double.TryParse on: {0} ({1} vs {2})", str, FloatUtils.GetBits(x), FloatUtils.GetBits(value))); - //} } } #endif diff --git a/src/Microsoft.ML.Core/Utilities/MathUtils.cs b/src/Microsoft.ML.Core/Utilities/MathUtils.cs index 8106ff5a2c..65090e1b85 100644 --- a/src/Microsoft.ML.Core/Utilities/MathUtils.cs +++ b/src/Microsoft.ML.Core/Utilities/MathUtils.cs @@ -257,11 +257,7 @@ public static Float SoftMax(Float[] inputs, int count) if (count == 1) return max; - - //else if (leng == 2) { - // return SoftMax(inputs[0], inputs[1]); - //} - + double intermediate = 0.0; Float cutoff = max - LogTolerance; diff --git a/src/Microsoft.ML.InternalStreams/InternalStreams.cs b/src/Microsoft.ML.InternalStreams/InternalStreams.cs index 76cd8413f6..d89420c721 100644 --- a/src/Microsoft.ML.InternalStreams/InternalStreams.cs +++ b/src/Microsoft.ML.InternalStreams/InternalStreams.cs @@ -523,44 +523,6 @@ private Win32() { } - //[DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - //static extern bool GetOverlappedResult( - // IntPtr hFile, - // IntPtr lpBuffer, - // ref OVERLAPPED lpOverlapped, - // out int lpNumberOfBytesTransferred, - // bool bWait); - - //[StructLayout(LayoutKind.Explicit)] - //struct OVERLAPPED - //{ - // //public ULONG_PTR Internal; - // //public ULONG_PTR InternalHigh; - // //public DWORD Offset; - // //public DWORD OffsetHigh; - // //public HANDLE hEvent; - // [FieldOffset(0)] - // public IntPtr Internal; - // [FieldOffset(4)] - // public IntPtr InternalHigh; - // [FieldOffset(8)] - // public uint Offset; - // [FieldOffset(12)] - // public uint OffsetHigh; - // [FieldOffset(16)] - // public IntPtr hEvent; - //} - - // [StructLayout(Packing = 4)] - // private struct FILE_STREAM_INFORMATION - // { - // ULONG NextEntry; - // ULONG NameLength; - // LARGE_INTEGER Size; - // LARGE_INTEGER AllocationSize; - // USHORT Name[1]; - // } - #endregion } #endregion DLLimport @@ -1297,17 +1259,6 @@ public static bool PathsEqual(string path1, string path2) return false; } return (path1C.Equals(path2C)); - - //if (path1 == null || path1.Length == 0 || path2 == null || path2.Length == 0) - //{ - // return path1 == path2; - //} - //path1 = path1.TrimEnd(pathSeparators); - //path2 = path2.TrimEnd(pathSeparators); - //if (path1.Length != path2.Length) return false; - //path1 = path1.Replace('\\', '/'); - //path2 = path2.Replace('\\', '/'); - //return string.Compare(path1, path2, !path1.StartsWith("cosmos:")) == 0; } /// @@ -2190,8 +2141,6 @@ public static bool DirectoryExists(string path) // InternalStore: if (InternalStoreUtility.IsInternalStore(path)) { - // return true if path is the main directory of a store: - //return false; return DirectoryExists(InternalStoreUtility.StorePath(path)); } @@ -2208,9 +2157,6 @@ public static bool DirectoryExists(string path) for (int i = 0; i < ZStreamIn.decompressionArchiveExtensions.Length; i++) { string ext = ZStreamIn.decompressionArchiveExtensions[i]; - //for (int i = 0; i < ZStreamIn.decompressionExtensions.Length; i++) - //{ - // string ext = ZStreamIn.decompressionExtensions[i]; if ((nameLower.EndsWith(ext) && File.Exists(path)) || File.Exists(path + ext)) { @@ -2341,18 +2287,15 @@ public static void CreateDirectory(string path) // check for special names: if (ZStreamIn.IsNullStream(path)) { - //throw new IOException("Cannot create directory for null stream"); return; } if (ZStreamIn.IsConsoleStream(path)) { - //throw new IOException("Cannot create directory for console stream"); return; } // check for clipboard: if (ClipboardReadStream.IsClipboardStream(path)) { - //throw new IOException("Cannot create directory for clipboard"); return; } @@ -2409,18 +2352,11 @@ public static void CreateDirectory(string path) //Console.WriteLine("Opening: " + path); using (Stream s = ZStreamOut.Open(path)) { - //Console.WriteLine("Trying to create: " + path); s.Close(); - //Console.WriteLine("Closed: " + path); if (s is CmdStream && ((CmdStream)s).ExitCode != 0) { - //Console.WriteLine("ExitCode: " + ((CmdStream)s).ExitCode); throw new IOException("Failed to create archive"); } - //else - //{ - // Console.WriteLine("Stream type: " + s.GetType().Name); - //} } try { @@ -2497,7 +2433,6 @@ public static string[] DirectoryEntries(string path) private static string[] DirectoryEntries(string path, bool allowFile, bool allowDirectory) { - //return ExpandWildcards(path + "*"); if (path == null || path.Length == 0) return new string[0]; @@ -2513,23 +2448,17 @@ private static string[] DirectoryEntries(string path, bool allowFile, bool allow // check for special names: if (ZStreamIn.IsNullStream(path)) { - //return new string[0]; - //return allowFile ? new string[] { ZStreamIn.nullStreamNames[0] } : new string[0]; return path.IndexOfAny(pathSeparators) < 0 ? new string[] { ZStreamIn.NullStreamName } : new string[0]; } if (ZStreamIn.IsConsoleStream(path)) { - //return new string[0]; - //return allowFile ? new string[] { ZStreamIn.consoleStreamNames[0] } : new string[0]; return path.IndexOfAny(pathSeparators) < 0 ? new string[] { ZStreamIn.ConsoleStreamName } : new string[0]; } // check for clipboard: if (ClipboardReadStream.IsClipboardStream(path)) { - //return new string[0]; - //return allowFile ? new string[] { "clip:" } : new string[0]; return path.IndexOfAny(pathSeparators) < 0 || string.Compare(path.TrimEnd(pathSeparators), "clip:", true) == 0 ? new string[] { "clip:" } : new string[0]; @@ -2661,7 +2590,6 @@ private static string[] DirectoryEntries(string path, bool allowFile, bool allow // check for compressed archives in path: // only one path segment is allowed to be an archive... // normalize path: - //if (fileName[fileName.Length - 1] != '\\') fileName = fileName + "\\"; string zfileName = path.Replace('/', '\\'); bool isUnc = zfileName.StartsWith("\\\\"); while (zfileName.IndexOf("\\\\") >= 0) @@ -2674,9 +2602,6 @@ private static string[] DirectoryEntries(string path, bool allowFile, bool allow string archPath = null; string inArch = null; - //for (int i = 0; i < ZStreamIn.decompressionArchiveExtensions.Length; i++) - //{ - // string ext = ZStreamIn.decompressionArchiveExtensions[i]; for (int i = 0; i < ZStreamIn.decompressionExtensions.Length; i++) { string ext = ZStreamIn.decompressionExtensions[i]; @@ -2705,9 +2630,6 @@ private static string[] DirectoryEntries(string path, bool allowFile, bool allow continue; if (Directory.Exists(partial)) continue; - //for (int c = 0; c < ZStreamIn.decompressionArchiveExtensions.Length; c++) - //{ - // string ext = ZStreamIn.decompressionArchiveExtensions[c]; for (int c = 0; c < ZStreamIn.decompressionExtensions.Length; c++) { string ext = ZStreamIn.decompressionExtensions[c]; @@ -2724,7 +2646,6 @@ private static string[] DirectoryEntries(string path, bool allowFile, bool allow } if (archPath != null) { - //Console.WriteLine(archPath + " :: " + inArch); // check for path in archive. inArch = inArch.Trim('\\'); if (inArch.Length == 0) @@ -2967,7 +2888,6 @@ private static StreamInfo[] DirectoryEntriesInfo(string path, bool allowFile, bo // check for compressed archives in path: // only one path segment is allowed to be an archive... // normalize path: - //if (fileName[fileName.Length - 1] != '\\') fileName = fileName + "\\"; string zfileName = path.Replace('/', '\\'); bool isUnc = zfileName.StartsWith("\\\\"); while (zfileName.IndexOf("\\\\") >= 0) @@ -2980,9 +2900,6 @@ private static StreamInfo[] DirectoryEntriesInfo(string path, bool allowFile, bo string archPath = null; string inArch = null; - //for (int i = 0; i < ZStreamIn.decompressionArchiveExtensions.Length; i++) - //{ - // string ext = ZStreamIn.decompressionArchiveExtensions[i]; for (int i = 0; i < ZStreamIn.decompressionExtensions.Length; i++) { string ext = ZStreamIn.decompressionExtensions[i]; @@ -3011,9 +2928,6 @@ private static StreamInfo[] DirectoryEntriesInfo(string path, bool allowFile, bo continue; if (Directory.Exists(partial)) continue; - //for (int c = 0; c < ZStreamIn.decompressionArchiveExtensions.Length; c++) - //{ - // string ext = ZStreamIn.decompressionArchiveExtensions[c]; for (int c = 0; c < ZStreamIn.decompressionExtensions.Length; c++) { string ext = ZStreamIn.decompressionExtensions[c]; @@ -3048,15 +2962,6 @@ private static StreamInfo[] DirectoryEntriesInfo(string path, bool allowFile, bo private static StreamInfo[] CockpitDirectoryEntriesInfo(string path, bool allowFile, bool allowDirectory) { - ////string pattern = searchPattern == null ? "*" : searchPattern; - //string pattern = "*"; - //pattern = "^" + System.Text.RegularExpressions.Regex.Escape(pattern) + "$"; - //pattern = pattern.Replace(@"\?", "."); - //pattern = pattern.Replace(@"\*", ".*"); - //System.Text.RegularExpressions.Regex nameRegex = new System.Text.RegularExpressions.Regex( - // pattern, - // System.Text.RegularExpressions.RegexOptions.IgnoreCase | System.Text.RegularExpressions.RegexOptions.CultureInvariant); - List res = new List(); string htmlDir; try @@ -3149,11 +3054,8 @@ private static StreamInfo[] CockpitDirectoryEntriesInfo(string path, bool allowF { if (allowFile) { - //if (nameRegex.IsMatch(filename)) - //{ string fullname = PathCombine(path, filename); res.Add(new StreamInfo(fullname, length, lastModified)); - //} } filename = null; length = -1; @@ -3176,15 +3078,12 @@ private static StreamInfo[] CockpitDirectoryEntriesInfo(string path, bool allowF if (line != "Parent directory\\") { - //if (nameRegex.IsMatch(filename)) - //{ string fullname = PathCombine(path, filename); if (!fullname.EndsWith("\\") && !fullname.EndsWith("/")) { fullname = fullname + "\\"; } res.Add(new StreamInfo(fullname, 0, DateTime.MinValue)); - //} } } filename = null; @@ -3357,7 +3256,6 @@ public static bool FileExists(string fileName) // check for compressed archives in path: // only one path segment is allowed to be an archive... // normalize path: - //if (fileName[fileName.Length - 1] != '\\') fileName = fileName + "\\"; fileName = fileName.Replace('/', '\\'); bool isUnc = fileName.StartsWith("\\\\"); while (fileName.IndexOf("\\\\") >= 0) @@ -3370,10 +3268,6 @@ public static bool FileExists(string fileName) string archPath = null; string inArch = null; - // check all compressed files, not just archives... - //for (int i = 0; i < ZStreamIn.decompressionArchiveExtensions.Length; i++) - //{ - // string ext = ZStreamIn.decompressionArchiveExtensions[i]; for (int i = 0; i < ZStreamIn.decompressionExtensions.Length; i++) { string ext = ZStreamIn.decompressionExtensions[i]; @@ -3402,9 +3296,6 @@ public static bool FileExists(string fileName) continue; if (Directory.Exists(partial)) continue; - //for (int c = 0; c < ZStreamIn.decompressionArchiveExtensions.Length; c++) - //{ - // string ext = ZStreamIn.decompressionArchiveExtensions[c]; for (int c = 0; c < ZStreamIn.decompressionExtensions.Length; c++) { string ext = ZStreamIn.decompressionExtensions[c]; @@ -4200,14 +4091,6 @@ public static void Copy(Stream source, Stream destination) if (source is UnbufferedStream) { UnbufferedStream us = (UnbufferedStream)source; - //unsafe - //{ - // byte* buffer; - // for (int c = us.Read(out buffer); c > 0; c = us.Read(out buffer)) - // { - // destination.Write(...); - // } - //} byte[] buffer; for (int c = us.Read(out buffer); c > 0; c = us.Read(out buffer)) { @@ -4367,7 +4250,6 @@ public static Thread ConsumeBackground(Stream stream) { if (stream == null || !stream.CanRead) return null; - //ThreadPool.UnsafeQueueUserWorkItem(new WaitCallback(ReadAll), stream); ParameterizedThreadStart consumeStart = new ParameterizedThreadStart(ReadAll); Thread consume = Utils.CreateBackgroundThread(consumeStart); consume.Start(stream); @@ -4392,7 +4274,6 @@ public static Thread ConsumeBackground(TextReader reader) { if (reader == null) return null; - //ThreadPool.UnsafeQueueUserWorkItem(new WaitCallback(ReadAllLines), reader); ParameterizedThreadStart consumeStart = new ParameterizedThreadStart(ReadAllLines); Thread consume = Utils.CreateBackgroundThread(consumeStart); consume.Start(reader); @@ -4429,14 +4310,11 @@ private static void ReadAllLines(object readerObj) TextReader reader = (TextReader)readerObj; try { - //while (reader.ReadLine() != null) - //{ - //} char[] buffer = new char[512]; int c; while ((c = reader.Read(buffer, 0, buffer.Length)) > 0) { - //Console.Error.Write(buffer, 0, c); + } } catch @@ -5385,16 +5263,6 @@ public static ulong Checksum(string fileName, ChecksumType type) using (Stream inFile = ZStreamIn.Open(fileName)) { // compute the basic crc - // for (int count = inFile.Read(buffer, 0, buffer.Length); count > 0; count = inFile.Read(buffer, 0, buffer.Length)) - // { - // length += count; - // for (int i = 0; i < count; i++) - // { - // // New checksum value - // crcZip = (crcZip >> 8) ^ zip_crctab[((int)crcZip ^ buffer[i]) & 0xff]; - // crc32 = (crc32 << 8) ^ crctab[(crc32 >> 24) ^ ( ((uint)buffer[i]) & 0xFF )]; - // } - // } switch (type) { case ChecksumType.Crc32: @@ -5754,8 +5622,8 @@ public static string[] ReadLines(string fileName) public class StreamInfo : IComparable { private readonly string _path; - private /*readonly*/ long _length; - private /*readonly*/ DateTime _lastWriteTime; + private long _length; + private DateTime _lastWriteTime; private string _canonicalPath; /// @@ -5937,15 +5805,6 @@ public string CanonicalPath } } - ///// - ///// Get the filename with DOS-style slashes ("\"). - ///// - ///// the filename with DOS-style slashes - //public string GetDosName() - //{ - // return Name.Replace('/', '\\'); - //} - /// /// Gets whether the item is a directory (not including archives, /// which can be accessed as directories). @@ -6262,19 +6121,7 @@ public static StreamReader Open(string fileName, Encoding encoding) s = ZStreamIn.Open(fileName); if (_bufferSize > 0) { - //// hack for console! - //if (fileName.Length > 0 && (fileName[0] == '-' || fileName[0] == '$') && - // (fileName.Length == 1 || (fileName.Length == 2 && - // (fileName[1] == '-' || fileName[1] == '$')))) - //{ - // //s = new BufferedStream(s, BUFFER_SIZE); - // StreamReader sr = new StreamReader(s, encoding, true); - // return sr; - //} - //else - //{ return new StreamReader(s, encoding, true, _bufferSize); - //} } else { @@ -6444,18 +6291,7 @@ public static StreamReader OpenBuffered(string fileName, Encoding encoding) s = ZStreamIn.OpenBuffered(fileName); if (_bufferSize > 0) { - //// hack for console! - //if (fileName.Length > 0 && (fileName[0] == '-' || fileName[0] == '$') && - // (fileName.Length == 1 || (fileName.Length == 2 && - // (fileName[1] == '-' || fileName[1] == '$')))) - //{ - // //s = new BufferedStream(s, BUFFER_SIZE); - // return new StreamReader(s, encoding, true); - //} - //else - //{ return new StreamReader(s, encoding, true, _bufferSize); - //} } else { @@ -6683,8 +6519,6 @@ public override void Close() /// protected override void Dispose(bool disposing) { - //if (disposing) - //{ if (!isClosed) { try @@ -6713,21 +6547,18 @@ protected override void Dispose(bool disposing) } } } - catch //(Exception e2) + catch { // ignore any problems... - //Console.WriteLine("!! (Dispose) Problem deleting in " + tempDir + ": " + e2.ToString()); } Directory.Delete(tempDir, true); tempDir = null; } - catch //(Exception e) + catch { // ignore any problems... - //Console.WriteLine("!! (Dispose) Problem deleting " + tempDir + ": " + e.ToString()); } } - //} base.Dispose (disposing); } #endif @@ -7880,10 +7711,6 @@ private static Stream Open(string fileName, bool buffered, bool bufferedFallback //// - .NET's gzip is very large (30% larger compressed files) //// - .NET's gzip breaks for files over 4 GB // could wrap to get length, maybe fix 4GB problem... - //if (archiveName.EndsWith(".gz", StringComparison.OrdinalIgnoreCase)) - //{ - // return GzipEncodeStream.GetLengthTag(archiveName); - //} s = new System.IO.Compression.GZipStream( new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), @@ -8178,16 +8005,6 @@ private static Stream Open(string fileName, bool buffered, bool bufferedFallback if (!bufferedFallback) throw; } - //if (BUFFER_SIZE > 0) - //{ - // return UnbufferedStream.Open(fileName, FileMode.Open, FileAccess.Read, FileShare.Read, - // true, async, 4 * BUFFER_SIZE); - //} - //else - //{ - // return UnbufferedStream.Open(fileName, FileMode.Open, FileAccess.Read, FileShare.Read, - // true, async, 64 * 1024); - //} #else throw new NotSupportedException("Unbuffered IO is not supported."); #endif @@ -8776,14 +8593,6 @@ private static Stream Open(string fileName, bool append, bool breakChunksAtLines return new MultiStream(fileName, true); } - // check for Azure Storage objects - //AzureStorageIO.AzureStorageIdentificationStructure azureStorageIdentificationStructure = - // new AzureStorageIO.AzureStorageIdentificationStructure(fileName); - //if (azureStorageIdentificationStructure.IsAzureStorageObject()) - //{ - // return AzureStorageIO.GetBlobStreamFromBlob(azureStorageIdentificationStructure) as Stream; - //} - if (fileNameLower.StartsWith("http:") || fileNameLower.StartsWith("https:")) { AzureStorageIO azureStorage = new AzureStorageIO(); @@ -8816,16 +8625,12 @@ private static Stream Open(string fileName, bool append, bool breakChunksAtLines int cIndex = fileName.LastIndexOf(':'); if (cIndex > 0) { - // does not really need to exist... - //if (File.Exists(fileName.Substring(0, cIndex))) - //{ if (cIndex > 1 || fileName.IndexOfAny(_pathSeparators, 2) < 0) { // named: return new NamedStream(fileName, true, append); } - //} } Stream s = null; @@ -8898,10 +8703,6 @@ private static Stream Open(string fileName, bool append, bool breakChunksAtLines // without a buffer, this is even worse... s = new BufferedStream(s, 32 * 1024); } - //if (s == null) - //{ - // throw new IOException("Cannot open file '" + fileName + "'"); - //} } break; @@ -8933,29 +8734,6 @@ private static Stream Open(string fileName, bool append, bool breakChunksAtLines } break; - // Doesn't work currently in 7-zip: - //case ".zip": - // { - // if (append) throw new ArgumentException("Cannot append to a zip stream.", "append"); - // try - // { - // s = new Z7zEncodeStream(fileName, "zip"); - // } - // catch (InvalidOperationException) - // { - // throw new InvalidOperationException("ZStreamOut requires 7za.exe to be in the " + - // "path for " + ext + " compression. " + - // "See http://7-zip.org"); - // } - // catch (System.ComponentModel.Win32Exception) - // { - // throw new InvalidOperationException("ZStreamOut requires 7za.exe to be in the " + - // "path for " + ext + " compression. " + - // "See http://7-zip.org"); - // } - // } - // break; - #if ENABLE_LZMA case ".lzma": { @@ -9007,12 +8785,8 @@ private static Stream Open(string fileName, bool append, bool breakChunksAtLines if (seg > 0) { archPath = zfileName.Substring(0, seg + ext.Length); - //if (File.Exists(archPath)) - //{ inArch = zfileName.Substring(seg + ext.Length).Trim('/', '\\'); break; - //} - //archPath = null; } } if (archPath != null) @@ -9509,25 +9283,6 @@ public static implicit operator Var(DateTime d) return new Var(d.ToString()); } - // array conversions might be nice... *** - - ///// - ///// Convert an array of values. - ///// - ///// the values to convert - ///// the converted array - ///// - ///// This is not very good as an implicit cast, because it can be expensive - ///// and it causes heap allocation. However, it is allowed for convenience. - ///// - //public static implicit operator string[](VarList vals) - //{ - // if (vals == null) return null; - // string[] res = new string[vals.Length]; - // vals.CopyTo(res, 0); - // return res; - //} - /// /// Convert an array of values. /// @@ -9835,24 +9590,6 @@ private static double ParseDouble(string s) } } - //private bool IsInteger() - //{ - // if (raw == null || raw.Length == 0) return false; - // int i = 0; - // if (raw[0] == '-') - // { - // - // } - // bool first = true; - // foreach (char c in raw) - // { - // first = false; - // } - //} - //private bool IsNumerical() - //{ - //} - /// Returns the hash code for this value. /// A 32-bit signed integer hash code. public override int GetHashCode() diff --git a/src/Microsoft.ML.InternalStreams/LowFragmentationStream.cs b/src/Microsoft.ML.InternalStreams/LowFragmentationStream.cs index 11a49bfced..4dd0603e06 100644 --- a/src/Microsoft.ML.InternalStreams/LowFragmentationStream.cs +++ b/src/Microsoft.ML.InternalStreams/LowFragmentationStream.cs @@ -570,9 +570,6 @@ public override void SetLength(long value) /// the maximum number of bytes to write public override void Write(byte[] buffer, int offset, int count) { - // what if it fails? - //try - //{ long next = Position + count; if (_length > 0 && next >= _length) { @@ -585,11 +582,6 @@ public override void Write(byte[] buffer, int offset, int count) _position = -1; base.Write(buffer, offset, count); _position = next; - //} - //catch - //{ - // position = base.Position; - //} } /// @@ -598,9 +590,6 @@ public override void Write(byte[] buffer, int offset, int count) /// the byte to write public override void WriteByte(byte value) { - // what if it fails? - //try - //{ long next = Position + 1; if (_length > 0 && next >= _length) { @@ -613,11 +602,6 @@ public override void WriteByte(byte value) _position = -1; base.WriteByte(value); _position = next; - //} - //catch - //{ - // position = base.Position; - //} } /// @@ -688,11 +672,6 @@ public override bool CanRead } } - // /// - // /// Get whether the stream can seek - true, but it really cannot. - // /// (CanSeek is stupidly checked by the FileStream Position property - // /// on get, not just set.) - // /// /// /// Get whether the stream can seek. /// @@ -700,8 +679,6 @@ public override bool CanSeek { get { - // do we need to force this to true, or can we rely on it? - //return true; return base.CanSeek; } } @@ -713,8 +690,6 @@ public override bool CanWrite { get { - // pass through... - //return true; return base.CanWrite; } } @@ -739,15 +714,12 @@ public override long Position { get { - // the efficiency concern is over VerifyOSHandlePosition when the handle is exposed... - //return base.Position; if (_position < 0) _position = base.Position; return _position; } set { - //throw new NotSupportedException("LowFragmentationStream cannot seek"); Seek(value); } } diff --git a/src/Microsoft.ML.InternalStreams/TableIO.cs b/src/Microsoft.ML.InternalStreams/TableIO.cs index 7341cc4a96..64eaa1fd22 100644 --- a/src/Microsoft.ML.InternalStreams/TableIO.cs +++ b/src/Microsoft.ML.InternalStreams/TableIO.cs @@ -3485,12 +3485,6 @@ private string columnName(int col) /// The row to write as an array of fields public void WriteRow(object[] fields) { - // widen if needed: - //while (fields.Length > dataTable.Columns.Count) - //{ - // DataColumn col = new DataColumn(); - // dataTable.Columns.Add(col); - //} OleDbCommand oleCmd = new OleDbCommand(); oleCmd.Connection = oleConn; string cmd = "INSERT INTO " + "[Sheet1$] "; // + "(FirstName, LastName) "; @@ -3527,22 +3521,6 @@ public void WriteRow(object[] fields) { throw new Exception("Excel insert error."); } - - //if (oleAdapter == null) return; - // widen if needed: - //while (fields.Length > dataTable.Columns.Count) - //{ - // DataColumn col = new DataColumn(); - // dataTable.Columns.Add(col); - //} - //dataTable.LoadDataRow(fields, false); - //for (int i = 0; i < fields.Length; i++) - //{ - // if (fields[i] == DBNull.Value) - // { - // fields[i] = null; - // } - //} } /// @@ -3560,11 +3538,6 @@ public int RowLength() /// public void Close() { - //if (oleAdapter != null) - //{ - // oleAdapter.Update(dataTable); - // oleAdapter = null; - //} if (oleConn != null) { oleConn.Close(); diff --git a/src/Microsoft.ML.InternalStreams/UnbufferedStream.cs b/src/Microsoft.ML.InternalStreams/UnbufferedStream.cs index 23442d8475..5826897e08 100644 --- a/src/Microsoft.ML.InternalStreams/UnbufferedStream.cs +++ b/src/Microsoft.ML.InternalStreams/UnbufferedStream.cs @@ -212,11 +212,6 @@ protected override void Dispose(bool disposing) private void ReleaseBuffer() { - //if (checkStream != null) - //{ - // checkStream.Close(); - // checkStream = null; - //} if (!_released) { if (_buffer != null) @@ -249,10 +244,7 @@ private void ReleaseBuffer() if (_buffer[j] != IntPtr.Zero) { - //Console.Write("(freeing... "); bool ret = IOUtil.Win32.VirtualFree(_buffer[j], IntPtr.Zero, /*MEM_DECOMMIT*/ IOUtil.Win32.FreeType.MEM_RELEASE); - //int count = Interlocked.Decrement(ref allocCount); - //Console.WriteLine("done: " + ret + " " + count + ")"); _buffer[j] = IntPtr.Zero; } } @@ -271,14 +263,6 @@ private void ReleaseBuffer() } if (_fillThread != null) { - // skip this - we close the handle - //try - //{ - // fillThread.Abort(); - //} - //catch - //{ - //} _fillThread = null; } } @@ -319,7 +303,6 @@ private void Reopen(long startPosition, bool allocate) } _fillThread = null; } - //if (allocate) ReleaseBuffer(); long startRemainder = (startPosition % _sectorSize); try { @@ -364,15 +347,6 @@ private void Reopen(long startPosition, bool allocate) if (_blockSize < _sectorSize) _blockSize = (int)_sectorSize; - // check on buffer alignment: - //mBuffer = new byte[blockSize + 16]; - //fixed (byte* mb = mBuffer) - //{ - // IntPtr tmp = new IntPtr(mb); - // Console.WriteLine(tmp.ToInt64()); - // Console.WriteLine((tmp.ToInt64() + 7) & ~7); - //} - _readMutex = new AutoResetEvent[parallelLevel]; _pullMutex = new AutoResetEvent[parallelLevel]; _readSize = new int[parallelLevel]; @@ -386,7 +360,6 @@ private void Reopen(long startPosition, bool allocate) completed = true; for (int i = 0; i < _buffer.Length; i++) { - //if (buffer[i] != IntPtr.Zero) throw new VirtualAllocException("Buffer double allocated"); _buffer[i] = IntPtr.Zero; #if REUSE_BUFFERS @@ -407,20 +380,7 @@ private void Reopen(long startPosition, bool allocate) if (_buffer[i] == IntPtr.Zero) { - //Console.WriteLine("allocated: " + - // (System.Diagnostics.Process.GetCurrentProcess().VirtualMemorySize / 1024 / 1024) + " MB"); - //Console.Write("(allocating... "); _buffer[i] = IOUtil.Win32.VirtualAlloc(IntPtr.Zero, new IntPtr(_blockSize + 8), IOUtil.Win32.AllocationType.MEM_RESERVE | IOUtil.Win32.AllocationType.MEM_COMMIT, IOUtil.Win32.Protect.PAGE_READWRITE); - //if (buffer[i] == IntPtr.Zero || buffer[i].ToInt64() < 0) - //{ - // Console.WriteLine("failed @" + (blockSize + 8) + ": " + allocCount + ")"); - // Console.ReadLine(); - //} - //else - //{ - // int acount = Interlocked.Increment(ref allocCount); - // Console.WriteLine("done: " + acount + ")"); - //} } if (_buffer[i] == IntPtr.Zero || _buffer[i].ToInt64() < 0) @@ -451,10 +411,7 @@ private void Reopen(long startPosition, bool allocate) if (_buffer[j] != IntPtr.Zero) { - //Console.Write("(freeing... "); bool ret = IOUtil.Win32.VirtualFree(_buffer[j], IntPtr.Zero, /*MEM_DECOMMIT*/ IOUtil.Win32.FreeType.MEM_RELEASE); - //int count = Interlocked.Decrement(ref allocCount); - //Console.WriteLine("done: " + ret + " " + count + ")"); _buffer[j] = IntPtr.Zero; } } @@ -469,8 +426,6 @@ private void Reopen(long startPosition, bool allocate) } // try reducing the buffer size: _blockSize = _blockSize / 2; - // collect to compact memory: - //GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); // try again: @@ -496,7 +451,6 @@ private void Reopen(long startPosition, bool allocate) } _alignedLimit = _length - (_unalignedTail == null ? 0 : _unalignedTail.Length); - //if (alignedLimit < 0) alignedLimit = 0; if (_parallel) { _fillThread = Utils.CreateBackgroundThread(FillBuffer); @@ -507,7 +461,6 @@ private void Reopen(long startPosition, bool allocate) if (startRemainder != 0) { - //Seek(startRemainder, SeekOrigin.Current); Skip(startRemainder); } } @@ -519,11 +472,7 @@ private void Reopen(long startPosition, bool allocate) { try { - // don't bother trying to save them? - //Console.Write("(freeing... "); bool ret = IOUtil.Win32.VirtualFree(_buffer[j], IntPtr.Zero, /*MEM_DECOMMIT*/ IOUtil.Win32.FreeType.MEM_RELEASE); - //int count = Interlocked.Decrement(ref allocCount); - //Console.WriteLine("done: " + ret + " " + count + ")"); _buffer[j] = IntPtr.Zero; } catch @@ -742,11 +691,9 @@ public override bool CanSeek private int _bufferGetIndex; private int _fillCount; private Thread _fillThread; - private /*readonly*/ AutoResetEvent[] _readMutex; - private /*readonly*/ AutoResetEvent[] _pullMutex; - //private readonly AutoResetEvent[] readFillMutex; - //private readonly AutoResetEvent[] readGetMutex; - private /*readonly*/ int[] _readSize; + private AutoResetEvent[] _readMutex; + private AutoResetEvent[] _pullMutex; + private int[] _readSize; private byte[] _unalignedTail; /// @@ -850,13 +797,11 @@ private void FillBuffer() private bool FillBufferPass() { - //Console.WriteLine("[" + totalRead + " / " + alignedLimit + " / " + length + "]"); // read the aligned amount only, specified by 'alignedLimit': if (_totalRead < _alignedLimit) { int curBlock = (int)Math.Min(_blockSize, _alignedLimit - _totalRead); - - //Console.WriteLine(" " + totalRead + " -> " + (totalRead + curBlock) + " / " + alignedLimit); + // wait for pull of last read: #if MONITOR Console.WriteLine(":: " + Path.GetFileName(fileName) + " :: " + "".PadLeft(4) + @@ -873,12 +818,8 @@ private bool FillBufferPass() { // read into buffer: int readBytes; - //long t = DateTime.UtcNow.Ticks; - //long startPos = IOUtil.Win32.Raw.GetFilePos(handle); bool readres = IOUtil.Win32.Raw.ReadFile(_handle, _alignedBuffer[_bufferFillIndex], (int)Math.Min(_blockSize, _alignedLimit - _totalRead), out readBytes); - //t = DateTime.UtcNow.Ticks - t; - //Console.WriteLine("core time: " + (t / (double)TimeSpan.TicksPerSecond).ToString("0.000")); if (!readres) { // try again?? *** @@ -927,14 +868,12 @@ private bool FillBufferPass() if (_totalRead < _alignedLimit || (_unalignedTail != null && _unalignedTail.Length != 0)) { - //Console.WriteLine("continue..."); _readMutex[_bufferFillIndex].Set(); _bufferFillIndex = (_bufferFillIndex + 1) % _readMutex.Length; return true; } else { - //Console.WriteLine(" totalRead: " + totalRead + " alignedLimit: " + alignedLimit); _done[_bufferFillIndex] = true; _readMutex[_bufferFillIndex].Set(); _bufferFillIndex = (_bufferFillIndex + 1) % _readMutex.Length; @@ -949,11 +888,6 @@ private bool FillBufferPass() } else if (_alignedLimit == _totalRead && _unalignedTail != null && _unalignedTail.Length != 0) { - //Console.Error.WriteLine(); - //Console.Error.WriteLine("Unaligned tail: " + alignedLimit + " + " + unalignedTail.Length + " => " + length); - - // wait for pull of last read: - //Console.WriteLine(" fill get : " + fillCount + " [" + bufferFillIndex + "]"); _pullMutex[_bufferFillIndex].WaitOne(); try @@ -963,7 +897,6 @@ private bool FillBufferPass() _totalRead += _unalignedTail.Length; CopyBuffer(_unalignedTail, _alignedBuffer[_bufferFillIndex], _unalignedTail.Length); _readSize[_bufferFillIndex] = _unalignedTail.Length; - //Console.WriteLine(" tail done"); _done[_bufferFillIndex] = true; } catch (Exception ex) @@ -971,9 +904,6 @@ private bool FillBufferPass() _pullMutex[_bufferFillIndex].Set(); throw ex; } - - //fillCount++; - //Console.WriteLine(" fill set : " + fillCount + " [" + bufferFillIndex + "]" + " *"); _readMutex[_bufferFillIndex].Set(); _bufferFillIndex = (_bufferFillIndex + 1) % _readMutex.Length; @@ -981,13 +911,8 @@ private bool FillBufferPass() } else { - //Console.Error.WriteLine(); - //Console.Error.WriteLine("No unaligned tail."); - _readSize[_bufferFillIndex] = 0; - //Console.WriteLine(" null done?"); _done[_bufferFillIndex] = true; - //Console.WriteLine(" fill set : " + fillCount + " [" + bufferFillIndex + "]" + " **"); _readMutex[_bufferFillIndex].Set(); _bufferFillIndex = (_bufferFillIndex + 1) % _readMutex.Length; return false; @@ -1003,17 +928,11 @@ private void GetBuffer() { if (_gotDone) { - //Console.WriteLine("GetBuffer done"); _currentBufferBottom = 0; _currentBufferLimit = 0; _currentBuffer = _alignedBuffer[0]; - //unsafe - //{ - // current = (byte*)currentBuffer.ToPointer(); - //} return; } - // Console.WriteLine("Copying buffer " + bufferIndex); IntPtr res = _alignedBuffer[_bufferGetIndex]; if (_getCount != 0) @@ -1044,7 +963,6 @@ private void GetBuffer() _currentBufferLimit = _readSize[_bufferGetIndex]; if (_done[_bufferGetIndex] || _currentBufferLimit == 0) { - //Console.WriteLine(" done : " + done[bufferGetIndex] + " " + currentBufferLimit); for (int i = 0; i < _readSize.Length; i++) { _readSize[i] = 0; @@ -1061,13 +979,9 @@ private void GetBuffer() _bufferGetIndex = (_bufferGetIndex + 1) % _readMutex.Length; - //unsafe - //{ - // current = (byte*)currentBuffer.ToPointer(); - //} } - private /*unsafe*/ void CopyBuffer(byte[] source, IntPtr dest, int count) + private void CopyBuffer(byte[] source, IntPtr dest, int count) { if (count <= 0) return; @@ -1083,11 +997,9 @@ private void GetBuffer() } } #endregion - - //// Marshal is incredibly slow!! - //Marshal.Copy(source, 0, dest, count); + } - private /*unsafe*/ void CopyBuffer(IntPtr source, int sourceStart, byte[] dest, int destStart, int count) + private void CopyBuffer(IntPtr source, int sourceStart, byte[] dest, int destStart, int count) { if (count <= 0) return; @@ -1139,8 +1051,6 @@ private void GetBuffer() } #endregion - //// Marshal is slow ?? - // Marshal.Copy(new IntPtr(source.ToInt64() + sourceStart), dest, destStart, count); } private IntPtr _currentBuffer; @@ -1171,7 +1081,6 @@ public int Read(out byte[] buffer) _sharedBuffer = new byte[_blockSize]; } buffer = _sharedBuffer; - //Console.WriteLine("(reading out into buffer[" + buffer.Length + "])"); int res = Read(buffer); // should we resize the array here? it might be convenient... if (res == 0) @@ -1195,6 +1104,7 @@ public int Read(byte[] buffer) { return Read(buffer, 0, buffer.Length); } + /// /// Reads a block of bytes from the stream and writes the data in a given buffer. /// @@ -1210,17 +1120,13 @@ public int Read(byte[] buffer) /// The buffer is null public override int Read(byte[] buffer, int offset, int count) { - //Console.WriteLine("Read into buffer.Length = " + buffer.Length + ", offset = " + offset + ", count = " + count); #if !OLD_DIRECT_READ Contracts.CheckValue(buffer, nameof(buffer)); Contracts.CheckParam(0 <= offset && offset <= buffer.Length, nameof(offset)); Contracts.CheckParam(offset <= offset + count && offset + count <= buffer.Length, nameof(count)); - - //Console.WriteLine("(reading " + offset + " , " + count + ")"); + int read = 0; - //unsafe - //{ while (count > 0) { if (_currentBufferLimit == _currentBufferBottom) @@ -1233,7 +1139,6 @@ public override int Read(byte[] buffer, int offset, int count) { CopyBuffer(_currentBuffer, _currentBufferBottom, buffer, offset, count); _currentBufferBottom += count; - //current += count; read += count; count = 0; break; @@ -1248,40 +1153,12 @@ public override int Read(byte[] buffer, int offset, int count) _currentBufferBottom = _currentBufferLimit; } } - //} - - //if (checkStream == null) - //{ - // checkStream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read); - //} - //byte[] cbuf = new byte[count]; - //int checkRead = checkStream.Read(cbuf, 0, cbuf.Length); - //if (checkRead != read) - //{ - // Console.WriteLine("!!! bytes read mismatch at " + fileName + ": " + checkStream.Position + " / " + Position + ": " + - // "read = " + read + "; checkRead = " + checkRead); - //} - //else - //{ - // Console.WriteLine(">>> bytes read match at " + fileName + ": " + checkStream.Position + " / " + Position + ": " + - // "read = " + read + "; checkRead = " + checkRead); - //} return read; #else - //if (done) return 0; if (overRead != null) return overRead.Read(buffer, offset, count); if (Position + count <= length) { -//////// return base.Read(buffer, offset, count); - - //int readBytes; - //bool readres; - //fixed (byte* alignedBuffer = mBuffer) - //{ - // readres = ReadFile(handle, new IntPtr(alignedBuffer), blockSize, out readBytes, IntPtr.Zero); - //} - //return readBytes; #if SYNCHRONOUS int readBytes; @@ -1407,22 +1284,10 @@ public override int Read(byte[] buffer, int offset, int count) overRead.Seek(pos, SeekOrigin.Begin); return first + Read(buffer, offset, count); } - //Console.WriteLine(" req: " + res); - if (res == 0) return 0; - res = Math.Min(res, Read(buffer, offset, res)); - //Console.WriteLine(" read: " + res); - //if (resized) - //{ - // ResizeFile(fileName, length); - //} return Math.Min(res, remaining); #endif } - //private FileStream checkStream = null; - - //private unsafe byte* current = null; - /// /// Read a block of bytes from the stream and advance the position. /// @@ -1437,13 +1302,6 @@ public unsafe int Read(out byte* buffer) if (_currentBufferLimit == _currentBufferBottom) { GetBuffer(); - //if (gotDone) - //{ - // //Console.WriteLine("(gotDone : currentBufferBottom = " + currentBufferBottom + - // // ", currentBufferLimit = " + currentBufferLimit + ")"); - // return 0; - //} - //if (currentBufferLimit == 0) return 0; } buffer = ((byte*)_currentBuffer.ToPointer()) + _currentBufferBottom; int count = _currentBufferLimit - _currentBufferBottom; @@ -1451,12 +1309,6 @@ public unsafe int Read(out byte* buffer) return count; } - // TODO: make ReadByte() fast! *** - - // private byte[] readByteBuffer = new byte[512]; - // private int readByteBufferPos = 0; - // private int readByteBufferCount = 0; - /// /// Retrieve the next byte in the stream and advance the position. /// @@ -1464,7 +1316,6 @@ public unsafe int Read(out byte* buffer) /// This is not as efficient as block reading, because of overhead issues. public override int ReadByte() { - //checkStream.ReadByte(); // check?? if (_currentBufferLimit == _currentBufferBottom) { GetBuffer(); @@ -1476,8 +1327,6 @@ public override int ReadByte() { return *(((byte*)_currentBuffer.ToPointer()) + _currentBufferBottom++); } - //// Marshal is incredibly slow!! - //return Marshal.ReadByte(currentBuffer, currentBufferBottom++); } /// @@ -1497,8 +1346,6 @@ public int Peek() { return *(((byte*)_currentBuffer.ToPointer()) + _currentBufferBottom); } - //// Marshal is incredibly slow!! - //return Marshal.ReadByte(currentBuffer, currentBufferBottom); } /// @@ -1509,7 +1356,6 @@ public bool Eof() { if (_currentBufferBottom < _currentBufferLimit) return false; - //if (gotDone) return true; // is this good enough? We want to be cheap... return (Position < Length); } diff --git a/src/Microsoft.ML.StandardLearners/Optimizer/OptimizationMonitor.cs b/src/Microsoft.ML.StandardLearners/Optimizer/OptimizationMonitor.cs index e97216e854..2f4d7974f4 100644 --- a/src/Microsoft.ML.StandardLearners/Optimizer/OptimizationMonitor.cs +++ b/src/Microsoft.ML.StandardLearners/Optimizer/OptimizationMonitor.cs @@ -169,10 +169,6 @@ public Float Tolerance public bool Terminate(Optimizer.OptimizerState state, out string message) { _unnormMeanImprovement = (state.LastValue - state.Value) + _lambda * _unnormMeanImprovement; - //if (state.Iter < 5) { - // message = "wait for 5..."; - // return false; - //} Float crit = _unnormMeanImprovement * (1 - _lambda) / (1 - MathUtils.Pow(_lambda, state.Iter)); message = string.Format("{0:0.000e0}", crit); diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index de0db6f3a9..15d81da88e 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -201,55 +201,6 @@ public void EntryPointCaching() Done(); } - //[Fact] - //public void EntryPointSchemaManipulation() - //{ - // var dv1_data = new[] - // { - // new Dv1 { Col1 = 11, Col2 = 21, Col3 = 31, Col4 = 41 }, - // new Dv1 { Col1 = 12, Col2 = 22, Col3 = 32, Col4 = 42 }, - // new Dv1 { Col1 = 13, Col2 = 23, Col3 = 33, Col4 = 43 }, - // }; - // var dv1 = Env.CreateDataView(dv1_data); - - // var concatOut = SchemaManipulation.ConcatColumns(Env, - // new ConcatTransform.Arguments { Column = new[] { ConcatTransform.Column.Parse("ColA:Col1,Col2") }, Data = dv1 }); - - // var postConcatDv = Env.CreateTransform("Concat{col=ColA:Col1,Col2}", dv1); - // CheckSameValues(concatOut.OutputData, postConcatDv); - - // var copyOut = SchemaManipulation.CopyColumns(Env, - // new CopyColumnsTransform.Arguments - // { - // Column = new[] { CopyColumnsTransform.Column.Parse("ColB:Col3"), CopyColumnsTransform.Column.Parse("ColC:Col4") }, - // Data = concatOut.OutputData, - // }); - - // var postCopyDv = Env.CreateTransform("Copy{col=ColB:Col3 col=ColC:Col4}", postConcatDv); - // CheckSameValues(copyOut.OutputData, postCopyDv); - - // var dropOut = SchemaManipulation.DropColumns(Env, - // new DropColumnsTransform.Arguments { Column = new[] { "Col1", "Col2", "Col3", "Col4" }, Data = copyOut.OutputData }); - - // var postDropDv = Env.CreateTransform("Drop{col=Col1 col=Col2 col=Col3 col=Col4}", postCopyDv); - // CheckSameValues(dropOut.OutputData, postDropDv); - - // var selectOut = SchemaManipulation.SelectColumns(Env, - // new DropColumnsTransform.KeepArguments { Column = new[] { "ColA", "ColB" }, Data = dropOut.OutputData }); - - // var postSelectDv = Env.CreateTransform("Keep{col=ColA col=ColB}", postDropDv); - - // CheckSameValues(selectOut.OutputData, postSelectDv); - - // var combinedModel = ModelOperations.CombineTransformModels(Env, - // new ModelOperations.CombineTransformModelsInput - // { - // Models = new[] { concatOut.Model, copyOut.Model, dropOut.Model, selectOut.Model } - // }).OutputModel; - // CheckSameValues(selectOut.OutputData, combinedModel.Apply(Env, dv1)); - // Done(); - //} - /// /// Helper function to get the type of build being used. /// @@ -600,63 +551,6 @@ public void EntryPointExecGraphCommand() cmd.Run(); } - //[Fact] - //public void EntryPointArrayOfVariables() - //{ - // string inputGraph = @" - // { - // ""Nodes"": [ - // { - // ""Name"": ""SchemaManipulation.ConcatColumns"", - // ""Inputs"": { - // ""Data"": ""$data1"", - // ""Column"": [{""Name"":""ColA"", ""Source"":[""Col1"", ""Col2""]}] - // }, - // ""Outputs"": { - // ""Model"": ""$model1"", - // ""OutputData"": ""$data2"" - // } - // }, - // { - // ""Name"": ""SchemaManipulation.CopyColumns"", - // ""Inputs"": { - // ""Data"": ""$data2"", - // ""Column"": [{""Name"":""ColB"", ""Source"":""Col3""}, {""Name"":""ColC"", ""Source"":""Col4""}] - // }, - // ""Outputs"": { - // ""Model"": ""$model2"", - // ""OutputData"": ""$data3"" - // } - // }, - // { - // ""Name"": ""ModelOperations.CombineTransformModels"", - // ""Inputs"": { - // Models: [""$model1"", ""$model2""] - // }, - // ""Outputs"": { - // OutputModel: ""$model3"" - // } - // } - // ] - // }"; - - // JObject graph = JObject.Parse(inputGraph); - // var catalog = ModuleCatalog.CreateInstance(Env); - // var runner = new GraphRunner(Env, catalog, graph[FieldNames.Nodes] as JArray); - - // var dv1_data = new[] - // { - // new Dv1 { Col1 = 11, Col2 = 21, Col3 = 31, Col4 = 41 }, - // new Dv1 { Col1 = 12, Col2 = 22, Col3 = 32, Col4 = 42 }, - // new Dv1 { Col1 = 13, Col2 = 23, Col3 = 33, Col4 = 43 }, - // }; - // var dv1 = Env.CreateDataView(dv1_data); - // runner.SetInput("data1", dv1); - // runner.RunAll(); - // var model = runner.GetOutput("model3"); - // Assert.NotNull(model); - //} - [Fact] public void EntryPointCalibrate() { From b635e938496ffc64a882b7c9cb5f7aefea001140 Mon Sep 17 00:00:00 2001 From: Daniel Drews Date: Tue, 10 Jul 2018 18:24:25 -0500 Subject: [PATCH 2/6] Round #2 --- src/Microsoft.ML.CpuMath/Thunk.cs | 244 +++++++++--------- .../DataLoadSave/Text/TextLoaderParser.cs | 6 +- .../Application/LogLossApplication.cs | 11 +- .../Application/WinLossSurplusApplication.cs | 1 - .../Dataset/FeatureFlock.cs | 30 +-- .../Dataset/SegmentIntArray.cs | 3 - .../Dataset/SparseIntArray.cs | 5 - src/Microsoft.ML.FastTree/FastTreeRanking.cs | 2 - .../Training/ScoreTracker.cs | 2 +- .../InternalStreams.cs | 172 +++--------- .../LowFragmentationStream.cs | 7 +- src/Microsoft.ML.InternalStreams/TableIO.cs | 151 +---------- .../UnbufferedStream.cs | 65 +---- .../FactorizationMachineInterface.cs | 6 +- .../Optimizer/Optimizer.cs | 2 - .../Algorithms/SmacSweeper.cs | 10 - .../UnitTests/TestEntryPoints.cs | 4 +- .../TestPredictors.cs | 2 +- .../BaseTestBaseline.cs | 4 +- test/Microsoft.ML.TestFramework/Learners.cs | 5 +- .../TestInitialization.cs | 8 - 21 files changed, 182 insertions(+), 558 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/Thunk.cs b/src/Microsoft.ML.CpuMath/Thunk.cs index bc23963bbe..b2df7760f6 100644 --- a/src/Microsoft.ML.CpuMath/Thunk.cs +++ b/src/Microsoft.ML.CpuMath/Thunk.cs @@ -17,155 +17,155 @@ internal unsafe static class Thunk public static extern bool ChkAvx(); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulA(bool add, /*const*/ float* pmat, /*const*/ float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulA(bool add, float* pmat, float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulX(bool add, /*const*/ float* pmat, /*const*/ float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulX(bool add, float* pmat, float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulPA(bool add, /*const*/ float* pmat, /*const*/ int* pposSrc, /*const*/ float* psrc, + public static extern void MatMulPA(bool add, float* pmat, int* pposSrc, float* psrc, int posMin, int iposMin, int iposLim, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulPX(bool add, /*const*/ float* pmat, /*const*/ int* pposSrc, /*const*/ float* psrc, + public static extern void MatMulPX(bool add, float* pmat, int* pposSrc, float* psrc, int posMin, int iposMin, int iposLim, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulRU(bool add, /*const*/ int* pstarts, /*const*/ int* pindices, /*const*/ float* pcoefs, - /*const*/ float* psrc, float* pdst, int crow); + public static extern void MatMulRU(bool add, int* pstarts, int* pindices, float* pcoefs, + float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulRX(bool add, /*const*/ int* pstarts, /*const*/ int* pindices, /*const*/ float* pcoefs, - /*const*/ float* psrc, float* pdst, int crow); + public static extern void MatMulRX(bool add, int* pstarts, int* pindices, float* pcoefs, + float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulCU(bool add, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, - /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow); + public static extern void MatMulCU(bool add, int* pmprowiv, int* pmprowcol, + int* pruns, float* pcoefs, float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulDU(bool add, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, /*const*/ int* pmprowrun, - /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow); + public static extern void MatMulDU(bool add, int* pmprowiv, int* pmprowcol, int* pmprowrun, + int* pruns, float* pcoefs, float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulCX(bool add, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, - /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow); + public static extern void MatMulCX(bool add, int* pmprowiv, int* pmprowcol, + int* pruns, float* pcoefs, float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulDX(bool add, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, /*const*/ int* pmprowrun, - /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow); + public static extern void MatMulDX(bool add, int* pmprowiv, int* pmprowcol, int* pmprowrun, + int* pruns, float* pcoefs, float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MeanU(bool add, /*const*/ int* pmprowcol, /*const*/ int* pmprowindices, /*const*/ int* pindices, - /*const*/ float* psrc, float* pdst, int crow); + public static extern void MeanU(bool add, int* pmprowcol, int* pmprowindices, int* pindices, + float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MaxU(bool add, /*const*/ int* pmprowcol, /*const*/ int* pmprowindices, /*const*/ int* pindices, - /*const*/ float* psrc, float* pdst, int crow); + public static extern void MaxU(bool add, int* pmprowcol, int* pmprowindices, int* pindices, + float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void RespNormU(bool add, float alpha, float beta, bool avgOverFullKernel, float offset, - /*const*/ int* pmprowcol, /*const*/ int* pmprowindices, /*const*/ int* pindices, - /*const*/ float* psrc, float* pdst, int crow); + int* pmprowcol, int* pmprowindices, int* pindices, + float* psrc, float* pdst, int crow); // These treat pmat as if it is stored in column-major order. Thus, crow and ccol are the numbers of rows // and columns from that perspective. Alternatively, crow is the number of rows in the transpose of pmat // (thought of as row-major order). [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranA(bool add, /*const*/ float* pmat, /*const*/ float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranA(bool add, float* pmat, float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranX(bool add, /*const*/ float* pmat, /*const*/ float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranX(bool add, float* pmat, float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranPA(bool add, /*const*/ float* pmat, /*const*/ int* pposSrc, /*const*/ float* psrc, + public static extern void MatMulTranPA(bool add, float* pmat, int* pposSrc, float* psrc, int posMin, int iposMin, int iposLim, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranPX(bool add, /*const*/ float* pmat, /*const*/ int* pposSrc, /*const*/ float* psrc, + public static extern void MatMulTranPX(bool add, float* pmat, int* pposSrc, float* psrc, int posMin, int iposMin, int iposLim, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranRU(bool add, /*const*/ int* pstarts, /*const*/ int* pindices, /*const*/ float* pcoefs, - /*const*/ float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranRU(bool add, int* pstarts, int* pindices, float* pcoefs, + float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranRX(bool add, /*const*/ int* pstarts, /*const*/ int* pindices, /*const*/ float* pcoefs, - /*const*/ float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranRX(bool add, int* pstarts, int* pindices, float* pcoefs, + float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranCU(bool add, /*const*/ int* pmpcoliv, /*const*/ int* pmpcolrow, - /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranCU(bool add, int* pmpcoliv, int* pmpcolrow, + int* pruns, float* pcoefs, float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranDU(bool add, /*const*/ int* pmpcoliv, /*const*/ int* pmpcolrow, /*const*/ int* pmpcolrun, - /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranDU(bool add, int* pmpcoliv, int* pmpcolrow, int* pmpcolrun, + int* pruns, float* pcoefs, float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranCX(bool add, /*const*/ int* pmpcoliv, /*const*/ int* pmpcolrow, - /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranCX(bool add, int* pmpcoliv, int* pmpcolrow, + int* pruns, float* pcoefs, float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranDX(bool add, /*const*/ int* pmpcoliv, /*const*/ int* pmpcolrow, /*const*/ int* pmpcolrun, - /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranDX(bool add, int* pmpcoliv, int* pmpcolrow, int* pmpcolrun, + int* pruns, float* pcoefs, float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MeanBackU(bool add, /*const*/ int* pmpcolrow, /*const*/ int* pmpcolindices, /*const*/ int* pindices, - /*const*/ float* psrc, float* pdst, int crow, int ccol); + public static extern void MeanBackU(bool add, int* pmpcolrow, int* pmpcolindices, int* pindices, + float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MaxBackU(bool add, /*const*/ int* pmpcolrow, /*const*/ int* pmpcolindices, /*const*/ int* pindices, - /*const*/ float* psrc, float* pdst, /*const*/ float* pval, int crow, int ccol); + public static extern void MaxBackU(bool add, int* pmpcolrow, int* pmpcolindices, int* pindices, + float* psrc, float* pdst, float* pval, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void RespNormBackU(bool add, float alpha, float beta, bool avgOverFullKernel, float offset, - /*const*/ int* pmpcolrow, /*const*/ int* pmpcolindices, /*const*/ int* pindices, - /*const*/ float* perrors, float* perrorsPrev, /*const*/ float* pvaluesPrev, int crow, int ccol); + int* pmpcolrow, int* pmpcolindices, int* pindices, + float* perrors, float* perrorsPrev, float* pvaluesPrev, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranA(float a, /*const*/ float* px, /*const*/ float* py, float* pmat, int crow, int ccol, float decay); + public static extern void AddXYTranA(float a, float* px, float* py, float* pmat, int crow, int ccol, float decay); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranX(float a, /*const*/ float* px, /*const*/ float* py, float* pmat, int crow, int ccol, float decay); + public static extern void AddXYTranX(float a, float* px, float* py, float* pmat, int crow, int ccol, float decay); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranPA(float a, /*const*/ float* px, /*const*/ int* pposY, /*const*/ float* pvaluesY, + public static extern void AddXYTranPA(float a, float* px, int* pposY, float* pvaluesY, int posMinY, int iposMinY, int iposLimY, float* pmat, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranPX(float a, /*const*/ float* px, /*const*/ int* pposY, /*const*/ float* pvaluesY, + public static extern void AddXYTranPX(float a, float* px, int* pposY, float* pvaluesY, int posMinY, int iposMinY, int iposLimY, float* pmat, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranRU(float a, /*const*/ float* px, /*const*/ float* py, - /*const*/ int* pstarts, /*const*/ int* pindices, float* pcoefs, int crow, float decay); + public static extern void AddXYTranRU(float a, float* px, float* py, + int* pstarts, int* pindices, float* pcoefs, int crow, float decay); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranRX(float a, /*const*/ float* px, /*const*/ float* py, - /*const*/ int* pstarts, /*const*/ int* pindices, float* pcoefs, int crow, float decay); + public static extern void AddXYTranRX(float a, float* px, float* py, + int* pstarts, int* pindices, float* pcoefs, int crow, float decay); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranCU(float a, /*const*/ float* px, /*const*/ float* py, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, - /*const*/ int* pruns, float* pcoefs, int crow); + public static extern void AddXYTranCU(float a, float* px, float* py, int* pmprowiv, int* pmprowcol, + int* pruns, float* pcoefs, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranDU(float a, /*const*/ float* px, /*const*/ float* py, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, - /*const*/ int* pmprowrun, /*const*/ int* pruns, float* pcoefs, int crow); + public static extern void AddXYTranDU(float a, float* px, float* py, int* pmprowiv, int* pmprowcol, + int* pmprowrun, int* pruns, float* pcoefs, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranCX(float a, /*const*/ float* px, /*const*/ float* py, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, - /*const*/ int* pruns, float* pcoefs, int crow); + public static extern void AddXYTranCX(float a, float* px, float* py, int* pmprowiv, int* pmprowcol, + int* pruns, float* pcoefs, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranDX(float a, /*const*/ float* px, /*const*/ float* py, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, - /*const*/ int* pmprowrun, /*const*/ int* pruns, float* pcoefs, int crow); + public static extern void AddXYTranDX(float a, float* px, float* py, int* pmprowiv, int* pmprowcol, + int* pmprowrun, int* pruns, float* pcoefs, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranMomA(float a, /*const*/ float* px, /*const*/ float* py, float* pmat, float momentum, float* pdel, int crow, int ccol); + public static extern void AddXYTranMomA(float a, float* px, float* py, float* pmat, float momentum, float* pdel, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranMomX(float a, /*const*/ float* px, /*const*/ float* py, float* pmat, float momentum, float* pdel, int crow, int ccol); + public static extern void AddXYTranMomX(float a, float* px, float* py, float* pmat, float momentum, float* pdel, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranGradA(/*const*/ float* px, /*const*/ float* py, float* pmat, float* paccGrads, float* paccUpdates, + public static extern void AddXYTranGradA(float* px, float* py, float* pmat, float* paccGrads, float* paccUpdates, float decay, float cond, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranGradX(/*const*/ float* px, /*const*/ float* py, float* pmat, float* paccGrads, float* paccUpdates, + public static extern void AddXYTranGradX(float* px, float* py, float* pmat, float* paccGrads, float* paccUpdates, float decay, float cond, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranGradRU(/*const*/ float* px, /*const*/ float* py, /*const*/ int* pstarts, /*const*/ int* pindices, + public static extern void AddXYTranGradRU(float* px, float* py, int* pstarts, int* pindices, float* pcoefs, float* paccGrads, float* paccUpdates, float decay, float cond, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranGradRX(/*const*/ float* px, /*const*/ float* py, /*const*/ int* pstarts, /*const*/ int* pindices, + public static extern void AddXYTranGradRX(float* px, float* py, int* pstarts, int* pindices, float* pcoefs, float* paccGrads, float* paccUpdates, float decay, float cond, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranGradPA(/*const*/ float* px, /*const*/ int* pposY, /*const*/ float* pvaluesY, + public static extern void AddXYTranGradPA(float* px, int* pposY, float* pvaluesY, int posMinY, int iposMinY, int iposLimY, float* pmat, float* paccGrads, float* paccUpdates, float decay, float cond, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranGradPX(/*const*/ float* px, /*const*/ int* pposY, /*const*/ float* pvaluesY, + public static extern void AddXYTranGradPX(float* px, int* pposY, float* pvaluesY, int posMinY, int iposMinY, int iposLimY, float* pmat, float* paccGrads, float* paccUpdates, float decay, float cond, int crow, int ccol); @@ -176,7 +176,7 @@ public static extern void AddXYTranGradPX(/*const*/ float* px, /*const*/ int* pp [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void ScaleX(float a, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ScaleSrcU(float a, /*const*/ float* ps, float* pd, int c); + public static extern void ScaleSrcU(float a, float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void ScaleAddU(float a, float b, float* pd, int c); @@ -187,85 +187,85 @@ public static extern void AddXYTranGradPX(/*const*/ float* px, /*const*/ int* pp [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void ScaleMaxNormTranU(float maxNorm, float* pmat, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ScaleMaxNormRU(float maxNorm, /*const*/ int* pstarts, float* pmat, int crow); + public static extern void ScaleMaxNormRU(float maxNorm, int* pstarts, float* pmat, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void ScaleMaxNormCU(float maxNorm, int kernCount, int kernSize, float* pmat); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleA(float a, /*const*/ float* ps, float* pd, int c); + public static extern void AddScaleA(float a, float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleU(float a, /*const*/ float* ps, float* pd, int c); + public static extern void AddScaleU(float a, float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleX(float a, /*const*/ float* ps, float* pd, int c); + public static extern void AddScaleX(float a, float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleSU(float a, /*const*/ float* ps, /*const*/ int* pi, float* pd, int c); + public static extern void AddScaleSU(float a, float* ps, int* pi, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleCopyU(float a, /*const*/ float* ps, /*const*/ float* pd, float* pr, int c); + public static extern void AddScaleCopyU(float a, float* ps, float* pd, float* pr, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleMomA(float a, /*const*/ float* ps, float* pd, float momentum, float* pdel, int c); + public static extern void AddScaleMomA(float a, float* ps, float* pd, float momentum, float* pdel, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleMomX(float a, /*const*/ float* ps, float* pd, float momentum, float* pdel, int c); + public static extern void AddScaleMomX(float a, float* ps, float* pd, float momentum, float* pdel, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleGradA(/*const*/ float* ps, float* pd, float* paccGrads, float* paccUpdates, + public static extern void AddScaleGradA(float* ps, float* pd, float* paccGrads, float* paccUpdates, float decay, float cond, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleGradX(/*const*/ float* ps, float* pd, float* paccGrads, float* paccUpdates, + public static extern void AddScaleGradX(float* ps, float* pd, float* paccGrads, float* paccUpdates, float decay, float cond, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleMultiA(int count, /*const*/ float* ps, float* pd, float* paccGrads, + public static extern void AddScaleMultiA(int count, float* ps, float* pd, float* paccGrads, float* paccUpdates, float decay, float cond, int size); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void AddScalarU(float a, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddU(/*const*/ float* ps, float* pd, int c); + public static extern void AddU(float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddA(/*const*/ float* ps, float* pd, int c); + public static extern void AddA(float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddX(/*const*/ float* ps, float* pd, int c); + public static extern void AddX(float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddSU(/*const*/ float* ps, /*const*/ int* pi, float* pd, int c); + public static extern void AddSU(float* ps, int* pi, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumA(/*const*/ float* ps, int c); + public static extern float SumA(float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumU(/*const*/ float* ps, int c); + public static extern float SumU(float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumX(/*const*/ float* ps, int c); + public static extern float SumX(float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumSqU(/*const*/ float* ps, int c); + public static extern float SumSqU(float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumSqDiffU(float mean, /*const*/ float* ps, int c); + public static extern float SumSqDiffU(float mean, float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumAbsU(/*const*/ float* ps, int c); + public static extern float SumAbsU(float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumAbsDiffU(float mean, /*const*/ float* ps, int c); + public static extern float SumAbsDiffU(float mean, float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float MulElementWiseU(/*const*/ float* ps1, /*const*/float* ps2, float* pd, int c); + public static extern float MulElementWiseU(float* ps1, float* ps2, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float MulElementWiseSU(/*const*/ float* ps1, /*const*/float* ps2, /*const*/ int* pi, float* pd, int c); + public static extern float MulElementWiseSU(float* ps1, float* ps2, int* pi, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float MaxAbsU(/*const*/ float* ps, int c); + public static extern float MaxAbsU(float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float MaxAbsDiffU(float mean, /*const*/ float* ps, int c); + public static extern float MaxAbsDiffU(float mean, float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float DotU(/*const*/ float* pa, /*const*/ float* pb, int c); + public static extern float DotU(float* pa, float* pb, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float DotSU(/*const*/ float* pa, /*const*/ float* pb, /*const*/ int* pi, int c); + public static extern float DotSU(float* pa, float* pb, int* pi, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float Dist2(/*const*/ float* px, /*const*/ float* py, int c); + public static extern float Dist2(float* px, float* py, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplySigmoidA(/*const*/ float* ps, float* pd, int c); + public static extern void ApplySigmoidA(float* ps, float* pd, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplySigmoidX(/*const*/ float* ps, float* pd, int c) + public static void ApplySigmoidX(float* ps, float* pd, int c) { ApplySigmoidA(ps, pd, c); } @@ -345,84 +345,84 @@ public static void ApplyBoundedRectifiedLinearX(float* ps, float* pd, int c) } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplySigmoidDerivativeA(/*const*/ float* pv, float* pg, int c); + public static extern void ApplySigmoidDerivativeA(float* pv, float* pg, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplySigmoidDerivativeX(/*const*/ float* pv, float* pg, int c) + public static void ApplySigmoidDerivativeX(float* pv, float* pg, int c) { ApplySigmoidDerivativeA(pv, pg, c); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplyRectifiedLinearDerivativeA(/*const*/ float* pv, float* pg, int c); + public static extern void ApplyRectifiedLinearDerivativeA(float* pv, float* pg, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplyRectifiedLinearDerivativeX(/*const*/ float* pv, float* pg, int c) + public static void ApplyRectifiedLinearDerivativeX(float* pv, float* pg, int c) { ApplyRectifiedLinearDerivativeA(pv, pg, c); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplySquareDerivativeA(/*const*/ float* px, /*const*/ float* py, float* pg, int c, bool drop); + public static extern void ApplySquareDerivativeA(float* px, float* py, float* pg, int c, bool drop); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplySquareDerivativeX(/*const*/ float* px, /*const*/ float* py, float* pg, int c, bool drop) + public static void ApplySquareDerivativeX(float* px, float* py, float* pg, int c, bool drop) { ApplySquareDerivativeA(px, py, pg, c, drop); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplySqrtDerivativeA(/*const*/ float* pv, float* pg, int c); + public static extern void ApplySqrtDerivativeA(float* pv, float* pg, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplySqrtDerivativeX(/*const*/ float* pv, float* pg, int c) + public static void ApplySqrtDerivativeX(float* pv, float* pg, int c) { ApplySqrtDerivativeA(pv, pg, c); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplySoftRectifiedLinearDerivativeU(/*const*/ float* px, /*const*/ float* py, float* pg, int c); + public static extern void ApplySoftRectifiedLinearDerivativeU(float* px, float* py, float* pg, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplySoftRectifiedLinearDerivativeA(/*const*/ float* px, /*const*/ float* py, float* pg, int c) + public static void ApplySoftRectifiedLinearDerivativeA(float* px, float* py, float* pg, int c) { ApplySoftRectifiedLinearDerivativeU(px, py, pg, c); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplySoftRectifiedLinearDerivativeX(/*const*/ float* px, /*const*/ float* py, float* pg, int c) + public static void ApplySoftRectifiedLinearDerivativeX(float* px, float* py, float* pg, int c) { ApplySoftRectifiedLinearDerivativeU(px, py, pg, c); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplyAbsDerivativeA(/*const*/ float* px, /*const*/ float* py, float* pg, int c, bool drop); + public static extern void ApplyAbsDerivativeA(float* px, float* py, float* pg, int c, bool drop); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplyAbsDerivativeX(/*const*/ float* px, /*const*/ float* py, float* pg, int c, bool drop) + public static void ApplyAbsDerivativeX(float* px, float* py, float* pg, int c, bool drop) { ApplyAbsDerivativeA(px, py, pg, c, drop); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplyTanhDerivativeA(/*const*/ float* pv, float* pg, int c); + public static extern void ApplyTanhDerivativeA(float* pv, float* pg, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplyTanhDerivativeX(/*const*/ float* pv, float* pg, int c) + public static void ApplyTanhDerivativeX(float* pv, float* pg, int c) { ApplyTanhDerivativeA(pv, pg, c); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplyBoundedRectifiedLinearDerivativeA(/*const*/ float* pv, float* pg, int c); + public static extern void ApplyBoundedRectifiedLinearDerivativeA(float* pv, float* pg, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplyBoundedRectifiedLinearDerivativeX(/*const*/ float* pv, float* pg, int c) + public static void ApplyBoundedRectifiedLinearDerivativeX(float* pv, float* pg, int c) { ApplyBoundedRectifiedLinearDerivativeA(pv, pg, c); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ZeroItemsU(float* pd, int c, /*const*/ int* pindices, int cindices); + public static extern void ZeroItemsU(float* pd, int c, int* pindices, int cindices); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ZeroMatrixItemsCore(float* pd, int c, int ccol, int cfltRow, /*const*/ int* pindices, int cindices); + public static extern void ZeroMatrixItemsCore(float* pd, int c, int ccol, int cfltRow, int* pindices, int cindices); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void SdcaL1UpdateU(float primalUpdate, /*const*/ float* ps, float threshold, float* pd1, float* pd2, int c); + public static extern void SdcaL1UpdateU(float primalUpdate, float* ps, float threshold, float* pd1, float* pd2, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void SdcaL1UpdateSU(float primalUpdate, /*const*/ float* ps, /*const*/ int* pi, float threshold, float* pd1, float* pd2, int c); + public static extern void SdcaL1UpdateSU(float primalUpdate, float* ps, int* pi, float threshold, float* pd1, float* pd2, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void ScaleAdadeltaU(float* mat, float* accGrads, float* accUpdates, float decay, float cond, float* grads, int size); @@ -435,7 +435,7 @@ public static void ApplyBoundedRectifiedLinearDerivativeX(/*const*/ float* pv, f // In CoreCLR we use Buffer.MemoryCopy directly instead of // plumbing our own version. [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MemCpy(void* dst, /*const*/ void* src, long count); + public static extern void MemCpy(void* dst, void* src, long count); #endif } } diff --git a/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderParser.cs b/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderParser.cs index 48e44b31e7..fde45c6f4f 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderParser.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderParser.cs @@ -137,9 +137,9 @@ private sealed class ParseStats private volatile int _cref; // Total number of rows, number of unparsable values, number of format errors. - private /*volatile*/ long _rowCount; - private /*volatile*/ long _badCount; - private /*volatile*/ long _fmtCount; + private long _rowCount; + private long _badCount; + private long _fmtCount; public ParseStats(IChannelProvider provider, int cref, long maxShow = MaxShow) { diff --git a/src/Microsoft.ML.FastTree/Application/LogLossApplication.cs b/src/Microsoft.ML.FastTree/Application/LogLossApplication.cs index b629cbdbba..2c2a254ad8 100644 --- a/src/Microsoft.ML.FastTree/Application/LogLossApplication.cs +++ b/src/Microsoft.ML.FastTree/Application/LogLossApplication.cs @@ -106,16 +106,7 @@ protected override void GetGradientInOneQuery(int query, int threadIndex) double delta = (_coef * labelDiff) / (1.0 + Math.Exp(_coef * labelDiff * (_scores[d1] - _scores[d2]))); _gradient[d1] += delta; - _gradient[d2] -= delta; - /* - double labelDiff = (labels[d1] - labels[d2]); - double margin = labelDiff * (_scores[d1] - _scores[d2]); - if (_coef - margin > 0) - { - _gradient[d1] += 0.01 * labelDiff; - _gradient[d2] -= 0.01 * labelDiff; - } - */ + _gradient[d2] -= delta; } } } diff --git a/src/Microsoft.ML.FastTree/Application/WinLossSurplusApplication.cs b/src/Microsoft.ML.FastTree/Application/WinLossSurplusApplication.cs index a36f9ccb76..5e73aae00c 100644 --- a/src/Microsoft.ML.FastTree/Application/WinLossSurplusApplication.cs +++ b/src/Microsoft.ML.FastTree/Application/WinLossSurplusApplication.cs @@ -120,7 +120,6 @@ protected override void GetGradientInOneQuery(int query, int threadIndex) double inverseMaxDCG = _inverseMaxDCGT[query]; - //int[] permutation = (threadIndex < 0 ? new int[numDocuments] : _permutationBuffers[threadIndex]); int[] permutation = _permutationBuffers[threadIndex]; short[] labels = Labels; diff --git a/src/Microsoft.ML.FastTree/Dataset/FeatureFlock.cs b/src/Microsoft.ML.FastTree/Dataset/FeatureFlock.cs index 2c58f55603..9b5bf9b9ae 100644 --- a/src/Microsoft.ML.FastTree/Dataset/FeatureFlock.cs +++ b/src/Microsoft.ML.FastTree/Dataset/FeatureFlock.cs @@ -698,20 +698,6 @@ public void FillSplitCandidatesCategoricalLowPopulation(LeastSquaresRegressionTr var binStats = virtualBins[i]; catFeatureCount += 1 + binStats.SubFeatures.Length; - /*int feature = features[i]; - int subfeature = feature - featureMin; - Contracts.Assert(0 <= subfeature && subfeature < Flock.Count); - Contracts.Assert(subfeature <= feature); - Contracts.Assert(learner.TrainData.FlockToFirstFeature(flock) == feature - subfeature); - Contracts.Assert(featureUseCount[feature] >= 0); - Contracts.Assert(Flock.BinCount(subfeature) == 2); - Contracts.Assert(GetMaxBorder(subfeature) == GetMinBorder(subfeature)); - - var binStats = GetBinStats(GetMinBorder(subfeature)); - sumGTTargets += binStats.SumTargets; - if (hasWeights) - sumGTWeights += binStats.SumWeights;*/ - sumGTTargets += binStats.SumTargets; gtCount += binStats.Count; docsInCurrentGroup += binStats.Count; @@ -933,21 +919,7 @@ public void FillSplitCandidatesCategoricalNeighborBundling(LeastSquaresRegressio { var binStats = virtualBins[i]; catFeatureCount += 1 + binStats.SubFeatures.Length; - - /*int feature = features[i]; - int subfeature = feature - featureMin; - Contracts.Assert(0 <= subfeature && subfeature < Flock.Count); - Contracts.Assert(subfeature <= feature); - Contracts.Assert(learner.TrainData.FlockToFirstFeature(flock) == feature - subfeature); - Contracts.Assert(featureUseCount[feature] >= 0); - Contracts.Assert(Flock.BinCount(subfeature) == 2); - Contracts.Assert(GetMaxBorder(subfeature) == GetMinBorder(subfeature)); - - var binStats = GetBinStats(GetMinBorder(subfeature)); - sumGTTargets += binStats.SumTargets; - if (hasWeights) - sumGTWeights += binStats.SumWeights;*/ - + sumGTTargets += binStats.SumTargets; gtCount += binStats.Count; docsInCurrentGroup += binStats.Count; diff --git a/src/Microsoft.ML.FastTree/Dataset/SegmentIntArray.cs b/src/Microsoft.ML.FastTree/Dataset/SegmentIntArray.cs index 3bbd33407b..02ae0f2c84 100644 --- a/src/Microsoft.ML.FastTree/Dataset/SegmentIntArray.cs +++ b/src/Microsoft.ML.FastTree/Dataset/SegmentIntArray.cs @@ -168,11 +168,8 @@ public static void StatsOfBestEncoding(uint[] ivalues, int bitsForMaxItem, bool max = 0; bits = TransitionCost; - //IEnumerator ienum = ivalues.GetEnumerator(); - //while (ienum.MoveNext()) for (int i = 0; i < ivalues.Length; ++i) { - //uint val = (uint)ienum.Current; uint val = (uint)ivalues[i]; if (val > max) max = val; diff --git a/src/Microsoft.ML.FastTree/Dataset/SparseIntArray.cs b/src/Microsoft.ML.FastTree/Dataset/SparseIntArray.cs index 133d35936b..fc360ddea4 100644 --- a/src/Microsoft.ML.FastTree/Dataset/SparseIntArray.cs +++ b/src/Microsoft.ML.FastTree/Dataset/SparseIntArray.cs @@ -546,13 +546,9 @@ public unsafe int this[int virtualIndex] { get { - //if (virtualIndex < _index) throw new Exception("Index must move forward"); - if (virtualIndex < _nextIndex) return 0; - //if (virtualIndex >= _array._length) throw new IndexOutOfRangeException(); - if (virtualIndex == _nextIndex) return _array._values[_pos]; @@ -561,7 +557,6 @@ public unsafe int this[int virtualIndex] { while (_pos < _array._values.Length) { - //_index = _nextIndex; _nextIndex += pDeltas[_pos]; if (virtualIndex < _nextIndex) return 0; diff --git a/src/Microsoft.ML.FastTree/FastTreeRanking.cs b/src/Microsoft.ML.FastTree/FastTreeRanking.cs index 70919fbdea..9b16802daa 100644 --- a/src/Microsoft.ML.FastTree/FastTreeRanking.cs +++ b/src/Microsoft.ML.FastTree/FastTreeRanking.cs @@ -579,7 +579,6 @@ private void SetupSecondaryGains(Arguments args) _secondaryMetricShare = 0.0; return; } - //for (int i = 0; i < _secondaryGains.Length; ++i) _secondaryGains[i] *= cmd.secondaryMetricShare; _secondaryInverseMaxDCGT = DCGCalculator.MaxDCG(_secondaryGains, Dataset.Boundaries, new int[] { args.lambdaMartMaxTruncation })[0].Select(d => 1.0 / d).ToArray(); } @@ -726,7 +725,6 @@ protected override void GetGradientInOneQuery(int query, int threadIndex) double inverseMaxDcg = _inverseMaxDcgt[query]; double secondaryInverseMaxDcg = _secondaryMetricShare == 0 ? 0.0 : _secondaryInverseMaxDcgt[query]; - //int[] permutation = (threadIndex < 0 ? new int[numDocuments] : _permutationBuffers[threadIndex]); int[] permutation = _permutationBuffers[threadIndex]; short[] labels = _labels; diff --git a/src/Microsoft.ML.FastTree/Training/ScoreTracker.cs b/src/Microsoft.ML.FastTree/Training/ScoreTracker.cs index cc83c05cf5..c65aace383 100644 --- a/src/Microsoft.ML.FastTree/Training/ScoreTracker.cs +++ b/src/Microsoft.ML.FastTree/Training/ScoreTracker.cs @@ -53,7 +53,7 @@ public void Initialize(ScoreTracker scores1, RegressionTree tree, DocumentPartit } //InitScores -initScores can be null in such case the scores are reinitialized to Zero - private void InitializeScores(double[] initScores /* = null */) + private void InitializeScores(double[] initScores) { if (initScores == null) { diff --git a/src/Microsoft.ML.InternalStreams/InternalStreams.cs b/src/Microsoft.ML.InternalStreams/InternalStreams.cs index d89420c721..c5196761e9 100644 --- a/src/Microsoft.ML.InternalStreams/InternalStreams.cs +++ b/src/Microsoft.ML.InternalStreams/InternalStreams.cs @@ -2,7 +2,6 @@ #define LZMA_PLAIN #define UNBUFFERED -//#define GZIP_UNBUFFERED using System; using System.Collections; @@ -206,9 +205,6 @@ public enum AllocationType : uint MEM_RESERVE = 0x2000, MEM_RESET = 0x80000, - //MEM_PRIVATE = 0x20000, - //MEM_MAPPED = 0x40000, - /// Can be combined with the types above: /// /// This can be combined with the allocation type. @@ -218,9 +214,6 @@ public enum AllocationType : uint /// This can be combined with the allocation type. /// MEM_LARGE_PAGES = 0x20000000, - //MEM_WRITE_WATCH = 0x200000, - //MEM_PHYSICAL = 0x400000, - //MEM_4MB_PAGES = 0x80000000, } public enum FreeType : uint @@ -268,13 +261,13 @@ public struct IO_STATUS_BLOCK // are these still uint on x64? *** // If this is the wrong size, the results are disastrous... - public uint /*ulong*/ /*IntPtr*/ /*NTSTATUS*/ Status; - public uint /*ulong*/ /*IntPtr*/ /*ULONG_PTR*/ Information; + public uint Status; + public uint Information; private IO_STATUS_BLOCK(bool d) { - Status = 0; //IntPtr.Zero; - Information = 0; //IntPtr.Zero; + Status = 0; + Information = 0; } } @@ -326,7 +319,7 @@ public enum FILE_INFORMATION_CLASS : int private struct FILE_ALLOCATION_INFORMATION { // is this too fragile?? *** - public long /*LARGE_INTEGER*/ AllocationSize; + public long AllocationSize; public FILE_ALLOCATION_INFORMATION(long length) { AllocationSize = length; @@ -355,10 +348,10 @@ public static extern IntPtr GetStdHandle( // this seems rather fragile. [DllImport("NTDLL", ExactSpelling = true, SetLastError = true)] public static extern int NtSetInformationFile( - IntPtr /*HANDLE*/ fileHandle, - ref IO_STATUS_BLOCK /*PIO_STATUS_BLOCK*/ ioStatusBlock, - ref long /*FILE_ALLOCATION_INFORMATION*/ /*IntPtr*/ /*PVOID*/ fileInformation, - uint /*ULONG*/ length, + IntPtr fileHandle, + ref IO_STATUS_BLOCK ioStatusBlock, + ref long fileInformation, + uint length, FILE_INFORMATION_CLASS fileInformationClass); [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] @@ -379,7 +372,7 @@ public static extern bool CloseHandle( public static extern bool SetFilePointerEx( IntPtr handle, long offset, - out long /*IntPtr*/ newPosition, + out long newPosition, SeekOrigin seekOrigin); [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] @@ -415,14 +408,13 @@ public static extern bool SetEndOfFile( [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] public static extern bool ReadFile( IntPtr hFile, - byte[] /*IntPtr*/ lpBuffer, + byte[] lpBuffer, int nNumberOfBytesToRead, out int nNumberOfBytesRead, - // ref OVERLAPPED lpOverlapped); IntPtr lpOverlapped); public static bool ReadFile( IntPtr hFile, - byte[] /*IntPtr*/ lpBuffer, + byte[] lpBuffer, int nNumberOfBytesToRead, out int nNumberOfBytesRead) { @@ -442,7 +434,6 @@ public static extern bool ReadFile( IntPtr lpBuffer, int nNumberOfBytesToRead, out int nNumberOfBytesRead, - // ref OVERLAPPED lpOverlapped); IntPtr lpOverlapped); public static bool ReadFile( IntPtr hFile, @@ -457,14 +448,13 @@ public static bool ReadFile( [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] public static extern bool WriteFile( IntPtr hFile, - byte[] /*IntPtr*/ lpBuffer, + byte[] lpBuffer, int nNumberOfBytesToWrite, out int nNumberOfBytesWritten, - // ref OVERLAPPED lpOverlapped); IntPtr lpOverlapped); public static bool WriteFile( IntPtr hFile, - byte[] /*IntPtr*/ lpBuffer, + byte[] lpBuffer, int nNumberOfBytesToWrite, out int nNumberOfBytesWritten) { @@ -1403,17 +1393,10 @@ public static string PathCorrectCase(string path) { break; } - } - else - { - //Console.WriteLine(" non-match: " + dirs[i] + " : " + - // dirs[i].Substring(dstart, next - pos) + - // " != " + path.Substring(pos, next - pos)); - } + } } if (found == null) { - //Console.WriteLine("No matching entries found: " + res.ToString() + " " + path.Substring(pos, next - pos)); res.Append(pathOrig.Substring(pos)); break; } @@ -1453,20 +1436,15 @@ public static string GetFileName(string path) // check for special names: if (ZStreamIn.IsNullStream(path)) { - //return null; - //return path; return ZStreamIn.NullStreamName; } if (ZStreamIn.IsConsoleStream(path)) { - //return path; return ZStreamIn.ConsoleStreamName; } // check for clipboard: if (ClipboardReadStream.IsClipboardStream(path)) { - //return null; - //return path; return "clip:"; } @@ -1603,20 +1581,15 @@ public static string GetName(string path) // check for special names: if (ZStreamIn.IsNullStream(path)) { - //return null; - //return path; return ZStreamIn.NullStreamName; } if (ZStreamIn.IsConsoleStream(path)) { - //return path; return ZStreamIn.ConsoleStreamName; } // check for clipboard: if (ClipboardReadStream.IsClipboardStream(path)) { - //return null; - //return path; return "clip:"; } @@ -2349,7 +2322,6 @@ public static void CreateDirectory(string path) { try { - //Console.WriteLine("Opening: " + path); using (Stream s = ZStreamOut.Open(path)) { s.Close(); @@ -2818,8 +2790,6 @@ private static StreamInfo[] DirectoryEntriesInfo(string path, bool allowFile, bo DateTime lastMod = DateTime.MinValue; try { - //DirectoryInfo f = new DirectoryInfo(dirs[i]); - //lastMod = f.LastWriteTime; lastMod = Directory.GetLastWriteTime(dirs[i]); } catch @@ -2944,7 +2914,6 @@ private static StreamInfo[] DirectoryEntriesInfo(string path, bool allowFile, bo } if (archPath != null) { - //Console.WriteLine(archPath + " :: " + inArch); // check for path in archive. inArch = inArch.Trim('\\'); if (inArch.Length == 0) @@ -3312,7 +3281,6 @@ public static bool FileExists(string fileName) } if (archPath != null) { - //Console.WriteLine(archPath + " :: " + inArch); // check for path in archive. inArch = inArch.Trim('\\'); if (inArch.Length == 0) @@ -3630,7 +3598,6 @@ public static void Delete(string fileName, bool recursive, bool includeCompresse // check for compressed archive as directory segment: // only one path segment is allowed to be an archive... // normalize path: - //if (fileName[fileName.Length - 1] != '\\') fileName = fileName + "\\"; string zfileName = fileName.Replace('/', '\\'); bool isUnc = zfileName.StartsWith("\\\\"); while (zfileName.IndexOf("\\\\") >= 0) @@ -4099,11 +4066,9 @@ public static void Copy(Stream source, Stream destination) } else { - //Console.Error.WriteLine("writing... " + source.GetType().Name + " -> " + destination.GetType().Name); byte[] buffer = new byte[64 * 1024]; for (int c = source.Read(buffer, 0, buffer.Length); c > 0; c = source.Read(buffer, 0, buffer.Length)) { - //Console.Error.WriteLine("writing " + c + "..."); destination.Write(buffer, 0, c); } } @@ -4592,10 +4557,6 @@ public static IEnumerable Lines(string fileName, bool skipBlank) /// /// The reading is performed through . /// - ///// fileName is null. - ///// fileName is invalid. - ///// fileName cannot be found. - ///// The utilities needed to open a stream are not available. public static string ReadFile(string fileName) { try @@ -4619,10 +4580,6 @@ public static string ReadFile(string fileName) /// /// The reading is performed through . /// - ///// fileName is null. - ///// fileName is invalid. - ///// fileName cannot be found. - ///// The utilities needed to open a stream are not available. public static string ReadFile(string fileName, System.Text.Encoding encoding) { try @@ -5339,7 +5296,6 @@ public static ulong Checksum(string fileName, ChecksumType type) byte[] buffer = new byte[4 * 1024 * 1024]; for (int count = inFile.Read(buffer, 0, buffer.Length); count > 0; count = inFile.Read(buffer, 0, buffer.Length)) { - //Console.WriteLine("read: " + count + " total: " + length); length += count; fixed (byte* bb = buffer) { @@ -5957,11 +5913,9 @@ public long Length #endif public class ZStreamReader //: StreamReader { - // private string tempDir = null; - // private static string fallbackExtension = ""; private static bool _defaultToLocalEncoding = false; - private static int _bufferSize = 32 * 1024; //-1; //1024*1024; //32768; + private static int _bufferSize = 32 * 1024; /// /// Get or set whether to allow fallback to the compression library if executables @@ -6125,7 +6079,7 @@ public static StreamReader Open(string fileName, Encoding encoding) } else { - return new StreamReader(s, encoding, true); //, BUFFER_SIZE); + return new StreamReader(s, encoding, true); } } catch @@ -6283,7 +6237,6 @@ public static StreamReader OpenBuffered(string fileName, Encoding encoding) { return new SqlTextReader(fileName); } - // hack for console! #endif Stream s = null; try @@ -6436,17 +6389,15 @@ public static StreamReader OpenUnbuffered(string fileName, Encoding encoding) } } } - catch //(Exception e2) + catch { // ignore any problems... - //Console.WriteLine("!! (Open_) Problem deleting in " + ztempDir + ": " + e2.ToString()); } Directory.Delete(ztempDir, true); } - catch //(Exception e) + catch { // ignore any problems... - //Console.WriteLine("!! (Open_) Problem deleting " + ztempDir + ": " + e.ToString()); } // should we open the original file, then? *** throw new Exception("unrar failed on file '" + fileName + "'"); @@ -6483,7 +6434,6 @@ public override void Close() { try { - //Console.WriteLine("(Close) Should be deleting: " + tempDir); try { string[] badFiles = Directory.GetFiles(tempDir); @@ -6495,18 +6445,16 @@ public override void Close() } } } - catch //(Exception e2) + catch { // ignore any problems... - //Console.WriteLine("!! (Close) Problem deleting " + tempDir + ": " + e2.ToString()); } Directory.Delete(tempDir, true); tempDir = null; } - catch //(Exception e) + catch { // ignore any problems... - //Console.WriteLine("!! (Close) Problem deleting " + tempDir + ": " + e.ToString()); } } } @@ -6535,7 +6483,6 @@ protected override void Dispose(bool disposing) { try { - //Console.WriteLine("(Dispose) Should be deleting: " + tempDir); try { string[] badFiles = Directory.GetFiles(tempDir); @@ -6612,7 +6559,7 @@ protected override void Dispose(bool disposing) /// Class to create StreamWriters given file paths. /// #endif - public class ZStreamWriter //: StreamWriter + public class ZStreamWriter { // backing field for UTF8Lenient private static readonly Encoding _utf8Lenient = new UTF8Encoding(false, false); @@ -6633,7 +6580,7 @@ public static Encoding UTF8Lenient //private static int compressionLevel = 1; private static bool _breakChunksAtLines = true; - private static int _bufferSize = 32 * 1024; //-1; //1024*1024; //32768; + private static int _bufferSize = 32 * 1024; /// /// Get or set whether the Open method should use a LowFragmentationStream for files. @@ -6980,12 +6927,7 @@ private static StreamWriter Open(string outFileName, bool append, bool breakChun sw = new StreamWriter(s, encoding); } sw.NewLine = WriteNewLine; -#if TLCFULLBUILD - if (string.Compare(outFileName, "clip:", true) == 0) - { - // we really want line-level flushing! *** - //sw.AutoFlush = true; - } +#if TLCFULLBUILD if (ZStreamIn.IsConsoleStream(outFileName)) { // match standard behavior, but not efficient in many cases! @@ -7168,7 +7110,7 @@ public class ZStreamIn private static string _fallbackExtension = ""; private static bool _defaultUnbuffered = false; - private static int _bufferSize = 32 * 1024; //-1; //64*1024; //32768; + private static int _bufferSize = 32 * 1024; /// /// Get or set extension to look to append when the given filename does not exist. @@ -7206,11 +7148,9 @@ public static bool AllowLibraryFallback internal static readonly string[] decompressionArchiveExtensions = new string[] { - // 7za: ".7z", ".zip", ".tar", - // 7z: ".cab", ".arj", ".rar", @@ -7219,14 +7159,12 @@ public static bool AllowLibraryFallback }; internal static readonly string[] decompressionExtensions = new string[] { - // 7za: ".gz", ".7z", ".zip", ".tar", ".bz2", ".z", - // 7z: ".cab", ".arj", ".rar", @@ -7418,7 +7356,6 @@ private static Stream Open(string fileName, bool buffered, bool bufferedFallback #if TLCFULLBUILD if (IsNullStream(fileName)) { - //return new NullStream(); return Stream.Null; } if (IsConsoleStream(fileName)) @@ -7429,8 +7366,6 @@ private static Stream Open(string fileName, bool buffered, bool bufferedFallback // Position also will not work. // Using a BufferedStream wrapper fixes the first problem, with some overhead. // A custom wrapper should probably be made, but it is a pain. - //return new BufferedStream(Console.OpenStandardInput(), 4096); - //return Console.OpenStandardInput(); // larger sizes somehow break line buffering, making it be every *two* lines... return new BufferedStream(Console.OpenStandardInput(), 1024); } @@ -7990,13 +7925,10 @@ private static Stream Open(string fileName, bool buffered, bool bufferedFallback { try { - //return new BufferedStream(new UnbufferedStream(fileName), 8 * 1024 * 1024); return new UnbufferedStream(fileName); } catch (UnbufferedStream.VirtualAllocException) { - // always fallback on memory allocation failure: ? - //return Open(fileName, true, async); throw; } } @@ -8038,15 +7970,6 @@ private static Stream Open(string fileName, bool buffered, bool bufferedFallback } } - // private static FileStream OpenUnbufferedStream(string filename) - // { - //#if UNBUFFERED - // return UnbufferedStream.Open(filename, FileMode.Open, FileAccess.Read, FileShare.None, true, false, BUFFER_SIZE); - //#else - // throw new NotSupportedException("Unbuffered IO is not supported."); - //#endif - // } - #if UNBUFFERED #if TLCFULLBUILD @@ -8216,7 +8139,7 @@ public class ZStreamOut private static bool _defaultLowFragmentation = true; private static bool _breakChunksAtLines = false; - private static int _bufferSize = 32 * 1024; //64*1024; //-1; //64*1024; //32768; + private static int _bufferSize = 32 * 1024; /// /// Get or set the compression level (0 - 9) used for compressed streams. @@ -8283,19 +8206,12 @@ public static bool BreakChunksAtLines internal static readonly string[] compressionArchiveExtensions = new string[] { - // 7za: ".7z" - //".zip", // broken - //".tar" // broken }; internal static readonly string[] compressionExtensions = new string[] { - // 7za: ".gz", ".7z", - //".zip", // broken - //".tar", // broken - //".bzip2", ".bz2", }; @@ -8532,27 +8448,9 @@ private static Stream Open(string fileName, bool append, bool breakChunksAtLines // terrible perf for ReadByte(), by default - too many managed->unmanaged transitions. // buffering, however, leads to problems with timing for some applications that expect // streaming results. - - //return Console.OpenStandardOutput(); - LineBufferedStream res = new LineBufferedStream(Console.OpenStandardOutput(), _bufferSize); res.LineBuffer = false; return res; - //return new BufferedStream(Console.OpenStandardOutput(), BUFFER_SIZE); - - //return new LineBufferedStream( - //new FileStream( - // new Microsoft.Win32.SafeHandles.SafeFileHandle(IOUtil.Win32.GetStdHandle(-11), false), - // FileAccess.Write, BUFFER_SIZE, false) - // ); - - //return new FileStream( - // new Microsoft.Win32.SafeHandles.SafeFileHandle(IOUtil.Win32.GetStdHandle(-11), false), - // FileAccess.Write, BUFFER_SIZE, false); - - //return new FileStream( - // new Microsoft.Win32.SafeHandles.SafeFileHandle(IOUtil.Win32.GetStdHandle(-11), false), - // FileAccess.Write, 1, false); } // check for clipboard: if (ClipboardReadStream.IsClipboardStream(fileName)) @@ -8643,7 +8541,6 @@ private static Stream Open(string fileName, bool append, bool breakChunksAtLines case ".gz": { // appending really could be enabled... *** - //bool failed = false; if (append) throw new ArgumentException("Cannot append to a gz stream.", "append"); try @@ -8652,11 +8549,11 @@ private static Stream Open(string fileName, bool append, bool breakChunksAtLines } catch (InvalidOperationException) { - //failed = true; + } catch (System.ComponentModel.Win32Exception) { - //failed = true; + } if (s == null) { @@ -8697,7 +8594,6 @@ private static Stream Open(string fileName, bool append, bool breakChunksAtLines //// - Seeking is not supported //// - Compression and decompression are not parallelized s = new System.IO.Compression.GZipStream( - //new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.Read), ZStreamOut.Open(fileName + "$"), System.IO.Compression.CompressionMode.Compress); // without a buffer, this is even worse... @@ -8706,11 +8602,7 @@ private static Stream Open(string fileName, bool append, bool breakChunksAtLines } break; - //case ".gz": - //case ".zip": // broken!! - //case ".tar": // broken!! case ".7z": - //case ".bzip2": case ".bz2": { if (append) @@ -8764,7 +8656,6 @@ private static Stream Open(string fileName, bool append, bool breakChunksAtLines // check for compressed archives in path: // only one path segment is allowed to be an archive... // normalize path: - //if (fileName[fileName.Length - 1] != '\\') fileName = fileName + "\\"; string zfileName = fileName.Replace('/', '\\'); bool isUnc = zfileName.StartsWith("\\\\"); while (zfileName.IndexOf("\\\\") >= 0) @@ -8791,7 +8682,6 @@ private static Stream Open(string fileName, bool append, bool breakChunksAtLines } if (archPath != null) { - //Console.WriteLine(archPath + " :: " + inArch); // check for path in archive. inArch = inArch.Trim('\\'); if (inArch.Length != 0) @@ -12133,9 +12023,6 @@ ulong IConvertible.ToUInt64(IFormatProvider provider) object IConvertible.ToType(Type conversionType, IFormatProvider provider) { - //throw new Exception("The method or operation is not implemented."); - // *** what do we really need here? - //return this; if (conversionType == typeof(Var)) { return (Var)this; @@ -12215,8 +12102,7 @@ public class NoPreambleEncoding : System.Text.Encoding { private readonly Encoding _baseEncoding; private static readonly byte[] _preamble = new byte[0]; - - // private backing field for UTF16 + private readonly Encoding _utf16 = new NoPreambleEncoding(Encoding.Unicode); /// diff --git a/src/Microsoft.ML.InternalStreams/LowFragmentationStream.cs b/src/Microsoft.ML.InternalStreams/LowFragmentationStream.cs index 4dd0603e06..673d54bc70 100644 --- a/src/Microsoft.ML.InternalStreams/LowFragmentationStream.cs +++ b/src/Microsoft.ML.InternalStreams/LowFragmentationStream.cs @@ -240,7 +240,6 @@ public static bool TrimOverextension(string fileName) byte[] buffer = new byte[1024 * 1024]; s.Seek(-1, SeekOrigin.End); int lastByte = s.ReadByte(); - //Console.WriteLine("last: " + lastByte); if (lastByte == 0) { long end = s.Length; @@ -297,7 +296,6 @@ private void Extend() } catch { - //Console.WriteLine("extend failed!"); // ignore? This could mean there is not enough space on disk! // If we leave the extendedLength set high, this instance will // harmlessly believe it has preallocated and just allow the writes @@ -317,9 +315,8 @@ private void Extend() } else { - //extendedLength = len; + } - //Seek(pos, SeekOrigin.Begin); } private void Truncate() @@ -426,7 +423,6 @@ public override void Flush() /// public void Reserve(long length) { - //if (length > base.Length) if (length > _extendedLength) { base.SetLength(length); @@ -548,7 +544,6 @@ public override void SetLength(long value) if (Position < value) { // must remember the true length... - //Console.WriteLine("SetLength setting length..."); _length = value; } else diff --git a/src/Microsoft.ML.InternalStreams/TableIO.cs b/src/Microsoft.ML.InternalStreams/TableIO.cs index 64eaa1fd22..c9d8207dc3 100644 --- a/src/Microsoft.ML.InternalStreams/TableIO.cs +++ b/src/Microsoft.ML.InternalStreams/TableIO.cs @@ -121,18 +121,6 @@ public interface ITableReader : ITableProcessor, IEnumerable, ITableRow /// The current row as an array of fields string[] ReadRow(int len); - /* - /// - /// Get the field at the column index. - /// - string this[int index] { get; } - - /// - /// Get the field at the column with the given header. - /// - string this[string header] { get; } - */ - /// /// Reset the reader to the beginning. /// @@ -419,8 +407,6 @@ public class CsvReader : ITableReader, IDisposable public CsvReader(TextReader tr) { _file = tr; - //if (file == null) - // Console.Out.WriteLine(" Error: CsvReader could not open null file!"); } /// @@ -429,7 +415,6 @@ public CsvReader(TextReader tr) /// the name of the file to read the table from public CsvReader(string fname) : this(ZStreamReader.Open(fname)) - // : this(new StreamReader(fname, Encoding.UTF8, true)) { } @@ -448,7 +433,6 @@ public CsvReader(Stream fstream) /// the name of the file to read the table from /// the encoding to use to interpet the file public CsvReader(string fname, Encoding encoding) - // : this(ZStreamReader.Open(fname, encoding)) : this(new StreamReader(fname, encoding, true)) { } @@ -834,7 +818,6 @@ private string ReadItemFromLine(ref int curPos) if (_file.Peek() == -1) break; // read a new row... - // rowNumber++; -- don't count it as a new row. _curLine = _file.ReadLine(); curPos = -1; res.Append('\n'); @@ -904,7 +887,6 @@ private string ReadItemFromLine(ref int curPos) { return ReadItemFromLine(ref curPos); } - //Console.Out.WriteLine(" Item: " + resStr); return resStr; } @@ -1159,21 +1141,17 @@ public void NextRow() (_curLine.Length == 0 || (_curLine[0] == ' ' && _curLine.Trim().Length == 0))) { - //Console.Out.WriteLine("(Skipping row as blank line...)"); continue; } ReadRowFromLine(); if (_skipBlankColumnsLines && RowBlank()) { - //Console.Out.WriteLine("(Skipping row with blank columns...)"); _curRow = null; _curRowArray = null; continue; } break; } - - //ReadRowFromLine(); } /// @@ -1321,8 +1299,6 @@ public class CsvWriter : ITableWriter, IDisposable public CsvWriter(TextWriter tr) { _file = tr; - //if (file == null) - // Console.Out.WriteLine(" Error: CsvReader could not open null file!"); _rowNumber = 1; } @@ -1653,27 +1629,7 @@ public void WriteRow(string[] items, int len) if (_skipBlankLines && _lineStart && items.Length == 0) return; - // if (len < 0 || len == items.Length) - // { - // file.WriteLine(string.Join(delimiter, items)); - // } - // else - // { - // if (items.Length < len) - // { - // file.Write(string.Join(delimiter, items)); - // // pad with empty strings: - // for (int i = items.Length; i < len; i++) - // { - // file.Write(delimiter); - // } - // file.WriteLine(); - // } - // else - // { - // file.WriteLine(string.Join(delimiter, items, 0, len)); - // } - // } + if (items.Length == 0) { if (len > 0) @@ -1755,45 +1711,17 @@ public void WriteRow(StringCollection items, int len) if (!_parseQuotes) { - // string[] itemsA = new string[items.Count]; - // items.CopyTo(itemsA, 0); - if (_normalizeWhitespace) { - // for (int i = 0; i < itemsA.Length; i++) - // { - // itemsA[i] = NormalizeWS(itemsA[i]); - // } for (int i = 0; i < items.Count; i++) { items[i] = NormalizeWS(items[i]); } } - - // if (skipBlankLines && lineStart && itemsA.Length == 0) return; + if (_skipBlankLines && _lineStart && items.Count == 0) return; - // if (len < 0 || len == itemsA.Length) - // { - // file.WriteLine(string.Join(delimiter, itemsA)); - // } - // else - // { - // if (itemsA.Length < len) - // { - // file.Write(string.Join(delimiter, itemsA)); - // // pad with empty strings: - // for (int i = itemsA.Length; i < len; i++) - // { - // file.Write(delimiter); - // } - // file.WriteLine(); - // } - // else - // { - // file.WriteLine(string.Join(delimiter, itemsA, 0, len)); - // } - // } + if (items.Count == 0) { if (len > 0) @@ -2020,8 +1948,6 @@ public XmlTableReader(XmlTextReader tr) _tableName = _file.Name; _file.Read(); } - - // NextRow(); // - don't do this! Might initialize too soon. } } @@ -2293,7 +2219,6 @@ public void NextRow() _headers[index] = givenHeader; } } - //currentRow.Add(name, val); _currentRow[name] = val; if (!_headerOrdered) { @@ -2356,12 +2281,10 @@ public void NextRow() _headers[index] = givenHeader; } } - //currentRow.Add(name, val); _currentRow[name] = val; if (!_headerOrdered) { // skip adding to the sequence for attributes! - //currentRowSequence.Add(val); } else { @@ -2382,7 +2305,7 @@ public void NextRow() val = _file.ReadString(); if (TrimWhitespace) val = val.Trim(); - //currentRow.Add("", val); + _currentRow[""] = val; } } @@ -2756,7 +2679,7 @@ public void WriteRow(string[] items, int len) val = ""; if (TrimWhitespace) val = val.Trim(); - ////file.WriteElementString(name, val); + if (SkipEmptyElements && val.Length == 0) { // just skip it... @@ -2776,9 +2699,7 @@ public void WriteRow(string[] items, int len) } } } - - // delay writing the end element: - //file.WriteEndElement(); + _elementEndPending = true; _openElementCount++; } @@ -2794,8 +2715,7 @@ public void NextRow() WriteRow((string[])_currentRowSequence.ToArray(typeof(string))); _currentRowSequence = null; } - - //currentRow = null; + _currentRowSequence = null; _currentCol = 0; } @@ -2833,7 +2753,7 @@ public string this[string header] { if (header == null) return; - //if (currentRow == null) currentRow = new Hashtable(); + string givenHeader = header; header = header.Trim(); if (IgnoreHeaderCase) @@ -3023,7 +2943,6 @@ public SpreadsheetReader(string fname) filename = fname; // determine format: xlsFormat = false; - //Console.WriteLine("Extension: " + Path.GetExtension(filename).ToLower()); if (Path.GetExtension(filename).ToLower() == ".xls") { xlsFormat = true; @@ -3034,47 +2953,28 @@ public SpreadsheetReader(string fname) if (xlsFormat) // Excel Format { connString += "\"" + Path.GetFullPath(filename) + "\"" + ";" + - // "Extended Properties=\"Excel 8.0;HDR=No\""; "Extended Properties=\"Excel 8.0;HDR=Yes\""; } else // assume CSV { connString += Path.GetDirectoryName(Path.GetFullPath(filename)) + ";" + - // "Extended Properties=\"text;HDR=No;FMT=Delimited\""; "Extended Properties=\"text;HDR=Yes;FMT=Delimited\""; } - //Console.WriteLine("Opening with: " + connString); string selectString = "SELECT * FROM "; if (xlsFormat) // Excel Format { selectString += "[Sheet1$]"; // hard-coded first sheet? *** - //selectString += "foo"; // hard-coded first sheet? *** } else // assume CSV { selectString += Path.GetFileName(filename); } - //Console.WriteLine("Selecting with: " + selectString); oleConn = new OleDbConnection(connString); oleConn.Open(); - //OleDbCommand openCmd = new OleDbCommand(selectString, oleConn); OleDbCommand openCmd = oleConn.CreateCommand(); openCmd.CommandText = selectString; - //openCmd.Connection.Open(); - //Console.WriteLine("Opened connection."); - //Console.WriteLine("Database: " + oleConn.Database); - //Console.WriteLine("Datasource: " + oleConn.DataSource); - - // start up reader: - //oleReader = openCmd.ExecuteReader(CommandBehavior.CloseConnection); - - //MessageBox.Show(connString, "Excel Connection String"); - //MessageBox.Show(selectString, "Excel Select String"); oleReader = openCmd.ExecuteReader(); - //Console.WriteLine("Started Reader."); - - //TestConnection(); // Initialize the row reading: isEof = !Next(); @@ -3357,9 +3257,6 @@ public class SpreadsheetWriter private string filename; private bool xlsFormat; private OleDbConnection oleConn; - //private OleDbDataAdapter oleAdapter; - //private DataTable dataTable; - //private string[] cols; // trim whitespace from each entry: private bool trimWhitespace = true; @@ -3383,7 +3280,6 @@ public SpreadsheetWriter(string fname) // determine format: xlsFormat = false; - //Console.WriteLine("Extension: " + Path.GetExtension(filename).ToLower()); if (Path.GetExtension(filename).ToLower() == ".xls") { xlsFormat = true; @@ -3395,7 +3291,6 @@ public SpreadsheetWriter(string fname) if (xlsFormat) // Excel Format { connString += filename + ";" + - // "Extended Properties=Excel 8.0;"; "Extended Properties=\"Excel 8.0;HDR=No\""; } else // assume CSV @@ -3403,7 +3298,6 @@ public SpreadsheetWriter(string fname) connString += Path.GetDirectoryName(Path.GetFullPath(filename)) + ";" + "Extended Properties=\"text;HDR=Yes;FMT=Delimited\""; } - //Console.WriteLine("Opening with: " + connString); string selectString = "SELECT * FROM "; if (xlsFormat) // Excel Format @@ -3419,35 +3313,6 @@ public SpreadsheetWriter(string fname) oleConn = new OleDbConnection(connString); oleConn.Open(); - //oleAdapter = new OleDbDataAdapter(selectString, connString); - //oleAdapter = new OleDbDataAdapter(selectString, oleConn); - //OleDbCommandBuilder oleCommandBuilder = new OleDbCommandBuilder(oleAdapter); - //oleConn = oleAdapter.SelectCommand.Connection; - //oleConn.Open(); - //dataTable = new DataTable("sheet"); - - //oleAdapter.InsertCommand = new OleDbCommand(); - //oleAdapter.InsertCommand.CommandText = "INSERT INTO [Sheet1$]"; - //// VALUES ('Other','Figimingle','c:\\images\\gardenhose.bmp')"; - //oleAdapter.InsertCommand.Connection = oleConn; - - //OleDbCommand openCmd = new OleDbCommand(selectString, oleConn); - ////OleDbCommand openCmd = oleConn.CreateCommand(); - ////openCmd.CommandText = selectString; - //openCmd.Connection.Open(); - //Console.WriteLine("Opened connection."); - //Console.WriteLine("Database: " + oleConn.Database); - //Console.WriteLine("Datasource: " + oleConn.DataSource); - - // start up reader: - //oleReader = openCmd.ExecuteReader(CommandBehavior.CloseConnection); - ////oleReader = openCmd.ExecuteReader(); - //Console.WriteLine("Started Reader."); - - //TestConnection(); - - // Initialize the column names: - //SetupColumnNames(); } private string sqlString(string item) diff --git a/src/Microsoft.ML.InternalStreams/UnbufferedStream.cs b/src/Microsoft.ML.InternalStreams/UnbufferedStream.cs index 5826897e08..8f005e9817 100644 --- a/src/Microsoft.ML.InternalStreams/UnbufferedStream.cs +++ b/src/Microsoft.ML.InternalStreams/UnbufferedStream.cs @@ -78,15 +78,12 @@ public static int BufferSize private string _fileName; private readonly uint _sectorSize; - private /*readonly*/ IntPtr[] _buffer; - private /*readonly*/ IntPtr[] _alignedBuffer; + private IntPtr[] _buffer; + private IntPtr[] _alignedBuffer; private readonly IntPtr _handle; - private /*readonly*/ int _blockSize; + private int _blockSize; private bool _parallel; private bool _released = true; - //private int bufferStart = 0; - //private int bufferEnd = 0; - //private byte[] mBuffer; #endregion @@ -244,7 +241,7 @@ private void ReleaseBuffer() if (_buffer[j] != IntPtr.Zero) { - bool ret = IOUtil.Win32.VirtualFree(_buffer[j], IntPtr.Zero, /*MEM_DECOMMIT*/ IOUtil.Win32.FreeType.MEM_RELEASE); + bool ret = IOUtil.Win32.VirtualFree(_buffer[j], IntPtr.Zero, IOUtil.Win32.FreeType.MEM_RELEASE); _buffer[j] = IntPtr.Zero; } } @@ -365,7 +362,6 @@ private void Reopen(long startPosition, bool allocate) #if REUSE_BUFFERS if (blockSize == initialBlockSize) { - //Console.WriteLine("locking..."); lock (standingBuffers) { if (standingBufferCount > 0) @@ -374,7 +370,6 @@ private void Reopen(long startPosition, bool allocate) buffer[i] = standingBuffers[standingBufferCount]; } } - //Console.WriteLine("locked."); } #endif @@ -411,7 +406,7 @@ private void Reopen(long startPosition, bool allocate) if (_buffer[j] != IntPtr.Zero) { - bool ret = IOUtil.Win32.VirtualFree(_buffer[j], IntPtr.Zero, /*MEM_DECOMMIT*/ IOUtil.Win32.FreeType.MEM_RELEASE); + bool ret = IOUtil.Win32.VirtualFree(_buffer[j], IntPtr.Zero, IOUtil.Win32.FreeType.MEM_RELEASE); _buffer[j] = IntPtr.Zero; } } @@ -472,7 +467,7 @@ private void Reopen(long startPosition, bool allocate) { try { - bool ret = IOUtil.Win32.VirtualFree(_buffer[j], IntPtr.Zero, /*MEM_DECOMMIT*/ IOUtil.Win32.FreeType.MEM_RELEASE); + bool ret = IOUtil.Win32.VirtualFree(_buffer[j], IntPtr.Zero, IOUtil.Win32.FreeType.MEM_RELEASE); _buffer[j] = IntPtr.Zero; } catch @@ -790,8 +785,6 @@ private void FillBuffer() { // ignore all exceptions on the filling thread... // may be needed, but will hide potential problems. - //System.Diagnostics.Debug.WriteLine(""); - //System.Diagnostics.Debug.WriteLine("FillBuffer exception: " + ex.ToString()); } } @@ -1014,40 +1007,6 @@ private void CopyBuffer(IntPtr source, int sourceStart, byte[] dest, int destSta { dest[destStart] = *(s++); } - - // simple pointer copy: - // byte* s = ((byte*)source.ToPointer()) + sourceStart; - // fixed (byte* dd = dest) - // { - // byte* d = dd + destStart; - // byte* dEnd = d + count; - // while (d != dEnd) - // { - // *(d++) = *(s++); - // } - // } - - // int pointer copy: - // int* s = (int*)(((byte*)source.ToPointer()) + sourceStart); - // fixed (byte* dd = dest) - // { - // int* d = (int*)(dd + destStart); - // int* dEnd = d + (count >> 2); - // while (d != dEnd) - // { - // *d = *s; - // d++; - // s++; - // } - // count = count & 3; - // if (count != 0) - // { - // for (int i = 0; i < count; i++) - // { - // *(((byte*)d) + i) = *(((byte*)s) + i); - // } - // } - // } } #endregion @@ -1256,20 +1215,13 @@ public override int Read(byte[] buffer, int offset, int count) if (count == 0) return 0; int remaining = (int)(length - Position); int paddedRemaining = (int)(base.Length - Position); - //Console.WriteLine("Partial read - requested " + count + ", left: " + remaining + - // " [" + paddedRemaining + "]"); int res = paddedRemaining; if (res == 0) return 0; //bool resized = false; if (res % sectorSize != 0) - { - //// 1. just truncate: - //res = (int)((res / sectorSize) * sectorSize); - - //// 2. can't resize, we've locked it: - //ResizeFile(fileName, ((length / sectorSize + 1) * sectorSize)); + { - //// 3. total hack: + //// total hack: int first = (int)((res / sectorSize) * sectorSize); if (first != 0) { @@ -1376,7 +1328,6 @@ public override bool IsAsync get { return false; - //return base.IsAsync; } } #endif diff --git a/src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachineInterface.cs b/src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachineInterface.cs index b5fdbd0262..eae140f2d2 100644 --- a/src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachineInterface.cs +++ b/src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachineInterface.cs @@ -31,12 +31,12 @@ private static bool Compat(AlignedArray a) } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void CalculateIntermediateVariablesNative(int fieldCount, int latentDim, int count, int* /*const*/ fieldIndices, int* /*const*/ featureIndices, - float* /*const*/ featureValues, float* /*const*/ linearWeights, float* /*const*/ latentWeights, float* latentSum, float* response); + public static extern void CalculateIntermediateVariablesNative(int fieldCount, int latentDim, int count, int* fieldIndices, int* featureIndices, + float* featureValues, float* linearWeights, float* latentWeights, float* latentSum, float* response); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void CalculateGradientAndUpdateNative(float lambdaLinear, float lambdaLatent, float learningRate, int fieldCount, int latentDim, float weight, - int count, int* /*const*/ fieldIndices, int* /*const*/ featureIndices, float* /*const*/ featureValues, float* /*const*/ latentSum, float slope, + int count, int* fieldIndices, int* featureIndices, float* featureValues, float* latentSum, float slope, float* linearWeights, float* latentWeights, float* linearAccumulatedSquaredGrads, float* latentAccumulatedSquaredGrads); public static void CalculateIntermediateVariables(int fieldCount, int latentDim, int count, int[] fieldIndices, int[] featureIndices, float[] featureValues, diff --git a/src/Microsoft.ML.StandardLearners/Optimizer/Optimizer.cs b/src/Microsoft.ML.StandardLearners/Optimizer/Optimizer.cs index ceab09074d..25117f2c14 100644 --- a/src/Microsoft.ML.StandardLearners/Optimizer/Optimizer.cs +++ b/src/Microsoft.ML.StandardLearners/Optimizer/Optimizer.cs @@ -493,8 +493,6 @@ internal virtual bool LineSearch(IChannel ch, bool force) }); } - //if (_newX.AlmostEquals(_x)) throw new PrematureConvergenceException(this, "Step size interval numerically zero."); - Value = Eval(ref _newX, ref _newGrad); GradientCalculations++; if (!FloatUtils.IsFinite(Value)) diff --git a/src/Microsoft.ML.Sweeper/Algorithms/SmacSweeper.cs b/src/Microsoft.ML.Sweeper/Algorithms/SmacSweeper.cs index 87420a265e..927ffc6b5d 100644 --- a/src/Microsoft.ML.Sweeper/Algorithms/SmacSweeper.cs +++ b/src/Microsoft.ML.Sweeper/Algorithms/SmacSweeper.cs @@ -197,16 +197,6 @@ private ParameterSet[] TreeOrderedCandidatesSearch(FastForestRegressionPredictor leafValueList.Add(val, Tuple.Create(i, j)); } } - - // Step 2: Go through, starting from best leaves. - - //ch.Info("Ha ha, we trained {0} trees", ensemble.NumTrees); - //// This is a pretty silly example of inspecting the tree. - //int count = ensemble.Trees.Sum(t => t.SplitFeatures.Take(t.NumNodes).Count(f => f == 5)); - //ch.Info("Our random forest ensemble used the feature with index 5, {0} times!!", count); - //double allLeavesSum = ensemble.Trees.Sum(t => t.LeafValues.Take(t.NumLeaves).Sum()); - //ch.Info("Our random forest, across all leaves, summed to {0}", allLeavesSum); - //int[] path = t.pathToLeaf(leafIndex); return null; } diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index 15d81da88e..30a2df8337 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -3318,9 +3318,7 @@ public void EntryPointLinearPredictorSummary() { var dataPath = GetDataPath("breast-cancer-withheader.txt"); var inputFile = new SimpleFileHandle(Env, dataPath, false, false); - - /*var dataView = ImportTextData.ImportText(Env, new ImportTextData.Input - { InputFile = inputFile, CustomSchema = "header+ col=Label:0 col=Features:Num:1-9"*/ + var dataView = ImportTextData.TextLoader(Env, new ImportTextData.LoaderInput() { Arguments = diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index 928f740b57..7446f39f80 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -90,7 +90,7 @@ public IList GetDatasetsForClassificationWeightingPredictorsTest() [TestCategory("Binary")] public void BinaryClassifierPerceptronTest() { - var binaryPredictors = new[] { TestLearners.perceptron, /*TestLearners.perceptron_reg*/ }; + var binaryPredictors = new[] { TestLearners.perceptron }; var binaryClassificationDatasets = GetDatasetsForBinaryClassifierBaseTest(); RunAllTests(binaryPredictors, binaryClassificationDatasets); Done(); diff --git a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs index 848fa15a44..f57b50bdab 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs @@ -269,8 +269,8 @@ protected void DoNotEverUseInvertPass() private static readonly Regex _matchDateTime = new Regex(@"[0-9]{1,4}[-/][0-9]{1,2}[-/][0-9]{1,4} [0-9]{1,2}:[0-9]{1,2}:[0-9]{1,4}(\.[0-9]+)?( [AP]M)?", RegexOptions.IgnoreCase | RegexOptions.Compiled); private static readonly Regex _matchTime = new Regex(@"[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?", RegexOptions.Compiled); private static readonly Regex _matchShortTime = new Regex(@"\([0-9]{2}:[0-9]{2}(\.[0-9]+)?\)", RegexOptions.Compiled); - private static readonly Regex _matchMemory = new Regex(@"memory usage\(MB\): [0-9]+" /*(s)\: [0-9]+"*/, RegexOptions.Compiled); - private static readonly Regex _matchElapsed = new Regex(@"Time elapsed\(s\): [0-9.]+" /*(s)\: [0-9\.]+"*/, RegexOptions.Compiled); + private static readonly Regex _matchMemory = new Regex(@"memory usage\(MB\): [0-9]+", RegexOptions.Compiled); + private static readonly Regex _matchElapsed = new Regex(@"Time elapsed\(s\): [0-9.]+", RegexOptions.Compiled); private static readonly Regex _matchTimes = new Regex(@"Instances caching time\(s\): [0-9\.]+", RegexOptions.Compiled); private static readonly Regex _matchUpdatesPerSec = new Regex(@", ([0-9\.]+|Infinity)M WeightUpdates/sec", RegexOptions.Compiled); private static readonly Regex _matchParameterT = new Regex(@"=PARAM:/t:[0-9]+", RegexOptions.Compiled); diff --git a/test/Microsoft.ML.TestFramework/Learners.cs b/test/Microsoft.ML.TestFramework/Learners.cs index 6ac7a3a212..9672c88fe8 100644 --- a/test/Microsoft.ML.TestFramework/Learners.cs +++ b/test/Microsoft.ML.TestFramework/Learners.cs @@ -35,16 +35,13 @@ public PredictorAndArgs(SubComponent trainer, string tag = null) } } - public /*static*/ class TestLearnersBase + public class TestLearnersBase { // This ensures that the needed assemblies are loaded! static TestLearnersBase() { bool ok = true; - //ok &= typeof(BinaryNeuralNetwork) != null; ok &= typeof(FastTreeBinaryClassificationTrainer) != null; - //ok &= typeof(OneClassSvmTrainer) != null; - //ok &= typeof(LDSvmTrainer) != null; Contracts.Check(ok, "Missing assemblies!"); } diff --git a/test/Microsoft.ML.TestFramework/TestInitialization.cs b/test/Microsoft.ML.TestFramework/TestInitialization.cs index b1debcfea4..ebe0eb0a79 100644 --- a/test/Microsoft.ML.TestFramework/TestInitialization.cs +++ b/test/Microsoft.ML.TestFramework/TestInitialization.cs @@ -199,14 +199,6 @@ public TestImageAnalyticsTransforms(ITestOutputHelper helper) } } - /* public partial class TestUtilities - { - public TestUtilities(ITestOutputHelper helper) - { - TestContext = new TestContext(helper); - } - }*/ - public partial class TestRepositoryReader : BaseTestBaseline { public TestRepositoryReader(ITestOutputHelper helper) From 97290b2ce2cdd3a72aee116578615b86b348ff2e Mon Sep 17 00:00:00 2001 From: Daniel Drews Date: Tue, 10 Jul 2018 19:54:52 -0500 Subject: [PATCH 3/6] Removing Microsoft.ML.InternalStreams per comment on #513 --- Microsoft.ML.sln | 7 - .../Microsoft.ML.Console.csproj | 1 - .../Microsoft.ML.FastTree.csproj | 1 - .../InternalStreams.cs | 12999 ---------------- .../LowFragmentationStream.cs | 722 - .../Microsoft.ML.InternalStreams.csproj | 17 - src/Microsoft.ML.InternalStreams/TableIO.cs | 3414 ---- .../UnbufferedStream.cs | 1626 -- .../Microsoft.ML.Sweeper.csproj | 1 - 9 files changed, 18788 deletions(-) delete mode 100644 src/Microsoft.ML.InternalStreams/InternalStreams.cs delete mode 100644 src/Microsoft.ML.InternalStreams/LowFragmentationStream.cs delete mode 100644 src/Microsoft.ML.InternalStreams/Microsoft.ML.InternalStreams.csproj delete mode 100644 src/Microsoft.ML.InternalStreams/TableIO.cs delete mode 100644 src/Microsoft.ML.InternalStreams/UnbufferedStream.cs diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index 36620a88a2..4847dd937e 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -36,8 +36,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Tests", "test\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.TestFramework", "test\Microsoft.ML.TestFramework\Microsoft.ML.TestFramework.csproj", "{B5989C06-4FFA-46C1-9D85-9366B34AB0A2}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.InternalStreams", "src\Microsoft.ML.InternalStreams\Microsoft.ML.InternalStreams.csproj", "{C4F7938F-7109-43C8-92A5-9BE47C7FF7D9}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Predictor.Tests", "test\Microsoft.ML.Predictor.Tests\Microsoft.ML.Predictor.Tests.csproj", "{6B047E09-39C9-4583-96F3-685D84CA4117}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.ResultProcessor", "src\Microsoft.ML.ResultProcessor\Microsoft.ML.ResultProcessor.csproj", "{3769FCC3-9AFF-4C37-97E9-6854324681DF}" @@ -150,10 +148,6 @@ Global {B5989C06-4FFA-46C1-9D85-9366B34AB0A2}.Debug|Any CPU.Build.0 = Debug|Any CPU {B5989C06-4FFA-46C1-9D85-9366B34AB0A2}.Release|Any CPU.ActiveCfg = Release|Any CPU {B5989C06-4FFA-46C1-9D85-9366B34AB0A2}.Release|Any CPU.Build.0 = Release|Any CPU - {C4F7938F-7109-43C8-92A5-9BE47C7FF7D9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C4F7938F-7109-43C8-92A5-9BE47C7FF7D9}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C4F7938F-7109-43C8-92A5-9BE47C7FF7D9}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C4F7938F-7109-43C8-92A5-9BE47C7FF7D9}.Release|Any CPU.Build.0 = Release|Any CPU {6B047E09-39C9-4583-96F3-685D84CA4117}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6B047E09-39C9-4583-96F3-685D84CA4117}.Debug|Any CPU.Build.0 = Debug|Any CPU {6B047E09-39C9-4583-96F3-685D84CA4117}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -218,7 +212,6 @@ Global {2F636A2C-062C-49F4-85F3-60DCADAB6A43} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {64BC22D3-1E76-41EF-94D8-C79E471FF2DD} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} {B5989C06-4FFA-46C1-9D85-9366B34AB0A2} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} - {C4F7938F-7109-43C8-92A5-9BE47C7FF7D9} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {6B047E09-39C9-4583-96F3-685D84CA4117} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} {3769FCC3-9AFF-4C37-97E9-6854324681DF} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {B7B593C5-FB8C-4ADA-A638-5B53B47D087E} = {09EADF06-BE25-4228-AB53-95AE3E15B530} diff --git a/src/Microsoft.ML.Console/Microsoft.ML.Console.csproj b/src/Microsoft.ML.Console/Microsoft.ML.Console.csproj index 1256bf75ba..382c5df61f 100644 --- a/src/Microsoft.ML.Console/Microsoft.ML.Console.csproj +++ b/src/Microsoft.ML.Console/Microsoft.ML.Console.csproj @@ -15,7 +15,6 @@ - diff --git a/src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj b/src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj index 2a701542f0..425ae1bf7d 100644 --- a/src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj +++ b/src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Microsoft.ML.InternalStreams/InternalStreams.cs b/src/Microsoft.ML.InternalStreams/InternalStreams.cs deleted file mode 100644 index c5196761e9..0000000000 --- a/src/Microsoft.ML.InternalStreams/InternalStreams.cs +++ /dev/null @@ -1,12999 +0,0 @@ -// owner: rragno - -#define LZMA_PLAIN -#define UNBUFFERED - -using System; -using System.Collections; -using System.Collections.Generic; -using System.IO; -using System.Runtime.InteropServices; -using System.Text; -using System.Threading; -using Microsoft.ML.Runtime; -using Microsoft.ML.Runtime.Internal.Utilities; - -// TODO: -// * Unbuffered writing -// * temporary file support (~ prefix?) -// * Store end emit Cosmos tools -// * Copy compression utils from a share -// * convert local UNC path to local path -// * reset and restart internal gzip as 4GB occurs -// * check encoding -// * Concatenation support for copy, wildcards -// * support compressed extensions in wildcard expansion -// * command piping support -// * C# execution support -// * XPRESS? (LZO / UCL?) -// * QuickLZ -// -// * make MultiStream RAID-like -// * parchive-like parity -// * IFilter -// * Compression support in Cosmos, HTTP -// * Positioned StreamReader fixes -// * Allow "_" as capital " " -// * filter stream -// * detect encoding for HTTP -// * Unicode for console -// * Fix zip support (possibly add zip.exe and unzip.exe as tools) -// * Complete archive directory support -// - Directory listings (especially for directories) -// - wildcard pattern integration -// - Directory removal -// - Directory moving -// * Pluggable decompression -// * built-in zip and gzip handling with J# -// * convert named streams to compressed archives -// * clipboard binary support -// * symbolic streams (temporaries?) -// * Support single file/URL for Cosmos config -// * Support programatic cosmos defaults -// * Support default cosmos path from settings or ini -// * I/O completion ports? -// * Examples - -// * T, Y, Concat stream composition (+ streamnames?) -// * Generalize provider interface! -// * shares, shortcuts, hardlinks -// * sparse files, NTFS compression, encryption -// * named pipes -// * shared memory -// * SharedDictionary support for shared memory or such -// * make TextReaders Enumerable -// * (check gzip size hack) -// * (defragmentation) - -//// GZip/Compression TODO: *** -//// - Support LN tag at the beginning -//// - Enable easy adding of tag to end of existing 4GB+ gzip files -//// - Enable seekable gzip files -//// - Enable opening the file and piping (potentially with the same Stream used to get the length) -//// - Enable appending by adding a new gzip file - -//// Unbuffered I/O TODO: *** -//// - investigate using FileStream with undocumented NoBuffering flag - -//// Other TODO: *** -//// - Make BoundedStream avoid using .Position! - -namespace Microsoft.ML.Runtime.Internal.IO -{ - #region IOUtil - /// - /// Utility functionality for handling paths and other I/O issues. - /// - public static class IOUtil - { - #region DLLimport - /// - /// Imports the Win32 APIs used by the library. - /// - internal class Win32 - { - #region Constants - - public static readonly IntPtr INVALID_HANDLE_VALUE = new IntPtr(-1); - public static readonly IntPtr NULL_HANDLE = IntPtr.Zero; - - [Flags] - public enum FileAccess : uint - { - QUERY = 0x00000000, - GENERIC_READ = 0x80000000, - GENERIC_WRITE = 0x40000000, - GENERIC_EXECUTE = 0x10000000, - GENERIC_ALL = 0x10000000, - } - - [Flags] - public enum FileShare : uint - { - FILE_SHARE_READ = 1, - FILE_SHARE_WRITE = 2, - FILE_SHARE_DELETE = 4, - FILE_SHARE_READ_AND_WRITE = 1 | 2, - } - - public enum CreationDisposition : uint - { - CREATE_NEW = 1, - CREATE_ALWAYS = 2, - OPEN_EXISTING = 3, - OPEN_ALWAYS = 4, - TRUNCATE_EXISTING = 5, - } - - [Flags] - public enum FileFlag : uint - { - FILE_FLAG_WRITE_THROUGH = 0x80000000, - FILE_FLAG_OVERLAPPED = 0x40000000, - FILE_FLAG_NO_BUFFERING = 0x20000000, - FILE_FLAG_RANDOM_ACCESS = 0x10000000, - FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000, - FILE_FLAG_DELETE_ON_CLOSE = 0x04000000, - FILE_FLAG_BACKUP_SEMANTICS = 0x02000000, - FILE_FLAG_POSIX_SEMANTICS = 0x01000000, - FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000, - FILE_FLAG_OPEN_NO_RECALL = 0x00100000, - FILE_FLAG_FIRST_PIPE_INSTANCE = 0x00080000, - } - - [Flags] - public enum FileAttributes : uint - { - FILE_ATTRIBUTE_READONLY = 0x00000001, - FILE_ATTRIBUTE_HIDDEN = 0x00000002, - FILE_ATTRIBUTE_SYSTEM = 0x00000004, - FILE_ATTRIBUTE_DIRECTORY = 0x00000010, - FILE_ATTRIBUTE_ARCHIVE = 0x00000020, - FILE_ATTRIBUTE_DEVICE = 0x00000040, - FILE_ATTRIBUTE_NORMAL = 0x00000080, - FILE_ATTRIBUTE_TEMPORARY = 0x00000100, - FILE_ATTRIBUTE_SPARSE_FILE = 0x00000200, - FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400, - FILE_ATTRIBUTE_COMPRESSED = 0x00000800, - FILE_ATTRIBUTE_OFFLINE = 0x00001000, - FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 0x00002000, - FILE_ATTRIBUTE_ENCRYPTED = 0x00004000, - } - - [Flags] - public enum FileFlagsAndAttributes : uint - { - FILE_FLAG_WRITE_THROUGH = 0x80000000, - FILE_FLAG_OVERLAPPED = 0x40000000, - FILE_FLAG_NO_BUFFERING = 0x20000000, - FILE_FLAG_RANDOM_ACCESS = 0x10000000, - FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000, - FILE_FLAG_DELETE_ON_CLOSE = 0x04000000, - FILE_FLAG_BACKUP_SEMANTICS = 0x02000000, - FILE_FLAG_POSIX_SEMANTICS = 0x01000000, - FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000, - FILE_FLAG_OPEN_NO_RECALL = 0x00100000, - FILE_FLAG_FIRST_PIPE_INSTANCE = 0x00080000, - - FILE_ATTRIBUTE_READONLY = 0x00000001, - FILE_ATTRIBUTE_HIDDEN = 0x00000002, - FILE_ATTRIBUTE_SYSTEM = 0x00000004, - FILE_ATTRIBUTE_DIRECTORY = 0x00000010, - FILE_ATTRIBUTE_ARCHIVE = 0x00000020, - FILE_ATTRIBUTE_DEVICE = 0x00000040, - FILE_ATTRIBUTE_NORMAL = 0x00000080, - FILE_ATTRIBUTE_TEMPORARY = 0x00000100, - FILE_ATTRIBUTE_SPARSE_FILE = 0x00000200, - FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400, - FILE_ATTRIBUTE_COMPRESSED = 0x00000800, - FILE_ATTRIBUTE_OFFLINE = 0x00001000, - FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 0x00002000, - FILE_ATTRIBUTE_ENCRYPTED = 0x00004000, - } - - public enum SeekOrigin : uint - { - FILE_BEGIN = 0, - FILE_CURRENT = 1, - FILE_END = 2, - } - - [Flags] - public enum AllocationType : uint - { - MEM_COMMIT = 0x1000, - MEM_RESERVE = 0x2000, - MEM_RESET = 0x80000, - - /// Can be combined with the types above: - /// - /// This can be combined with the allocation type. - /// - MEM_TOP_DOWN = 0x100000, - /// - /// This can be combined with the allocation type. - /// - MEM_LARGE_PAGES = 0x20000000, - } - - public enum FreeType : uint - { - MEM_DECOMMIT = 0x4000, - MEM_RELEASE = 0x8000, - MEM_FREE = 0x10000, - } - - [Flags] - public enum Protect : uint - { - PAGE_NONE = 0x00000000, - PAGE_NOACCESS = 0x00000001, - PAGE_READONLY = 0x00000002, - PAGE_READWRITE = 0x00000004, - PAGE_WRITECOPY = 0x00000008, - PAGE_EXECUTE = 0x00000010, - PAGE_EXECUTE_READ = 0x00000020, - PAGE_EXECUTE_READWRITE = 0x00000040, - PAGE_EXECUTE_WRITECOPY = 0x00000080, - PAGE_GUARD = 0x00000100, - PAGE_NOCACHE = 0x00000200, - PAGE_WRITECOMBINE = 0x00000400, - } - - /// - /// Access for the mapped file. - /// - public enum MapAccess : uint - { - FILE_MAP_COPY = 0x00000001, - FILE_MAP_WRITE = 0x00000002, - FILE_MAP_READ = 0x00000004, - FILE_MAP_ALL_ACCESS = 0x0000001f, - } - - #endregion - - #region Structures - - public struct IO_STATUS_BLOCK - { - public static readonly IO_STATUS_BLOCK NullBlock = new IO_STATUS_BLOCK(true); - - // are these still uint on x64? *** - // If this is the wrong size, the results are disastrous... - public uint Status; - public uint Information; - - private IO_STATUS_BLOCK(bool d) - { - Status = 0; - Information = 0; - } - } - - public enum FILE_INFORMATION_CLASS : int - { - FileDirectoryInformation = 1, - FileFullDirectoryInformation, - FileBothDirectoryInformation, - FileBasicInformation, - FileStandardInformation, - FileInternalInformation, - FileEaInformation, - FileAccessInformation, - FileNameInformation, - FileRenameInformation, - FileLinkInformation, - FileNamesInformation, - FileDispositionInformation, - FilePositionInformation, - FileFullEaInformation, - FileModeInformation, - FileAlignmentInformation, - FileAllInformation, - FileAllocationInformation, - FileEndOfFileInformation, - FileAlternateNameInformation, - FileStreamInformation, - FilePipeInformation, - FilePipeLocalInformation, - FilePipeRemoteInformation, - FileMailslotQueryInformation, - FileMailslotSetInformation, - FileCompressionInformation, - FileCopyOnWriteInformation, - FileCompletionInformation, - FileMoveClusterInformation, - FileQuotaInformation, - FileReparsePointInformation, - FileNetworkOpenInformation, - FileObjectIdInformation, - FileTrackingInformation, - FileOleDirectoryInformation, - FileContentIndexInformation, - FileInheritContentIndexInformation, - FileOleInformation, - FileMaximumInformation - }; - - private struct FILE_ALLOCATION_INFORMATION - { - // is this too fragile?? *** - public long AllocationSize; - public FILE_ALLOCATION_INFORMATION(long length) - { - AllocationSize = length; - } - } - - #endregion - - public static bool NT_SUCCESS(uint status) - { - return (status <= 0x3FFFFFFF); - } - - [DllImport("NTDLL", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern uint NtQueryInformationFile( - IntPtr handle, - ref IO_STATUS_BLOCK ioStatusBlock, - byte[] fileInformation, - uint length, - FILE_INFORMATION_CLASS fileInformationClass); - - [DllImport("kernel32.dll", SetLastError = true)] - public static extern IntPtr GetStdHandle( - int nStdHandle); - - // this seems rather fragile. - [DllImport("NTDLL", ExactSpelling = true, SetLastError = true)] - public static extern int NtSetInformationFile( - IntPtr fileHandle, - ref IO_STATUS_BLOCK ioStatusBlock, - ref long fileInformation, - uint length, - FILE_INFORMATION_CLASS fileInformationClass); - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern IntPtr CreateFile( - string fileName, - FileAccess desiredAccess, - FileShare shareMode, - IntPtr securityAttributes, - CreationDisposition creationDisposition, - FileFlagsAndAttributes flagsAndAttributes, - IntPtr templateFile); - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern bool CloseHandle( - IntPtr hObject); - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern bool SetFilePointerEx( - IntPtr handle, - long offset, - out long newPosition, - SeekOrigin seekOrigin); - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern bool GetFileSizeEx( - IntPtr handle, - out long newPosition); - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern bool DeleteFile( - string fileName); - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern IntPtr VirtualAlloc( - IntPtr lpAddress, - IntPtr dwSize, - AllocationType flAllocationType, - Protect flProtect); - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern bool VirtualFree( - IntPtr lpAddress, - IntPtr dwSize, - FreeType dwFreeType); - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern bool FlushFileBuffers( - IntPtr handle); - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern bool SetEndOfFile( - IntPtr handle); - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern bool ReadFile( - IntPtr hFile, - byte[] lpBuffer, - int nNumberOfBytesToRead, - out int nNumberOfBytesRead, - IntPtr lpOverlapped); - public static bool ReadFile( - IntPtr hFile, - byte[] lpBuffer, - int nNumberOfBytesToRead, - out int nNumberOfBytesRead) - { - return ReadFile(hFile, lpBuffer, nNumberOfBytesToRead, out nNumberOfBytesRead, IntPtr.Zero); - } - - // there is likely a better way to do this... - public class Raw - { - private Raw() - { - } - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern bool ReadFile( - IntPtr hFile, - IntPtr lpBuffer, - int nNumberOfBytesToRead, - out int nNumberOfBytesRead, - IntPtr lpOverlapped); - public static bool ReadFile( - IntPtr hFile, - IntPtr lpBuffer, - int nNumberOfBytesToRead, - out int nNumberOfBytesRead) - { - return ReadFile(hFile, lpBuffer, nNumberOfBytesToRead, out nNumberOfBytesRead, IntPtr.Zero); - } - } - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern bool WriteFile( - IntPtr hFile, - byte[] lpBuffer, - int nNumberOfBytesToWrite, - out int nNumberOfBytesWritten, - IntPtr lpOverlapped); - public static bool WriteFile( - IntPtr hFile, - byte[] lpBuffer, - int nNumberOfBytesToWrite, - out int nNumberOfBytesWritten) - { - return WriteFile(hFile, lpBuffer, nNumberOfBytesToWrite, out nNumberOfBytesWritten, IntPtr.Zero); - } - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto, BestFitMapping = false)] - public static extern bool GetDiskFreeSpace( - string path, - out uint sectorsPerCluster, - out uint bytesPerSector, - out uint numberOfFreeClusters, - out uint totalNumberOfClusters); - - [DllImport("KERNEL32")] - public static extern bool GetDiskFreeSpaceEx( - string lpDirectoryName, - out UInt64 lpFreeBytesAvailable, - out UInt64 lpTotalNumberOfBytes, - out UInt64 lpTotalNumberOfFreeBytes); - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto)] - public static extern IntPtr CreateFileMapping( - IntPtr hFile, - IntPtr lpAttributes, - Protect flProtect, - int dwMaximumSizeLow, - int dwMaximumSizeHigh, - string lpName); - - [DllImport("KERNEL32", SetLastError = true)] - public static extern bool FlushViewOfFile( - IntPtr lpBaseAddress, - int dwNumBytesToFlush); - - [DllImport("KERNEL32", SetLastError = true)] - public static extern IntPtr MapViewOfFile( - IntPtr hFileMappingObject, - int dwDesiredAccess, - int dwFileOffsetHigh, - int dwFileOffsetLow, - int dwNumBytesToMap); - - [DllImport("KERNEL32", SetLastError = true, CharSet = CharSet.Auto)] - public static extern IntPtr OpenFileMapping( - int dwDesiredAccess, - bool bInheritHandle, - string lpName); - - [DllImport("KERNEL32", SetLastError = true)] - public static extern bool UnmapViewOfFile( - IntPtr lpBaseAddress); - - #region Unused - private Win32() - { - } - - #endregion - } - #endregion DLLimport - -#if TLCFULLBUILD - #region Disk Info - - /// - /// Get the amount of space free on a volume. - /// - /// The drive, share, directory, or file located on the volume - /// the free space on the given volume - public static long DiskFree(string path) - { - // Could support cosmos... *** - if (path == null || path.Length == 0) - return 0; - path = Directory.GetDirectoryRoot(path); - ulong freeBytesAvailable; - ulong totalNumberOfBytes; - ulong totalNumberOfFreeBytes; - if (!Win32.GetDiskFreeSpaceEx(path, out freeBytesAvailable, out totalNumberOfBytes, out totalNumberOfFreeBytes)) - { - freeBytesAvailable = 0; - } - return (long)freeBytesAvailable; - } - - #endregion - - #region Path Manipulation - - internal static readonly char[] pathSeparators = new char[] { '/', '\\' }; - - /// - /// Combine a base path with another partial path - /// - /// the base path to start in - /// the path to combine with the base path - /// the combined path - /// - /// If subPath is an absolute path, it will be returned. If either path is - /// empty or null, the other path will be returned. - /// - public static string PathCombine(string basePath, string subPath) - { - if (basePath == null) - return subPath; - if (subPath == null) - return basePath; - if (basePath.Length == 0) - return subPath; - if (subPath.Length == 0) - return basePath; - - string nameLower = subPath.ToLower(); - - // check for special names: - if (ZStreamIn.IsNullStream(subPath)) - { - return subPath; - } - if (ZStreamIn.IsConsoleStream(subPath)) - { - return subPath; - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(subPath)) - { - return subPath; - } - - // check for cosmos: - if (nameLower.StartsWith("cosmos:")) - { - return subPath; - } - - // check for Cockpit: - if (nameLower.StartsWith("cockpit:")) - { - return subPath; - } - - // check for Multistream: - if (nameLower.StartsWith("multi:")) - { - return subPath; - } - if (nameLower.StartsWith("filelist:")) - { - return subPath; - } - - // check for HTTP: - if (nameLower.StartsWith("http://") || nameLower.StartsWith("https://")) - { - return subPath; - } - - if (subPath[0] == '/' || subPath[0] == '\\') - { - if (subPath.Length == 1 || (subPath.Length == 2 && subPath[1] == '.')) - { - return PathRoot(basePath); - } - if (subPath[1] == '/' || subPath[1] == '\\') - { - return subPath; - } - basePath = PathRoot(basePath); - subPath = subPath.Substring(1); - } - - // check for SQL: - if (SqlTextReader.IsSqlTextReader(subPath)) - { - return subPath; - } - - // check for InternalStore: - if (InternalStoreUtility.IsInternalStore(subPath)) - { - return subPath; - } - - // examine basePath - - if (InternalStoreUtility.IsInternalStore(basePath)) - { - // is this best? - return basePath.Substring(0, basePath.IndexOf(':') + 1) + - PathCombine(InternalStoreUtility.StorePath(basePath), subPath); - } - - // check for SQL: - if (SqlTextReader.IsSqlTextReader(basePath)) - { - // is this best? *** - if (!basePath.EndsWith("/")) - basePath = basePath + "/"; - return basePath + subPath; - } - - // check for Multistream: - if (string.Compare(basePath, 0, "multi:", 0, "multi:".Length, true) == 0) - { - // is this best? *** - return basePath.Substring(0, "multi:".Length) + - PathCombine(basePath.Substring("multi:".Length), subPath); - } - if (string.Compare(basePath, 0, "filelist:", 0, "filelist:".Length, true) == 0) - { - // is this best? *** - return basePath.Substring(0, "filelist:".Length) + - PathCombine(basePath.Substring("filelist:".Length), subPath); - } - - return ReducePath(Path.Combine(basePath, subPath)); - } - - /// - /// Find the root element of a path. - /// - /// the path to analyze - /// the root element of the path, or the empty string if none exists - /// - ///

- /// This is only based on the text of the path string. - ///

- ///

- /// The path returned will be canonical, so it can be compared for equality with - /// other roots. - ///

- /// - public static string PathRoot(string path) - { - if (path == null || path.Length == 0) - return ""; - string nameLower = path.ToLower(); - - // check for special names: - if (ZStreamIn.IsNullStream(path)) - { - //return ""; - // should it be this? The length then won't match... *** - return ZStreamIn.NullStreamName; - } - if (ZStreamIn.IsConsoleStream(path)) - { - //return ""; - // should it be this? The length then won't match... *** - return ZStreamIn.ConsoleStreamName; - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(path)) - { - // should this be empty, or "clip:" ? - return "clip:"; - } - - // check for cosmos: - if (nameLower.StartsWith("cosmos:")) - { - if (nameLower.Length < "cosmos://".Length) - return "cosmos:"; - int s = nameLower.IndexOfAny(pathSeparators, "cosmos://".Length); - if (s < 0) - { - return nameLower; - } - s = nameLower.IndexOfAny(pathSeparators, s + 1); - if (s < 0) - { - return nameLower; - } - return nameLower.Substring(0, s); - } - - // check for Cockpit: - if (nameLower.StartsWith("cockpit:")) - { - // what is right here? - if (path.Length < "cockpit://aa".Length) - return path; - int s = path.IndexOfAny(pathSeparators, "cockpit://a".Length); - if (s < 0) - { - return path; - } - return path.Substring(0, s); - } - - // check for Multistream: - if (nameLower.StartsWith("multi:") || nameLower.StartsWith("filelist:")) - { - // what is right here? *** - if (nameLower.Length < "multi:a".Length) - return path; - int s = path.IndexOfAny(pathSeparators, "multi:a".Length); - if (s < 0) - { - return path; - } - return path.Substring(0, s); - } - if (nameLower.StartsWith("filelist:")) - { - // what is right here? *** - if (nameLower.Length < "filelist:a".Length) - return path; - int s = path.IndexOfAny(pathSeparators, "filelist:a".Length); - if (s < 0) - { - return path; - } - return path.Substring(0, s); - } - - // check for HTTP: - if (nameLower.StartsWith("http://") || nameLower.StartsWith("https://")) - { - // what is right here? - int s = path.IndexOfAny(pathSeparators, "http://a".Length); - if (s < 0) - { - return path; - } - return path.Substring(0, s); - } - - // check for SQL: - if (SqlTextReader.IsSqlTextReader(path)) - { - return SqlTextReader.PathRoot(path); - } - - // InternalStore: - if (InternalStoreUtility.IsInternalStore(path)) - { - return InternalStoreUtility.PathRoot(path); - //return "store:" + PathRoot(InternalStoreUtility.StorePath(path)); - } - - if (nameLower.IndexOfAny(pathSeparators) < 0) - { - if (nameLower.Length == 2 && nameLower[1] == ':') - { - //return nameLower + "\\"; - return nameLower + "/"; - } - return ""; - } - - if (path[0] == ':') - return ""; - try - { - return Path.GetPathRoot(path).ToLower().Replace('\\', '/'); - } - catch - { - return ""; - } - } - - /// - /// Find the parent of a given file or directory, based on the path. - /// - /// the original path to start from - /// the path to the parent directory, or null if none exists - /// - /// The parent is determined based only on the given path information. - /// Additionally, a file in the current directory will return ".", but - /// this may be misleading if the path is misinterpretted. The current - /// directory, ".", will return "..", but that might not be a valid directory. - /// - public static string PathParent(string path) - { - if (path == null || path.Length == 0) - return null; - - string nameLower = path.ToLower(); - - // check for special names: - if (ZStreamIn.IsNullStream(path)) - { - return path.IndexOfAny(pathSeparators) < 0 ? null : ZStreamIn.NullStreamName; - } - if (ZStreamIn.IsConsoleStream(path)) - { - return path.IndexOfAny(pathSeparators) < 0 ? null : ZStreamIn.ConsoleStreamName; - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(path)) - { - return path.IndexOfAny(pathSeparators) < 0 || string.Compare(path, "clip://", true) == 0 ? - null : "clip:"; - } - - // check for Cosmos: - if (nameLower.StartsWith("cosmos://")) - { - if (nameLower.Length < "cosmos://a/b/c".Length) - return null; - int s = nameLower.IndexOfAny(pathSeparators, "cosmos://".Length); - if (s < 0) - { - return null; - } - s = nameLower.IndexOfAny(pathSeparators, s + 1); - if (s < 0) - { - return null; - } - int e = nameLower.Length - 1; - if (nameLower[e] == '/' || nameLower[e] == '\\') - e--; - if (e <= s) - return null; - e = nameLower.LastIndexOfAny(pathSeparators, e); - return path.Substring(0, e + 1); - } - - // check for Cockpit: - if (nameLower.StartsWith("cockpit:")) - { - if (nameLower.Length < "cockpit://a/b".Length) - return null; - int s = nameLower.IndexOfAny(pathSeparators, "cockpit://a".Length); - if (s < 0) - { - return null; - } - int e = nameLower.Length - 1; - if (nameLower[e] == '/' || nameLower[e] == '\\') - e--; - if (e <= s) - return null; - e = nameLower.LastIndexOfAny(pathSeparators, e); - return path.Substring(0, e + 1); - } - - // check for Multistream: - if (nameLower.StartsWith("multi:")) - { - // what is right here? *** - if (nameLower.Length < "multi:a/b".Length) - return null; - int s = nameLower.IndexOfAny(pathSeparators, "multi:a/b".Length); - if (s < 0) - { - return null; - } - int e = nameLower.Length - 1; - if (nameLower[e] == '/' || nameLower[e] == '\\') - e--; - if (e <= s) - return null; - e = nameLower.LastIndexOfAny(pathSeparators, e); - return path.Substring(0, e + 1); - } - if (nameLower.StartsWith("filelist:")) - { - // what is right here? *** - if (nameLower.Length < "filelist:a/b".Length) - return null; - int s = nameLower.IndexOfAny(pathSeparators, "filelist:a/b".Length); - if (s < 0) - { - return null; - } - int e = nameLower.Length - 1; - if (nameLower[e] == '/' || nameLower[e] == '\\') - e--; - if (e <= s) - return null; - e = nameLower.LastIndexOfAny(pathSeparators, e); - return path.Substring(0, e + 1); - } - - // check for HTTP: - if (nameLower.StartsWith("http://") || nameLower.StartsWith("https://")) - { - if (nameLower.Length < "http://a/b".Length) - return null; - int s = nameLower.IndexOfAny(pathSeparators, "http://a".Length); - if (s < 0) - { - return null; - } - int e = nameLower.Length - 1; - if (nameLower[e] == '/' || nameLower[e] == '\\') - e--; - if (e <= s) - return null; - e = nameLower.LastIndexOfAny(pathSeparators, e); - return path.Substring(0, e + 1); - } - - // check for SQL: - if (SqlTextReader.IsSqlTextReader(path)) - { - string s = SqlTextReader.PathRoot(path); - if (s == null || s.Length == 0 || s.Length <= path.Length - 1) - return null; - return s; - } - - // InternalStore: - if (InternalStoreUtility.IsInternalStore(path)) - { - // should this just always return null? - //return null; - return path.Substring(0, path.IndexOf(':') + 1) + PathParent(InternalStoreUtility.StorePath(path)); - } - - if (nameLower == "." || nameLower == "./" || nameLower == ".\\") - return ".."; - - // filename with no path separators: - if (nameLower.IndexOfAny(pathSeparators) < 0) - { - // check for a drive specification: - if (nameLower.Length == 2 && nameLower[1] == ':') - { - return null; - } - // a simple filename, in the current directory? - return "."; - } - // drive root: - if (nameLower.Length == 1) - return null; - - // path with separators: - // UNC: - if ((nameLower[0] == '/' || nameLower[0] == '\\') && - (nameLower[1] == '/' || nameLower[1] == '\\')) - { - if (nameLower.Length <= 3) - return null; - // machine names do not count as directories!! - // \\machine\share\path - int s = nameLower.IndexOfAny(pathSeparators, 2); - if (s < 0 || s == nameLower.Length - 1) - return null; - s = nameLower.IndexOfAny(pathSeparators, s + 1); - if (s < 0 || s == nameLower.Length - 1) - return null; - int e = nameLower.Length - 1; - if (nameLower[e] == '/' || nameLower[e] == '\\') - e--; - if (e <= s) - return null; - e = nameLower.LastIndexOfAny(pathSeparators, e); - return path.Substring(0, e + 1); - } - - // simple file path: - { - int e = nameLower.Length - 1; - if (nameLower[e] == '/' || nameLower[e] == '\\') - e--; - e = nameLower.LastIndexOfAny(pathSeparators, e); - if (e < 0) - return "."; - return path.Substring(0, e + 1); - } - } - - /// - /// Gets the full path of the file or directory in a standard form. - /// - /// the original path - /// the full path, with a standard casing and presentation - /// - ///

- /// A trailing slash will be preserved. In general, the canonical path for a directory - /// includes the trailing slash, but it would be expensive to check for existance - /// and insert the slash when it is omitted (and incorrect, in the case of directory - /// paths for directories that do not yet exist). - ///

- ///

- /// ignores a trailing slash, to account for this effect. - ///

- /// - public static string GetCanonicalPath(string path) - { - if (path == null || path.Length == 0) - return ""; - path = path.Trim(); - if (path.Length == 0) - return ""; - path = path.Replace('\\', '/'); - // remove repeated slashes? - string nameLower = path.ToLower(); - - // check for special names: - if (ZStreamIn.IsNullStream(path)) - { - return ZStreamIn.NullStreamName; - } - if (ZStreamIn.IsConsoleStream(path)) - { - return ZStreamIn.ConsoleStreamName; - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(path)) - { - return "clip:"; - } - - // check for cosmos: - if (string.Compare(path, 0, "cosmos://", 0, "cosmos://".Length, true) == 0) - { - string root = IOUtil.PathRoot(path); - path = root.ToLower() + - path.Substring(root.Length).Replace("//", "/"); - - return ReducePath(path); - } - - // check for Cockpit: - if (string.Compare(path, 0, "cockpit:", 0, "cockpit:".Length, true) == 0) - { - // should we trim a trailing slash in this case?? - string root = IOUtil.PathRoot(path); - path = "cockpit:" + root.Substring("cockpit:".Length).Replace('\\', '/').ToUpper() + - path.Substring(root.Length).Replace('\\', '/').ToLower(); - return ReducePath(path); - } - - // check for Multistream: - if (string.Compare(path, 0, "multi:", 0, "multi:".Length, true) == 0) - { - // should we trim a trailing slash in this case?? - string root = IOUtil.PathRoot(path); - path = "multi:" + root.Substring("multi:".Length).Replace('\\', '/').ToUpper() + - path.Substring(root.Length).Replace('\\', '/').ToLower(); - return ReducePath(path); - } - if (string.Compare(path, 0, "filelist:", 0, "filelist:".Length, true) == 0) - { - // should we trim a trailing slash in this case?? - string root = IOUtil.PathRoot(path); - path = "multi:" + root.Substring("filelist:".Length).Replace('\\', '/').ToUpper() + - path.Substring(root.Length).Replace('\\', '/').ToLower(); - return ReducePath(path); - } - - // check for HTTP: - if (string.Compare(path, 0, "http://", 0, "http://".Length, true) == 0 || - string.Compare(path, 0, "https://", 0, "https://".Length, true) == 0) - { - // should we trim a trailing slash in this case?? - string root = IOUtil.PathRoot(path); - if (root.EndsWith(":80")) - { - path = root.Substring(0, root.Length - 3).ToLower() + - path.Substring(root.Length); - } - else - { - path = root.ToLower() + - path.Substring(root.Length); - } - return ReducePath(path); - } - - // check for SQL: - if (SqlTextReader.IsSqlTextReader(path)) - { - return SqlTextReader.GetCanonicalPath(path); - } - - // InternalStore: - if (InternalStoreUtility.IsInternalStore(path)) - { - return InternalStoreUtility.GetCanonicalPath(path); - } - - try - { - return ReducePath(Path.GetFullPath(path).Replace('\\', '/').ToLower()); - } - catch - { - path = path.ToLower(); - if (path.StartsWith("//")) - { - path = "/" + path.Substring(1).Replace("//", "/"); - } - else - { - path = path.Replace("//", "/"); - } - return ReducePath(path); - } - } - - private static string ReducePath(string path) - { - try - { - if (path == null) - return null; - if (path.Length == 0 || path == ".") - return path; - bool noBack = (path.IndexOf('/') < 0) && (path.IndexOf('\\') >= 0); - path = path.Replace('\\', '/'); - path = path.Replace("/./", "/"); - if (path.StartsWith("./")) - path = path.Substring(2); - if (path.EndsWith("/.")) - path = path.Substring(0, path.Length - 1); - if (path.Length == 0 || path == ".") - return path; - int i = 0; - while ((i = path.IndexOf("/../", i)) >= 0) - { - if (i == 0) - { - // malformed... skip. - i += 4; - } - else - { - int s = path.LastIndexOf('/', i - 1); - if (s < 0) - s = -1; - path = path.Substring(0, s + 1) + path.Substring(i + 4); - i = s; - } - if (i < 0 || i >= path.Length) - break; - } - if (path.Length > 3 && path.EndsWith("/..")) - { - int s = path.LastIndexOf('/', path.Length - 4); - if (s < 0) - { - path = "."; - } - else - { - path = path.Substring(0, s + 1); - } - } - if (path.Length == 0) - path = "."; - if (noBack) - path = path.Replace('/', '\\'); - return path; - } - catch - { - return path; - } - } - - /// - /// Determine whether two paths refer to the same location. - /// - /// the first path to compare - /// the second path to compare - /// true if the paths refer to the same entity; false otherwise - /// - /// This is equivalent to the expression - /// (GetCanonicalPath(path1).TrimEnd('/').Equals(GetCanonicalPath(path2).TrimEnd('/'))). - /// It will not find all paths that lead to the same location (for example, a - /// UNC share path may refer to a local file system path, or a hardlink or junction - /// may exist). - /// - public static bool PathsEqual(string path1, string path2) - { - if (path1 == null || path2 == null) - return path1 == path2; - string path1C = GetCanonicalPath(path1); - string path2C = GetCanonicalPath(path2); - if (path1C.Length != path2C.Length) - { - if (path2C.Length < path1C.Length) - { - string tmp = path1C; - path1C = path2C; - path2C = tmp; - } - if (path2C.Length - path1C.Length == 1 && - (path2C[path2C.Length - 1] == '/' || path2C[path2C.Length - 1] == '\\')) - { - return string.CompareOrdinal(path2C, 0, path1C, 0, path1C.Length) == 0; - } - return false; - } - return (path1C.Equals(path2C)); - } - - /// - /// Gets the given path with the correct case. - /// - /// the original path - /// the path with the recorded casing - /// - ///

- /// For case-sensitive file systems, such as Cosmos, this will fall back - /// to a case-insensitive match. - ///

- /// - public static string PathCorrectCase(string path) - { - if (path == null || path.Length == 0) - return path; - path = path.Trim(); - if (path.Length == 0) - return ""; - string pathOrig = path; - path = path.Replace('\\', '/'); - // remove repeated slashes? - string nameLower = path.ToLower(); - - // check for special names: - if (ZStreamIn.IsNullStream(path)) - { - return nameLower; - } - if (ZStreamIn.IsConsoleStream(path)) - { - return nameLower; - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(path)) - { - return nameLower; - } - - string root = IOUtil.PathRoot(path); - - // check for Cockpit: - if (string.Compare(path, 0, "cockpit:", 0, "cockpit:".Length, true) == 0) - { - // should we trim a trailing slash in this case?? - path = "cockpit:" + root.Substring("cockpit:".Length).Replace('\\', '/').ToUpper() + - path.Substring(root.Length).Replace('\\', '/'); - return path; - } - - // check for MultiStream: - if (string.Compare(path, 0, "multi:", 0, "multi:".Length, true) == 0) - { - // should we trim a trailing slash in this case?? - path = "multi:" + root.Substring("multi:".Length).Replace('\\', '/').ToUpper() + - path.Substring(root.Length).Replace('\\', '/'); - return path; - } - if (string.Compare(path, 0, "filelist:", 0, "filelist:".Length, true) == 0) - { - // should we trim a trailing slash in this case?? - path = "multi:" + root.Substring("filelist:".Length).Replace('\\', '/').ToUpper() + - path.Substring(root.Length).Replace('\\', '/'); - return path; - } - - // check for HTTP: - if (string.Compare(path, 0, "http://", 0, "http://".Length, true) == 0 || - string.Compare(path, 0, "https://", 0, "https://".Length, true) == 0) - { - // should we trim a trailing slash in this case?? - if (root.EndsWith(":80")) - { - path = root.Substring(0, root.Length - 3).ToLower() + - path.Substring(root.Length); - } - else - { - path = root.ToLower() + - path.Substring(root.Length); - } - return path; - } - - // check for SQL: - if (SqlTextReader.IsSqlTextReader(path)) - { - return SqlTextReader.GetCanonicalPath(path); - } - - StringBuilder res = new StringBuilder(path.Length); - int pos = 0; - - // cosmos and normal file system: - res.Append(root.ToLower()); - if ((root.EndsWith("\\") || root.EndsWith("/")) && - pathOrig.Length >= root.Length) - { - res[root.Length - 1] = pathOrig[root.Length - 1]; - } - pos = root.Length; - while (pos < path.Length) - { - int next = path.IndexOf('/', pos); - if (next < 0) - next = path.Length; - if (next == pos) - { - res.Append(pathOrig[pos]); - pos++; - continue; - } - string[] dirs = DirectoryEntries(pathOrig.Substring(0, pos)); - if (dirs == null || dirs.Length == 0) - { - //Console.WriteLine("No entries found: " + res.ToString()); - res.Append(pathOrig.Substring(pos)); - break; - } - string found = null; - for (int i = 0; i < dirs.Length; i++) - { - int len = dirs[i].Length; - if (dirs[i][dirs[i].Length - 1] == '/') - len--; - int dstart = dirs[i].LastIndexOfAny(pathSeparators, len - 1); - if (dstart < 0) - { - dstart = 0; - } - else - { - dstart++; - len = len - dstart; - } - if (len == next - pos && - string.Compare(path, pos, dirs[i], dstart, next - pos, true) == 0) - { - found = dirs[i].Substring(dstart, next - pos); - if (string.Compare(path, pos, dirs[i], dstart, next - pos, false) == 0) - { - break; - } - } - } - if (found == null) - { - res.Append(pathOrig.Substring(pos)); - break; - } - - res.Append(found); - if (next < pathOrig.Length) - { - res.Append(pathOrig[next]); - } - pos = next + 1; - } - - if (IsFileSystemPath(res.ToString())) - { - res.Replace('/', '\\'); - } - return res.ToString(); - } - - /// - /// Get the filename, without any path or protocol information. - /// - /// the path or stream name - /// the name of the file, or null if the name does not exist, or the empty - /// string if the path is a directory - /// - /// This may fail if the path is a directory, since it operates only on the string. In - /// that case, it could return the name of the last directory, if the path does not end on a - /// directory seperator character. However, it will attempt to - /// return an empty string in the case of directories. - /// - public static string GetFileName(string path) - { - if (path == null || path.Length == 0) - return null; - - // check for special names: - if (ZStreamIn.IsNullStream(path)) - { - return ZStreamIn.NullStreamName; - } - if (ZStreamIn.IsConsoleStream(path)) - { - return ZStreamIn.ConsoleStreamName; - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(path)) - { - return "clip:"; - } - - string pathLower = path.ToLower(); - // check for Cosmos: - if (pathLower.StartsWith("cosmos:")) - { - if (path[path.Length - 1] == '/' || path[path.Length - 1] == '\\') - { - // a directory... - return ""; - } - int lastSep = path.LastIndexOfAny(pathSeparators); - if (lastSep < "cosmos://a".Length) - { - return ""; - } - return path.Substring(lastSep + 1); - } - - // check for Cockpit: - if (pathLower.StartsWith("cockpit:")) - { - if (path[path.Length - 1] == '/' || path[path.Length - 1] == '\\') - { - // a directory... - return ""; - } - int lastSep = path.LastIndexOfAny(pathSeparators); - if (lastSep < "cockpit://a".Length) - { - return ""; - } - return path.Substring(lastSep + 1); - } - - // check for Multistream: - if (pathLower.StartsWith("multi:")) - { - if (path[path.Length - 1] == '/' || path[path.Length - 1] == '\\') - { - // a directory... - return ""; - } - int lastSep = path.LastIndexOfAny(pathSeparators); - if (lastSep < "multi:a".Length) - { - return ""; - } - return path.Substring(lastSep + 1); - } - if (pathLower.StartsWith("filelist:")) - { - if (path[path.Length - 1] == '/' || path[path.Length - 1] == '\\') - { - // a directory... - return ""; - } - int lastSep = path.LastIndexOfAny(pathSeparators); - if (lastSep < "filelist:a".Length) - { - return ""; - } - return path.Substring(lastSep + 1); - } - - // check for HTTP: - if (pathLower.StartsWith("http:") || pathLower.StartsWith("https:")) - { - if (path[path.Length - 1] == '/' || path[path.Length - 1] == '\\') - { - // a directory... - return ""; - } - int lastSep = path.LastIndexOfAny(pathSeparators); - if (lastSep < "http://a".Length) - { - return ""; - } - return path.Substring(lastSep + 1); - } - - // check for SQL: - if (SqlTextReader.IsSqlTextReader(path)) - { - return SqlTextReader.GetFileName(path); - } - - // InternalStore: - if (InternalStoreUtility.IsInternalStore(path)) - { - // maybe not right? *** - // return: - // - whole path? - // - inner path of directory? - // - directory name alone? < - // only return it here if explicitly repeated?? - path = path.TrimEnd(pathSeparators); - int e = path.LastIndexOfAny(pathSeparators); - if (e < 0) - { - e = path.IndexOf(':'); - } - if (e < 0) - return null; - return path.Substring(e + 1).TrimEnd(pathSeparators); - } - - // plain file: - try - { - return Path.GetFileName(path); - } - catch - { - return null; - } - } - - /// - /// Get the filename or directory name, without any path or protocol information. - /// - /// the file or directory name - /// the name of the file or directory, or null if the name does not exist - /// - /// Unlike , this method attempts to return the name - /// for directories as well. This still operates only on the path string. - /// - public static string GetName(string path) - { - if (path == null || path.Length == 0) - return null; - - // check for special names: - if (ZStreamIn.IsNullStream(path)) - { - return ZStreamIn.NullStreamName; - } - if (ZStreamIn.IsConsoleStream(path)) - { - return ZStreamIn.ConsoleStreamName; - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(path)) - { - return "clip:"; - } - - path = path.TrimEnd(pathSeparators); - string pathLower = path.ToLower(); - // check for Cosmos: - if (pathLower.StartsWith("cosmos:")) - { - int lastSep = path.LastIndexOfAny(pathSeparators); - if (lastSep < "cosmos://a".Length) - { - return ""; - } - return path.Substring(lastSep + 1); - } - - // check for Cockpit: - if (pathLower.StartsWith("cockpit:")) - { - int lastSep = path.LastIndexOfAny(pathSeparators); - if (lastSep < "cockpit://a".Length) - { - return ""; - } - return path.Substring(lastSep + 1); - } - - // check for Multistream: - if (pathLower.StartsWith("multi:")) - { - int lastSep = path.LastIndexOfAny(pathSeparators); - if (lastSep < "multi:a".Length) - { - return ""; - } - return path.Substring(lastSep + 1); - } - if (pathLower.StartsWith("filelist:")) - { - int lastSep = path.LastIndexOfAny(pathSeparators); - if (lastSep < "filelist:a".Length) - { - return ""; - } - return path.Substring(lastSep + 1); - } - - // check for HTTP: - if (pathLower.StartsWith("http:") || pathLower.StartsWith("https:")) - { - int lastSep = path.LastIndexOfAny(pathSeparators); - if (lastSep < "http://a".Length) - { - return ""; - } - return path.Substring(lastSep + 1); - } - - // check for SQL: - if (SqlTextReader.IsSqlTextReader(path)) - { - return SqlTextReader.GetName(path); - } - - // InternalStore: - if (InternalStoreUtility.IsInternalStore(path)) - { - // maybe not right? *** - // return: - // - whole path? - // - inner path of directory? - // - directory name alone? < - int e = path.LastIndexOfAny(pathSeparators); - if (e < 0) - { - e = path.IndexOf(':'); - } - if (e < 0) - return null; - return path.Substring(e + 1).TrimEnd(pathSeparators); - } - - // plain file: - try - { - return Path.GetFileName(path); - } - catch - { - return null; - } - } - - private static bool PathAncestor(string ancestor, string path2) - { - if (ancestor == null || ancestor.Length == 0 || path2 == null || path2.Length == 0) - { - return false; - } - ancestor = ancestor.TrimEnd(pathSeparators); - path2 = path2.TrimEnd(pathSeparators); - if (ancestor.Length > path2.Length) - return false; - ancestor = ancestor.Replace('\\', '/'); - path2 = path2.Replace('\\', '/'); - if (string.Compare(ancestor, 0, path2, 0, ancestor.Length, !ancestor.StartsWith("cosmos:")) != 0) - { - return false; - } - if (ancestor.Length == path2.Length || path2[ancestor.Length] == '/') - { - return true; - } - return false; - } - - /// - /// Determine if a path refers to a true file system path, rather than - /// a stream name. - /// - /// the path to test - /// true if path is a file system file or directory, false otherwise - /// - ///

- /// The result is based on the path string alone; it does not reflect whether the - /// given item actually exists. - ///

- ///

- /// For compressed files, a result of true merely means that the path refers - /// to a compressed file on a file system. Reading or writing the file without - /// InternalStreams will still reslt in different behavior, since the raw file will be - /// read. Similarly, a named stream will not work with standard .NET file - /// operations. - ///

- /// - private static bool IsFileSystemPath(string path) - { - if (path == null || path.Length == 0) - return false; - if (path[path.Length - 1] == '$') - return false; - if (path[0] == '/' || path[0] == '\\') - return true; - if (path.Length == 2 && path[1] == ':') - return true; - if (path.Length > 2 && path[1] == ':' && - (path[2] == '/' || path[2] == '\\')) - return true; - - string nameLower = path.ToLower(); - - // check for special names: - if (ZStreamIn.IsNullStream(path)) - { - return false; - } - if (ZStreamIn.IsConsoleStream(path)) - { - return false; - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(path)) - { - return false; - } - - // check for cosmos: - if (nameLower.StartsWith("cosmos:")) - { - return false; - } - - // check for Cockpit: - if (nameLower.StartsWith("cockpit:")) - { - return false; - } - - // check for Multistream: - if (nameLower.StartsWith("multi:")) - { - return false; - } - if (nameLower.StartsWith("filelist:")) - { - return false; - } - - // check for HTTP: - if (nameLower.StartsWith("http://") || nameLower.StartsWith("https://")) - { - return false; - } - - // check for SqlStream: - if (SqlTextReader.IsSqlTextReader(path)) - { - return false; - } - - // InternalStore: - if (InternalStoreUtility.IsInternalStore(path)) - { - return false; - } - - return true; - } - - private static readonly char[] _wildChars = new char[] { '*', '?' }; - private static readonly char[] _wildPlusChars = new char[] { '*', '?', '+' }; - - /// - /// Expand an extended wildcard pattern into a set of file paths. - /// - /// the pattern to expand - /// the set of file paths matching the pattern - /// - /// The wildcard pattern accepts the standard "*" and "?" placeholders. - /// "..." also refers to a recursive search over subdirectories. - /// "+" can also be used to make a union of several filenames or patterns. - /// In addition to filenames, HTTP URLs, nul, null, $, - /// -, and Cosmos stream names are all recognized as elements. - /// Names of files that do not exist will be excluded. - /// - public static string[] ExpandWildcards(string pattern) - { - if (pattern == null || (pattern.IndexOfAny(_wildPlusChars) < 0 && pattern.IndexOf("...") < 0)) - { - if (FileExists(pattern)) - { - return new string[] { pattern }; - } - else - { - return new string[0]; - } - } - List matchList = new List(); - bool disjoint = false; - int filePatternCount = 0; - string[] patterns; - // try to avoid bad splitting? - if (string.Compare(pattern, 0, "http:", 0, 5, true) == 0 || - string.Compare(pattern, 0, "https:", 0, 6, true) == 0) - { - patterns = new string[] { pattern }; - } - else - { - patterns = pattern.Split('+'); - } - foreach (string pat in patterns) - { - // hard-code in special types?? - if (pat.Length == 0) - continue; - string patLower = pat.ToLower(); - if (pat == "$" || pat == "-" || - patLower.StartsWith("cosmos:") || - patLower.StartsWith("cockpit:") || - patLower.StartsWith("http:") || patLower.StartsWith("https:") || - InternalStoreUtility.IsInternalStore(pat) || - SqlTextReader.IsSqlTextReader(pat) || - patLower == "nul" || patLower == "null") - { - // hack for at least some wildcards in cosmos, cockpit: - if (patLower.StartsWith("cosmos:") && - (pat.IndexOfAny(_wildChars) >= 0 || pat.IndexOf("...") >= 0)) - { - matchList.AddRange(Cosmos.DirectoryEntries(pat, true, false, false)); - } - else if (patLower.StartsWith("cockpit:") && - (pat.EndsWith("/*") || pat.EndsWith("\\*"))) - { - matchList.AddRange(DirectoryFiles(pat.Substring(0, pat.Length - 1))); - } - else if ((patLower.StartsWith("http:") || patLower.StartsWith("https:")) && - pat.IndexOf("*") > pat.LastIndexOf("/")) - { - //Is this a wildcard blob path? get all the blobs under this directory - int index = pat.LastIndexOf("/"); - string filter = pat.Substring(index + 1); - AzureStorageIO azureIO = new AzureStorageIO(); - List blobsInPath = azureIO.ListBlobsInPath(pat.Remove(index), filter); - matchList.AddRange(blobsInPath); - } - else - { - if (FileExists(pat)) - matchList.Add(pat); - } - } - else if (patLower.StartsWith("multi:") || patLower.StartsWith("filelist:")) - { - // add in multistream completions?? - if (patLower.StartsWith("multi:")) - { - patLower = patLower.Substring("multi:".Length); - } - else - { - patLower = patLower.Substring("filelist:".Length); - } - string[] mList = ExpandWildcards(patLower); - for (int i = 0; i < mList.Length; i++) - { - if (mList[i].Length == 0) - continue; - matchList.Add("multi:" + mList[i]); - } - } - else - { - filePatternCount++; - int prepatternCount = matchList.Count; - if (pat.IndexOfAny(_wildChars) >= 0 || pat.IndexOf("...") >= 0) - { - // compressed extensions are not automatically used! *** - int recursiveIndex = pat.IndexOf("..."); - if (recursiveIndex >= 0) - { - string left = pat.Substring(0, recursiveIndex); - string right = pat.Substring(recursiveIndex + 3); - right = right.TrimStart('\\', '/'); - if (right.Length == 0) - right = "*"; - string path = left; - bool pathEmpty = (path == null || path.Length == 0); - if (pathEmpty) - path = "."; - Stack dirsLeft = new Stack(); - dirsLeft.Push(path); - while (dirsLeft.Count != 0) - { - string dir = (string)dirsLeft.Pop(); - - // watch for lack of access: - try - { - // this is actually incorrect, for 3-char extensions: *** - string[] files = Directory.GetFiles(dir, right); - if (pathEmpty) - { - for (int i = 0; i < files.Length; i++) - { - if (files[i].StartsWith("./") || files[i].StartsWith(".\\")) - files[i] = files[i].Substring(2); - } - } - matchList.AddRange(files); - - string[] subs = Directory.GetDirectories(dir); - for (int s = subs.Length - 1; s >= 0; s--) - { - dirsLeft.Push(subs[s]); - } - } - catch - { - // ignore - } - } - } - else - { - try - { - string path = Path.GetDirectoryName(pat); - bool pathEmpty = !(pat.StartsWith("./") || pat.StartsWith(".\\")); - if (path == null || path.Length == 0) - path = "."; - // watch for lack of access: - try - { - string[] files = Directory.GetFiles(path, Path.GetFileName(pat)); - if (pathEmpty) - { - for (int i = 0; i < files.Length; i++) - { - if (files[i].StartsWith("./") || files[i].StartsWith(".\\")) - files[i] = files[i].Substring(2); - } - } - matchList.AddRange(files); - } - catch - { - // ignore - } - } - catch - { - // ignore bad path? - } - } - } - else - { - // what to do?? Filter to only those that exist?? *** - if (!FileExists(pat)) - continue; - matchList.Add(pat); - } - if (filePatternCount > 1 && matchList.Count != prepatternCount) - { - disjoint = true; - } - } - } - if (disjoint || true) - { - // remove duplicates, very inefficiently - but it is simple, preserves - // the order, uses no additional memory, and is case-insensitive...: - for (int i = 0; i < matchList.Count - 1; i++) - { - for (int j = i + 1; j < matchList.Count; j++) - { - bool caseInsensitive = !(string.Compare((string)matchList[i], 0, "cosmos:", 0, "cosmos:".Length, true) == 0); - { - if (string.Compare((string)matchList[i], (string)matchList[j], caseInsensitive) == 0) - { - matchList.RemoveAt(j); - j--; - } - } - } - } - } - return matchList.ToArray(); - } - - #endregion - - #region Directory Operations - - /// - /// Determine if a directory or archive exists. - /// - /// the directory to look for - /// true if the directory exists, false otherwise - /// - /// This will only detect directories on file paths, Cosmos, and - /// compressed archives. - /// - public static bool DirectoryExists(string path) - { - if (path == null || path.Length == 0) - return false; - string nameLower = path.ToLower(); - - // check for special names: - if (ZStreamIn.IsNullStream(path)) - { - // allow self-directory: - return path.IndexOfAny(pathSeparators) < 0; - } - if (ZStreamIn.IsConsoleStream(path)) - { - // allow self-directory: - return path.IndexOfAny(pathSeparators) < 0; - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(path)) - { - return path.IndexOfAny(pathSeparators) < 0 || - string.Compare(path, "clip://", true) == 0; - } - - // check for cosmos: - if (nameLower.StartsWith("cosmos:")) - { - return Cosmos.DirectoryExists(path); - } - - // check for Cockpit: - if (nameLower.StartsWith("cockpit:")) - { - // what is right here? - string parent = PathParent(path); - if (parent == null) - { - // assume it exists?? - return true; - } - string name = GetName(path); - string[] dirs = DirectoryEntries(parent, false, true); - for (int i = 0; i < dirs.Length; i++) - { - string dir = GetName(dirs[i]).Trim(pathSeparators); - if (string.Compare(name, dir, true) == 0) - { - return true; - } - } - return false; - } - - // check for Multistream: - if (nameLower.StartsWith("multi:")) - { - return false; - } - if (nameLower.StartsWith("filelist:")) - { - return false; - } - - // check for HTTP: - if (nameLower.StartsWith("http://") || nameLower.StartsWith("https://")) - { - // what is right here? - return false; //HttpStream.Exists(path); - } - - // check for SqlStream: - if (SqlTextReader.IsSqlTextReader(path)) - { - return SqlTextReader.ExistsDatabase(path); - } - - // InternalStore: - if (InternalStoreUtility.IsInternalStore(path)) - { - return DirectoryExists(InternalStoreUtility.StorePath(path)); - } - - if (Directory.Exists(path)) - return true; - - if ((path.StartsWith("\\\\") || path.StartsWith("//")) && - !Directory.Exists(Path.GetPathRoot(path))) - { - return false; - } - - // check for compressed archive: - for (int i = 0; i < ZStreamIn.decompressionArchiveExtensions.Length; i++) - { - string ext = ZStreamIn.decompressionArchiveExtensions[i]; - if ((nameLower.EndsWith(ext) && File.Exists(path)) || - File.Exists(path + ext)) - { - return true; - } - } - - // check for compressed archives in path: - // only one path segment is allowed to be an archive... - // normalize path: - if (path[path.Length - 1] != '\\') - path = path + "\\"; - path = path.Replace('/', '\\'); - bool isUnc = path.StartsWith("\\\\"); - while (path.IndexOf("\\\\") >= 0) - { - path = path.Replace("\\\\", "\\"); - } - if (isUnc) - path = "\\" + path; - nameLower = path.ToLower(); - - string archPath = null; - string inArch = null; - // should non-archives be considered as one-file directories?? - for (int i = 0; i < ZStreamIn.decompressionArchiveExtensions.Length; i++) - { - string ext = ZStreamIn.decompressionArchiveExtensions[i]; - int seg = nameLower.IndexOf(ext + "\\"); - if (seg > 0) - { - archPath = path.Substring(0, seg + ext.Length); - if (File.Exists(archPath)) - { - inArch = path.Substring(seg + ext.Length); - break; - } - archPath = null; - } - } - if (archPath == null) - { -#if !QUICK_ARCHIVE_SEARCH - // add in extension to each segment... - string[] segs = path.Split('\\'); - for (int i = 0; i < segs.Length; i++) - { - if (segs[i].Length == 0) - continue; - string partial = string.Join("\\", segs, 0, i + 1); - if (partial.Length == 2 && partial[1] == ':') - continue; - if (Directory.Exists(partial)) - continue; - for (int c = 0; c < ZStreamIn.decompressionArchiveExtensions.Length; c++) - { - string ext = ZStreamIn.decompressionArchiveExtensions[c]; - if (File.Exists(partial + ext)) - { - archPath = partial + ext; - inArch = string.Join("\\", segs, i + 1, segs.Length - i - 1); - break; - } - } - // quit when parent will not exist - break; -#else - // add in extension to last segment... - string[] segs = path.TrimEnd('\\').Split('\\'); - for (int i = 0; i < segs.Length; i++) - { - if (segs[i].Length == 0) continue; - string partial = string.Join("\\", segs, 0, segs.Length - 1); - for (int c = 0; c < ZStreamIn.decompressionArchiveExtensions.Length; c++) - { - string ext = ZStreamIn.decompressionArchiveExtensions[c]; - if (File.Exists(partial + ext)) - { - archPath = partial + ext; - inArch = segs[segs.Length - 1]; - break; - } - } - if (archPath != null) break; - // quit when parent will not exist - if (!Directory.Exists(partial)) break; -#endif - } - } - if (archPath != null) - { - // check for path in archive. - inArch = inArch.Trim('\\'); - if (inArch.Length == 0) - return true; - // Must check directories... - if (Z7zDecodeStream.Exists(archPath, inArch + "/*")) - return true; - if (!Z7zDecodeStream.Exists7z && Path.GetExtension(archPath).ToLower() == ".rar") - { - if (RarDecodeStream.Exists(archPath, inArch + "/*")) - return true; - } - } - - return false; - } - - /// - /// Create a directory or archive. - /// - /// the directory or archive name to create - /// - ///

- /// Archives will be created empty. - ///

- ///

- /// When a directory already exists, this method will silently do nothing. - ///

- /// - /// The directory cannot be created. - public static void CreateDirectory(string path) - { - if (path == null || path.Length == 0) - throw new IOException("Cannot create directory for empty path"); - string nameLower = path.ToLower(); - - // check for special names: - if (ZStreamIn.IsNullStream(path)) - { - return; - } - if (ZStreamIn.IsConsoleStream(path)) - { - return; - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(path)) - { - return; - } - - // check for cosmos: - if (nameLower.StartsWith("cosmos:")) - { - Cosmos.CreateDirectory(path); - return; - } - - // check for Cockpit: - if (nameLower.StartsWith("cockpit:")) - { - throw new IOException("Cannot create directory for Cockpit"); - } - - // check for Multistream: - if (nameLower.StartsWith("multi:") || nameLower.StartsWith("filelist:")) - { - throw new IOException("Cannot create directory for Multistream"); - } - - // check for HTTP: - if (nameLower.StartsWith("http://") || nameLower.StartsWith("https://")) - { - throw new IOException("Cannot create directory for HTTP"); - } - - // check for SqlStream: - if (SqlTextReader.IsSqlTextReader(path)) - { - throw new IOException("Cannot create directory for SQL"); - } - - // check for InternalStore: - if (InternalStoreUtility.IsInternalStore(path)) - { - throw new IOException("Cannot create directory for InternalStore"); - } - - if (Directory.Exists(path)) - return; - - // check for compressed archive: - for (int i = 0; i < ZStreamOut.compressionArchiveExtensions.Length; i++) - { - string ext = ZStreamOut.compressionArchiveExtensions[i]; - if ((nameLower.EndsWith(ext))) - { - if (!File.Exists(path)) - { - try - { - using (Stream s = ZStreamOut.Open(path)) - { - s.Close(); - if (s is CmdStream && ((CmdStream)s).ExitCode != 0) - { - throw new IOException("Failed to create archive"); - } - } - try - { - Delete(path + "/" + Path.GetFileNameWithoutExtension(path)); - } - catch - { - // ignore - } - } - catch (Exception ex) - { - Console.WriteLine(ex); - throw new IOException("Failed to create archive"); - } - } - return; - } - } - - try - { - Directory.CreateDirectory(path); - } - catch - { - throw new IOException("Failed to create directory"); - } - } - - /// - /// Get the paths to files within a directory. - /// - /// the directory to look in - /// the set of paths for the files in that directory - /// - /// This will silently return the empty list if there are any problems. - /// - public static string[] DirectoryFiles(string path) - { - return DirectoryEntries(path, true, false); - } - - /// - /// Get the paths to directories within a directory. - /// - /// the directory to look in - /// the set of paths for the directories in that directory - /// - /// This will silently return the empty list if there are any problems. - /// - public static string[] DirectoryDirectories(string path) - { - return DirectoryEntries(path, false, true); - } - - /// - /// Get the paths to files and directories within a directory. - /// - /// the directory to look in - /// the set of paths for the files and directories in that directory - /// - ///

- /// Directories will be distinguished by ending with "/". - ///

- ///

- /// This will silently return the empty list if there are any problems. - ///

- /// - public static string[] DirectoryEntries(string path) - { - return DirectoryEntries(path, true, true); - } - - private static string[] DirectoryEntries(string path, bool allowFile, bool allowDirectory) - { - - if (path == null || path.Length == 0) - return new string[0]; - if (!allowFile && !allowDirectory) - return new string[0]; - if (path[path.Length - 1] != '\\' && path[path.Length - 1] != '/') - { - path = path + "/"; - } - - string pathLower = path.ToLower(); - - // check for special names: - if (ZStreamIn.IsNullStream(path)) - { - return path.IndexOfAny(pathSeparators) < 0 ? - new string[] { ZStreamIn.NullStreamName } : new string[0]; - } - if (ZStreamIn.IsConsoleStream(path)) - { - return path.IndexOfAny(pathSeparators) < 0 ? - new string[] { ZStreamIn.ConsoleStreamName } : new string[0]; - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(path)) - { - return path.IndexOfAny(pathSeparators) < 0 || - string.Compare(path.TrimEnd(pathSeparators), "clip:", true) == 0 ? - new string[] { "clip:" } : new string[0]; - } - - // check for cosmos: - if (pathLower.StartsWith("cosmos:")) - { - return Cosmos.DirectoryEntries(path, allowFile, allowDirectory, false); - } - - // check for Cockpit: - if (pathLower.StartsWith("cockpit:")) - { - StreamInfo[] entries = DirectoryEntriesInfo(path, allowFile, allowDirectory); - if (entries == null) - return null; - string[] res = new string[entries.Length]; - for (int i = 0; i < res.Length; i++) - { - res[i] = entries[i].Path; - } - return res; - } - - // check for Multistream: - if (pathLower.StartsWith("multi:") || pathLower.StartsWith("filelist:")) - { - return new string[0]; - } - - // check for HTTP: - if (pathLower.StartsWith("http://") || pathLower.StartsWith("https://")) - { - return new string[0]; - } - - // check for SqlStream: - if (SqlTextReader.IsSqlTextReader(path)) - { - return SqlTextReader.DatabaseTablePaths(path); - } - - // check for InternalStore: - if (InternalStoreUtility.IsInternalStore(path)) - { - return InternalStoreUtility.DirectoryEntries(path); - } - - if (Directory.Exists(path)) - { - try - { - if (!allowDirectory) - { - return Directory.GetFiles(path); - } - else if (!allowFile) - { - string[] resDirs = Directory.GetDirectories(path); - for (int i = 0; i < resDirs.Length; i++) - { - resDirs[i] = resDirs[i] + "/"; - } - return resDirs; - } - else - { - string[] res = Directory.GetFiles(path); - string[] resDirs = Directory.GetDirectories(path); - if (resDirs.Length != 0) - { - string[] resFiles = res; - res = new string[res.Length + resDirs.Length]; - for (int i = 0; i < resDirs.Length; i++) - { - res[i] = resDirs[i] + "/"; - } - if (resFiles.Length != 0) - { - Array.Copy(resFiles, 0, res, resDirs.Length, resFiles.Length); - } - } - return res; - } - } - catch - { - return new string[0]; - } - } - - if ((path.StartsWith("\\\\") || path.StartsWith("//")) && - !Directory.Exists(Path.GetPathRoot(path))) - { - return new string[0]; - } - - // check for compressed forms: - string pathBase = pathLower.Substring(0, path.Length - 1); - for (int i = 0; i < ZStreamIn.decompressionExtensions.Length; i++) - { - string ext = ZStreamIn.decompressionExtensions[i]; - if (pathBase.EndsWith(ext) && File.Exists(pathBase)) - { - string[] res = Z7zDecodeStream.DirectoryEntries(pathBase, null, allowFile, allowDirectory); - if (res.Length == 0 && !Z7zDecodeStream.Exists7z && ext == ".rar") - { - res = RarDecodeStream.DirectoryEntries(pathBase, null, allowFile, allowDirectory); - } - return res; - } - else - { - string pathBaseExt = pathBase + ext; - if (File.Exists(pathBaseExt)) - { - string[] res = Z7zDecodeStream.DirectoryEntries(pathBaseExt, null, allowFile, allowDirectory); - if (res.Length == 0 && !Z7zDecodeStream.Exists7z && ext == ".rar") - { - res = RarDecodeStream.DirectoryEntries(pathBaseExt, null, allowFile, allowDirectory); - } - return res; - } - } - } - - // check for compressed archive as directory segment: - // check for compressed archives in path: - // only one path segment is allowed to be an archive... - // normalize path: - string zfileName = path.Replace('/', '\\'); - bool isUnc = zfileName.StartsWith("\\\\"); - while (zfileName.IndexOf("\\\\") >= 0) - { - zfileName = zfileName.Replace("\\\\", "\\"); - } - if (isUnc) - zfileName = "\\" + zfileName; - string znameLower = zfileName.ToLower(); - - string archPath = null; - string inArch = null; - for (int i = 0; i < ZStreamIn.decompressionExtensions.Length; i++) - { - string ext = ZStreamIn.decompressionExtensions[i]; - int seg = znameLower.IndexOf(ext + "\\"); - if (seg > 0) - { - archPath = zfileName.Substring(0, seg + ext.Length); - if (File.Exists(archPath)) - { - inArch = zfileName.Substring(seg + ext.Length).Trim('/', '\\'); - break; - } - archPath = null; - } - } - if (archPath == null) - { - // add in extension to each segment... - string[] segs = zfileName.Split('\\'); - for (int i = 0; i < segs.Length; i++) - { - if (segs[i].Length == 0) - continue; - string partial = string.Join("\\", segs, 0, i + 1); - if (partial.Length == 2 && partial[1] == ':') - continue; - if (Directory.Exists(partial)) - continue; - for (int c = 0; c < ZStreamIn.decompressionExtensions.Length; c++) - { - string ext = ZStreamIn.decompressionExtensions[c]; - if (File.Exists(partial + ext)) - { - archPath = partial + ext; - inArch = string.Join("\\", segs, i + 1, segs.Length - i - 1).Trim('/', '\\'); - break; - } - } - // quit when parent will not exist - break; - } - } - if (archPath != null) - { - // check for path in archive. - inArch = inArch.Trim('\\'); - if (inArch.Length == 0) - return new string[0]; - string[] res = Z7zDecodeStream.DirectoryEntries(archPath, inArch + "\\*", allowFile, allowDirectory); - if (res.Length == 0 && !Z7zDecodeStream.Exists7z && Path.GetExtension(archPath).ToLower() == ".rar") - { - res = RarDecodeStream.DirectoryEntries(archPath, inArch + "\\*", allowFile, allowDirectory); - } - return res; - } - - return new string[0]; - } - - /// - /// Get the objects for files within a directory. - /// - /// the directory to look in - /// the set of objects for the files in that directory - /// - /// This will silently return the empty list if there are any problems. - /// - public static StreamInfo[] DirectoryFilesInfo(string path) - { - return DirectoryEntriesInfo(path, true, false); - } - - /// - /// Get the objects for files and directories within a directory. - /// - /// the directory to look in - /// the set of objects for the files and directories in that directory - /// - ///

- /// This will silently return the empty list if there are any problems. - ///

- /// - public static StreamInfo[] DirectoryEntriesInfo(string path) - { - return DirectoryEntriesInfo(path, true, true); - } - - private static StreamInfo[] DirectoryEntriesInfo(string path, bool allowFile, bool allowDirectory) - { - // is it worth keeping this distinct from DirectoryEntries for efficiency?? **** - - if (path == null || path.Length == 0) - return new StreamInfo[0]; - if (!allowFile && !allowDirectory) - return new StreamInfo[0]; - if (path[path.Length - 1] != '\\' && path[path.Length - 1] != '/') - { - path = path + (path.IndexOf('/') < 0 ? "\\" : "/"); - } - - string pathLower = path.ToLower(); - - // check for special names: - if (ZStreamIn.IsNullStream(path)) - { - return path.IndexOfAny(pathSeparators) < 0 ? - new StreamInfo[] { new StreamInfo(ZStreamIn.NullStreamName, 0, DateTime.MinValue) } : new StreamInfo[0]; - } - if (ZStreamIn.IsConsoleStream(path)) - { - return path.IndexOfAny(pathSeparators) < 0 ? - new StreamInfo[] { new StreamInfo(ZStreamIn.ConsoleStreamName, 0, DateTime.MinValue) } : new StreamInfo[0]; - } - - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(path)) - { - return path.IndexOfAny(pathSeparators) < 0 || - string.Compare(path.TrimEnd(pathSeparators), "clip:", true) == 0 ? - new StreamInfo[] { new StreamInfo("clip:", 0, DateTime.MinValue) } : new StreamInfo[0]; - } - - // check for cosmos: - if (pathLower.StartsWith("cosmos:")) - { - return Cosmos.DirectoryEntriesInfo(path, allowFile, allowDirectory, false); - } - - // check for Cockpit: - if (pathLower.StartsWith("cockpit:")) - { - return CockpitDirectoryEntriesInfo(path, allowFile, allowDirectory); - } - - // check for Multistream: - if (pathLower.StartsWith("multi:") || pathLower.StartsWith("filelist:")) - { - return new StreamInfo[0]; - } - - // check for HTTP: - if (pathLower.StartsWith("http://") || pathLower.StartsWith("https://")) - { - return new StreamInfo[0]; - } - - // check for SqlStream: - if (SqlTextReader.IsSqlTextReader(path)) - { - if (allowFile) - { - string[] files = SqlTextReader.DatabaseTablePaths(path); - StreamInfo[] res = new StreamInfo[files.Length]; - for (int i = 0; i < res.Length; i++) - { - long len = 0; - DateTime lastMod = DateTime.MinValue; - res[i] = new StreamInfo(files[i], len, lastMod); - } - return res; - } - return new StreamInfo[0]; - } - - // check for InternalStore: - if (InternalStoreUtility.IsInternalStore(path)) - { - return InternalStoreUtility.DirectoryEntriesInfo(path); - } - - if (Directory.Exists(path)) - { - try - { - StreamInfo[] res = null; - if (allowFile) - { - string[] files = Directory.GetFiles(path); - res = new StreamInfo[files.Length]; - for (int i = 0; i < res.Length; i++) - { - long len = 0; - DateTime lastMod = DateTime.MinValue; - try - { - FileInfo f = new FileInfo(files[i]); - len = f.Length; - lastMod = f.LastWriteTime; - } - catch - { - // ignore? - } - res[i] = new StreamInfo(files[i], len, lastMod); - } - } - if (!allowDirectory) - return res; - - string[] dirs = Directory.GetDirectories(path); - if (res == null || res.Length == 0) - { - res = new StreamInfo[dirs.Length]; - } - else - { - StreamInfo[] resOld = res; - res = new StreamInfo[resOld.Length + dirs.Length]; - Array.Copy(resOld, 0, res, res.Length - resOld.Length, resOld.Length); - } - for (int i = 0; i < dirs.Length; i++) - { - long len = 0; - DateTime lastMod = DateTime.MinValue; - try - { - lastMod = Directory.GetLastWriteTime(dirs[i]); - } - catch - { - // ignore? - } - if (dirs[i].Length == 0) - { - dirs[i] = ".\\"; - } - else - { - if (dirs[i][dirs[i].Length - 1] != '\\' && - dirs[i][dirs[i].Length - 1] != '/') - { - dirs[i] = dirs[i] + "\\"; - } - } - res[i] = new StreamInfo(dirs[i], len, lastMod); - } - - return res; - } - catch - { - return new StreamInfo[0]; - } - } - - if ((path.StartsWith("\\\\") || path.StartsWith("//")) && - !Directory.Exists(Path.GetPathRoot(path))) - { - return new StreamInfo[0]; - } - - // check for compressed forms: - string pathBase = pathLower.Substring(0, path.Length - 1); - for (int i = 0; i < ZStreamIn.decompressionExtensions.Length; i++) - { - string ext = ZStreamIn.decompressionExtensions[i]; - if (pathBase.EndsWith(ext) && File.Exists(pathBase)) - { - StreamInfo[] res = Z7zDecodeStream.DirectoryEntriesInfo(pathBase, null, allowFile, allowDirectory); - if (res.Length == 0 && !Z7zDecodeStream.Exists7z && ext == ".rar") - { - res = RarDecodeStream.DirectoryEntriesInfo(pathBase, null, allowFile, allowDirectory); - } - return res; - } - else - { - string pathBaseExt = pathBase + ext; - if (File.Exists(pathBaseExt)) - { - StreamInfo[] res = Z7zDecodeStream.DirectoryEntriesInfo(pathBaseExt, null, allowFile, allowDirectory); - if (res.Length == 0 && !Z7zDecodeStream.Exists7z && ext == ".rar") - { - res = RarDecodeStream.DirectoryEntriesInfo(pathBaseExt, null, allowFile, allowDirectory); - } - return res; - } - } - } - - // check for compressed archive as directory segment: - // check for compressed archives in path: - // only one path segment is allowed to be an archive... - // normalize path: - string zfileName = path.Replace('/', '\\'); - bool isUnc = zfileName.StartsWith("\\\\"); - while (zfileName.IndexOf("\\\\") >= 0) - { - zfileName = zfileName.Replace("\\\\", "\\"); - } - if (isUnc) - zfileName = "\\" + zfileName; - string znameLower = zfileName.ToLower(); - - string archPath = null; - string inArch = null; - for (int i = 0; i < ZStreamIn.decompressionExtensions.Length; i++) - { - string ext = ZStreamIn.decompressionExtensions[i]; - int seg = znameLower.IndexOf(ext + "\\"); - if (seg > 0) - { - archPath = zfileName.Substring(0, seg + ext.Length); - if (File.Exists(archPath)) - { - inArch = zfileName.Substring(seg + ext.Length).Trim('/', '\\'); - break; - } - archPath = null; - } - } - if (archPath == null) - { - // add in extension to each segment... - string[] segs = zfileName.Split('\\'); - for (int i = 0; i < segs.Length; i++) - { - if (segs[i].Length == 0) - continue; - string partial = string.Join("\\", segs, 0, i + 1); - if (partial.Length == 2 && partial[1] == ':') - continue; - if (Directory.Exists(partial)) - continue; - for (int c = 0; c < ZStreamIn.decompressionExtensions.Length; c++) - { - string ext = ZStreamIn.decompressionExtensions[c]; - if (File.Exists(partial + ext)) - { - archPath = partial + ext; - inArch = string.Join("\\", segs, i + 1, segs.Length - i - 1).Trim('/', '\\'); - break; - } - } - // quit when parent will not exist - break; - } - } - if (archPath != null) - { - // check for path in archive. - inArch = inArch.Trim('\\'); - if (inArch.Length == 0) - return new StreamInfo[0]; - StreamInfo[] res = Z7zDecodeStream.DirectoryEntriesInfo(archPath, inArch + "\\*", allowFile, allowDirectory); - if (res.Length == 0 && !Z7zDecodeStream.Exists7z && Path.GetExtension(archPath).ToLower() == ".rar") - { - res = RarDecodeStream.DirectoryEntriesInfo(archPath, inArch + "\\*", allowFile, allowDirectory); - } - return res; - } - - return new StreamInfo[0]; - } - - private static StreamInfo[] CockpitDirectoryEntriesInfo(string path, bool allowFile, bool allowDirectory) - { - List res = new List(); - string htmlDir; - try - { - if (!path.EndsWith("/") && !path.EndsWith("\\")) - { - path = path + '\\'; - } - htmlDir = ReadFile(path); - } - catch - { - // throw exception? - return new StreamInfo[0]; - } - if (htmlDir == null || htmlDir.Length == 0) - return new StreamInfo[0]; - htmlDir = htmlDir.Replace("
", "
"); - - Hashtable filenames = new Hashtable(); - -#if !DISABLE_XML - System.Xml.XmlTextReader xmlReader = null; - try - { - // apparently, not IDisposable in .NET 1.1: - //using (System.Xml.XmlTextReader xmlReader = new System.Xml.XmlTextReader(new StringReader(htmlDir))) - xmlReader = new System.Xml.XmlTextReader(new StringReader(htmlDir)); - { - string filename = null; - long length = -1; - DateTime lastModified = DateTime.MinValue; - bool isDirectory = false; - - // Parse node by node - while (xmlReader.Read()) - { - if (xmlReader.IsStartElement("tr")) - { - filename = null; - length = -1; - lastModified = DateTime.MinValue; - isDirectory = false; - } - else if (xmlReader.IsStartElement("td")) - { - xmlReader.Read(); - if (xmlReader.IsStartElement("a")) - { - // Store the anchor text value - xmlReader.Read(); - filename = xmlReader.Value; - } - else - { - try - { - string val = xmlReader.Value; - if (string.Compare(val.Trim(), " ", true) == 0) - { - isDirectory = true; - } - else - { - if (val.Length != 0) - { - if (char.IsDigit(val[0])) - { - // Attempt to convert to a long - length = Convert.ToInt64(val); - } - else - { - // Attempt to convert to a timestamp - lastModified = Convert.ToDateTime(val); - } - } - } - } - catch - { - // Skip this item - } - } - } - // When all the data has been captured, record the entry - if (filename != null) - { - if (length >= 0 && lastModified != DateTime.MinValue) - { - if (allowFile) - { - string fullname = PathCombine(path, filename); - res.Add(new StreamInfo(fullname, length, lastModified)); - } - filename = null; - length = -1; - lastModified = DateTime.MinValue; - isDirectory = false; - } - else if (isDirectory) - { - if (allowDirectory) - { - string line = filename; - line = line.Replace('\t', ' '); - int oldLen = -1; - while (line.Length != oldLen) - { - oldLen = line.Length; - line = line.Replace(" ", " "); - } - line = line.Trim(); - - if (line != "Parent directory\\") - { - string fullname = PathCombine(path, filename); - if (!fullname.EndsWith("\\") && !fullname.EndsWith("/")) - { - fullname = fullname + "\\"; - } - res.Add(new StreamInfo(fullname, 0, DateTime.MinValue)); - } - } - filename = null; - length = -1; - lastModified = DateTime.MinValue; - isDirectory = false; - } - } - } - } - } - catch - { - // throw exception? - return new StreamInfo[0]; - } - finally - { - try - { - if (xmlReader != null) - xmlReader.Close(); - } - catch - { - // ignore - } - } -#endif - return res.ToArray(); - } - - #endregion - - #region File Operations - - /// - /// Determine if a file exists, including in forms with compression extensions, HTTP URLs, and - /// Cosmos streams. - /// - /// the original filename - /// true if a file with a name of fileName or its compressed variations exists, false otherwise - /// - /// This is useful for detecting whether a file can be opened by - /// ZStreamIn.Open(), since it will look for compressed forms and other variations. - /// - public static bool FileExists(string fileName) - { - if (fileName == null || fileName.Length == 0) - return false; - string nameLower = fileName.ToLower(); - - // check for special names: - if (ZStreamIn.IsNullStream(fileName)) - { - return true; - } - if (ZStreamIn.IsConsoleStream(fileName)) - { - return true; - } - - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(fileName)) - { - return ClipboardReadStream.FileExists(); - } - - // check for cosmos: - if (nameLower.StartsWith("cosmos:")) - { - // no compression support, anyway... - fileName = fileName.TrimEnd('$'); - return Cosmos.FileExists(fileName); - } - - // check for Cockpit: - if (nameLower.StartsWith("cockpit:")) - { - // what is right here? - string parent = PathParent(fileName); - if (parent == null) - { - // assume it exists?? - return true; - } - string name = GetName(fileName); - string[] dirs = DirectoryEntries(parent, true, false); - for (int i = 0; i < dirs.Length; i++) - { - string dir = GetName(dirs[i]).Trim(pathSeparators); - if (string.Compare(name, dir, true) == 0) - { - return true; - } - } - return false; - } - - // check for Multistream: - if (nameLower.StartsWith("multi:")) - { - fileName = fileName.Substring("multi:".Length); - return FileExists(fileName); - } - if (nameLower.StartsWith("filelist:")) - { - fileName = fileName.Substring("filelist:".Length); - return FileExists(fileName); - } - - // check for HTTP: - if (nameLower.StartsWith("http://") || nameLower.StartsWith("https://")) - { - AzureStorageIO azureStorage = new AzureStorageIO(); - if (azureStorage.BlockBlobExistsByUri(fileName)) - { - return true; - } - return HttpStream.Exists(fileName); - } - - // check for SQL: - if (SqlTextReader.IsSqlTextReader(fileName)) - { - return SqlTextReader.Exists(fileName); - } - - // check for InternalStore: - if (InternalStoreUtility.IsInternalStore(fileName)) - { - return InternalStoreUtility.Exists(fileName); - } - // remove trailing "$" - if (fileName[fileName.Length - 1] == '$') - { - fileName = fileName.Substring(0, fileName.Length - 1); - } - - if (File.Exists(fileName)) - return true; - - // check for compressed forms: - for (int i = 0; i < ZStreamIn.decompressionExtensions.Length; i++) - { - if (File.Exists(fileName + ZStreamIn.decompressionExtensions[i])) - { - return true; - } - } - - // check for named stream: - // should this be based on file existance, first? - int cIndex = fileName.LastIndexOf(':'); - if (cIndex > 0) - { - if (File.Exists(fileName.Substring(0, cIndex))) - { - if (cIndex > 1 || - fileName.IndexOfAny(pathSeparators, 2) < 0) - { - // named: - return (Array.IndexOf(IOUtil.GetNamedStreams(fileName.Substring(0, cIndex)), - fileName.Substring(cIndex + 1)) >= 0); - } - } - } - - // check for compressed archive as directory segment: - // check for compressed archives in path: - // only one path segment is allowed to be an archive... - // normalize path: - fileName = fileName.Replace('/', '\\'); - bool isUnc = fileName.StartsWith("\\\\"); - while (fileName.IndexOf("\\\\") >= 0) - { - fileName = fileName.Replace("\\\\", "\\"); - } - if (isUnc) - fileName = "\\" + fileName; - nameLower = fileName.ToLower(); - - string archPath = null; - string inArch = null; - for (int i = 0; i < ZStreamIn.decompressionExtensions.Length; i++) - { - string ext = ZStreamIn.decompressionExtensions[i]; - int seg = nameLower.IndexOf(ext + "\\"); - if (seg > 0) - { - archPath = fileName.Substring(0, seg + ext.Length); - if (File.Exists(archPath)) - { - inArch = fileName.Substring(seg + ext.Length).Trim('/', '\\'); - break; - } - archPath = null; - } - } - if (archPath == null) - { - // add in extension to each segment... - string[] segs = fileName.Split('\\'); - for (int i = 0; i < segs.Length; i++) - { - if (segs[i].Length == 0) - continue; - string partial = string.Join("\\", segs, 0, i + 1); - if (partial.Length == 2 && partial[1] == ':') - continue; - if (Directory.Exists(partial)) - continue; - for (int c = 0; c < ZStreamIn.decompressionExtensions.Length; c++) - { - string ext = ZStreamIn.decompressionExtensions[c]; - if (File.Exists(partial + ext)) - { - archPath = partial + ext; - inArch = string.Join("\\", segs, i + 1, segs.Length - i - 1).Trim('/', '\\'); - break; - } - } - // quit when parent will not exist - break; - } - } - if (archPath != null) - { - // check for path in archive. - inArch = inArch.Trim('\\'); - if (inArch.Length == 0) - return false; - if (Z7zDecodeStream.Exists(archPath, inArch)) - return true; - if (!Z7zDecodeStream.Exists7z && Path.GetExtension(archPath).ToLower() == ".rar") - { - if (RarDecodeStream.Exists(archPath, inArch)) - return true; - } - } - - return false; - } - - /// - /// Find the full path for a command, if it exists in the environment path. - /// - /// the command to look for - /// the full path, or null if it is not found - /// - ///

- /// This will always look in the current PATH directories, as well as in - /// the current directory. The current directory takes precedence. - ///

- ///

- /// The command may have an extension included; it will also be tried - /// with ".exe", ".bat", and ".cmd", in that order, in each path - /// directory. This gives similar results to the standard command line. - ///

- /// - public static string FindInPath(string cmd) - { - return FindInPath(cmd, false); - } - /// - /// Find the full path for a command, if it exists in the environment path. - /// - /// the command to look for - /// - /// if true, also check the directory of the calling assembly. - /// - /// the full path, or null if it is not found - /// - ///

- /// This will always look in the current PATH directories, as well as in - /// the current directory. The current directory takes precedence. - ///

- ///

- /// If is true, it takes - /// precedence over any other directory. - ///

- ///

- /// The command may have an extension included; it will also be tried - /// with ".exe", ".cmd", and ".bat", in that order, in each path - /// directory. This gives similar results to the standard command line. - ///

- /// - public static string FindInPath(string cmd, bool includeAssemblyDirectory) - { - if (cmd == null || cmd.Length == 0) - return null; - // caller's directory: - if (includeAssemblyDirectory) - { - string dir = Path.GetDirectoryName(System.Reflection.Assembly.GetCallingAssembly().Location); - string path = Path.Combine(dir, cmd); - if (File.Exists(path)) - return path; - string extPath; - extPath = path + ".exe"; - if (File.Exists(extPath)) - return extPath; - extPath = path + ".cmd"; - if (File.Exists(extPath)) - return extPath; - extPath = path + ".bat"; - if (File.Exists(extPath)) - return extPath; - } - // current directory: - { - string dir = Environment.CurrentDirectory; - string path = Path.Combine(dir, cmd); - if (File.Exists(path)) - return path; - string extPath; - extPath = path + ".exe"; - if (File.Exists(extPath)) - return extPath; - extPath = path + ".cmd"; - if (File.Exists(extPath)) - return extPath; - extPath = path + ".bat"; - if (File.Exists(extPath)) - return extPath; - } - // PATH: - { - string[] paths = Environment.GetEnvironmentVariable("PATH").Trim(';').Split(';'); - for (int i = 0; i < paths.Length; i++) - { - string dir = paths[i].Trim(); - if (dir.Length == 0) - continue; - string path = Path.Combine(dir, cmd); - if (File.Exists(path)) - return path; - string extPath; - extPath = path + ".exe"; - if (File.Exists(extPath)) - return extPath; - extPath = path + ".cmd"; - if (File.Exists(extPath)) - return extPath; - extPath = path + ".bat"; - if (File.Exists(extPath)) - return extPath; - } - } - return null; - } - - /// - /// Delete a stream or directory, if it exists. - /// - /// the stream or directory to delete - /// - ///

- /// This will silently do nothing if the file already does not exist. - ///

- ///

- /// If used on a directory, the directory must already be empty. - ///

- ///

- /// Note that this will not delete all compressed versions of the given filename, - /// without explicitly using an overload of this method. - ///

- /// - /// fileName is null. - /// fileName is invalid. - /// An I/O error has occurred, or the file cannot be deleted. - /// The stream does not support deleting. - public static void Delete(string fileName) - { - Delete(fileName, false); - } - /// - /// Delete a file or stream, if it exists. - /// - /// the file or stream to delete - /// if true, delete all files and subdirectories if - /// fileName is a directory; otherwise, fileName must be empty if it is a directory - /// - ///

- /// This will silently do nothing if the file already does not exist. - ///

- ///

- /// When including compressed files, this respects the setting of . - ///

- /// - /// fileName is null. - /// fileName is invalid. - /// An I/O error has occurred, or the file cannot be deleted. - /// The stream does not support deleting. - public static void Delete(string fileName, bool recursive) - { - Delete(fileName, recursive, false); - } - /// - /// Delete a file or stream, if it exists. - /// - /// the file or stream to delete - /// if true, delete all files and subdirectories if - /// fileName is a directory; otherwise, fileName must be empty if it is a directory - /// if true, also delete any compressed versions of the - /// given name; otherwise, only delete the given name - /// - ///

- /// This will silently do nothing if the file already does not exist. - ///

- ///

- /// When including compressed files, this respects the setting of . - ///

- /// - /// fileName is null. - /// fileName is invalid. - /// An I/O error has occurred, or the file cannot be deleted. - /// The stream does not support deleting. - public static void Delete(string fileName, bool recursive, bool includeCompressedVersions) - { - Contracts.CheckNonEmpty(fileName, nameof(fileName)); - - // check for special names: - string fileNameLower = fileName.ToLower(); - if (ZStreamIn.IsNullStream(fileName)) - { - // just succeed... - return; - } - if (ZStreamIn.IsConsoleStream(fileName)) - { - throw new NotSupportedException("Console streams cannot be deleted."); - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(fileName)) - { - throw new NotSupportedException("Clipboard streams cannot be deleted."); - } - - // check for Cosmos: - if (fileNameLower.StartsWith("cosmos:")) - { - Cosmos.Delete(fileName, recursive); - // don't look for compressed versions? - return; - } - - // check for Cockpit: - if (fileNameLower.StartsWith("cockpit:")) - { - throw new NotSupportedException("Cockpit streams cannot be deleted."); - } - - // check for Multistream: - if (fileNameLower.StartsWith("multi:")) - { - fileName = fileName.Substring("multi:".Length); - Delete(fileName); - } - if (fileNameLower.StartsWith("filelist:")) - { - fileName = fileName.Substring("filelist:".Length); - Delete(fileName); - } - - //REVIEW: We can actually delete a blob, however, it is blocked by this code - // check for HTTP: - if (fileNameLower.StartsWith("http:") || fileNameLower.StartsWith("https:")) - { - throw new NotSupportedException("HTTP streams or blobs cannot be deleted."); - } - - // check for SQL: - if (SqlTextReader.IsSqlTextReader(fileName)) - { - throw new NotSupportedException("Cannot delete a SQL table."); - } - - // check for InternalStore: - if (InternalStoreUtility.IsInternalStore(fileName)) - { - throw new NotSupportedException("Cannot delete a InternalStore."); - } - - // remove trailing "$" - if (fileName[fileName.Length - 1] == '$') - { - fileName = fileName.Substring(0, fileName.Length - 1); - } - - // check for named stream: - // should this be based on file existance, first? - int cIndex = fileName.LastIndexOf(':'); - if (cIndex > 0) - { - if (File.Exists(fileName.Substring(0, cIndex))) - { - if (cIndex > 1 || - fileName.IndexOfAny(pathSeparators, 2) < 0) - { - // named: - NamedStream.Delete(fileName); - return; - } - } - } - - bool deleted = false; - // plain file?: - // (handle the mess of exceptions that are normally generated) - try - { - if (Directory.Exists(fileName)) - { - Directory.Delete(fileName, recursive); - deleted = true; - } - else if (File.Exists(fileName)) - { - File.Delete(fileName); - deleted = true; - } - } - catch (UnauthorizedAccessException ex) - { - throw new IOException("Cannot delete: " + fileName, ex); - } - catch (PathTooLongException ex) - { - throw new ArgumentException("Path is too long: " + fileName, "fileName", ex); - } - catch (DirectoryNotFoundException ex) - { - throw new IOException("Directory does not exist: " + fileName, ex); - } - catch (NotSupportedException ex) - { - throw new IOException("Cannot delete: " + fileName, ex); - } - - if (!deleted) - { - // check for compressed archive as directory segment: - // only one path segment is allowed to be an archive... - // normalize path: - string zfileName = fileName.Replace('/', '\\'); - bool isUnc = zfileName.StartsWith("\\\\"); - while (zfileName.IndexOf("\\\\") >= 0) - { - zfileName = zfileName.Replace("\\\\", "\\"); - } - if (isUnc) - zfileName = "\\" + zfileName; - string zfileNameLower = zfileName.ToLower(); - - string archPath = null; - string inArch = null; - // should this be archives only? - for (int i = 0; i < ZStreamIn.decompressionExtensions.Length; i++) - { - string ext = ZStreamIn.decompressionExtensions[i]; - int seg = zfileNameLower.IndexOf(ext + "\\"); - if (seg > 0) - { - archPath = zfileName.Substring(0, seg + ext.Length); - if (File.Exists(archPath)) - { - inArch = zfileName.Substring(seg + ext.Length).Trim('\\'); - break; - } - archPath = null; - } - } - if (archPath == null) - { - // add in extension to each segment... - string[] segs = zfileName.Split('\\'); - for (int i = 0; i < segs.Length; i++) - { - if (segs[i].Length == 0) - continue; - string partial = string.Join("\\", segs, 0, i + 1); - if (partial.Length == 2 && partial[1] == ':') - continue; - if (Directory.Exists(partial)) - continue; - // should this be archives only? - for (int c = 0; c < ZStreamIn.decompressionExtensions.Length; c++) - { - string ext = ZStreamIn.decompressionExtensions[c]; - if (File.Exists(partial + ext)) - { - archPath = partial + ext; - inArch = string.Join("\\", segs, i + 1, segs.Length - i - 1).Trim('/', '\\'); - break; - } - } - // quit when parent will not exist - break; - } - } - if (archPath != null) - { - // found compressed archive for path segment - // check for path in archive. - inArch = inArch.Trim('\\'); - if (inArch.Length != 0) - { - // slow: - if (Z7zDecodeStream.Exists(archPath, inArch)) - { - // allow exception to throw if it occurs: - if (recursive) - { - Z7zEncodeStream.Delete(archPath, inArch); - } - else - { - if (Z7zDecodeStream.Exists(archPath, inArch + "\\*")) - { - throw new IOException("Cannot delete non-empty directory in archive when not recursive: " + - fileName); - } - Z7zEncodeStream.Delete(archPath, inArch); - } - } - // skip compressed versions operation! - return; - } - } - } - - // check for compressed versions, if needed: - if (includeCompressedVersions) - { - if (ZStreamIn.FallbackExtension != null) - { - if (ZStreamIn.FallbackExtension.Length != 0) - { - if (File.Exists(fileName + ZStreamIn.FallbackExtension)) - { - Delete(fileName + ZStreamIn.FallbackExtension, false); - } - } - else - { - for (int i = 0; i < ZStreamIn.decompressionExtensions.Length; i++) - { - if (File.Exists(fileName + ZStreamIn.decompressionExtensions[i])) - { - Delete(fileName + ZStreamIn.decompressionExtensions[i], false); - } - } - } - } - } - } - - /// - /// Get the length of the stream in bytes, or -1 if it cannot be obtained. - /// - /// the path or stream name - /// the length of the stream in bytes, or -1 if it cannot be obtained - public static long GetLength(string path) - { - // this could be more efficient! *** - // force buffered? - try - { - using (Stream str = ZStreamIn.OpenBuffered(path)) - { - return str.Length; - } - } - catch - { - return -1; - } - } - - /// - /// Set the length of a file, truncating or padding as needed. - /// - /// the file to alter - /// the desired length, in bytes - /// - /// Note that this only operates on normal files, and cannot be successfully used on compressed - /// files or special streams. - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// An I/O error has occurred. - /// The stream does not support both writing and seeking. - /// Attempted to set the value parameter to less than 0. - public static void ResizeFile(string fileName, long length) - { - using (FileStream pad = new FileStream(fileName, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite)) - { - pad.SetLength(length); - } - } - - /// - /// Get the named streams present in the given file. - /// - /// the file to inspect - /// the list of named streams available for the file, excluding the default stream - /// - ///

- /// Named streams only exist on NTFS file systems, generally, and might not be - /// preserved as files are moved around in various forms. Most applications can - /// only access the default stream, but command-line redirection will write and read - /// from named streams when specified. Named streams are specified with the syntax - /// "filename:streamname". This syntax will also work when opening files with - /// , , - /// , and . - ///

- ///

- /// Note that this will not automatically fallback to compressed variations of - /// the given filename. - ///

- /// - public static string[] GetNamedStreams(string fileName) - { - return NamedStream.GetNamedStreams(fileName); - } - - #endregion - - #region Reading and Writing - - /// - /// Copy one file or directory to another location. - /// - /// the file or stream to copy (potentially with a wildcard pattern) - /// the destination, either as a directory or a file - /// - ///

- /// It is best to put a trailing slash on directories in order to ensure that they are not - /// treated as files. - ///

- ///

- /// If a wildcard pattern is used, the files will all be copied into the destination. A - /// will be thrown if no files match. - /// If there are multiple matches, the destination should be a directory or a continuous stream - /// (such as the console or the null stream), since the effect will be as if each copy overwrote - /// the last one, otherwise. - ///

- /// - /// source or destination is null. - /// source or destination is invalid. - /// source cannot be found. - /// An I/O error has occurred. - /// The destination does not support writing. - public static void Copy(string source, string destination) - { - Copy(source, destination, false); - } - - /// - /// Copy one file or directory to another location, as lines of text, with any implicit conversion. - /// - /// the file or stream to copy (potentially with a wildcard pattern) - /// the destination, either as a directory or a file - /// - ///

- /// It is best to put a trailing slash on directories in order to ensure that they are not - /// treated as files. - ///

- ///

- /// If a wildcard pattern is used, the files will all be copied into the destination. A - /// will be thrown if no files match. - /// If there are multiple matches, the destination should be a directory or a continuous stream - /// (such as the console or the null stream), since the effect will be as if each copy overwrote - /// the last one, otherwise. - ///

- /// - /// source or destination is null. - /// source or destination is invalid. - /// source cannot be found. - /// An I/O error has occurred. - /// The destination does not support writing. - public static void CopyLines(string source, string destination) - { - Copy(source, destination, true); - } - - private static void Copy(string source, string destination, bool translateText) - { - Contracts.CheckNonEmpty(source, nameof(source)); - Contracts.CheckNonEmpty(destination, nameof(destination)); - - string[] files = ExpandWildcards(source); - if (files.Length == 0) - throw new FileNotFoundException("No files match the pattern: " + source); - - // this should be better encapsulated... - - if (files.Length > 1) - { - for (int i = 0; i < files.Length; i++) - { - Copy(files[i], destination); - } - return; - } - source = files[0]; - - string fullDestination = null; - - if (FileExists(destination)) - { - fullDestination = destination; - } - else if (DirectoryExists(destination)) - { - fullDestination = destination.TrimEnd(pathSeparators) + "/" + GetFileName(source.TrimEnd(pathSeparators)); - } - else if (DirectoryExists(Path.GetDirectoryName(destination))) - { - // is that correct??? - fullDestination = destination; - } - else - { - // check for special names: - string nameLower = destination.ToLower(); - - if (ZStreamIn.IsNullStream(destination)) - { - fullDestination = destination; - } - else if (ZStreamIn.IsConsoleStream(destination)) - { - fullDestination = destination; - } - else if (ClipboardReadStream.IsClipboardStream(destination)) - { - // clipboard - fullDestination = destination; - } - else if (nameLower.StartsWith("cosmos:")) - { - // cosmos - if (destination[destination.Length - 1] == '/' || destination[destination.Length - 1] == '\\') - { - fullDestination = destination + GetFileName(source.TrimEnd(pathSeparators)); - } - else - { - fullDestination = destination; - } - } - else if (nameLower.StartsWith("cockpit:")) - { - // Cockpit - throw new NotSupportedException("Cockpit streams cannot be written to."); - } - else if (nameLower.StartsWith("http://") || nameLower.StartsWith("https://")) - { - // HTTP - throw new NotSupportedException("HTTP streams cannot be written to."); - } - else if (InternalStoreUtility.IsInternalStore(nameLower)) - { - // InternalStore: - throw new NotSupportedException("Cannot copy to a InternalStore."); - } - else if (SqlTextReader.IsSqlTextReader(nameLower)) - { - // SQL: - //throw new NotSupportedException("Cannot copy to a SQL table."); - fullDestination = destination; - } - else - { - // default? - fullDestination = destination; - } - } - // what about Multistream? *** - - // this could be optimized, of course... - // should we special-case normal file copy to File.Copy()? *** - if (PathsEqual(source, fullDestination)) - return; - if (source[source.Length - 1] == '/' || source[source.Length - 1] == '\\' || DirectoryExists(source)) - { - // recursive copy? - // not for compressed files... - bool copyCompressed = false; - try - { - if (File.Exists(source.TrimEnd(pathSeparators))) - { - source = source.TrimEnd(pathSeparators) + "$"; - fullDestination = fullDestination.TrimEnd(pathSeparators) + "$"; - copyCompressed = true; - } - } - catch - { - // ignore - } - if (!copyCompressed) - { - // we should really try and avoid infinite loops... - if (PathAncestor(source, fullDestination)) - { - throw new ArgumentException("Cannot copy to a subdirectory of the source", "destination"); - } - CreateDirectory(fullDestination.TrimEnd(pathSeparators)); - foreach (string entry in DirectoryEntries(source)) - { - Copy(entry, fullDestination + "/"); - } - return; - } - } - - if (translateText) - { - // this could be optimized, of course... - using (StreamReader reader = ZStreamReader.Open(source)) - { - using (StreamWriter writer = ZStreamWriter.Open(fullDestination)) - { - // preallocate: - try - { - long length = reader.BaseStream.Length; - if (length > 0) - { - if (writer.BaseStream is LowFragmentationStream) - { - ((LowFragmentationStream)writer.BaseStream).Reserve(length); - } - } - } - catch - { - } - - for (string line = reader.ReadLine(); line != null; line = reader.ReadLine()) - { - writer.WriteLine(line); - } - } - } - } - else - { - using (Stream fileIn = ZStreamIn.Open(source)) - { - using (Stream fileOut = ZStreamOut.Open(fullDestination)) - { - // preallocate: - // bad for compression... we really should give the length there - try - { - long length = fileIn.Length; - if (length > 0) - { - if (fileOut is FileStream || fileOut is LowFragmentationStream) - { - fileOut.SetLength(length); - } - } - } - catch - { - } - - byte[] buffer = new byte[256 * 1024]; - for (int count = fileIn.Read(buffer, 0, buffer.Length); count > 0; count = fileIn.Read(buffer, 0, buffer.Length)) - { - fileOut.Write(buffer, 0, count); - } - } - } - } - } - - /// - /// Copy one Stream to another. - /// - /// the Stream to copy - /// the destination Stream - /// - /// The source and destination are both left open. - /// - /// source or destination is null. - /// An I/O error has occurred. - public static void Copy(Stream source, Stream destination) - { - if (source == null) - throw new ArgumentException("source cannot be null", "source"); - if (destination == null) - throw new ArgumentException("destination cannot be null", "destination"); - try - { - if (source is UnbufferedStream) - { - UnbufferedStream us = (UnbufferedStream)source; - byte[] buffer; - for (int c = us.Read(out buffer); c > 0; c = us.Read(out buffer)) - { - destination.Write(buffer, 0, c); - } - } - else - { - byte[] buffer = new byte[64 * 1024]; - for (int c = source.Read(buffer, 0, buffer.Length); c > 0; c = source.Read(buffer, 0, buffer.Length)) - { - destination.Write(buffer, 0, c); - } - } - } - catch (Exception ex) - { - throw new IOException("Could not copy source to destination", ex); - } - } - - /// - /// Copy a TextReader to a TextWriter, as lines of text, with any implicit conversion. - /// - /// the TextReader to copy - /// the destination TextWriter - /// - /// The source and destination are both left open. - /// - /// source or destination is null. - /// An I/O error has occurred. - public static void CopyLines(TextReader source, TextWriter destination) - { - if (source == null) - throw new ArgumentException("source cannot be null", "source"); - if (destination == null) - throw new ArgumentException("destination cannot be null", "destination"); - try - { - for (string line = source.ReadLine(); line != null; line = source.ReadLine()) - { - destination.WriteLine(line); - } - } - catch (Exception ex) - { - throw new IOException("Could not copy source to destination", ex); - } - } - - /// - /// Create a stream that is the concatenation of a set of streams. - /// - /// the name of the stream to create - /// the names of the streams to concatenate - /// - ///

- /// This will overwrite the destination. - ///

- ///

- /// For Cosmos files, this maps to a . - ///

- ///

- /// Wildcard patterns are allowed. - ///

- /// - /// The join could not be completed. - /// The destination or some sources are not valid stream names. - public static void Concatenate(string destination, params string[] sources) - { - if (destination == null || destination.Length == 0) - throw new ArgumentException("destination cannot be empty", "destination"); - if (sources == null) - sources = new string[0]; - bool allCosmos = (string.Compare(destination, 0, "cosmos://", 0, "cosmos://".Length, true) == 0); - if (allCosmos) - { - for (int i = 0; i < sources.Length; i++) - { - if (sources[i] == null || string.Compare(sources[i], 0, "cosmos://", 0, "cosmos://".Length, true) != 0) - { - allCosmos = false; - break; - } - } - } - if (allCosmos) - { - Cosmos.Concatenate(destination, sources); - return; - } - List expanded = null; - for (int i = 0; i < sources.Length; i++) - { - if (sources[i] == null || sources[i].Length == 0) - throw new ArgumentException("sources[" + i + "] cannot be empty", "sources"); - if (sources[i].IndexOf('*') >= 0 || sources[i].IndexOf('?') >= 0 || - sources[i].IndexOf("...") >= 0) - { - if (expanded == null) - { - expanded = new List(); - for (int j = 0; j < i; j++) - { - expanded.Add(sources[j]); - } - } - expanded.AddRange(IOUtil.ExpandWildcards(sources[i])); - } - else - { - if (expanded != null) - expanded.Add(sources[i]); - } - } - if (expanded != null) - { - sources = expanded.ToArray(); - } - - try - { - using (StreamWriter sw = ZStreamWriter.Open(destination)) - { - for (int i = 0; i < sources.Length; i++) - { - using (StreamReader sr = ZStreamReader.Open(sources[i])) - { - CopyLines(sr, sw); - } - } - } - } - catch (Exception ex) - { - throw new IOException("Could not concatenate all sources", ex); - } - } - - /// - /// Consume all bytes from a stream, in the background. - /// - /// the stream to consume - /// the thread that is consuming the data (which can be ignored) - /// - ///

- /// This method ignores all errors. It is useful for tasks such as ignoring - /// StandardError from a process. - ///

- ///

- /// The stream will be closed when the end is reached. - ///

- /// - public static Thread ConsumeBackground(Stream stream) - { - if (stream == null || !stream.CanRead) - return null; - ParameterizedThreadStart consumeStart = new ParameterizedThreadStart(ReadAll); - Thread consume = Utils.CreateBackgroundThread(consumeStart); - consume.Start(stream); - return consume; - } - - /// - /// Consume all lines from a stream, in the background. - /// - /// the TextReader to consume - /// the thread that is consuming the data (which can be ignored) - /// - ///

- /// This method ignores all errors. It is useful for tasks such as ignoring - /// StandardError from a process. - ///

- ///

- /// The stream will be closed when the end is reached. - ///

- /// - public static Thread ConsumeBackground(TextReader reader) - { - if (reader == null) - return null; - ParameterizedThreadStart consumeStart = new ParameterizedThreadStart(ReadAllLines); - Thread consume = Utils.CreateBackgroundThread(consumeStart); - consume.Start(reader); - return consume; - } - - private static void ReadAll(object streamObj) - { - Stream stream = (Stream)streamObj; - try - { - byte[] buffer = new byte[1024]; - while (stream.Read(buffer, 0, buffer.Length) > 0) - { - } - } - catch - { - } - finally - { - try - { - stream.Close(); - } - catch - { - } - } - } - - private static void ReadAllLines(object readerObj) - { - TextReader reader = (TextReader)readerObj; - try - { - char[] buffer = new char[512]; - int c; - while ((c = reader.Read(buffer, 0, buffer.Length)) > 0) - { - - } - } - catch - { - } - finally - { - try - { - reader.Close(); - } - catch - { - } - } - } - - #region Line Enumeration - -#if OLD_LINES - /// - /// Get an enumerator for the lines in the file or stream fileName - /// - /// the file to read (or URL, or Cosmos stream...) - /// an IEnumerable that generates an IEnumerator for the lines in fileName - /// - /// The reading is performed through . - /// - /// This allows the lines in a stream to be read as: - /// - /// foreach (string line in IOUtil.Lines("file.txt")) - /// { - /// ... - /// } - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static IEnumerable Lines(string fileName) - { - return Lines(fileName, false); - } - /// - /// Get an enumerator for the lines in the file or stream fileName - /// - /// the file to read (or URL, or Cosmos stream...) - /// if true, skip blank lines; if false, read all lines - /// an IEnumerable that generates an IEnumerator for the lines in fileName - /// - /// The reading is performed through . - /// - /// This allows the lines in a stream to be read as: - /// - /// foreach (string line in IOUtil.Lines("file.txt")) - /// { - /// ... - /// } - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static IEnumerable Lines(string fileName, bool skipBlank) - { - return new LineEnumerable(fileName, skipBlank); - } - - private class LineEnumerable : IEnumerable - { - private string fileName; - private bool skipBlank; - - public LineEnumerable(string fileName, bool skipBlank) - { - this.fileName = fileName; - this.skipBlank = skipBlank; - } - - public IEnumerator GetEnumerator() - { - return new LineEnumerator(fileName, skipBlank); - } - } - - /// - /// Enumerator to read through the lines in a StreamReader. - /// - private class LineEnumerator : IEnumerator, IDisposable - { - private string fileName; - private StreamReader reader; - private string line = null; - private bool skipBlank = false; - - /// - /// Create a new enumerator to read through the lines. - /// - /// the file to read lines from - public LineEnumerator(string fileName) - : this(fileName, false) - { - } - /// - /// Create a new enumerator to read through the lines. - /// - /// the file to read lines from - /// if true, skip blank lines; if false, read all lines - public LineEnumerator(string fileName, bool skipBlank) - { - this.fileName = fileName; - this.skipBlank = skipBlank; - this.reader = ZStreamReader.Open(fileName); - } - ~LineEnumerator() - { - Dispose(); - } - /// - /// Return the enumerator to the initial state. - /// - public void Reset() - { - //// can we do this? What about the BOM? - //reader.DiscardBufferedData(); - //reader.BaseStream.Seek(0, SeekOrigin.Begin); - //// should there be an IResetable? - try - { - reader.Close(); - } - catch - { - } - reader = ZStreamReader.Open(fileName); - line = null; - } - /// - /// Get the current line of the file. - /// - public string Current - { - get - { - return line; - } - } - - /// - /// Get the current line of the file. - /// - object IEnumerator.Current - { - get - { - return ((LineEnumerator)this).Current; - } - } - - /// - /// Move the enumerator to the next line. - /// - /// true if the next line exists, or false if at the end of the file - public bool MoveNext() - { - do - { - line = reader.ReadLine(); - } - while (skipBlank && (line != null && line.TrimEnd().Length == 0)); - - return line != null; - } - #region IDisposable Members - - public void Dispose() - { - try - { - reader.Close(); - } - catch - { - } - GC.SuppressFinalize(this); - } - - #endregion - } -#else - /// - /// Get an enumerator for the lines in the file or stream fileName - /// - /// the file to read (or URL, or Cosmos stream...) - /// an IEnumerable that generates an IEnumerator for the lines in fileName - /// - /// The reading is performed through . - /// - /// This allows the lines in a stream to be read as: - /// - /// foreach (string line in IOUtil.Lines("file.txt")) - /// { - /// ... - /// } - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static IEnumerable Lines(string fileName) - { - return Lines(fileName, false); - } - /// - /// Get an enumerator for the lines in the file or stream fileName - /// - /// the file to read (or URL, or Cosmos stream...) - /// if true, skip blank lines; if false, read all lines - /// an IEnumerable that generates an IEnumerator for the lines in fileName - /// - /// The reading is performed through . - /// - /// This allows the lines in a stream to be read as: - /// - /// foreach (string line in IOUtil.Lines("file.txt")) - /// { - /// ... - /// } - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static IEnumerable Lines(string fileName, bool skipBlank) - { - using (StreamReader reader = ZStreamReader.Open(fileName)) - { - for (string line = reader.ReadLine(); line != null; line = reader.ReadLine()) - { - if (skipBlank) - { - bool foundNonSpace = false; - foreach (char c in line) - { - if (c != ' ' && c != '\t' && c != '\r' && c != '\n' && c != '\v') - { - foundNonSpace = true; - break; - } - } - if (!foundNonSpace) - continue; - } - yield return line; - } - } - } -#endif - - #endregion - - /// - /// Read the file or stream specified by fileName. - /// - /// the file to read (or URL, or Cosmos stream...) - /// the text of fileName, or null if it cannot be read - /// - /// The reading is performed through . - /// - public static string ReadFile(string fileName) - { - try - { - using (StreamReader sr = ZStreamReader.Open(fileName)) - { - return sr.ReadToEnd(); - } - } - catch - { - return null; - } - } - /// - /// Read the file or stream specified by fileName. - /// - /// the file to read (or URL, or Cosmos stream...) - /// the encoding to use for reading the text - /// the text of fileName, or null if it cannot be read - /// - /// The reading is performed through . - /// - public static string ReadFile(string fileName, System.Text.Encoding encoding) - { - try - { - using (StreamReader sr = ZStreamReader.Open(fileName, encoding)) - { - return sr.ReadToEnd(); - } - } - catch - { - return null; - } - } - - /// - /// Write text to the file specified by fileName. - /// - /// the text to write - /// the file to write - /// - /// The writing is performed through . - /// - /// fileName is null. - /// fileName is invalid. - /// The utilities needed to open a stream are not available. - /// An error occurs while writing. - public static void WriteFile(string text, string fileName) - { - using (StreamWriter sr = ZStreamWriter.Open(fileName)) - { - if (text != null) - { - sr.Write(text); - } - } - } - /// - /// Write text to the file specified by fileName. - /// - /// the text to write - /// the file to write - /// if true, append to the file; otherwise, create or overwrite - /// - /// The writing is performed through . - /// - /// fileName is null. - /// fileName is invalid. - /// Append is specified, and fileName cannot be found. - /// The utilities needed to open a stream are not available. - /// An error occurs while writing. - public static void WriteFile(string text, string fileName, bool append) - { - using (StreamWriter sr = ZStreamWriter.Open(fileName, append)) - { - if (text != null) - { - sr.Write(text); - } - } - } - - /// - /// Read the file or stream specified by fileName, as a byte array. - /// - /// the file to read (or URL, or Cosmos stream...) - /// the bytes of fileName, or null if it cannot be read - /// - /// The reading is performed through . - /// - public static byte[] ReadBytes(string fileName) - { - try - { - long len = GetLength(fileName); - if (len >= 0) - { - byte[] res = new byte[len]; - using (Stream sr = ZStreamIn.Open(fileName)) - { - sr.Read(res, 0, res.Length); - return res; - } - } - else - { - // might just be forward-only... - // need double the memory, then... - List buffers = new List(); - long totalLength = 0; - using (Stream sr = ZStreamIn.Open(fileName)) - { - byte[] buf = new byte[256 * 1024]; - int count; - while ((count = sr.Read(buf, 0, buf.Length)) > 0) - { - buffers.Add(buf); - totalLength += count; - if (count < buf.Length) - break; - buf = new byte[256 * 1024]; - } - byte[] res = new byte[totalLength]; - int cur = 0; - for (int i = 0; i < buffers.Count - 1; i++) - { - Buffer.BlockCopy((byte[])buffers[i], 0, res, cur, buf.Length); - cur += buf.Length; - buffers[i] = null; - } - if (buffers.Count != 0) - { - Buffer.BlockCopy((byte[])buffers[buffers.Count - 1], 0, res, cur, count); - } - - return res; - } - } - } - catch - { - return null; - } - } - - /// - /// Write bytes to the file specified by fileName. - /// - /// the bytes to write - /// the file to write - /// - /// This is an alias for . - /// - /// - /// - /// The writing is performed through . - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. - /// An error occurs while writing. - public static void WriteFile(byte[] data, string fileName) - { - WriteBytes(data, fileName); - } - /// - /// Write bytes to the file specified by fileName. - /// - /// the bytes to write - /// the file to write - /// if true, append to the file; otherwise, create or overwrite - /// - /// - /// This is an alias for . - /// - /// - /// The writing is performed through . - /// - /// - /// fileName is null. - /// fileName is invalid. - /// Append is specified, and fileName cannot be found. - /// The utilities needed to open a stream are not available. - /// An error occurs while writing. - public static void WriteFile(byte[] data, string fileName, bool append) - { - WriteBytes(data, fileName, append); - } - /// - /// Write bytes to the file specified by fileName. - /// - /// the bytes to write - /// the file to write - /// - /// The writing is performed through . - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. - /// An error occurs while writing. - public static void WriteBytes(byte[] data, string fileName) - { - using (Stream sr = ZStreamOut.Open(fileName)) - { - if (data != null) - { - sr.Write(data, 0, data.Length); - } - } - } - /// - /// Write bytes to the file specified by fileName. - /// - /// the bytes to write - /// the file to write - /// if true, append to the file; otherwise, create or overwrite - /// - /// The writing is performed through . - /// - /// fileName is null. - /// fileName is invalid. - /// Append is specified, and fileName cannot be found. - /// The utilities needed to open a stream are not available. - /// An error occurs while writing. - public static void WriteBytes(byte[] data, string fileName, bool append) - { - using (Stream sr = ZStreamOut.Open(fileName, append)) - { - if (data != null) - { - sr.Write(data, 0, data.Length); - } - } - } - - /// - /// Read the lines of the file or stream specified by fileName. - /// - /// the file to read (or URL, or Cosmos stream...) - /// the lines of fileName, or null if it cannot be read - /// - /// The reading is performed through . - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static string[] ReadLines(string fileName) - { - try - { - using (StreamReader sr = ZStreamReader.Open(fileName)) - { - return ReadLines(sr); - } - } - catch - { - return null; - } - } - /// - /// Read the lines of the file or stream specified by fileName. - /// - /// the file to read (or URL, or Cosmos stream...) - /// the encoding to use for reading the text - /// the lines of fileName, or null if it cannot be read - /// - /// The reading is performed through . - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static string[] ReadLines(string fileName, System.Text.Encoding encoding) - { - try - { - using (StreamReader sr = ZStreamReader.Open(fileName, encoding)) - { - return ReadLines(sr); - } - } - catch - { - return null; - } - } - /// - /// Read the lines of the StreamReader specified. - /// - /// the TextReader to read - /// the lines of input, or null if it cannot be read - public static string[] ReadLines(TextReader input) - { - try - { - List lines = new List(); - for (string line = input.ReadLine(); line != null; line = input.ReadLine()) - { - lines.Add(line); - } - if (lines.Count > 0 && ((string)lines[lines.Count - 1]).Length == 0) - lines.RemoveAt(lines.Count - 1); - return lines.ToArray(); - } - catch - { - return null; - } - } - - /// - /// Write the lines to the file or stream specified by fileName. - /// - /// the lines to write - /// the file to write - /// - /// - /// This is an alias for . - /// - /// - /// The writing is performed through . - /// - /// - /// fileName is null. - /// fileName is invalid. - /// The utilities needed to open the stream are not available. - /// An error occurs while writing. - public static void WriteFile(string[] lines, string fileName) - { - WriteLines(lines, fileName); - } - /// - /// Write the lines to the file or stream specified by fileName. - /// - /// the lines to write - /// the file to write - /// if true, append to the stream; otherwise, overwrite - /// - /// - /// This is an alias for . - /// - /// - /// The writing is performed through . - /// - /// - /// fileName is null. - /// fileName is invalid. - /// The utilities needed to open the stream are not available. - /// An error occurs while writing. - public static void WriteFile(string[] lines, string fileName, bool append) - { - WriteLines(lines, fileName, append); - } - /// - /// Write the lines to the file or stream specified by fileName. - /// - /// the lines to write - /// the file to write - /// - /// The writing is performed through . - /// - /// fileName is null. - /// fileName is invalid. - /// The utilities needed to open the stream are not available. - /// An error occurs while writing. - public static void WriteLines(string[] lines, string fileName) - { - WriteLines(lines, fileName, false); - } - /// - /// Write the lines to the file or stream specified by fileName. - /// - /// the lines to write - /// the file to write - /// if true, append to the stream; otherwise, overwrite - /// - /// The writing is performed through . - /// - /// fileName is null. - /// fileName is invalid. - /// The utilities needed to open the stream are not available. - /// An error occurs while writing. - public static void WriteLines(string[] lines, string fileName, bool append) - { - using (StreamWriter sw = ZStreamWriter.Open(fileName, append)) - { - WriteLines(lines, sw); - } - } - /// - /// Write the lines to the specified TextWriter. - /// - /// the lines to write - /// the TextWriter to write to - /// - ///

- /// The writing is performed through . - ///

- ///

- /// The output will not be closed. - ///

- /// - /// An error occurs while writing. - public static void WriteLines(string[] lines, TextWriter output) - { - if (lines == null) - lines = new string[0]; - try - { - for (int i = 0; i < lines.Length; i++) - { - output.Write(lines[i]); - } - } - catch (Exception ex) - { - throw new IOException("Cannot write lines to output", ex); - } - } - - /// - /// Read a file compiled as Embedded Content in the calling assembly. - /// - /// The name of the file to read, without any namespace - /// The contents of the file, or null if it does not exist - public static string ReadResource(string name) - { - try - { - using (System.IO.StreamReader str = ReadResourceReader(name)) - { - if (str == null) - return null; - return str.ReadToEnd(); - } - } - catch - { - return null; - } - } - - /// - /// Read a file compiled as Embedded Content in the calling assembly. - /// - /// The name of the file to read, without any namespace - /// A StreamReader for reading the file, or null if it does not exist - public static StreamReader ReadResourceReader(string name) - { - Stream st = ReadResourceStream(name); - if (st == null) - return null; - return new StreamReader(st); - } - - /// - /// Read a file compiled as Embedded Content in the calling assembly. - /// - /// The name of the file to read, without any namespace - /// A Stream for reading the file, or null if it does not exist - public static Stream ReadResourceStream(string name) - { - // not a streamname, because it is really a code-level issue... - if (name == null || name.Length == 0) - return null; - try - { - string[] names = System.Reflection.Assembly.GetCallingAssembly().GetManifestResourceNames(); - string fullName = null; - string dotName = "." + name; - foreach (string s in names) - { - // this is a hack... *** - if (s == name || s.EndsWith(dotName, StringComparison.OrdinalIgnoreCase)) - { - fullName = s; - break; - } - } - if (fullName == null) - return null; - System.Reflection.Assembly assem = System.Reflection.Assembly.GetExecutingAssembly(); - return assem.GetManifestResourceStream(name); - } - catch - { - return null; - } - } - - #endregion - - #region Checksum, Etc - #region Tables - static readonly uint[] _crctab = - { - 0x0, - 0x04C11DB7, 0x09823B6E, 0x0D4326D9, 0x130476DC, 0x17C56B6B, - 0x1A864DB2, 0x1E475005, 0x2608EDB8, 0x22C9F00F, 0x2F8AD6D6, - 0x2B4BCB61, 0x350C9B64, 0x31CD86D3, 0x3C8EA00A, 0x384FBDBD, - 0x4C11DB70, 0x48D0C6C7, 0x4593E01E, 0x4152FDA9, 0x5F15ADAC, - 0x5BD4B01B, 0x569796C2, 0x52568B75, 0x6A1936C8, 0x6ED82B7F, - 0x639B0DA6, 0x675A1011, 0x791D4014, 0x7DDC5DA3, 0x709F7B7A, - 0x745E66CD, 0x9823B6E0, 0x9CE2AB57, 0x91A18D8E, 0x95609039, - 0x8B27C03C, 0x8FE6DD8B, 0x82A5FB52, 0x8664E6E5, 0xBE2B5B58, - 0xBAEA46EF, 0xB7A96036, 0xB3687D81, 0xAD2F2D84, 0xA9EE3033, - 0xA4AD16EA, 0xA06C0B5D, 0xD4326D90, 0xD0F37027, 0xDDB056FE, - 0xD9714B49, 0xC7361B4C, 0xC3F706FB, 0xCEB42022, 0xCA753D95, - 0xF23A8028, 0xF6FB9D9F, 0xFBB8BB46, 0xFF79A6F1, 0xE13EF6F4, - 0xE5FFEB43, 0xE8BCCD9A, 0xEC7DD02D, 0x34867077, 0x30476DC0, - 0x3D044B19, 0x39C556AE, 0x278206AB, 0x23431B1C, 0x2E003DC5, - 0x2AC12072, 0x128E9DCF, 0x164F8078, 0x1B0CA6A1, 0x1FCDBB16, - 0x018AEB13, 0x054BF6A4, 0x0808D07D, 0x0CC9CDCA, 0x7897AB07, - 0x7C56B6B0, 0x71159069, 0x75D48DDE, 0x6B93DDDB, 0x6F52C06C, - 0x6211E6B5, 0x66D0FB02, 0x5E9F46BF, 0x5A5E5B08, 0x571D7DD1, - 0x53DC6066, 0x4D9B3063, 0x495A2DD4, 0x44190B0D, 0x40D816BA, - 0xACA5C697, 0xA864DB20, 0xA527FDF9, 0xA1E6E04E, 0xBFA1B04B, - 0xBB60ADFC, 0xB6238B25, 0xB2E29692, 0x8AAD2B2F, 0x8E6C3698, - 0x832F1041, 0x87EE0DF6, 0x99A95DF3, 0x9D684044, 0x902B669D, - 0x94EA7B2A, 0xE0B41DE7, 0xE4750050, 0xE9362689, 0xEDF73B3E, - 0xF3B06B3B, 0xF771768C, 0xFA325055, 0xFEF34DE2, 0xC6BCF05F, - 0xC27DEDE8, 0xCF3ECB31, 0xCBFFD686, 0xD5B88683, 0xD1799B34, - 0xDC3ABDED, 0xD8FBA05A, 0x690CE0EE, 0x6DCDFD59, 0x608EDB80, - 0x644FC637, 0x7A089632, 0x7EC98B85, 0x738AAD5C, 0x774BB0EB, - 0x4F040D56, 0x4BC510E1, 0x46863638, 0x42472B8F, 0x5C007B8A, - 0x58C1663D, 0x558240E4, 0x51435D53, 0x251D3B9E, 0x21DC2629, - 0x2C9F00F0, 0x285E1D47, 0x36194D42, 0x32D850F5, 0x3F9B762C, - 0x3B5A6B9B, 0x0315D626, 0x07D4CB91, 0x0A97ED48, 0x0E56F0FF, - 0x1011A0FA, 0x14D0BD4D, 0x19939B94, 0x1D528623, 0xF12F560E, - 0xF5EE4BB9, 0xF8AD6D60, 0xFC6C70D7, 0xE22B20D2, 0xE6EA3D65, - 0xEBA91BBC, 0xEF68060B, 0xD727BBB6, 0xD3E6A601, 0xDEA580D8, - 0xDA649D6F, 0xC423CD6A, 0xC0E2D0DD, 0xCDA1F604, 0xC960EBB3, - 0xBD3E8D7E, 0xB9FF90C9, 0xB4BCB610, 0xB07DABA7, 0xAE3AFBA2, - 0xAAFBE615, 0xA7B8C0CC, 0xA379DD7B, 0x9B3660C6, 0x9FF77D71, - 0x92B45BA8, 0x9675461F, 0x8832161A, 0x8CF30BAD, 0x81B02D74, - 0x857130C3, 0x5D8A9099, 0x594B8D2E, 0x5408ABF7, 0x50C9B640, - 0x4E8EE645, 0x4A4FFBF2, 0x470CDD2B, 0x43CDC09C, 0x7B827D21, - 0x7F436096, 0x7200464F, 0x76C15BF8, 0x68860BFD, 0x6C47164A, - 0x61043093, 0x65C52D24, 0x119B4BE9, 0x155A565E, 0x18197087, - 0x1CD86D30, 0x029F3D35, 0x065E2082, 0x0B1D065B, 0x0FDC1BEC, - 0x3793A651, 0x3352BBE6, 0x3E119D3F, 0x3AD08088, 0x2497D08D, - 0x2056CD3A, 0x2D15EBE3, 0x29D4F654, 0xC5A92679, 0xC1683BCE, - 0xCC2B1D17, 0xC8EA00A0, 0xD6AD50A5, 0xD26C4D12, 0xDF2F6BCB, - 0xDBEE767C, 0xE3A1CBC1, 0xE760D676, 0xEA23F0AF, 0xEEE2ED18, - 0xF0A5BD1D, 0xF464A0AA, 0xF9278673, 0xFDE69BC4, 0x89B8FD09, - 0x8D79E0BE, 0x803AC667, 0x84FBDBD0, 0x9ABC8BD5, 0x9E7D9662, - 0x933EB0BB, 0x97FFAD0C, 0xAFB010B1, 0xAB710D06, 0xA6322BDF, - 0xA2F33668, 0xBCB4666D, 0xB8757BDA, 0xB5365D03, 0xB1F740B4 - }; - - static readonly uint[] _zipCrctab = - { - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, - 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, - 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, - 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, - 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, - 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, - 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, - 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, - 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, - 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, - 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, - 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, - 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, - 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, - 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, - 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, - 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, - 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, - 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, - 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, - 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, - 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, - 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, - 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, - 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, - 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, - 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, - 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, - 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, - 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, - 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, - 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, - 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, - 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, - 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, - 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, - 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, - 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, - 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, - 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, - 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, - 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d - }; - #endregion - - /// - /// The types of checksums that can be calculated. - /// - public enum ChecksumType - { - /// - /// The checksum used in the ZIP and gzip file formats. - /// - Zip, - /// - /// Standard CRC32. - /// - Crc32, - /// - /// The first bytes of the SHA1 hash. - /// - SHA1Prefix, - /// - /// The first bytes of the SHA256 hash. - /// - SHA256Prefix, - } - - /// - /// Calculate a standard ZIP/gzip checksum for the specified file. - /// - /// the file to find the checksum of - /// the checksum, as an integer - /// - /// To convert this to a string as displayed for ZIP and gzip, - /// use ToString("X8"). - /// - public static uint Checksum(string fileName) - { - unchecked - { - return (uint)(Checksum(fileName, ChecksumType.Zip) & 0xFFFFFFFF); - } - } - - /// - /// Calculate a standard CRC32 checksum for the specified file. - /// - /// the file to find the checksum of - /// the type of checksum to calculate - /// the checksum, as a 64-bit integer - /// - /// For checksums smaller than 64-bits, the high bits of the return - /// value will be zero. - /// - public static ulong Checksum(string fileName, ChecksumType type) - { - long length = 0; - - using (Stream inFile = ZStreamIn.Open(fileName)) - { - // compute the basic crc - switch (type) - { - case ChecksumType.Crc32: - default: - unsafe - { - uint crc32 = 0; - if (inFile is UnbufferedStream) - { - UnbufferedStream inFileU = (UnbufferedStream)inFile; - byte* buffer; - for (int count = inFileU.Read(out buffer); count > 0; count = inFileU.Read(out buffer)) - { - length += count; - byte* bEnd = buffer + count; - while (buffer != bEnd) - { - crc32 = (crc32 << 8) ^ _crctab[(crc32 >> 24) ^ (((uint)(*buffer)) & 0xFF)]; - buffer++; - } - } - } - else - { - byte[] buffer = new byte[4 * 1024 * 1024]; - for (int count = inFile.Read(buffer, 0, buffer.Length); count > 0; count = inFile.Read(buffer, 0, buffer.Length)) - { - length += count; - fixed (byte* bb = buffer) - { - byte* b = bb; - byte* bEnd = bb + count; - while (b != bEnd) - { - crc32 = (crc32 << 8) ^ _crctab[(crc32 >> 24) ^ (((uint)(*b)) & 0xFF)]; - b++; - } - } - } - } - long adj = length; - while (adj != 0) - { - crc32 = (crc32 << 8) ^ - _crctab[((crc32 >> 24) ^ adj) & 0xFF]; - adj >>= 8; - } - crc32 = ~crc32 & 0xFFFFFFFF; - return crc32; - } - - case ChecksumType.Zip: - unsafe - { - uint crcZip = 0xffffffff; - if (inFile is UnbufferedStream) - { - UnbufferedStream inFileU = (UnbufferedStream)inFile; - byte* buffer; - for (int count = inFileU.Read(out buffer); count > 0; count = inFileU.Read(out buffer)) - { - length += count; - byte* bEnd = buffer + count; - while (buffer != bEnd) - { - // New checksum value - crcZip = (crcZip >> 8) ^ _zipCrctab[((int)crcZip ^ (*buffer)) & 0xff]; - buffer++; - } - } - } - else - { - byte[] buffer = new byte[4 * 1024 * 1024]; - for (int count = inFile.Read(buffer, 0, buffer.Length); count > 0; count = inFile.Read(buffer, 0, buffer.Length)) - { - length += count; - fixed (byte* bb = buffer) - { - byte* b = bb; - byte* bEnd = bb + count; - while (b != bEnd) - { - // New checksum value - crcZip = (crcZip >> 8) ^ _zipCrctab[((int)crcZip ^ (*b)) & 0xff]; - b++; - } - } - } - } - // fix up the crc - crcZip = crcZip ^ 0xffffffff; - return crcZip; - } - - case ChecksumType.SHA1Prefix: - { - System.Security.Cryptography.SHA1Managed sha = new System.Security.Cryptography.SHA1Managed(); - byte[] hash = sha.ComputeHash(inFile); - return BitConverter.ToUInt64(hash, 0); - } - - case ChecksumType.SHA256Prefix: - { - System.Security.Cryptography.SHA256Managed sha = new System.Security.Cryptography.SHA256Managed(); - byte[] hash = sha.ComputeHash(inFile); - return BitConverter.ToUInt64(hash, 0); - } - } - } - } - #endregion -#else - #region File-only operations - - /// - /// Set the length of a file, truncating or padding as needed. - /// - /// the file to alter - /// the desired length, in bytes - /// - /// Note that this only operates on normal files, and cannot be successfully used on compressed - /// files or special streams. - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// An I/O error has occurred. - /// The stream does not support both writing and seeking. - /// Attempted to set the value parameter to less than 0. - public static void ResizeFile(string fileName, long length) - { - using (FileStream pad = new FileStream(fileName, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite)) - { - pad.SetLength(length); - } - } - - /// - /// Determine if a file exists. - /// - /// the original filename - /// true if a file with a name of fileName exists, false otherwise - public static bool FileExists(string fileName) - { - return !string.IsNullOrEmpty(fileName) && File.Exists(fileName); - } - - private static readonly char[] wildChars = new char[] { '*', '?' }; - private static readonly char[] wildPlusChars = new char[] { '*', '?', '+' }; - - /// - /// Expand an extended wildcard pattern into a set of file paths. - /// - /// the pattern to expand - /// the set of file paths matching the pattern - /// - /// The wildcard pattern accepts the standard "*" and "?" placeholders. - /// "..." also refers to a recursive search over subdirectories. - /// "+" can also be used to make a union of several filenames or patterns. - /// In addition to filenames, HTTP URLs, nul, null, $, - /// -, and Cosmos stream names are all recognized as elements. - /// Names of files that do not exist will be excluded. - /// - public static string[] ExpandWildcards(string pattern) - { - if (pattern == null || (pattern.IndexOfAny(wildPlusChars) < 0 && pattern.IndexOf("...") < 0)) - { - if (FileExists(pattern)) - { - return new string[] { pattern }; - } - else - { - return new string[0]; - } - } - List matchList = new List(); - bool disjoint = false; - int filePatternCount = 0; - string[] patterns = pattern.Split('+'); - foreach (string pat in patterns) - { - // hard-code in special types?? - if (pat.Length == 0) - continue; - string patLower = pat.ToLower(); - - filePatternCount++; - int prepatternCount = matchList.Count; - if (pat.IndexOfAny(wildChars) >= 0 || pat.IndexOf("...") >= 0) - { - // compressed extensions are not automatically used! *** - int recursiveIndex = pat.IndexOf("..."); - if (recursiveIndex >= 0) - { - string left = pat.Substring(0, recursiveIndex); - string right = pat.Substring(recursiveIndex + 3); - right = right.TrimStart('\\', '/'); - if (right.Length == 0) - right = "*"; - string path = left; - bool pathEmpty = (path == null || path.Length == 0); - if (pathEmpty) - path = "."; - Stack dirsLeft = new Stack(); - dirsLeft.Push(path); - while (dirsLeft.Count != 0) - { - string dir = (string)dirsLeft.Pop(); - - // watch for lack of access: - try - { - // this is actually incorrect, for 3-char extensions: *** - string[] files = Directory.GetFiles(dir, right); - if (pathEmpty) - { - for (int i = 0; i < files.Length; i++) - { - if (files[i].StartsWith("./") || files[i].StartsWith(".\\")) - files[i] = files[i].Substring(2); - } - } - matchList.AddRange(files); - - string[] subs = Directory.GetDirectories(dir); - for (int s = subs.Length - 1; s >= 0; s--) - { - dirsLeft.Push(subs[s]); - } - } - catch - { - // ignore - } - } - } - else - { - try - { - string path = Path.GetDirectoryName(pat); - bool pathEmpty = !(pat.StartsWith("./") || pat.StartsWith(".\\")); - if (path == null || path.Length == 0) - path = "."; - // watch for lack of access: - try - { - string[] files = Directory.GetFiles(path, Path.GetFileName(pat)); - if (pathEmpty) - { - for (int i = 0; i < files.Length; i++) - { - if (files[i].StartsWith("./") || files[i].StartsWith(".\\")) - files[i] = files[i].Substring(2); - } - } - matchList.AddRange(files); - } - catch - { - // ignore - } - } - catch - { - // ignore bad path? - } - } - } - else - { - // what to do?? Filter to only those that exist?? *** - if (!FileExists(pat)) - continue; - matchList.Add(pat); - } - if (filePatternCount > 1 && matchList.Count != prepatternCount) - { - disjoint = true; - } - } - if (disjoint || true) - { - // remove duplicates, very inefficiently - but it is simple, preserves - // the order, uses no additional memory, and is case-insensitive...: - for (int i = 0; i < matchList.Count - 1; i++) - { - for (int j = i + 1; j < matchList.Count; j++) - { - if (string.Compare((string)matchList[i], (string)matchList[j], true) == 0) - { - matchList.RemoveAt(j); - j--; - } - } - } - } - return matchList.ToArray(); - } - - /// - /// Read the lines of the file or stream specified by fileName. - /// - /// the file to read (or URL, or Cosmos stream...) - /// the lines of fileName, or null if it cannot be read - /// - /// The reading is performed through . - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static string[] ReadLines(string fileName) - { - try - { - using (StreamReader input = ZStreamReader.Open(fileName)) - { - try - { - List lines = new List(); - for (string line = input.ReadLine(); line != null; line = input.ReadLine()) - { - lines.Add(line); - } - if (lines.Count > 0 && ((string)lines[lines.Count - 1]).Length == 0) - lines.RemoveAt(lines.Count - 1); - return lines.ToArray(); - } - catch - { - return null; - } - } - } - catch - { - return null; - } - } - #endregion -#endif - } - -#if TLCFULLBUILD - /// - /// Class representing file or directory information. - /// - /// - /// This is similar to the purpose of and its - /// subclasses, and . - /// However, they are not easy to effectively subclass. - /// - public class StreamInfo : IComparable - { - private readonly string _path; - private long _length; - private DateTime _lastWriteTime; - private string _canonicalPath; - - /// - /// Create a new StreamInfo. - /// - /// the name of the file or directory - a directory - /// must end with "/" or "\" - /// the length of the item - /// the last write time of the item - public StreamInfo(string path, long length, DateTime lastWriteTime) - { - _length = length; - _lastWriteTime = lastWriteTime; - _canonicalPath = null; - _path = path; - if (_path == null) - { - _path = ""; - } - else - { - _path = path; - } - } - - /// - /// Create a new StreamInfo for an existing file or directory. - /// - /// the path to the file or directory - /// The given path does not exist. - public StreamInfo(string path) - { - if (path == null || path.Length == 0) - { - throw new FileNotFoundException("No directory or file for empty path"); - } - try - { - _canonicalPath = null; - _path = path; - // this is not really efficient: - if (IOUtil.FileExists(path)) - { - // this only works for file systems!! - FileInfo f = new FileInfo(path); - _length = f.Length; - _lastWriteTime = f.LastWriteTime; - } - else if (IOUtil.DirectoryExists(path)) - { - // this only works for file systems!! - if (path[path.Length - 1] != '/' && path[path.Length - 1] != '\\') - { - _path = _path + "/"; - } - _length = 0; - _lastWriteTime = Directory.GetLastWriteTime(path); - } - else - { - throw new FileNotFoundException("No directory or file: " + path); - } - } - catch - { - throw new FileNotFoundException("Cannot access directory or file: " + path); - } - } - - /// - /// Create a new StreamInfo for an existing file or directory. - /// - /// the path to the file or directory - /// dummy parameter to indicate delaying the fetch of metadata - /// The given path does not exist. - internal StreamInfo(string path, bool delay) - { - _canonicalPath = null; - _path = path; - _length = -1; - _lastWriteTime = DateTime.MinValue; - } - - /// - /// Compare two instances, based on the canonical path. - /// - /// the instance to compare to - /// negative if this instance is less than obj; 0 if equal; - /// postive otherwise - public int CompareTo(object obj) - { - StreamInfo other = obj as StreamInfo; - if (other == null) - return -1; - return CanonicalPath.CompareTo(other.CanonicalPath); - } - - /// - /// Determine if this instance is equal to another. - /// - /// the instance to compare to - /// true if obj is a StreamInfo referring to the same canonical path; - /// false otherwise - public override bool Equals(object obj) - { - if (obj is StreamInfo) - { - return CanonicalPath.Equals(((StreamInfo)obj).CanonicalPath); - } - - return false; - } - - /// - /// Get the hashcode, based on the canonical path. - /// - /// the hash code - public override int GetHashCode() - { - return CanonicalPath.GetHashCode(); - } - - /// - /// Get the . - /// - /// the canonical name - public override string ToString() - { - return CanonicalPath; - } - - /// - /// Gets the full path of the file or directory. - /// - public string Path - { - get { return _path; } - } - - /// - /// Gets the name of this item, with no parent path or salshes. - /// - public string Name - { - get - { - if (_path.Length == 0) - return ""; - int end = _path[_path.Length - 1] == '/' || _path[_path.Length - 1] == '\\' ? - _path.Length - 1 : _path.Length; - // what if it ends in multiple slashes? *** - int start = _path.LastIndexOfAny(_pathSeperators, end - 1); - if (start < 0) - { - start = 0; - } - else - { - start++; - } - return _path.Substring(start, end - start); - } - } - - private static readonly char[] _pathSeperators = new char[] { '/', '\\' }; - - /// - /// Gets the full path of the file or directory in a standard form. - /// - public string CanonicalPath - { - get - { - if (_canonicalPath == null) - { - _canonicalPath = IOUtil.GetCanonicalPath(_path); - } - return _canonicalPath; - } - } - - /// - /// Gets whether the item is a directory (not including archives, - /// which can be accessed as directories). - /// - public bool IsDirectory - { - get - { - return Path.Length != 0 && CanonicalPath[CanonicalPath.Length - 1] == '/'; - } - } - /// - /// Gets whether the item is a compressed file or archive. - /// - public bool IsCompressed - { - get - { - string ext = System.IO.Path.GetExtension(Path); - if (ext.Length == 0) - return false; - ext = ext.ToLower(); - return Array.IndexOf(ZStreamIn.decompressionExtensions, ext) >= 0; - } - } - - /// - /// Gets the last modification time. - /// - public DateTime LastWriteTime - { - get - { - if (_lastWriteTime == DateTime.MinValue) - { - if (IsDirectory) - { - if (Directory.Exists(_path)) - { - _lastWriteTime = Directory.GetLastWriteTime(_path); - } - else - { - // not actually a file... - } - } - else - { - if (File.Exists(_path)) - { - _lastWriteTime = File.GetLastWriteTime(_path); - } - else - { - // not actually a file... - } - } - } - return _lastWriteTime; - } - } - - /// - /// Gets the length of the file in bytes, or 0 for directories. - /// - public long Length - { - get - { - if (_length < 0) - { - if (IsDirectory) - { - _length = 0; - } - else - { - if (File.Exists(_path)) - { - FileInfo f = new FileInfo(_path); - _length = f.Length; - } - else - { - // this shouldn't happen... - } - } - } - return _length; - } - } - } - -#endif // TLCFULLBUILD - #endregion - - #region ZStreams - -#if TLCFULLBUILD - /// - /// Class to create StreamReaders that automatically decompress based on the file extensions. - /// - /// - ///

- /// Compressed files are recognized by extension and automatically decompressed. - /// Filenames that do not exist are checked to see if compressed versions exist; if so, - /// the compressed file is silently opened. (For example, "doc.txt.gz" will be used if - /// "doc.txt" is requested but does not exist). To read compressed files directly, - /// without decompression, append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console input. - /// - /// - /// URLs starting with "http://" or "https://" are downloaded with HTTP. - /// - /// - /// Names starting with "cosmos://" are fetched as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for reading text. - /// - /// - /// The form "sql:server/db/table" or "sql:server/db/{query}" refers to SQL Server. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// Compression support relies on executable utilities to be in the path. - /// See for 7z.exe and 7za.exe (for many formats - - /// .7z, .gz, .zip, .rar, .bz2, .cab, .arj), for gzip.exe - /// (for .gz), or for unrar.exe (for .rar). - /// - /// -#else - /// - /// Class to create StreamReaders given file paths. - /// -#endif - public class ZStreamReader //: StreamReader - { - private static bool _defaultToLocalEncoding = false; - - private static int _bufferSize = 32 * 1024; - - /// - /// Get or set whether to allow fallback to the compression library if executables - /// are not found in the path. false by default. Using the fallback may result in - /// slower performance and larger files. This setting is shared with ZStreamIn, - /// ZStreamOut, ZStreamReader, and ZStreamWriter. - /// - public static bool AllowLibraryFallback - { - get { return ZStreamIn.AllowLibraryFallback; } - set { ZStreamIn.AllowLibraryFallback = value; } - } - - private ZStreamReader() - { - } - - /// - /// Get or set whether to default to the local encoding, rather than a lenient UTF8. - /// - public static bool DefaultToLocalEncoding - { - get { return _defaultToLocalEncoding; } - set { _defaultToLocalEncoding = value; } - } - - /// - /// Get or set extension to look to append when the given filename does not exist. - /// If set to empty string (the default), try all known extensions; - /// if set to null, disable. - /// This is mapped to ZStreamIn.FallbackExtension. - /// - public static string FallbackExtension - { - //get { return fallbackExtension; } - //set { fallbackExtension = value; } - get { return ZStreamIn.FallbackExtension; } - set { ZStreamIn.FallbackExtension = value; } - } - -#if TLCFULLBUILD - /// - /// Open the specified file. - /// - /// name of the file to open - /// A StreamReader for the (possibly uncompressed) text - /// - ///

- /// Compressed files are recognized by extension and automatically decompressed. - /// Filenames that do not exist are checked to see if compressed versions exist; if so, - /// the compressed file is silently opened. (For example, "doc.txt.gz" will be used if - /// "doc.txt" is requested but does not exist). To read compressed files directly, - /// without decompression, append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console input. - /// - /// - /// URLs starting with "http://" or "https://" are downloaded with HTTP. - /// - /// - /// Names starting with "cosmos://" are fetched as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for reading text. - /// - /// - /// The form "sql:server/db/table" or "sql:server/db/{query}" refers to SQL Server. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. -#endif - public static StreamReader Open(string fileName) - { - Encoding enc = DefaultToLocalEncoding ? Encoding.Default : ZStreamWriter.UTF8Lenient; - return Open(fileName, enc); - } - -#if TLCFULLBUILD - /// - /// Open the specified file. - /// - /// name of the file to open - /// the encoding to use - /// A StreamReader for the (possibly uncompressed) text - /// - ///

- /// Compressed files are recognized by extension and automatically decompressed. - /// Filenames that do not exist are checked to see if compressed versions exist; if so, - /// the compressed file is silently opened. (For example, "doc.txt.gz" will be used if - /// "doc.txt" is requested but does not exist). To read compressed files directly, - /// without decompression, append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console input. - /// - /// - /// URLs starting with "http://" or "https://" are downloaded with HTTP. - /// - /// - /// Names starting with "cosmos://" are fetched as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for reading text. - /// - /// - /// The form "sql:server/db/table" or "sql:server/db/{query}" refers to SQL Server. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. -#endif - public static StreamReader Open(string fileName, Encoding encoding) - { -#if TLCFULLBUILD - // check for SqlStream - we don't want to open this as a Stream: - if (SqlTextReader.IsSqlTextReader(fileName)) - { - return new SqlTextReader(fileName); - } -#endif - // hack for console! It seems to break otherwise, at times, at least in 1.1... - Stream s = null; - try - { - s = ZStreamIn.Open(fileName); - if (_bufferSize > 0) - { - return new StreamReader(s, encoding, true, _bufferSize); - } - else - { - return new StreamReader(s, encoding, true); - } - } - catch - { - if (s != null) - { - try - { - s.Close(); - } - catch - { - } - } - throw; - } - } - -#if UNBUFFERED - -#if TLCFULLBUILD - /// - /// Open the specified file with normal file caching. - /// - /// name of the file to open - /// A StreamReader for the (possibly uncompressed) text - /// - /// - /// This method opens the file with system caching, regardless of the setting of - /// . - /// - ///

- /// Compressed files are recognized by extension and automatically decompressed. - /// Filenames that do not exist are checked to see if compressed versions exist; if so, - /// the compressed file is silently opened. (For example, "doc.txt.gz" will be used if - /// "doc.txt" is requested but does not exist). To read compressed files directly, - /// without decompression, append a "$" to the filename. - ///

- /// - /// There are several special filenames: - /// - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console input. - /// - /// - /// URLs starting with "http://" or "https://" are downloaded with HTTP. - /// - /// - /// Names starting with "cosmos://" are fetched as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for reading text. - /// - /// - /// The form "sql:server/db/table" or "sql:server/db/{query}" refers to SQL Server. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. -#endif - public static StreamReader OpenBuffered(string fileName) - { - Encoding enc = DefaultToLocalEncoding ? Encoding.Default : ZStreamWriter.UTF8Lenient; - return OpenBuffered(fileName, enc); - } - -#if TLCFULLBUILD - /// - /// Open the specified file with normal file caching. - /// - /// name of the file to open - /// the encoding to use - /// A StreamReader for the (possibly uncompressed) text - /// - /// - /// This method opens the file with system caching, regardless of the setting of - /// . - /// - ///

- /// Compressed files are recognized by extension and automatically decompressed. - /// Filenames that do not exist are checked to see if compressed versions exist; if so, - /// the compressed file is silently opened. (For example, "doc.txt.gz" will be used if - /// "doc.txt" is requested but does not exist). To read compressed files directly, - /// without decompression, append a "$" to the filename. - ///

- /// - /// There are several special filenames: - /// - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console input. - /// - /// - /// URLs starting with "http://" or "https://" are downloaded with HTTP. - /// - /// - /// Names starting with "cosmos://" are fetched as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for reading text. - /// - /// - /// The form "sql:server/db/table" or "sql:server/db/{query}" refers to SQL Server. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. -#endif - public static StreamReader OpenBuffered(string fileName, Encoding encoding) - { -#if TLCFULLBUILD - // check for Azure Storage objects - AzureStorageIO azureStorage = new AzureStorageIO(); - string fileNameLower = fileName.ToLower(); - - //REVIEW: Standardize on a naming convention or employ some other way to distinguish azure vs general http streams. - // check for HTTP: - if (fileNameLower.StartsWith("http://") || fileNameLower.StartsWith("https://")) - { - if (azureStorage.BlockBlobExistsByUri(fileNameLower)) - { - return new StreamReader(azureStorage.GetBlobStream(fileNameLower), encoding); - } - } - - // check for SqlStream - we don't want to open this as a Stream: - if (SqlTextReader.IsSqlTextReader(fileName)) - { - return new SqlTextReader(fileName); - } -#endif - Stream s = null; - try - { - s = ZStreamIn.OpenBuffered(fileName); - if (_bufferSize > 0) - { - return new StreamReader(s, encoding, true, _bufferSize); - } - else - { - return new StreamReader(s, encoding, true); //, BUFFER_SIZE); - } - } - catch - { - if (s != null) - { - try - { - s.Close(); - } - catch - { - } - } - throw; - } - } - - /// - /// Open the specified file (with unbuffered I/O, if possible). - /// - /// name of the file to open - /// A StreamReader for the (possibly uncompressed) text - /// - /// - /// Unbuffered I/O can give better performance, especially on fast RAID arrays. - /// It does not use the system file cache. This is the only way to get speeds over - /// 60 MB/sec or more on reading (350 MB/sec or more is possible on a good array). - /// - /// - /// While compressed files and special stream names will be understood, unbuffered I/O will - /// not be enabled on anything but simple files. - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static StreamReader OpenUnbuffered(string fileName) - { - Encoding enc = DefaultToLocalEncoding ? Encoding.Default : ZStreamWriter.UTF8Lenient; - return OpenUnbuffered(fileName, enc); - } - /// - /// Open the specified file (with unbuffered I/O, if possible). - /// - /// name of the file to open - /// the encoding to use - /// A StreamReader for the (possibly uncompressed) text - /// - /// - /// Unbuffered I/O can give better performance, especially on fast RAID arrays. - /// It does not use the system file cache. This is the only way to get speeds over - /// 60 MB/sec or more on reading (350 MB/sec or more is possible on a good array). - /// - /// - /// While compressed files and special stream names will be understood, unbuffered I/O will - /// not be enabled on anything but simple files. - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static StreamReader OpenUnbuffered(string fileName, Encoding encoding) - { -#if TLCFULLBUILD - // check for Azure Storage objects - AzureStorageIO azureStorage = new AzureStorageIO(); - string fileNameLower = fileName.ToLower(); - if (fileNameLower.StartsWith("http://") || fileNameLower.StartsWith("https://")) - { - if (azureStorage.BlockBlobExistsByUri(fileNameLower)) - { - return new StreamReader(azureStorage.GetBlobStream(fileNameLower), encoding); - } - } - - // check for SqlStream - we don't want to open this as a Stream: - if (SqlTextReader.IsSqlTextReader(fileName)) - { - return new SqlTextReader(fileName); - } -#endif - Stream s = null; - try - { - s = ZStreamIn.OpenUnbuffered(fileName); - return new StreamReader(s, encoding, true, 64 * 1024); - } - catch - { - if (s != null) - { - try - { - s.Close(); - } - catch - { - } - } - throw; - } - } - -#endif - -#if TEMP_FILE_RAR - ///////////////////////// - //// Old temp file code: - // assume only one file inside! - System.CodeDom.Compiler.TempFileCollection tempCollection = new System.CodeDom.Compiler.TempFileCollection(); - tempCollection.KeepFiles = true; - ztempDir = tempCollection.BasePath; - Directory.CreateDirectory(ztempDir); - string fullName = "\"" + (new FileInfo(fileName)).FullName + "\""; - System.Diagnostics.ProcessStartInfo procInfo = new System.Diagnostics.ProcessStartInfo("unrar", "e -y -o+ -inul -dh " + fullName); - procInfo.WindowStyle = System.Diagnostics.ProcessWindowStyle.Hidden; - procInfo.CreateNoWindow = true; - procInfo.UseShellExecute = false; - procInfo.WorkingDirectory = ztempDir; - System.Diagnostics.Process proc = System.Diagnostics.Process.Start(procInfo); - proc.WaitForExit(); - string[] fileNames = Directory.GetFiles(ztempDir); - if (fileNames.Length == 0) - { - try - { - try - { - string[] badFiles = Directory.GetFiles(ztempDir); - if (badFiles != null) - { - for (int i = 0; i < badFiles.Length; i++) - { - File.Delete(Path.Combine(ztempDir, badFiles[i])); - } - } - } - catch - { - // ignore any problems... - } - Directory.Delete(ztempDir, true); - } - catch - { - // ignore any problems... - } - // should we open the original file, then? *** - throw new Exception("unrar failed on file '" + fileName + "'"); - } - s = new FileStream(fileNames[0], FileMode.Open, FileAccess.Read); - - private ZStreamReader(Stream s, string ztempDir, Encoding e, bool detectEncoding, int bufferSize) - : base(s, e, detectEncoding, bufferSize) - { - tempDir = ztempDir; - } - - /// - /// Destroy the ZStreamReader. - /// - ~ZStreamReader() - { - Dispose(true); - } - - /// - /// Close the StreamReader. - /// - public override void Close() - { - try - { - base.Close(); - isClosed = true; - } - finally - { - if (tempDir != null) - { - try - { - try - { - string[] badFiles = Directory.GetFiles(tempDir); - if (badFiles != null) - { - for (int i = 0; i < badFiles.Length; i++) - { - File.Delete(Path.Combine(tempDir, badFiles[i])); - } - } - } - catch - { - // ignore any problems... - } - Directory.Delete(tempDir, true); - tempDir = null; - } - catch - { - // ignore any problems... - } - } - } - } - - private bool isClosed = false; - /// - /// Release all resources - /// - /// - protected override void Dispose(bool disposing) - { - if (!isClosed) - { - try - { - base.Close(); - isClosed = true; - } - catch - { - // ignore - probably already closed or some such. - } - } - if (tempDir != null) - { - try - { - try - { - string[] badFiles = Directory.GetFiles(tempDir); - if (badFiles != null) - { - for (int i = 0; i < badFiles.Length; i++) - { - File.Delete(Path.Combine(tempDir, badFiles[i])); - } - } - } - catch - { - // ignore any problems... - } - Directory.Delete(tempDir, true); - tempDir = null; - } - catch - { - // ignore any problems... - } - } - base.Dispose (disposing); - } -#endif - } - -#if TLCFULLBUILD - /// - /// Class to create StreamWriters that automatically compress based on the file extensions. - /// - /// - ///

- /// Compressed files are recognized by extension and automatically compressed. - /// To write to a file with a compression extension directly, without compression, - /// append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console output. Note that this is line buffered, - /// which may cause problems if the data is needed immediately and is not as fast as full buffering. - /// - /// - /// Names starting with "cosmos://" are stored as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for writing text. - /// - /// - /// The form "sql:server/db/table" refers to SQL Server, for an existing table. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// Compression support relies on executable utilities to be in the path. - /// See for 7z.exe and 7za.exe (for - /// .7z, .gz), for gzip.exe - /// (for .gz). - /// - /// -#else - /// - /// Class to create StreamWriters given file paths. - /// -#endif - public class ZStreamWriter - { - // backing field for UTF8Lenient - private static readonly Encoding _utf8Lenient = new UTF8Encoding(false, false); - - /// - /// A lenient UTF8 encoding that ignores problems and skips the BOM. - /// - public static Encoding UTF8Lenient - { - get - { - return _utf8Lenient; - } - } - - private const string WriteNewLine = "\r\n"; - private static Encoding _writeEncoding = UTF8Lenient; - //private static int compressionLevel = 1; - private static bool _breakChunksAtLines = true; - - private static int _bufferSize = 32 * 1024; - - /// - /// Get or set whether the Open method should use a LowFragmentationStream for files. - /// true, by default. - /// - /// - /// The has strong advantages, increasing write - /// speed and decreasing fragmentation. - /// - public static bool DefaultLowFragmentation - { - get { return ZStreamOut.DefaultLowFragmentation; } - set { ZStreamOut.DefaultLowFragmentation = value; } - } - - /// - /// Get or set the encoding that is used for new StreamWriters. - /// - public static Encoding WriteEncoding - { - get { return _writeEncoding; } - set { _writeEncoding = value; } - } - -#if TLCFULLBUILD - /// - /// Get or set whether to break at line boundaries when using chunked streams, - /// such as or . - /// True by default, unlike . - /// - /// - /// This will not necessarily have any effect. It is currently unimplemented in - /// , and most streams have no concept of chunks. - /// -#else - /// - /// Get or set whether to break at line boundaries when using chunked streams. - /// True by default, unlike . - /// - /// - /// This will not necessarily have any effect. Most streams have no concept of chunks. - /// -#endif - public static bool BreakChunksAtLines - { - get { return _breakChunksAtLines; } - set { _breakChunksAtLines = value; } - } - - /// - /// Get or set whether to allow fallback to the compression library if executables - /// are not found in the path. false by default. Using the fallback may result in - /// slower performance and larger files. This setting is shared with ZStreamIn, - /// ZStreamOut, ZStreamReader, and ZStreamWriter. - /// - public static bool AllowLibraryFallback - { - get { return ZStreamIn.AllowLibraryFallback; } - set { ZStreamIn.AllowLibraryFallback = value; } - } - - private ZStreamWriter() - { - } - -#if TLCFULLBUILD - /// - /// Open the given file, accepting special stream names and decompressing by extension. - /// - /// file to write to - /// A StreamWriter for the file - /// - ///

- /// Compressed files are recognized by extension and automatically compressed. - /// To write to a file with a compression extension directly, without compression, - /// append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console output. Note that this is line buffered, - /// which may cause problems if the data is needed immediately and is not as fast as full buffering. - /// - /// - /// Names starting with "cosmos://" are stored as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for writing text. - /// - /// - /// The form "sql:server/db/table" refers to SQL Server, for an existing table. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// The utilities needed to open a stream are not available. -#endif - public static StreamWriter Open(string outFileName) - { - return Open(outFileName, false); - } - -#if TLCFULLBUILD - /// - /// Open the given file, accepting special stream names and decompressing by extension. - /// - /// file to write to - /// Encoding for writing - /// A StreamWriter for the file - /// - ///

- /// Compressed files are recognized by extension and automatically compressed. - /// To write to a file with a compression extension directly, without compression, - /// append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console output. Note that this is line buffered, - /// which may cause problems if the data is needed immediately and is not as fast as full buffering. - /// - /// - /// Names starting with "cosmos://" are stored as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for writing text. - /// - /// - /// The form "sql:server/db/table" refers to SQL Server, for an existing table. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// The utilities needed to open a stream are not available. -#endif - public static StreamWriter Open(string outFileName, Encoding encoding) - { - return Open(outFileName, false, encoding); - } - -#if TLCFULLBUILD - /// - /// Open the given file, accepting special stream names and decompressing by extension. - /// - /// file to write to - /// if true, append; if false, overwrite - /// A StreamWriter for the file - /// - ///

- /// Compressed files are recognized by extension and automatically compressed. - /// To write to a file with a compression extension directly, without compression, - /// append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console output. Note that this is line buffered, - /// which may cause problems if the data is needed immediately and is not as fast as full buffering. - /// - /// - /// Names starting with "cosmos://" are stored as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for writing text. - /// - /// - /// The form "sql:server/db/table" refers to SQL Server, for an existing table. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// Append is specified, and fileName cannot be found. - /// The utilities needed to open a stream are not available. -#endif - public static StreamWriter Open(string outFileName, bool append) - { - return Open(outFileName, append, BreakChunksAtLines); - } - -#if TLCFULLBUILD - /// - /// Open the given file, accepting special stream names and decompressing by extension. - /// - /// file to write to - /// if true, append; if false, overwrite - /// Encoding for writing - /// A StreamWriter for the file - /// - ///

- /// Compressed files are recognized by extension and automatically compressed. - /// To write to a file with a compression extension directly, without compression, - /// append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console output. Note that this is line buffered, - /// which may cause problems if the data is needed immediately and is not as fast as full buffering. - /// - /// - /// Names starting with "cosmos://" are stored as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for writing text. - /// - /// - /// The form "sql:server/db/table" refers to SQL Server, for an existing table. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// Append is specified, and fileName cannot be found. - /// The utilities needed to open a stream are not available. -#endif - public static StreamWriter Open(string outFileName, bool append, Encoding encoding) - { - return Open(outFileName, append, BreakChunksAtLines, false, encoding); - } - -#if TLCFULLBUILD - /// - /// Open the given file, accepting special stream names and decompressing by extension. - /// - /// file to write to - /// if true, append; if false, overwrite - /// if true, break at line boundaries when using chunked streams - /// A StreamWriter for the file - /// - ///

- /// Compressed files are recognized by extension and automatically compressed. - /// To write to a file with a compression extension directly, without compression, - /// append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console output. Note that this is line buffered, - /// which may cause problems if the data is needed immediately and is not as fast as full buffering. - /// - /// - /// Names starting with "cosmos://" are stored as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for writing text. - /// - /// - /// The form "sql:server/db/table" refers to SQL Server, for an existing table. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// Append is specified, and fileName cannot be found. - /// The utilities needed to open a stream are not available. -#endif - public static StreamWriter Open(string outFileName, bool append, bool breakChunksAtLines) - { - return Open(outFileName, append, breakChunksAtLines, false); - } - - private static StreamWriter Open(string outFileName, bool append, bool breakChunksAtLines, bool unbuffered) - { - return Open(outFileName, append, breakChunksAtLines, unbuffered, WriteEncoding); - } - - private static StreamWriter Open(string outFileName, bool append, bool breakChunksAtLines, bool unbuffered, Encoding encoding) - { - // check for SqlStream - we don't want to open this as a Stream: -#if TLCFULLBUILD - if (SqlTextReader.IsSqlTextReader(outFileName)) - { - return new SqlTextWriter(outFileName); - } -#endif - Stream s = unbuffered ? ZStreamOut.OpenUnbuffered(outFileName, append, breakChunksAtLines) - : ZStreamOut.Open(outFileName, append, breakChunksAtLines); - - StreamWriter sw; - int size = _bufferSize; - if (string.Compare(outFileName, "clip:", true) == 0) - { - size = 4; - } - if (size > 0) - { - sw = new StreamWriter(s, encoding, size); - } - else - { - sw = new StreamWriter(s, encoding); - } - sw.NewLine = WriteNewLine; -#if TLCFULLBUILD - if (ZStreamIn.IsConsoleStream(outFileName)) - { - // match standard behavior, but not efficient in many cases! - // we might really want line-level flushing! *** - sw.AutoFlush = true; - ((LineBufferedStream)sw.BaseStream).LineBuffer = true; - } -#endif - return sw; - } - -#if UNBUFFERED - - /// - /// Open the given file (unbuffered, if possible). - /// - /// file to write to - /// Encoding for writing - /// A StreamWriter for the file - /// - /// - /// Unbuffered I/O can give better performance, especially on fast RAID arrays. - /// It does not use the system file cache. However, for writing, this currently - /// has no effect. - /// - /// - /// fileName is null. - /// fileName is invalid. - /// The utilities needed to open a stream are not available. - public static StreamWriter OpenUnbuffered(string outFileName, Encoding encoding) - { - return OpenUnbuffered(outFileName, false, encoding); - } - - /// - /// Open the given file (unbuffered, if possible). - /// - /// file to write to - /// A StreamWriter for the file - /// - /// - /// Unbuffered I/O can give better performance, especially on fast RAID arrays. - /// It does not use the system file cache. However, for writing, this currently - /// has no effect. - /// - /// - /// fileName is null. - /// fileName is invalid. - /// The utilities needed to open a stream are not available. - public static StreamWriter OpenUnbuffered(string outFileName) - { - return OpenUnbuffered(outFileName, false); - } - - /// - /// Open the given file (unbuffered, if possible). - /// - /// file to write to - /// if true, append; if false, overwrite - /// A StreamWriter for the file - /// - /// - /// Unbuffered I/O can give better performance, especially on fast RAID arrays. - /// It does not use the system file cache. However, for writing, this currently - /// has no effect. - /// - /// - /// fileName is null. - /// fileName is invalid. - /// Append is specified, and fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static StreamWriter OpenUnbuffered(string outFileName, bool append) - { - return OpenUnbuffered(outFileName, append, BreakChunksAtLines); - } - - /// - /// Open the given file (unbuffered, if possible). - /// - /// file to write to - /// if true, append; if false, overwrite - /// Encoding for writing - /// A StreamWriter for the file - /// - /// - /// Unbuffered I/O can give better performance, especially on fast RAID arrays. - /// It does not use the system file cache. However, for writing, this currently - /// has no effect. - /// - /// - /// fileName is null. - /// fileName is invalid. - /// Append is specified, and fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static StreamWriter OpenUnbuffered(string outFileName, bool append, Encoding encoding) - { - return Open(outFileName, append, _breakChunksAtLines, true, encoding); - } - - /// - /// Open the given file (unbuffered, if possible). - /// - /// file to write to - /// if true, append; if false, overwrite - /// if true, break at line boundaries when using chunked streams - /// A StreamWriter for the file - /// - /// - /// Unbuffered I/O can give better performance, especially on fast RAID arrays. - /// It does not use the system file cache. However, for writing, this currently - /// has no effect. - /// - /// - /// fileName is null. - /// fileName is invalid. - /// Append is specified, and fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static StreamWriter OpenUnbuffered(string outFileName, bool append, bool breakChunksAtLines) - { - return Open(outFileName, append, breakChunksAtLines, true); - } - -#endif - } - -#if TLCFULLBUILD - /// - /// Class to create input Streams that automatically decompress based on the file extensions. - /// - /// - ///

- /// Compressed files are recognized by extension and automatically decompressed. - /// Filenames that do not exist are checked to see if compressed versions exist; if so, - /// the compressed file is silently opened. (For example, "doc.txt.gz" will be used if - /// "doc.txt" is requested but does not exist). To read compressed files directly, - /// without decompression, append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console input. - /// - /// - /// URLs starting with "http://" or "https://" are downloaded with HTTP. - /// - /// - /// Names starting with "cosmos://" are fetched as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for reading text. - /// - /// - /// The form "sql:server/db/table" or "sql:server/db/{query}" refers to SQL Server. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// Compression support relies on executable utilities to be in the path. - /// See for 7z.exe and 7za.exe (for many formats - - /// .7z, .gz, .zip, .rar, .bz2, .cab, .arj), for gzip.exe - /// (for .gz), or for unrar.exe (for .rar). - /// - /// -#else - /// - /// Class to create input Streams given file paths. - /// -#endif - public class ZStreamIn - { - private static string _fallbackExtension = ""; - private static bool _defaultUnbuffered = false; - - private static int _bufferSize = 32 * 1024; - - /// - /// Get or set extension to look to append when the given filename does not exist. - /// If set to empty string (the default), try all known extensions; - /// if set to null, disable. - /// - public static string FallbackExtension - { - get { return _fallbackExtension; } - set { _fallbackExtension = value; } - } - - /// - /// Get or set whether the Open method should use unbuffered I/O whenever possible. - /// false, by default. - /// - public static bool DefaultUnbuffered - { - get { return _defaultUnbuffered; } - set { _defaultUnbuffered = value; } - } - - /// - /// Get or set whether to allow fallback to the compression library if executables - /// are not found in the path. false by default. Using the fallback may result in - /// slower performance and larger files. This setting is shared with ZStreamIn, - /// ZStreamOut, ZStreamReader, and ZStreamWriter. - /// - public static bool AllowLibraryFallback - { - get { return _allowLibraryFallback; } - set { _allowLibraryFallback = value; } - } - private static bool _allowLibraryFallback = true; - - internal static readonly string[] decompressionArchiveExtensions = new string[] - { - ".7z", - ".zip", - ".tar", - ".cab", - ".arj", - ".rar", - ".lzh", - ".chm" - }; - internal static readonly string[] decompressionExtensions = new string[] - { - ".gz", - ".7z", - ".zip", - ".tar", - ".bz2", - ".z", - ".cab", - ".arj", - ".rar", - ".lzh", - ".chm" - }; - internal static readonly string NullStreamName = "nul"; - private static readonly string[] _nullStreamNames = new string[] - { - "nul", - "null" - }; - internal static bool IsNullStream(string fileName) - { - if (fileName == null || fileName.Length == 0) - return false; - for (int i = 0; i < _nullStreamNames.Length; i++) - { - int len = _nullStreamNames[i].Length; - if (string.Compare(fileName, 0, _nullStreamNames[i], 0, len, true) == 0) - { - if (fileName.Length == len) - return true; - - if (fileName[len] == '\\' || - fileName[len] == '/') - { - len++; - // should collapse repeated slashes... - while (len < fileName.Length && - (fileName[len] == '\\' || fileName[len] == '/')) - { - len++; - } - if (len == fileName.Length) - return true; - for (int j = 0; i < _nullStreamNames.Length; j++) - { - if (fileName.Length - len == _nullStreamNames[j].Length && - string.Compare(fileName, len, _nullStreamNames[j], 0, _nullStreamNames[j].Length, true) == 0) - { - return true; - } - } - } - } - } - return false; - } - internal static readonly string ConsoleStreamName = "$"; - private static readonly string[] _consoleStreamNames = new string[] - { - "$", - "-" - }; - internal static bool IsConsoleStream(string fileName) - { - if (fileName == null || fileName.Length == 0) - return false; - for (int i = 0; i < _consoleStreamNames.Length; i++) - { - int len = _consoleStreamNames[i].Length; - if (string.Compare(fileName, 0, _consoleStreamNames[i], 0, len, true) == 0) - { - if (fileName.Length == len) - return true; - - if (fileName[len] == '\\' || - fileName[len] == '/') - { - len++; - // should collapse repeated slashes... - while (len < fileName.Length && - (fileName[len] == '\\' || fileName[len] == '/')) - { - len++; - } - if (len == fileName.Length) - return true; - for (int j = 0; i < _consoleStreamNames.Length; j++) - { - if (fileName.Length - len == _consoleStreamNames[j].Length && - string.Compare(fileName, len, _consoleStreamNames[j], 0, _consoleStreamNames[j].Length, true) == 0) - { - return true; - } - } - } - } - } - return false; - } - - /// - /// Gets the set of extensions (such as ".gz") that are accepted for decompression. - /// - public static string[] DecompressionExtensions - => (string[])decompressionExtensions.Clone(); - - /// - /// Gets the set of extensions (such as ".gz") that are accepted for decompression - /// as archives that can act as directories. - /// - public static string[] DecompressionArchiveExtensions - => (string[])decompressionArchiveExtensions.Clone(); - - private static readonly char[] _pathSeparators = new char[] { '/', '\\' }; - - private ZStreamIn() - { - } - -#if TLCFULLBUILD - private static bool _gzipFailure; - private static bool _gzip7ZFailure; - private static bool _full7ZFailure; -#endif - -#if TLCFULLBUILD - /// - /// Open the specified file. - /// - /// name of the file to open - /// A Stream for the (possibly uncompressed) data - /// - ///

- /// Compressed files are recognized by extension and automatically decompressed. - /// Filenames that do not exist are checked to see if compressed versions exist; if so, - /// the compressed file is silently opened. (For example, "doc.txt.gz" will be used if - /// "doc.txt" is requested but does not exist). To read compressed files directly, - /// without decompression, append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console input. - /// - /// - /// URLs starting with "http://" or "https://" are downloaded with HTTP. - /// - /// - /// Names starting with "cosmos://" are fetched as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for reading text. - /// - /// - /// The form "sql:server/db/table" or "sql:server/db/{query}" refers to SQL Server. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. -#endif - public static Stream Open(string fileName) - { - if (DefaultUnbuffered) - { - return Open(fileName, false, true, false); - } - else - { - return Open(fileName, true); - } - } - - private static Stream Open(string fileName, bool buffered) - { - return Open(fileName, buffered, false, false); - } - private static Stream Open(string fileName, bool buffered, bool bufferedFallback, bool async) - { - Contracts.CheckNonEmpty(fileName, nameof(fileName)); - - // check for special names: - string fileNameLower = fileName.ToLower(); -#if TLCFULLBUILD - if (IsNullStream(fileName)) - { - return Stream.Null; - } - if (IsConsoleStream(fileName)) - { - // note that ReadByte() and WriteByte() are pathetic, and allocate an array! *** - // They also make too many managed->unmanaged transitions. - // This is 200x slower than it should be. - // Position also will not work. - // Using a BufferedStream wrapper fixes the first problem, with some overhead. - // A custom wrapper should probably be made, but it is a pain. - // larger sizes somehow break line buffering, making it be every *two* lines... - return new BufferedStream(Console.OpenStandardInput(), 1024); - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(fileName)) - { - return new ClipboardReadStream(); - } - - // check for Cosmos: - if (fileNameLower.StartsWith("cosmos:")) - { - try - { - // no compression support, anyway... - fileName = fileName.TrimEnd('$'); - return new CosmosReadStream(fileName); - } - catch (InvalidOperationException) - { - throw new InvalidOperationException("ZStreamIn requires cosmos.cmd or cosmos.exe to be in the " + - "path for Cosmos reading."); - } - catch (System.ComponentModel.Win32Exception) - { - throw new InvalidOperationException("ZStreamIn requires cosmos.cmd or cosmos.exe to be in the " + - "path for Cosmos reading."); - } - } - - // check for Cockpit: - if (fileNameLower.StartsWith("cockpit:") || fileNameLower.StartsWith("cockpit:")) - { - if (fileName.Length <= "cockpit:".Length) - throw new ArgumentException("Invalid cockpit streamname", "fileName"); - // should we URL-encode? - fileName = fileName.Substring("cockpit:".Length); - fileName = fileName.Replace('/', '\\'); - - string cockpitServer = "cockpit.search.msn.com:81"; - - // check for forced cockpit server: - int hostStart = 0; - while (hostStart < fileName.Length && fileName[hostStart] == '\\') - hostStart++; - if (hostStart < fileName.Length - 2) - { - int hostEnd = hostStart + 1; - while (hostEnd < fileName.Length && fileName[hostEnd] != '\\') - hostEnd++; - int div = fileName.IndexOf('@', hostStart + 1, hostEnd - hostStart - 1); - if (div > 0) - { - // extract cockpit server! - cockpitServer = fileName.Substring(div + 1, hostEnd - div - 1); - fileName = fileName.Substring(0, div) + fileName.Substring(hostEnd); - } - } - - // ls is really for internal use... - string cmd = "get"; - if (fileName.EndsWith("\\") || fileName.EndsWith("/")) - cmd = "ls"; - // force searchmsn proxy!! - fileName = "http://" + cockpitServer + "@searchmsn/files?cmd=" + cmd + "&path=" + - fileName; - fileNameLower = fileName.ToLower(); - } - - // check for Multistream: - if (fileNameLower.StartsWith("multi:")) - { - fileName = fileName.Substring("multi:".Length); - return new MultiStream(fileName); - } - if (fileNameLower.StartsWith("filelist:")) - { - fileName = fileName.Substring("filelist:".Length); - return new MultiStream(fileName); - } - - // check for HTTP: - if (fileNameLower.StartsWith("http:") || fileNameLower.StartsWith("https:")) - { - // check for Azure Storage objects - AzureStorageIO azureStorage = new AzureStorageIO(); - if (azureStorage.BlockBlobExistsByUri(fileNameLower)) - { - return azureStorage.GetBlobStream(fileNameLower); - } - return new HttpStream(fileName); - } - - // check for SqlStream: - if (SqlTextReader.IsSqlTextReader(fileName)) - { - return new SqlTextReader(fileName).CreateStream(); - } - - // check for InternalStore: - if (InternalStoreUtility.IsInternalStore(fileName)) - { - try - { - fileName = fileName.TrimEnd('$'); - return new InternalStoreStream(fileName); - } - catch (InvalidOperationException) - { - throw new InvalidOperationException("ZStreamIn requires tstore.exe to be in the " + - "path for InternalStore reading."); - } - catch (System.ComponentModel.Win32Exception) - { - throw new InvalidOperationException("ZStreamIn requires tstore.exe to be in the " + - "path for InternalStore reading."); - } - } - - // remove trailing "$" - bool forceRaw = false; - if (fileName[fileName.Length - 1] == '$') - { - forceRaw = true; - fileName = fileName.Substring(0, fileName.Length - 1); - } - - // check for named stream: - // should this be based on file existance, first? - int cIndex = fileName.LastIndexOf(':'); - if (cIndex > 0) - { - if (File.Exists(fileName.Substring(0, cIndex))) - { - if (cIndex > 1 || - fileName.IndexOfAny(_pathSeparators, 2) < 0) - { - // named: - return new NamedStream(fileName, false); - } - } - } - - Stream s = null; - try - { - if (!forceRaw) - { - // check for compressed versions, if needed: - if (FallbackExtension != null) - { - if (!File.Exists(fileName)) - { - if (FallbackExtension.Length != 0) - { - if (File.Exists(fileName + FallbackExtension)) - { - fileName = fileName + FallbackExtension; - } - } - else - { - for (int i = 0; i < decompressionExtensions.Length; i++) - { - if (File.Exists(fileName + decompressionExtensions[i])) - { - fileName = fileName + decompressionExtensions[i]; - break; - } - } - } - } - } - - string ext = Path.GetExtension(fileName).ToLower(); - switch (ext) - { - case ".gz": - { - // try using gzip, fall back to 7zip, and fall back to SharpZipLib: - if (!_gzipFailure) - { -#if GZIP_UNBUFFERED - if (DefaultUnbuffered) - { - Stream sRaw = null; - try - { - sRaw = OpenUnbuffered(fileName); - try - { - s = new GzipDecodeStream(sRaw); - } - catch (InvalidOperationException) - { - gzipFailure = true; - try - { - if (sRaw != null) sRaw.Close(); - } - catch - { - } - } - catch (System.ComponentModel.Win32Exception) - { - gzipFailure = true; - try - { - if (sRaw != null) sRaw.Close(); - } - catch - { - } - } - } - catch - { - try - { - if (sRaw != null) sRaw.Close(); - } - catch - { - } - } - } - else - { - try - { - s = new GzipDecodeStream(fileName); - } - catch (InvalidOperationException) - { - gzipFailure = true; - } - catch (System.ComponentModel.Win32Exception) - { - gzipFailure = true; - } - } -#else - try - { - s = new GzipDecodeStream(fileName); - } - catch (InvalidOperationException) - { - _gzipFailure = true; - } - catch (System.ComponentModel.Win32Exception) - { - _gzipFailure = true; - } -#endif - } - if (s == null && !_gzip7ZFailure) - { - try - { - s = new Z7zDecodeStream(fileName); //, true); - } - catch (InvalidOperationException) - { - _gzip7ZFailure = true; - } - catch (System.ComponentModel.Win32Exception) - { - _gzip7ZFailure = true; - } - } - if (AllowLibraryFallback && s == null) - { - // this could support unbuffered, Cosmos, etc... *** - //// NOTE: - //// - .NET's gzip is very slow (30% or more longer) - //// - .NET's gzip is very large (30% larger compressed files) - //// - .NET's gzip breaks for files over 4 GB - // could wrap to get length, maybe fix 4GB problem... - - s = new System.IO.Compression.GZipStream( - new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), - System.IO.Compression.CompressionMode.Decompress); - } - if (s == null) - { - throw new InvalidOperationException("ZStreamIn requires gzip.exe, 7za.exe, or 7z.exe to be in the " + - "path for " + ext + " decompression, unless AllowLibraryFallback is set. " + - "See http://7-zip.org or http://gnuwin32.sourceforge.net/packages/gzip.htm"); - } - } - break; - - //case ".gz": - case ".7z": - case ".zip": - //case ".bzip2": - case ".bz2": - case ".z": - case ".tar": - { - try - { - s = new Z7zDecodeStream(fileName, true); - } - catch (InvalidOperationException) - { - throw new InvalidOperationException("ZStreamIn requires 7z.exe or 7za.exe to be in the " + - "path for " + ext + " decompression. " + - "See http://7-zip.org"); - } - catch (System.ComponentModel.Win32Exception) - { - throw new InvalidOperationException("ZStreamIn requires 7z.exe or 7za.exe to be in the " + - "path for " + ext + " decompression. " + - "See http://7-zip.org"); - } - } - break; - - //case ".rar": - //case ".rpm": - //case ".deb": - case ".cab": - case ".arj": - case ".lzh": - case ".chm": - { - try - { - s = new Z7zDecodeStream(fileName, true); - } - catch (InvalidOperationException) - { - throw new InvalidOperationException("ZStreamIn requires 7z.exe to be in the " + - "path for " + ext + " decompression. " + - "See http://7-zip.org"); - } - catch (System.ComponentModel.Win32Exception) - { - throw new InvalidOperationException("ZStreamIn requires 7z.exe to be in the " + - "path for " + ext + " decompression. " + - "See http://7-zip.org"); - } - } - break; - - case ".rar": - { - if (!_full7ZFailure) - { - try - { - s = new Z7zDecodeStream(fileName, true); - } - catch (InvalidOperationException) - { - _full7ZFailure = true; - } - catch (System.ComponentModel.Win32Exception) - { - _full7ZFailure = true; - } - } - if (s == null) - { - try - { - s = new RarDecodeStream(fileName); - } - catch (InvalidOperationException) - { - //unrarFailure = true; - throw new InvalidOperationException("ZStreamIn requires unrar.exe or 7z.exe to be in the " + - "path for " + ext + " decompression. " + - "See http://7-zip.org or http://rarsoft.com"); - } - catch (System.ComponentModel.Win32Exception) - { - //unrarFailure = true; - throw new InvalidOperationException("ZStreamIn requires unrar.exe or 7z.exe to be in the " + - "path for " + ext + " decompression. " + - "See http://7-zip.org or http://rarsoft.com"); - } - } - } - break; - -#if ENABLE_LZMA - case ".lzma": - { - try - { - s = new LzmaDecodeStream(fileName, false); - } - catch (InvalidOperationException) - { - throw new InvalidOperationException("ZStreamIn requires lzma.exe to be in the " + - "path for " + ext + " decompression."); - } - catch (System.ComponentModel.Win32Exception) - { - throw new InvalidOperationException("ZStreamIn requires lzma.exe to be in the " + - "path for " + ext + " decompression."); - } - } - break; -#endif - } - - if (s == null && !File.Exists(fileName)) - { - // check for compressed archive as directory segment: - // *** TODO ! - // check for compressed archives in path: - // only one path segment is allowed to be an archive... - // normalize path: - //if (fileName[fileName.Length - 1] != '\\') fileName = fileName + "\\"; - string zfileName = fileName.Replace('/', '\\'); - bool isUnc = zfileName.StartsWith("\\\\"); - while (zfileName.IndexOf("\\\\") >= 0) - { - zfileName = zfileName.Replace("\\\\", "\\"); - } - if (isUnc) - zfileName = "\\" + zfileName; - string zfileNameLower = zfileName.ToLower(); - - string archPath = null; - string inArch = null; - // should this really be only archives?? *** - for (int i = 0; i < ZStreamIn.decompressionExtensions.Length; i++) - { - ext = ZStreamIn.decompressionExtensions[i]; - int seg = zfileNameLower.IndexOf(ext + "\\"); - if (seg > 0) - { - archPath = zfileName.Substring(0, seg + ext.Length); - if (File.Exists(archPath)) - { - inArch = zfileName.Substring(seg + ext.Length).Trim('/', '\\'); - break; - } - archPath = null; - } - } - if (archPath == null) - { - // add in extension to each segment... - string[] segs = zfileName.Split('\\'); - for (int i = 0; i < segs.Length; i++) - { - if (segs[i].Length == 0) - continue; - string partial = string.Join("\\", segs, 0, i + 1); - if (partial.Length == 2 && partial[1] == ':') - continue; - if (Directory.Exists(partial)) - continue; - // should this really be only archives?? *** - for (int c = 0; c < ZStreamIn.decompressionExtensions.Length; c++) - { - ext = ZStreamIn.decompressionExtensions[c]; - if (File.Exists(partial + ext)) - { - archPath = partial + ext; - inArch = string.Join("\\", segs, i + 1, segs.Length - i - 1).Trim('/', '\\'); - break; - } - } - // quit when parent will not exist - break; - } - } - if (archPath != null) - { - //Console.WriteLine(archPath + " :: " + inArch); - // check for path in archive. - inArch = inArch.Trim('\\'); - if (inArch.Length != 0) - { - // optimize for gzip and support unrar.exe: - if (Path.GetExtension(archPath).ToLower() == ".gz") - { - try - { - return Open(archPath); - } - catch (InvalidOperationException) - { - } - } - try - { - s = new Z7zDecodeStream(archPath, inArch); - } - catch (InvalidOperationException) - { - } - catch (System.ComponentModel.Win32Exception) - { - } - if (s == null && !Z7zDecodeStream.Exists7z && - Path.GetExtension(archPath).ToLower() == ".rar") - { - try - { - s = new RarDecodeStream(archPath, inArch); - } - catch (InvalidOperationException) - { - } - catch (System.ComponentModel.Win32Exception) - { - } - } - if (s == null) - { - throw new InvalidOperationException("ZStreamIn requires 7z.exe or 7za.exe to be in the " + - "path for " + ext + " decompression. " + - "See http://7-zip.org"); - } - } - } - } - } -#else - Stream s = null; - try - { -#endif - if (s == null) - { - // don't use unbuffered for small files: - if (!buffered && bufferedFallback) - { - try - { - long len = (new FileInfo(fileName)).Length; - if (len <= 2 * 8 * 1024 * 1024) - { - buffered = true; - } - } - catch - { - } - } - // unbuffered: - if (!buffered) - { -#if UNBUFFERED - // assume sequential and not async? *** - // why are we ignoring the async flag? - try - { - try - { - return new UnbufferedStream(fileName); - } - catch (UnbufferedStream.VirtualAllocException) - { - throw; - } - } - catch - { - if (!bufferedFallback) - throw; - } -#else - throw new NotSupportedException("Unbuffered IO is not supported."); -#endif - } - - // buffered: - if (_bufferSize > 0) - { - s = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, _bufferSize); - } - else - { - s = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - } - } - return s; - } - catch - { - if (s != null) - { - try - { - s.Close(); - } - catch - { - } - } - throw; - } - } - -#if UNBUFFERED - -#if TLCFULLBUILD - /// - /// Open the specified file with normal file caching. - /// - /// name of the file to open - /// A Stream for the (possibly uncompressed) data - /// - /// - /// This method opens the file with system caching, regardless of the setting of - /// . - /// - /// - /// - ///

- /// Compressed files are recognized by extension and automatically decompressed. - /// Filenames that do not exist are checked to see if compressed versions exist; if so, - /// the compressed file is silently opened. (For example, "doc.txt.gz" will be used if - /// "doc.txt" is requested but does not exist). To read compressed files directly, - /// without decompression, append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console input. - /// - /// - /// URLs starting with "http://" or "https://" are downloaded with HTTP. - /// - /// - /// Names starting with "cosmos://" are fetched as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for reading text. - /// - /// - /// The form "sql:server/db/table" or "sql:server/db/{query}" refers to SQL Server. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. -#endif - public static Stream OpenBuffered(string fileName) - { - return Open(fileName, true); - } - - /// - /// Open the specified file (with unbuffered I/O, if possible). - /// - /// name of the file to open - /// A Stream for the data - /// - /// - /// Unbuffered I/O can give better performance, especially on fast RAID arrays. - /// It does not use the system file cache. This is the only way to get speeds over - /// 60 MB/sec or more on reading (350 MB/sec or more is possible on a good array). - /// - /// - /// While compressed files and special stream names will be understood, unbuffered I/O will - /// not be enabled on anything but simple files. - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static Stream OpenUnbuffered(string fileName) - { - return OpenUnbuffered(fileName, false); - } - /// - /// Open the specified file (with unbuffered I/O, if possible). - /// - /// name of the file to open - /// whether to use asynchronous I/O - /// A Stream for the data - /// - /// - /// Unbuffered I/O can give better performance, especially on fast RAID arrays. - /// It does not use the system file cache. This is the only way to get speeds over - /// 60 MB/sec or more on reading (350 MB/sec or more is possible on a good array). - /// - /// - /// While compressed files and special stream names will be understood, unbuffered I/O will - /// not be enabled on anything but simple files. - /// - /// - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// The utilities needed to open a stream are not available. - public static Stream OpenUnbuffered(string fileName, bool async) - { - return Open(fileName, false, false, async); - } - -#endif - } - -#if TLCFULLBUILD - /// - /// Class to create output Streams that automatically compress based on the file extensions. - /// - /// - ///

- /// Compressed files are recognized by extension and automatically compressed. - /// To write to a file with a compression extension directly, without compression, - /// append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console output. Note that this is fully buffered, - /// which may cause problems if the data is needed immediately - /// ( uses line buffering). - /// - /// - /// Names starting with "cosmos://" are stored as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for writing text. - /// - /// - /// The form "sql:server/db/table" refers to SQL Server, for an existing table. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// Compression support relies on executable utilities to be in the path. - /// See for 7z.exe and 7za.exe (for - /// .7z, .gz), for gzip.exe - /// (for .gz). - /// - /// -#else - /// - /// Class to create output Streams given file paths. - /// -#endif - public class ZStreamOut - { - private static int _compressionLevel = 6; - private static bool _defaultLowFragmentation = true; - private static bool _breakChunksAtLines = false; - - private static int _bufferSize = 32 * 1024; - - /// - /// Get or set the compression level (0 - 9) used for compressed streams. - /// - /// - ///

- /// The default is 1, which is the worst (but fastest) compression. - /// Setting a higher level can significantly improve the compression ratio, - /// especially for tighter compression methods (such as 7z), but the time - /// needed will increase. - ///

- ///

- /// If file size is a problem, raising this value can help. - ///

- ///

- /// A setting of 0 represents storing without compression for methods that - /// support this. - ///

- /// - public static int CompressionLevel - { - get { return _compressionLevel; } - set { _compressionLevel = Math.Max(0, Math.Min(9, value)); } - } - - /// - /// Get or set whether the Open method should use a LowFragmentationStream for files. - /// true, by default. - /// - /// - /// The has strong advantages, increasing write - /// speed and decreasing fragmentation. - /// - public static bool DefaultLowFragmentation - { - get { return _defaultLowFragmentation; } - set { _defaultLowFragmentation = value; } - } - -#if TLCFULLBUILD - /// - /// Get or set whether to break at line boundaries when using chunked streams, - /// such as or . False by default, - /// unlike . - /// - /// - /// This will not necessarily have any effect. It is currently unimplemented in - /// , and most streams have no concept of chunks. - /// -#else - /// - /// Get or set whether to break at line boundaries when using chunked streams. - /// False by default. - /// - /// - /// This will not necessarily have any effect. Most streams have no concept of chunks. - /// -#endif - public static bool BreakChunksAtLines - { - get { return _breakChunksAtLines; } - set { _breakChunksAtLines = value; } - } - - internal static readonly string[] compressionArchiveExtensions = new string[] - { - ".7z" - }; - internal static readonly string[] compressionExtensions = new string[] - { - ".gz", - ".7z", - ".bz2", - }; - - /// - /// Gets the set of extensions (such as ".gz") that are accepted for compression. - /// - public static string[] CompressionExtensions - { - get - { - return (string[])compressionExtensions.Clone(); - } - } - - /// - /// Gets the set of extensions (such as ".7z") that are accepted for multi-file - /// archive compression. - /// - public static string[] CompressionArchiveExtensions - { - get - { - return (string[])compressionArchiveExtensions.Clone(); - } - } - - /// - /// Get whether to allow fallback to the compression library if executables - /// are not found in the path. false by default. Using the fallback may result in - /// slower performance and larger files. This setting is shared with ZStreamIn, - /// ZStreamOut, ZStreamReader, and ZStreamWriter. - /// - public static bool AllowLibraryFallback - { - get { return ZStreamIn.AllowLibraryFallback; } - set { ZStreamIn.AllowLibraryFallback = value; } - } - - private static readonly char[] _pathSeparators = new char[] { '/', '\\' }; - - private ZStreamOut() - { - } - -#if TLCFULLBUILD - /// - /// Open the given file, accepting special stream names and decompressing by extension. - /// - /// file to write to - /// A Stream for the file - /// - ///

- /// Compressed files are recognized by extension and automatically compressed. - /// To write to a file with a compression extension directly, without compression, - /// append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console output. Note that this is fully buffered, - /// which may cause problems if the data is needed immediately - /// ( uses line buffering). - /// - /// - /// Names starting with "cosmos://" are stored as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for writing text. - /// - /// - /// The form "sql:server/db/table" refers to SQL Server, for an existing table. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid. - /// The utilities needed to open a stream are not available. -#endif - public static Stream Open(string fileName) - { - return Open(fileName, false); - } - -#if TLCFULLBUILD - /// - /// Open the given file, accepting special stream names and decompressing by extension. - /// - /// file to write to - /// if true, append; if false, overwrite - /// A Stream for the file - /// - ///

- /// Compressed files are recognized by extension and automatically compressed. - /// To write to a file with a compression extension directly, without compression, - /// append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console output. Note that this is fully buffered, - /// which may cause problems if the data is needed immediately - /// ( uses line buffering). - /// - /// - /// Names starting with "cosmos://" are stored as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for writing text. - /// - /// - /// The form "sql:server/db/table" refers to SQL Server, for an existing table. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid, or appending cannot be done. - /// The utilities needed to open a stream are not available. - /// Append is specified, and fileName cannot be found. -#endif - public static Stream Open(string fileName, bool append) - { - return Open(fileName, append, BreakChunksAtLines); - } - public static Stream Open(string fileName, bool append, out string pathFull) - { - return Open(fileName, append, BreakChunksAtLines, out pathFull); - } -#if TLCFULLBUILD - /// - /// Open the given file, accepting special stream names and decompressing by extension. - /// - /// file to write to - /// if true, append; if false, overwrite - /// if true, break at line boundaries when using chunked streams - /// A Stream for the file - /// - ///

- /// Compressed files are recognized by extension and automatically compressed. - /// To write to a file with a compression extension directly, without compression, - /// append a "$" to the filename. - ///

- /// There are several special filenames: - /// - /// - /// The special names "nul" and "null" refer to an empty stream. - /// - /// - /// The special names "-" and "$" refer to the console output. Note that this is fully buffered, - /// which may cause problems if the data is needed immediately - /// ( uses line buffering). - /// - /// - /// Names starting with "cosmos://" are stored as Cosmos streams. - /// - /// - /// The name "clip:" refers to the clipboard, for writing text. - /// - /// - /// The form "sql:server/db/table" refers to SQL Server, for an existing table. - /// - /// - /// "multi:filename" or "filelist:filename" refers to a - /// list of other files. - /// - /// - /// Names ending with ":streamname" open the NTFS named stream "streamname". - /// - /// - /// - /// fileName is null. - /// fileName is invalid, or appending cannot be done. - /// The utilities needed to open a stream are not available. - /// Append is specified, and fileName cannot be found. -#endif - public static Stream Open(string fileName, bool append, bool breakChunksAtLines) - { - string pathFull; - return Open(fileName, append, breakChunksAtLines, true, out pathFull); - } - public static Stream Open(string fileName, bool append, bool breakChunksAtLines, out string pathFull) - { - return Open(fileName, append, breakChunksAtLines, true, out pathFull); - } - - /// - /// Open the given file, accepting special stream names and decompressing by extension. - /// - /// file to write to - /// if true, append; if false, overwrite - /// if true, use buffered IO; if false, use unbuffered IO - /// if true, break at line boundaries when using chunked streams - /// A Stream for the file - /// fileName is null. - /// fileName is invalid, or appending cannot be done. - /// The utilities needed to open a stream are not available. - /// Append is specified, and fileName cannot be found. - private static Stream Open(string fileName, bool append, bool breakChunksAtLines, bool buffered, out string pathFull) - { - Contracts.CheckNonEmpty(fileName, nameof(fileName)); - - pathFull = null; - - // check for special names: - string fileNameLower = fileName.ToLower(); -#if TLCFULLBUILD - if (ZStreamIn.IsNullStream(fileName)) - { - //return new NullStream(true); - return Stream.Null; - } - if (ZStreamIn.IsConsoleStream(fileName)) - { - // buffer size is ignored for console streams! - // terrible perf for ReadByte(), by default - too many managed->unmanaged transitions. - // buffering, however, leads to problems with timing for some applications that expect - // streaming results. - LineBufferedStream res = new LineBufferedStream(Console.OpenStandardOutput(), _bufferSize); - res.LineBuffer = false; - return res; - } - // check for clipboard: - if (ClipboardReadStream.IsClipboardStream(fileName)) - { - return new ClipboardWriteStream(append); - } - - // check for Cosmos: - if (fileNameLower.StartsWith("cosmos:")) - { - try - { - // no compression support, anyway... - fileName = fileName.TrimEnd('$'); - return new CosmosWriteStream(fileName, append, breakChunksAtLines); - } - catch (InvalidOperationException) - { - throw new InvalidOperationException("ZStreamIn requires cosmos.cmd to be in the " + - "path for Cosmos reading."); - } - catch (System.ComponentModel.Win32Exception) - { - throw new InvalidOperationException("ZStreamIn requires cosmos.cmd to be in the " + - "path for Cosmos reading."); - } - } - - // check for Multistream: - if (fileNameLower.StartsWith("multi:")) - { - fileName = fileName.Substring("multi:".Length); - return new MultiStream(fileName, true); - } - if (fileNameLower.StartsWith("filelist:")) - { - fileName = fileName.Substring("filelist:".Length); - return new MultiStream(fileName, true); - } - - if (fileNameLower.StartsWith("http:") || fileNameLower.StartsWith("https:")) - { - AzureStorageIO azureStorage = new AzureStorageIO(); - return azureStorage.GetBlobStreamForWriting(fileNameLower); - - } - - // check for SqlStream: - if (SqlTextReader.IsSqlTextReader(fileName)) - { - return new SqlTextWriter(fileName).CreateStream(); - } - - // check for InternalStore: - if (InternalStoreUtility.IsInternalStore(fileName)) - { - throw new NotSupportedException("Cannot write to a InternalStore."); - } - - // remove trailing "$" - bool forceRaw = false; - if (fileName[fileName.Length - 1] == '$') - { - forceRaw = true; - fileName = fileName.Substring(0, fileName.Length - 1); - } - - // check for named stream: - // should this be based on file existance, first? - int cIndex = fileName.LastIndexOf(':'); - if (cIndex > 0) - { - if (cIndex > 1 || - fileName.IndexOfAny(_pathSeparators, 2) < 0) - { - // named: - return new NamedStream(fileName, true, append); - } - } - - Stream s = null; - if (!forceRaw) - { - string ext = Path.GetExtension(fileName).ToLower(); - // use special cases for efficiency: - switch (ext) - { - case ".gz": - { - // appending really could be enabled... *** - if (append) - throw new ArgumentException("Cannot append to a gz stream.", "append"); - try - { - s = new GzipEncodeStream(fileName, CompressionLevel); - } - catch (InvalidOperationException) - { - - } - catch (System.ComponentModel.Win32Exception) - { - - } - if (s == null) - { - try - { - s = new Z7zEncodeStream(fileName, Z7zEncodeStream.CompressionFormat.Gzip, CompressionLevel); - } - catch (InvalidOperationException) - { - if (!AllowLibraryFallback) - { - // should fallback here if SharpZip is used!! *** - throw new InvalidOperationException("ZStreamOut requires 7za.exe, 7z.exe, or gzip.exe to be in the " + - "path for " + ext + " compression, unless AllowLibraryFallback is set. " + - "See http://7-zip.org"); - } - } - catch (System.ComponentModel.Win32Exception) - { - if (!AllowLibraryFallback) - { - // should fallback here if SharpZip is used!! *** - throw new InvalidOperationException("ZStreamOut requires 7za.exe, 7z.exe, or gzip.exe to be in the " + - "path for " + ext + " compression, unless AllowLibraryFallback is set. " + - "See http://7-zip.org"); - } - } - } - if (AllowLibraryFallback && s == null) - { - // this could support unbuffered, Cosmos, etc... *** - //// NOTE: - //// - .NET's gzip is very slow (30% or more longer) - //// - .NET's gzip is very large (30% larger compressed files) - //// - .NET's gzip breaks for files over 4 GB - //// - Huffman trees are hard-coded and poor - binary data often *inflates* - //// - Position and Length are not supported - //// - Seeking is not supported - //// - Compression and decompression are not parallelized - s = new System.IO.Compression.GZipStream( - ZStreamOut.Open(fileName + "$"), - System.IO.Compression.CompressionMode.Compress); - // without a buffer, this is even worse... - s = new BufferedStream(s, 32 * 1024); - } - } - break; - - case ".7z": - case ".bz2": - { - if (append) - throw new ArgumentException("Cannot append to a 7z stream.", "append"); - try - { - s = new Z7zEncodeStream(fileName, CompressionLevel); - } - catch (InvalidOperationException) - { - throw new InvalidOperationException("ZStreamOut requires 7z.exe or 7za.exe to be in the " + - "path for " + ext + " compression. " + - "See http://7-zip.org"); - } - catch (System.ComponentModel.Win32Exception) - { - throw new InvalidOperationException("ZStreamOut requires 7z.exe or 7za.exe to be in the " + - "path for " + ext + " compression. " + - "See http://7-zip.org"); - } - } - break; - -#if ENABLE_LZMA - case ".lzma": - { - if (append) throw new ArgumentException("Cannot append to an lzma stream.", "append"); - try - { - s = new LzmaEncodeStream(fileName, true); - } - catch (InvalidOperationException) - { - throw new InvalidOperationException("ZStreamOut requires lzma.exe to be in the " + - "path for " + ext + " compression."); - } - catch (System.ComponentModel.Win32Exception) - { - throw new InvalidOperationException("ZStreamOut requires lzma.exe to be in the " + - "path for " + ext + " compression."); - } - } - break; -#endif - } - - if (s == null) - { - // check for compressed archive as directory segment: - // *** TODO ! - // check for compressed archives in path: - // only one path segment is allowed to be an archive... - // normalize path: - string zfileName = fileName.Replace('/', '\\'); - bool isUnc = zfileName.StartsWith("\\\\"); - while (zfileName.IndexOf("\\\\") >= 0) - { - zfileName = zfileName.Replace("\\\\", "\\"); - } - if (isUnc) - zfileName = "\\" + zfileName; - string zfileNameLower = zfileName.ToLower(); - - string archPath = null; - string inArch = null; - // this could be a problem... We can't write to non-archives in this way... - for (int i = 0; i < ZStreamIn.decompressionArchiveExtensions.Length; i++) - { - ext = ZStreamIn.decompressionArchiveExtensions[i]; - int seg = zfileNameLower.IndexOf(ext + "\\"); - if (seg > 0) - { - archPath = zfileName.Substring(0, seg + ext.Length); - inArch = zfileName.Substring(seg + ext.Length).Trim('/', '\\'); - break; - } - } - if (archPath != null) - { - // check for path in archive. - inArch = inArch.Trim('\\'); - if (inArch.Length != 0) - { - // what about unrar, etc? **** - try - { - s = new Z7zEncodeStream(archPath, inArch, CompressionLevel); - } - catch (InvalidOperationException) - { - throw new InvalidOperationException("ZStreamIn requires 7z.exe or 7za.exe to be in the " + - "path for " + ext + " compression. " + - "See http://7-zip.org"); - } - catch (System.ComponentModel.Win32Exception) - { - throw new InvalidOperationException("ZStreamIn requires 7z.exe or 7za.exe to be in the " + - "path for " + ext + " compression. " + - "See http://7-zip.org"); - } - } - } - } - } -#else - Stream s = null; -#endif - if (s == null) - { - // Report the full path. - pathFull = Path.GetFullPath(fileName); - - if (DefaultLowFragmentation) - { - if (_bufferSize > 0) - { - s = new LowFragmentationStream(fileName, append, _bufferSize); - } - else - { - s = new LowFragmentationStream(fileName, append); - } - } - else - { - if (_bufferSize > 0) - { - s = new FileStream(fileName, append ? FileMode.Append : FileMode.Create, FileAccess.Write, FileShare.Read, _bufferSize); - } - else - { - s = new FileStream(fileName, append ? FileMode.Append : FileMode.Create, FileAccess.Write, FileShare.Read); - } - } - } - return s; - } - -#if UNBUFFERED - - /// - /// Open the given file (unbuffered, if possible). - /// - /// file to write to - /// A Stream for the file - /// - /// - /// Unbuffered I/O can give better performance, especially on fast RAID arrays. - /// It does not use the system file cache. However, for writing, this currently - /// has no effect. - /// - /// - public static Stream OpenUnbuffered(string fileName) - { - return OpenUnbuffered(fileName, false); - } - - /// - /// Open the given file (unbuffered, if possible). - /// - /// file to write to - /// if true, append; if false, overwrite - /// A Stream for the file - /// - /// - /// Unbuffered I/O can give better performance, especially on fast RAID arrays. - /// It does not use the system file cache. However, for writing, this currently - /// has no effect. - /// - /// - public static Stream OpenUnbuffered(string fileName, bool append) - { - return OpenUnbuffered(fileName, append, BreakChunksAtLines); - } - - /// - /// Open the given file (unbuffered, if possible). - /// - /// file to write to - /// if true, append; if false, overwrite - /// if true, break at line boundaries when using chunked streams - /// A Stream for the file - /// - /// - /// Unbuffered I/O can give better performance, especially on fast RAID arrays. - /// It does not use the system file cache. However, for writing, this currently - /// has no effect. - /// - /// - public static Stream OpenUnbuffered(string fileName, bool append, bool breakChunksAtLines) - { - string pathFull; - return Open(fileName, append, breakChunksAtLines, false, out pathFull); - } - -#endif - } - - #endregion - - #region Var - /// - /// Convenience type to represent a value that should be easily converted. - /// - /// - /// - /// Var instance are immutable. - /// - /// - /// In the current form, Var is not very efficient - the value is always stored - /// internally as a string, and it is a reference type. - /// - /// - /// A Var will implicitly convert to and from primitive numerical types, as well as - /// , , and . It can also be compared for - /// equality or order with those types. Comparisons are performed by converting the - /// Var to the type of the other value (for example, comparing to 123 will cause a - /// numerical comparison, while "123" will cause a string comparison). - /// - /// - /// Numbers will be parsed properly if they contain commas, are in exponential notation, - /// or have surrounding whitespace. Hex numbers can be specified by starting with "0x" - /// or "-0x". - /// - /// - /// String methods can be used directly, without casting the value. - /// - /// - public class Var : - IEquatable, IEquatable, IEquatable, IEquatable, - IEquatable, IEquatable, IEquatable, IEquatable, - IEquatable, IEquatable, IEquatable, IEquatable, - IEquatable, IEquatable, IEquatable, IEquatable, - IComparable, IComparable, IComparable, IComparable, - IComparable, IComparable, IComparable, IComparable, - IComparable, IComparable, IComparable, IComparable, - IComparable, IComparable, IComparable, IComparable, - IEnumerable, - IComparable, IEnumerable, IConvertible - { - private string _raw; - - /// - /// Create a new Var, based on the string representation. - /// - /// the string representation - public Var(string raw) - { - _raw = raw; - } - - /// - /// Convert the specified Var to a string. - /// - /// the value to convert - /// the string version of the value - public static implicit operator string(Var v) - { - return v._raw; - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(string s) - { - return new Var(s); - } - - /// - /// Convert the specified Var to an int. - /// - /// the value to convert - /// the int version of the value - public static implicit operator int(Var v) - { - return ParseInt32(v._raw); - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(int n) - { - return new Var(n.ToString()); - } - - /// - /// Convert the specified Var to a uint. - /// - /// the value to convert - /// the uint version of the value - public static implicit operator uint(Var v) - { - return ParseUInt32(v._raw); - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(uint n) - { - return new Var(n.ToString()); - } - - /// - /// Convert the specified Var to a long. - /// - /// the value to convert - /// the long version of the value - public static implicit operator long(Var v) - { - return ParseInt64(v._raw); - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(long n) - { - return new Var(n.ToString()); - } - - /// - /// Convert the specified Var to a ulong. - /// - /// the value to convert - /// the ulong version of the value - public static implicit operator ulong(Var v) - { - return ParseUInt64(v._raw); - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(ulong n) - { - return new Var(n.ToString()); - } - - /// - /// Convert the specified Var to a short. - /// - /// the value to convert - /// the short version of the value - public static implicit operator short(Var v) - { - return ParseInt16(v._raw); - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(short n) - { - return new Var(n.ToString()); - } - - /// - /// Convert the specified Var to a ushort. - /// - /// the value to convert - /// the ushort version of the value - public static implicit operator ushort(Var v) - { - return ParseUInt16(v._raw); - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(ushort n) - { - return new Var(n.ToString()); - } - - /// - /// Convert the specified Var to a byte. - /// - /// the value to convert - /// the byte version of the value - public static implicit operator byte(Var v) - { - return ParseByte(v._raw); - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(byte n) - { - return new Var(n.ToString()); - } - - /// - /// Convert the specified Var to a sbyte. - /// - /// the value to convert - /// the sbyte version of the value - public static implicit operator sbyte(Var v) - { - return ParseSByte(v._raw); - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(sbyte n) - { - return new Var(n.ToString()); - } - - /// - /// Convert the specified Var to a float. - /// - /// the value to convert - /// the float version of the value - public static implicit operator float(Var v) - { - return ParseSingle(v._raw); - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(float n) - { - return new Var(n.ToString("R")); - } - - /// - /// Convert the specified Var to a double. - /// - /// the value to convert - /// the double version of the value - public static implicit operator double(Var v) - { - return ParseDouble(v._raw); - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(double n) - { - return new Var(n.ToString("R")); - } - - /// - /// Convert the specified Var to a char. - /// - /// the value to convert - /// the char version of the value - public static implicit operator char(Var v) - { - return v._raw == null || v._raw.Length == 0 ? '\0' : v._raw[0]; - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(char c) - { - return new Var(c.ToString()); - } - - /// - /// Convert the specified Var to a decimal. - /// - /// the value to convert - /// the decimal version of the value - public static implicit operator decimal(Var v) - { - return v._raw == null || v._raw.Length == 0 ? decimal.Zero : decimal.Parse(v._raw); - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(decimal n) - { - return new Var(n.ToString()); - } - - /// - /// Convert the specified Var to a bool. - /// - /// the value to convert - /// the bool version of the value - public static implicit operator bool(Var v) - { - return v._raw != null && v._raw != "0" && string.Compare(v._raw, "false", true) != 0; - } - /// - /// Convert the specified value to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(bool b) - { - return new Var(b ? "1" : null); - } - - /// - /// Convert the specified Var to a Guid. - /// - /// the value to convert - /// the Guid version of the value - public static implicit operator Guid(Var v) - { - //return v.raw == null || v.raw.Length == 0 ? Guid.Empty : new Guid(v.raw); - if (v._raw == null || v._raw.Length == 0) - return Guid.Empty; - - string s = v._raw; - if (s.Length == 32) - { - // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - s = s.Substring(0, 8) + "-" + s.Substring(8, 4) + "-" + s.Substring(12, 4) + "-" + s.Substring(16, 4) + "-" + s.Substring(20); - // OK, maybe that's not the most efficient... - } - return new Guid(s); - } - /// - /// Convert the specified Guid to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(Guid g) - { - return new Var(g.ToString()); - } - - /// - /// Convert the specified Var to a DateTime. - /// - /// the value to convert - /// the DateTime version of the value - public static implicit operator DateTime(Var v) - { - return v._raw == null || v._raw.Length == 0 ? DateTime.MinValue : - DateTime.Parse(v._raw, null, - System.Globalization.DateTimeStyles.AllowWhiteSpaces | - System.Globalization.DateTimeStyles.NoCurrentDateDefault - ); - } - /// - /// Convert the specified DateTime to a Var. - /// - /// the value to convert - /// the Var version of the value - public static implicit operator Var(DateTime d) - { - return new Var(d.ToString()); - } - - /// - /// Convert an array of values. - /// - /// the array to convert - /// the converted array - public static string[] Convert(Var[] vals) - { - if (vals == null) - return null; - string[] res = new string[vals.Length]; - for (int i = 0; i < vals.Length; i++) - { - res[i] = vals[i] == null ? null : (string)vals[i]; - } - return res; - } - /// - /// Convert an array of values. - /// - /// the array to convert - /// the converted array - public static Var[] Convert(string[] vals) - { - if (vals == null) - return null; - Var[] res = new Var[vals.Length]; - for (int i = 0; i < vals.Length; i++) - { - res[i] = new Var(vals[i]); - } - return res; - } - - /// - /// Return the string representation of this value. - /// - /// the string representation of this value - public override string ToString() - { - // leave null? - return _raw; - } - - private static long ParseInt64(string s) - { - // could be many times faster... - if (s == null) - return 0; - s = s.Trim(); - if (s.Length == 0) - return 0; - try - { - if (s.Length > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) - { - return System.Convert.ToInt64(s, 16); - } - if (s.Length > 3 && s[0] == '-' && s[1] == '0' && (s[2] == 'x' || s[2] == 'X')) - { - return -System.Convert.ToInt64(s.Substring(3), 16); - } - return long.Parse(s, System.Globalization.NumberStyles.AllowThousands | System.Globalization.NumberStyles.AllowExponent | System.Globalization.NumberStyles.AllowLeadingSign | System.Globalization.NumberStyles.AllowDecimalPoint); - } - catch (Exception ex) - { - if (ex is InvalidCastException || ex is FormatException || ex is OverflowException) - throw Contracts.ExceptDecode("String cannot be converted to integer: '{0}'", s); - throw; - } - } - private static ulong ParseUInt64(string s) - { - // could be many times faster... - if (s == null) - return 0; - s = s.Trim(); - if (s.Length == 0) - return 0; - try - { - if (s.Length > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) - { - return System.Convert.ToUInt64(s, 16); - } - return ulong.Parse(s, System.Globalization.NumberStyles.AllowThousands | System.Globalization.NumberStyles.AllowExponent | System.Globalization.NumberStyles.AllowLeadingSign | System.Globalization.NumberStyles.AllowDecimalPoint); - } - catch (Exception ex) - { - if (ex is InvalidCastException || ex is FormatException || ex is OverflowException) - throw Contracts.ExceptDecode("String cannot be converted to integer: '{0}'", s); - throw; - } - } - private static int ParseInt32(string s) - { - // could be many times faster... - if (s == null) - return 0; - s = s.Trim(); - if (s.Length == 0) - return 0; - try - { - if (s.Length > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) - { - return System.Convert.ToInt32(s, 16); - } - if (s.Length > 3 && s[0] == '-' && s[1] == '0' && (s[2] == 'x' || s[2] == 'X')) - { - checked - { - return (int)(-System.Convert.ToInt64(s.Substring(3), 16)); - } - } - return int.Parse(s, System.Globalization.NumberStyles.AllowThousands | System.Globalization.NumberStyles.AllowExponent | System.Globalization.NumberStyles.AllowLeadingSign | System.Globalization.NumberStyles.AllowDecimalPoint); - } - catch (Exception ex) - { - if (ex is InvalidCastException || ex is FormatException || ex is OverflowException) - throw Contracts.ExceptDecode("String cannot be converted to integer: '{0}'", s); - throw; - } - } - private static uint ParseUInt32(string s) - { - // could be many times faster... - if (s == null) - return 0; - s = s.Trim(); - if (s.Length == 0) - return 0; - try - { - if (s.Length > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) - { - return System.Convert.ToUInt32(s, 16); - } - return uint.Parse(s, System.Globalization.NumberStyles.AllowThousands | System.Globalization.NumberStyles.AllowExponent | System.Globalization.NumberStyles.AllowLeadingSign | System.Globalization.NumberStyles.AllowDecimalPoint); - } - catch (Exception ex) - { - if (ex is InvalidCastException || ex is FormatException || ex is OverflowException) - throw Contracts.ExceptDecode("String cannot be converted to integer: '{0}'", s); - throw; - } - } - private static short ParseInt16(string s) - { - // could be many times faster... - if (s == null) - return 0; - s = s.Trim(); - if (s.Length == 0) - return 0; - try - { - if (s.Length > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) - { - return System.Convert.ToInt16(s, 16); - } - if (s.Length > 3 && s[0] == '-' && s[1] == '0' && (s[2] == 'x' || s[2] == 'X')) - { - checked - { - return (short)(-System.Convert.ToInt32(s.Substring(3), 16)); - } - } - return short.Parse(s, System.Globalization.NumberStyles.AllowThousands | System.Globalization.NumberStyles.AllowExponent | System.Globalization.NumberStyles.AllowLeadingSign | System.Globalization.NumberStyles.AllowDecimalPoint); - } - catch (Exception ex) - { - if (ex is InvalidCastException || ex is FormatException || ex is OverflowException) - throw Contracts.ExceptDecode("String cannot be converted to integer: '{0}'", s); - throw; - } - } - private static ushort ParseUInt16(string s) - { - // could be many times faster... - if (s == null) - return 0; - s = s.Trim(); - if (s.Length == 0) - return 0; - try - { - if (s.Length > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) - { - return System.Convert.ToUInt16(s, 16); - } - return ushort.Parse(s, System.Globalization.NumberStyles.AllowThousands | System.Globalization.NumberStyles.AllowExponent | System.Globalization.NumberStyles.AllowLeadingSign | System.Globalization.NumberStyles.AllowDecimalPoint); - } - catch (Exception ex) - { - if (ex is InvalidCastException || ex is FormatException || ex is OverflowException) - throw Contracts.ExceptDecode("String cannot be converted to integer: '{0}'", s); - throw; - } - } - private static sbyte ParseSByte(string s) - { - // could be many times faster... - if (s == null) - return 0; - s = s.Trim(); - if (s.Length == 0) - return 0; - try - { - if (s.Length > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) - { - return System.Convert.ToSByte(s, 16); - } - if (s.Length > 3 && s[0] == '-' && s[1] == '0' && (s[2] == 'x' || s[2] == 'X')) - { - checked - { - return (sbyte)(-System.Convert.ToInt32(s.Substring(3), 16)); - } - } - return sbyte.Parse(s, System.Globalization.NumberStyles.AllowThousands | System.Globalization.NumberStyles.AllowExponent | System.Globalization.NumberStyles.AllowLeadingSign | System.Globalization.NumberStyles.AllowDecimalPoint); - } - catch (Exception ex) - { - if (ex is InvalidCastException || ex is FormatException || ex is OverflowException) - throw Contracts.ExceptDecode("String cannot be converted to integer: '{0}'", s); - throw; - } - } - private static byte ParseByte(string s) - { - // could be many times faster... - if (s == null) - return 0; - s = s.Trim(); - if (s.Length == 0) - return 0; - try - { - if (s.Length > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) - { - return System.Convert.ToByte(s, 16); - } - return byte.Parse(s, System.Globalization.NumberStyles.AllowThousands | System.Globalization.NumberStyles.AllowExponent | System.Globalization.NumberStyles.AllowLeadingSign | System.Globalization.NumberStyles.AllowDecimalPoint); - } - catch (Exception ex) - { - if (ex is InvalidCastException || ex is FormatException || ex is OverflowException) - throw Contracts.ExceptDecode("String cannot be converted to integer: '{0}'", s); - throw; - } - } - private static float ParseSingle(string s) - { - // could be many times faster... - if (s == null) - return 0.0F; - s = s.Trim(); - if (s.Length == 0) - return 0.0F; - try - { - if ((s.Length > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) || - (s.Length > 3 && s[0] == '-' && s[1] == '0' && (s[2] == 'x' || s[2] == 'X'))) - { - checked - { - return (float)ParseInt64(s); - } - } - return float.Parse(s, System.Globalization.NumberStyles.AllowThousands | System.Globalization.NumberStyles.AllowExponent | System.Globalization.NumberStyles.AllowLeadingSign | System.Globalization.NumberStyles.AllowDecimalPoint); - } - catch (Exception ex) - { - if (ex is InvalidCastException || ex is FormatException || ex is OverflowException) - throw Contracts.ExceptDecode("String cannot be converted to integer: '{0}'", s); - throw; - } - } - private static double ParseDouble(string s) - { - // could be many times faster... - if (s == null) - return 0.0; - s = s.Trim(); - if (s.Length == 0) - return 0.0; - try - { - if ((s.Length > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) || - (s.Length > 3 && s[0] == '-' && s[1] == '0' && (s[2] == 'x' || s[2] == 'X'))) - { - checked - { - return (double)ParseInt64(s); - } - } - return double.Parse(s, System.Globalization.NumberStyles.AllowThousands | System.Globalization.NumberStyles.AllowExponent | System.Globalization.NumberStyles.AllowLeadingSign | System.Globalization.NumberStyles.AllowDecimalPoint); - } - catch (Exception ex) - { - if (ex is InvalidCastException || ex is FormatException || ex is OverflowException) - throw Contracts.ExceptDecode("String cannot be converted to integer: '{0}'", s); - throw; - } - } - - /// Returns the hash code for this value. - /// A 32-bit signed integer hash code. - public override int GetHashCode() - { - // this may be bad - numbers have many string representations... *** - return _raw == null ? 0 : _raw.GetHashCode(); - } - - #region Comparisons - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public override bool Equals(object obj) - { - if (obj is Var) - { - return Equals((Var)obj); - } - else if (obj is string) - { - return Equals((string)obj); - } - else if (obj is double) - { - return Equals((double)obj); - } - else if (obj is float) - { - return Equals((float)obj); - } - else if (obj is int) - { - return Equals((int)obj); - } - else if (obj is uint) - { - return Equals((uint)obj); - } - else if (obj is long) - { - return Equals((long)obj); - } - else if (obj is ulong) - { - return Equals((ulong)obj); - } - else if (obj is short) - { - return Equals((short)obj); - } - else if (obj is ushort) - { - return Equals((ushort)obj); - } - else if (obj is byte) - { - return Equals((byte)obj); - } - else if (obj is sbyte) - { - return Equals((sbyte)obj); - } - else if (obj is char) - { - return Equals((char)obj); - } - else if (obj is decimal) - { - return Equals((decimal)obj); - } - else if (obj is Guid) - { - return Equals((Guid)obj); - } - else if (obj is DateTime) - { - return Equals((DateTime)obj); - } - return false; - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(Var obj) - { - if (_raw == null) - { - return obj == null || obj._raw == null; - } - return Equals(obj._raw); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(string obj) - { - return string.CompareOrdinal(_raw, obj) == 0; - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(double obj) - { - return ((double)this).Equals(obj); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(float obj) - { - return ((double)this).Equals(obj); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(int obj) - { - return ((long)this).Equals(obj); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(uint obj) - { - return ((long)this).Equals(obj); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(long obj) - { - return ((long)this).Equals(obj); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(ulong obj) - { - return ((ulong)this).Equals(obj); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(short obj) - { - return ((long)this).Equals(obj); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(ushort obj) - { - return ((long)this).Equals(obj); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(byte obj) - { - return ((long)this).Equals(obj); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(sbyte obj) - { - return ((long)this).Equals(obj); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(char obj) - { - return ((char)this).Equals(obj); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(decimal obj) - { - return ((decimal)this).Equals(obj); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(Guid obj) - { - return ((Guid)this).Equals(obj); - } - /// Determines whether this instance and a specified value are the same. - /// true if obj is the same as this instance; otherwise, false. - /// the value to compare to - public bool Equals(DateTime obj) - { - return ((DateTime)this).Equals(obj); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, Var v2) - { - // this may be bad - numbers have many string representations... *** - if ((object)v1 == null) - return (object)v2 == null; - if ((object)v2 == null) - return false; - return v1.Equals(v2); - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, Var v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, long v2) - { - if ((object)v1 == null) - return v2 == 0; - return (long)v1 == v2; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, long v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(long v1, Var v2) - { - return v2 == v1; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(long v1, Var v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, ulong v2) - { - if ((object)v1 == null) - return v2 == 0; - return (ulong)v1 == v2; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, ulong v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(ulong v1, Var v2) - { - return v2 == v1; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(ulong v1, Var v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, int v2) - { - if ((object)v1 == null) - return v2 == 0; - return (long)v1 == v2; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, int v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(int v1, Var v2) - { - return v2 == v1; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(int v1, Var v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, uint v2) - { - if ((object)v1 == null) - return v2 == 0; - return (long)v1 == v2; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, uint v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(uint v1, Var v2) - { - return v2 == v1; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(uint v1, Var v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, short v2) - { - if ((object)v1 == null) - return v2 == 0; - return (long)v1 == v2; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, short v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(short v1, Var v2) - { - return v2 == v1; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(short v1, Var v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, ushort v2) - { - if ((object)v1 == null) - return v2 == 0; - return (long)v1 == v2; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, ushort v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(ushort v1, Var v2) - { - return v2 == v1; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(ushort v1, Var v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, byte v2) - { - if ((object)v1 == null) - return v2 == 0; - return (long)v1 == v2; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, byte v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(byte v1, Var v2) - { - return v2 == v1; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(byte v1, Var v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, sbyte v2) - { - if ((object)v1 == null) - return v2 == 0; - return (long)v1 == v2; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, sbyte v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(sbyte v1, Var v2) - { - return v2 == v1; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(sbyte v1, Var v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, decimal v2) - { - if ((object)v1 == null) - return v2 == 0; - return (decimal)v1 == v2; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, decimal v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(decimal v1, Var v2) - { - return v2 == v1; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(decimal v1, Var v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, float v2) - { - if ((object)v1 == null) - return v2 == 0; - return (double)v1 == v2; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, float v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(float v1, Var v2) - { - return v2 == v1; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(float v1, Var v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, double v2) - { - if ((object)v1 == null) - return v2 == 0; - return (double)v1 == v2; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, double v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(double v1, Var v2) - { - return v2 == v1; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(double v1, Var v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, Guid v2) - { - if ((object)v1 == null) - return v2 == Guid.Empty; - return (Guid)v1 == v2; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, Guid v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(Var v1, DateTime v2) - { - if ((object)v1 == null) - return v2 == DateTime.MinValue; - return (DateTime)v1 == v2; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(Var v1, DateTime v2) - { - return !(v1 == v2); - } - /// Determines whether the specified values are the same. - /// true if v1 is the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator ==(DateTime v1, Var v2) - { - return v2 == v1; - } - /// Determines whether the specified values are not the same. - /// true if v1 is not the same as v2; otherwise, false. - /// the first value - /// the second value - public static bool operator !=(DateTime v1, Var v2) - { - return !(v1 == v2); - } - - int IComparable.CompareTo(object obj) - { - if (obj == null) - return _raw == null ? 0 : 1; - if (obj is Var) - { - return CompareTo((Var)obj); - } - else if (obj is string) - { - return CompareTo((string)obj); - } - else if (obj is double) - { - return CompareTo((double)obj); - } - else if (obj is float) - { - return CompareTo((float)obj); - } - else if (obj is int) - { - return CompareTo((int)obj); - } - else if (obj is uint) - { - return CompareTo((uint)obj); - } - else if (obj is long) - { - return CompareTo((long)obj); - } - else if (obj is ulong) - { - return CompareTo((ulong)obj); - } - else if (obj is short) - { - return CompareTo((short)obj); - } - else if (obj is ushort) - { - return CompareTo((ushort)obj); - } - else if (obj is byte) - { - return CompareTo((byte)obj); - } - else if (obj is sbyte) - { - return CompareTo((sbyte)obj); - } - else if (obj is char) - { - return CompareTo((char)obj); - } - else if (obj is decimal) - { - return CompareTo((decimal)obj); - } - else if (obj is Guid) - { - return CompareTo((Guid)obj); - } - else if (obj is DateTime) - { - return CompareTo((DateTime)obj); - } - return -1; - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(Var obj) - { - // possibly should use numerical if both are numerical... - if (obj == null) - return _raw == null ? 0 : 1; - return CompareTo(obj._raw); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(string obj) - { - return string.CompareOrdinal(_raw, obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(double obj) - { - return ((double)this).CompareTo(obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(float obj) - { - return ((double)this).CompareTo(obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(int obj) - { - return ((long)this).CompareTo(obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(uint obj) - { - return ((long)this).CompareTo(obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(long obj) - { - return ((long)this).CompareTo(obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(ulong obj) - { - return ((ulong)this).CompareTo(obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(short obj) - { - return ((long)this).CompareTo(obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(ushort obj) - { - return ((long)this).CompareTo(obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(byte obj) - { - return ((long)this).CompareTo(obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(sbyte obj) - { - return ((long)this).CompareTo(obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(char obj) - { - return ((char)this).CompareTo(obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(decimal obj) - { - return ((decimal)this).CompareTo(obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(Guid obj) - { - return ((Guid)this).CompareTo(obj); - } - /// Compares this instance with a specified value. - /// - /// A 32-bit signed integer indicating the lexical relationship between the two comparands. - /// negative if this instance is less than obj, zero if this instance is equal to obj, - /// positive if this instance is greater than obj. - /// - /// the value to compare to - public int CompareTo(DateTime obj) - { - return ((DateTime)this).CompareTo(obj); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, Var v2) - { - if ((object)v2 == null) - return false; - if ((object)v1 == null) - return v2._raw != null; - return v1.CompareTo(v2) < 0; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, Var v2) - { - if ((object)v1 == null) - return false; - if ((object)v2 == null) - return v1._raw != null; - return v2.CompareTo(v1) < 0; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, Var v2) - { - return !(v1 > v2); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, long v2) - { - long vv1 = ((object)v1 == null) ? 0 : (long)v1; - return vv1 < v2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, long v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, long v2) - { - long vv1 = ((object)v1 == null) ? 0 : (long)v1; - return vv1 > v2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, long v2) - { - return !(v1 > v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(long v1, Var v2) - { - long vv2 = ((object)v2 == null) ? 0 : (long)v2; - return v1 < vv2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(long v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(long v1, Var v2) - { - long vv2 = ((object)v2 == null) ? 0 : (long)v2; - return v1 > vv2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(long v1, Var v2) - { - return !(v1 > v2); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, ulong v2) - { - ulong vv1 = ((object)v1 == null) ? 0 : (ulong)v1; - return vv1 < v2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, ulong v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, ulong v2) - { - ulong vv1 = ((object)v1 == null) ? 0 : (ulong)v1; - return vv1 > v2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, ulong v2) - { - return !(v1 > v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(ulong v1, Var v2) - { - ulong vv2 = ((object)v2 == null) ? 0 : (ulong)v2; - return v1 < vv2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(ulong v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(ulong v1, Var v2) - { - ulong vv2 = ((object)v2 == null) ? 0 : (ulong)v2; - return v1 > vv2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(ulong v1, Var v2) - { - return !(v1 > v2); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, int v2) - { - int vv1 = ((object)v1 == null) ? 0 : (int)v1; - return vv1 < v2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, int v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, int v2) - { - int vv1 = ((object)v1 == null) ? 0 : (int)v1; - return vv1 > v2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, int v2) - { - return !(v1 > v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(int v1, Var v2) - { - int vv2 = ((object)v2 == null) ? 0 : (int)v2; - return v1 < vv2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(int v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(int v1, Var v2) - { - int vv2 = ((object)v2 == null) ? 0 : (int)v2; - return v1 > vv2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(int v1, Var v2) - { - return !(v1 > v2); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, uint v2) - { - uint vv1 = ((object)v1 == null) ? 0 : (uint)v1; - return vv1 < v2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, uint v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, uint v2) - { - uint vv1 = ((object)v1 == null) ? 0 : (uint)v1; - return vv1 > v2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, uint v2) - { - return !(v1 > v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(uint v1, Var v2) - { - uint vv2 = ((object)v2 == null) ? 0 : (uint)v2; - return v1 < vv2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(uint v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator >(uint v1, Var v2) - { - uint vv2 = ((object)v2 == null) ? 0 : (uint)v2; - return v1 > vv2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(uint v1, Var v2) - { - return !(v1 > v2); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, short v2) - { - short vv1 = ((object)v1 == null) ? (short)0 : (short)v1; - return vv1 < v2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, short v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, short v2) - { - short vv1 = ((object)v1 == null) ? (short)0 : (short)v1; - return vv1 > v2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, short v2) - { - return !(v1 > v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(short v1, Var v2) - { - short vv2 = ((object)v2 == null) ? (short)0 : (short)v2; - return v1 < vv2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(short v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(short v1, Var v2) - { - short vv2 = ((object)v2 == null) ? (short)0 : (short)v2; - return v1 > vv2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(short v1, Var v2) - { - return !(v1 > v2); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, ushort v2) - { - ushort vv1 = ((object)v1 == null) ? (ushort)0 : (ushort)v1; - return vv1 < v2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, ushort v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, ushort v2) - { - ushort vv1 = ((object)v1 == null) ? (ushort)0 : (ushort)v1; - return vv1 > v2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, ushort v2) - { - return !(v1 > v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(ushort v1, Var v2) - { - ushort vv2 = ((object)v2 == null) ? (ushort)0 : (ushort)v2; - return v1 < vv2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(ushort v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(ushort v1, Var v2) - { - ushort vv2 = ((object)v2 == null) ? (ushort)0 : (ushort)v2; - return v1 > vv2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(ushort v1, Var v2) - { - return !(v1 > v2); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, byte v2) - { - byte vv1 = ((object)v1 == null) ? (byte)0 : (byte)v1; - return vv1 < v2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, byte v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, byte v2) - { - byte vv1 = ((object)v1 == null) ? (byte)0 : (byte)v1; - return vv1 > v2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, byte v2) - { - return !(v1 > v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(byte v1, Var v2) - { - byte vv2 = ((object)v2 == null) ? (byte)0 : (byte)v2; - return v1 < vv2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(byte v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(byte v1, Var v2) - { - byte vv2 = ((object)v2 == null) ? (byte)0 : (byte)v2; - return v1 > vv2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(byte v1, Var v2) - { - return !(v1 > v2); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, sbyte v2) - { - sbyte vv1 = ((object)v1 == null) ? (sbyte)0 : (sbyte)v1; - return vv1 < v2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, sbyte v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, sbyte v2) - { - sbyte vv1 = ((object)v1 == null) ? (sbyte)0 : (sbyte)v1; - return vv1 > v2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, sbyte v2) - { - return !(v1 > v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(sbyte v1, Var v2) - { - sbyte vv2 = ((object)v2 == null) ? (sbyte)0 : (sbyte)v2; - return v1 < vv2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(sbyte v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(sbyte v1, Var v2) - { - sbyte vv2 = ((object)v2 == null) ? (sbyte)0 : (sbyte)v2; - return v1 > vv2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(sbyte v1, Var v2) - { - return !(v1 > v2); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, decimal v2) - { - decimal vv1 = ((object)v1 == null) ? 0 : (decimal)v1; - return vv1 < v2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, decimal v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, decimal v2) - { - decimal vv1 = ((object)v1 == null) ? 0 : (decimal)v1; - return vv1 > v2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, decimal v2) - { - return !(v1 > v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(decimal v1, Var v2) - { - decimal vv2 = ((object)v2 == null) ? 0 : (decimal)v2; - return v1 < vv2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(decimal v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(decimal v1, Var v2) - { - decimal vv2 = ((object)v2 == null) ? 0 : (decimal)v2; - return v1 > vv2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(decimal v1, Var v2) - { - return !(v1 > v2); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, Guid v2) - { - Guid vv1 = ((object)v1 == null) ? Guid.Empty : (Guid)v1; - return vv1.CompareTo(v2) < 0; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, Guid v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, Guid v2) - { - Guid vv1 = ((object)v1 == null) ? Guid.Empty : (Guid)v1; - return vv1.CompareTo(v2) > 0; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, Guid v2) - { - return !(v1 > v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Guid v1, Var v2) - { - Guid vv2 = ((object)v2 == null) ? Guid.Empty : (Guid)v2; - return v1.CompareTo(vv2) < 0; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Guid v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Guid v1, Var v2) - { - Guid vv2 = ((object)v2 == null) ? Guid.Empty : (Guid)v2; - return v1.CompareTo(vv2) > 0; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Guid v1, Var v2) - { - return !(v1 > v2); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, DateTime v2) - { - DateTime vv1 = ((object)v1 == null) ? DateTime.MinValue : (DateTime)v1; - return vv1.CompareTo(v2) < 0; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, DateTime v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, DateTime v2) - { - DateTime vv1 = ((object)v1 == null) ? DateTime.MinValue : (DateTime)v1; - return vv1.CompareTo(v2) > 0; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, DateTime v2) - { - return !(v1 > v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(DateTime v1, Var v2) - { - DateTime vv2 = ((object)v2 == null) ? DateTime.MinValue : (DateTime)v2; - return v1.CompareTo(vv2) < 0; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(DateTime v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(DateTime v1, Var v2) - { - DateTime vv2 = ((object)v2 == null) ? DateTime.MinValue : (DateTime)v2; - return v1.CompareTo(vv2) > 0; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(DateTime v1, Var v2) - { - return !(v1 > v2); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, float v2) - { - float vv1 = ((object)v1 == null) ? 0 : (float)v1; - return vv1 < v2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, float v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, float v2) - { - float vv1 = ((object)v1 == null) ? 0 : (float)v1; - return vv1 > v2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, float v2) - { - return !(v1 > v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(float v1, Var v2) - { - float vv2 = ((object)v2 == null) ? 0 : (float)v2; - return v1 < vv2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(float v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(float v1, Var v2) - { - float vv2 = ((object)v2 == null) ? 0 : (float)v2; - return v1 > vv2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(float v1, Var v2) - { - return !(v1 > v2); - } - - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(Var v1, double v2) - { - double vv1 = ((object)v1 == null) ? 0 : (double)v1; - return vv1 < v2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(Var v1, double v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(Var v1, double v2) - { - double vv1 = ((object)v1 == null) ? 0 : (double)v1; - return vv1 > v2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(Var v1, double v2) - { - return !(v1 > v2); - } - /// Compares two values. - /// true if v1 < v2; false otherwise - /// the first value - /// the second value - public static bool operator <(double v1, Var v2) - { - double vv2 = ((object)v2 == null) ? 0 : (double)v2; - return v1 < vv2; - } - /// Compares two values. - /// true if v1 >= v2; false otherwise - /// the first value - /// the second value - public static bool operator >=(double v1, Var v2) - { - return !(v1 < v2); - } - /// Compares two values. - /// true if v1 > v2; false otherwise - /// the first value - /// the second value - public static bool operator >(double v1, Var v2) - { - double vv2 = ((object)v2 == null) ? 0 : (double)v2; - return v1 > vv2; - } - /// Compares two values. - /// true if v1 <= v2; false otherwise - /// the first value - /// the second value - public static bool operator <=(double v1, Var v2) - { - return !(v1 > v2); - } - #endregion - - /// - /// Negate the value. - /// - /// the value to negate - /// the negated value - public static Var operator -(Var v) - { - if (v == null) - return null; - if (v._raw == null || v._raw.Length == 0) - return new Var(v._raw); - string s = v._raw.Trim(); - if (s.Length != 0) - { - s = (s[0] == '-') ? s.Substring(1) : "-" + s; - } - return new Var(s); - } - - /// - /// Negate the value. - /// - /// the value to negate - /// the negated value - public static Var operator !(Var v) - { - if (v == null) - return null; - return (Var)(!((bool)v)); - } - - /// - /// Increment the value. - /// - /// the value to increment - /// the incremented value - public static Var operator ++(Var v) - { - if (v == null) - return null; - if (v._raw == null || v._raw.Length == 0 || v._raw == "0") - return (Var)1; - // how to guess the type? *** - //char? - // ugly... - if (v._raw.Length == 1 && (v._raw[0] < '0' || v._raw[0] > '9')) - { - return (Var)((char)v + 1); - } - // decimal? ** - // ulong? ** - // integer? - Var res = (long)v + 1; - return res; - // double? ** - } - /// - /// Decrement the value. - /// - /// the value to decrement - /// the decremented value - public static Var operator --(Var v) - { - if (v == null) - return null; - if (v._raw == null || v._raw.Length == 0 || v._raw == "0") - return (Var)(-1); - // how to guess the type? *** - //char? - // ugly... - if (v._raw.Length == 1 && (v._raw[0] < '0' || v._raw[0] > '9')) - { - return (Var)((char)v - 1); - } - // decimal? ** - // ulong? ** - // integer? - return (Var)((long)v - 1); - // double? ** - } - - #region String Methods - /// Concatenates a specified separator between each element of a specified array, yielding a single concatenated string. - /// A consisting of the elements of value interspersed with the separator string. - /// A . - /// An array of . - /// value is null. - public static string Join(string separator, Var[] value) - { - Contracts.CheckValue(value, nameof(value)); - return Join(separator, value, 0, value.Length); - } - - /// Concatenates a specified separator between each element of a specified array, yielding a single concatenated string. Parameters specify the first array element and number of elements to use. - /// A object consisting of the strings in value joined by separator. Or, if count is zero, value has no elements, or separator and all the elements of value are . - /// The number of elements of value to use. - /// A . - /// An array of . - /// The first array element in value to use. - /// startIndex or count is less than 0.-or- startIndex plus count is greater than the number of elements in value. - /// value is null. - public static unsafe string Join(string separator, Var[] value, int startIndex, int count) - { - if (separator == null) - separator = ""; - Contracts.CheckValue(value, nameof(value)); - if (startIndex < 0) - throw Contracts.ExceptParam(nameof(startIndex), "Must be non-negative."); - if (count < 0) - throw Contracts.ExceptParam(nameof(count), "Must be non-negative."); - if (startIndex > value.Length - count) - throw Contracts.ExceptParam(nameof(startIndex)); - if (count == 0) - return ""; - if (separator.Length == 0 && startIndex == 0 && count == value.Length) - return Concat(value); - int length = 0; - int num2 = (startIndex + count) - 1; - for (int i = startIndex; i <= num2; i++) - { - if (value[i] != null) - { - string s = (string)value[i]; - if (s != null) - length += s.Length; - } - } - length += (count - 1) * separator.Length; - if ((length < 0) || ((length + 1) < 0)) - throw Contracts.Process(new InsufficientMemoryException()); - if (length == 0) - return ""; - StringBuilder sb = new StringBuilder(length); - if (value[startIndex] != null) - sb.Append((string)value[startIndex]); - for (int j = startIndex + 1; j <= num2; j++) - { - sb.Append(separator); - if (value[j] != null) - sb.Append((string)value[j]); - } - return sb.ToString(); - } - - /// Concatenates the elements of a specified array. - /// The concatenated elements of values. - /// An array of instances. - /// values is null. - public static string Concat(params Var[] values) - { - return string.Concat(Convert(values)); - } - - /// Returns a value indicating whether the specified object occurs within this string. - /// true if the value parameter occurs within this string, or if value is the empty string (""); otherwise, false. - /// The object to seek. - /// value is null. - public bool Contains(string value) { return _raw.Contains(value); } - - /// Copies a specified number of characters from a specified position in this instance to a specified position in an array of Unicode characters. - /// The number of characters in this instance to copy to destination. - /// An array element in destination. - /// A character position in this instance. - /// An array of Unicode characters. - /// destination is null. - /// sourceIndex, destinationIndex, or count is negative -or- count is greater than the length of the substring from startIndex to the end of this instance -or- count is greater than the length of the subarray from destinationIndex to the end of destination - public void CopyTo(int sourceIndex, char[] destination, int destinationIndex, int count) { _raw.CopyTo(sourceIndex, destination, destinationIndex, count); } - - /// Determines whether the end of this instance matches the specified string. - /// true if value matches the end of this instance; otherwise, false. - /// A to compare to. - /// value is null. - public bool EndsWith(string value) { return _raw.EndsWith(value); } - - /// Determines whether the end of this string matches the specified string when compared using the specified comparison option. - /// true if the value parameter matches the end of this string; otherwise, false. - /// One of the values that determines how this string and value are compared. - /// A object to compare to. - /// comparisonType is not a value. - /// value is null. - public bool EndsWith(string value, StringComparison comparisonType) { return _raw.EndsWith(value, comparisonType); } - - /// Determines whether the end of this string matches the specified string when compared using the specified culture. - /// true if the value parameter matches the end of this string; otherwise, false. - /// Cultural information that determines how this instance and value are compared. If culture is null, the current culture is used. - /// true to ignore case when comparing this instance and value; otherwise, false. - /// A object to compare to. - /// value is null. - public bool EndsWith(string value, bool ignoreCase, System.Globalization.CultureInfo culture) { return _raw.EndsWith(value, ignoreCase, culture); } - - /// Determines whether this string and a specified object have the same value. A parameter specifies the culture, case, and sort rules used in the comparison. - /// true if the value of the value parameter is the same as this string; otherwise, false. - /// One of the values. - /// A object. - /// This string is null. - /// comparisonType is not a value. - public bool Equals(string value, StringComparison comparisonType) { return _raw.Equals(value, comparisonType); } - - /// Retrieves an object that can iterate through the individual characters in this string. - /// A object. - public CharEnumerator GetEnumerator() { return _raw.GetEnumerator(); } - - /// Reports the index of the first occurrence of the specified Unicode character in this string. - /// The index position of value if that character is found, or -1 if it is not. - /// A Unicode character to seek. - public int IndexOf(char value) { return _raw.IndexOf(value); } - - /// Reports the index of the first occurrence of the specified in this instance. - /// The index position of value if that string is found, or -1 if it is not. If value is , the return value is 0. - /// The to seek. - /// value is null. - public int IndexOf(string value) { return _raw.IndexOf(value); } - - /// Reports the index of the first occurrence of the specified Unicode character in this string. The search starts at a specified character position. - /// The index position of value if that character is found, or -1 if it is not. - /// A Unicode character to seek. - /// The search starting position. - /// startIndex is less than zero or specifies a position beyond the end of this instance. - public int IndexOf(char value, int startIndex) { return _raw.IndexOf(value, startIndex); } - - /// Reports the index of the first occurrence of the specified in this instance. The search starts at a specified character position. - /// The index position of value if that string is found, or -1 if it is not. If value is , the return value is startIndex. - /// The to seek. - /// The search starting position. - /// startIndex is negative.-or- startIndex specifies a position not within this instance. - /// value is null. - public int IndexOf(string value, int startIndex) { return _raw.IndexOf(value, startIndex); } - - /// Reports the index of the first occurrence of the specified string in the current object. A parameter specifies the type of search to use for the specified string. - /// The index position of the value parameter if that string is found, or -1 if it is not. If value is , the return value is 0. - /// One of the values. - /// The object to seek. - /// comparisonType is not a valid value. - /// value is null. - public int IndexOf(string value, StringComparison comparisonType) { return _raw.IndexOf(value, comparisonType); } - - /// Reports the index of the first occurrence of the specified character in this instance. The search starts at a specified character position and examines a specified number of character positions. - /// The index position of value if that character is found, or -1 if it is not. - /// The number of character positions to examine. - /// A Unicode character to seek. - /// The search starting position. - /// count or startIndex is negative.-or- count + startIndex specifies a position beyond the end of this instance. - public int IndexOf(char value, int startIndex, int count) { return _raw.IndexOf(value, startIndex, count); } - - /// Reports the index of the first occurrence of the specified in this instance. The search starts at a specified character position and examines a specified number of character positions. - /// The index position of value if that string is found, or -1 if it is not. If value is , the return value is startIndex. - /// The number of character positions to examine. - /// The to seek. - /// The search starting position. - /// count or startIndex is negative.-or- count plus startIndex specify a position not within this instance. - /// value is null. - public int IndexOf(string value, int startIndex, int count) { return _raw.IndexOf(value, startIndex, count); } - - /// Reports the index of the first occurrence of the specified string in the current object. Parameters specify the starting search position in the current string and the type of search to use for the specified string. - /// The index position of the value parameter if that string is found, or -1 if it is not. If value is , the return value is 0. - /// One of the values. - /// The object to seek. - /// The search starting position. - /// startIndex is negative, or specifies a position that is not within this instance. - /// comparisonType is not a valid value. - /// value is null. - public int IndexOf(string value, int startIndex, StringComparison comparisonType) { return _raw.IndexOf(value, startIndex, comparisonType); } - - /// Reports the index of the first occurrence of the specified string in the current object. Parameters specify the starting search position in the current string, the number of characters in the current string to search, and the type of search to use for the specified string. - /// The index position of the value parameter if that string is found, or -1 if it is not. If value is , the return value is 0. - /// The number of character positions to examine. - /// One of the values. - /// The object to seek. - /// The search starting position. - /// count or startIndex is negative.-or- count plus startIndex specify a position that is not within this instance. - /// comparisonType is not a valid value. - /// value is null. - public int IndexOf(string value, int startIndex, int count, StringComparison comparisonType) { return _raw.IndexOf(value, startIndex, count, comparisonType); } - - /// Reports the index of the first occurrence in this instance of any character in a specified array of Unicode characters. - /// The index position of the first occurrence in this instance where any character in anyOf was found; otherwise, -1 if no character in anyOf was found. - /// A Unicode character array containing one or more characters to seek. - /// anyOf is null. - /// 2 - public int IndexOfAny(char[] anyOf) { return _raw.IndexOfAny(anyOf); } - - /// Reports the index of the first occurrence in this instance of any character in a specified array of Unicode characters. The search starts at a specified character position. - /// The index position of the first occurrence in this instance where any character in anyOf was found; otherwise, -1 if no character in anyOf was found. - /// A Unicode character array containing one or more characters to seek. - /// The search starting position. - /// startIndex is negative.-or- startIndex is greater than the number of characters in this instance. - /// anyOf is null. - public int IndexOfAny(char[] anyOf, int startIndex) { return _raw.IndexOfAny(anyOf, startIndex); } - - /// Reports the index of the first occurrence in this instance of any character in a specified array of Unicode characters. The search starts at a specified character position and examines a specified number of character positions. - /// The index position of the first occurrence in this instance where any character in anyOf was found; otherwise, -1 if no character in anyOf was found. - /// The number of character positions to examine. - /// A Unicode character array containing one or more characters to seek. - /// The search starting position. - /// count or startIndex is negative.-or- count + startIndex is greater than the number of characters in this instance. - /// anyOf is null. - public int IndexOfAny(char[] anyOf, int startIndex, int count) { return _raw.IndexOfAny(anyOf, startIndex, count); } - - /// Inserts a specified instance of at a specified index position in this instance. - /// A new equivalent to this instance but with value inserted at position startIndex. - /// The to insert. - /// The index position of the insertion. - /// startIndex is negative or greater than the length of this instance. - /// value is null. - public string Insert(int startIndex, string value) { return _raw.Insert(startIndex, value); } - - /// Indicates whether this string is in Unicode normalization form C. - /// true if this string is in normalization form C; otherwise, false. - public bool IsNormalized() { return _raw.IsNormalized(); } - - /// Indicates whether this string is in the specified Unicode normalization form. - /// true if this string is in the normalization form specified by the normalizationForm parameter; otherwise, false. - /// A Unicode normalization form. - public bool IsNormalized(NormalizationForm normalizationForm) { return _raw.IsNormalized(normalizationForm); } - - /// Reports the index position of the last occurrence of a specified Unicode character within this instance. - /// The index position of value if that character is found, or -1 if it is not. - /// A Unicode character to seek. - /// value is null. - public int LastIndexOf(char value) { return _raw.LastIndexOf(value); } - - /// Reports the index position of the last occurrence of a specified within this instance. - /// The index position of value if that string is found, or -1 if it is not. If value is , the return value is the last index position in value. - /// A to seek. - /// value is null. - public int LastIndexOf(string value) { return _raw.LastIndexOf(value); } - - /// Reports the index position of the last occurrence of a specified Unicode character within this instance. The search starts at a specified character position. - /// The index position of value if that character is found, or -1 if it is not. - /// A Unicode character to seek. - /// The starting position of a substring within this instance. - /// startIndex is less than zero or greater than the length of this instance. - /// value is null. - public int LastIndexOf(char value, int startIndex) { return _raw.LastIndexOf(value, startIndex); } - - /// Reports the index position of the last occurrence of a specified within this instance. The search starts at a specified character position. - /// The index position of value if that string is found, or -1 if it is not. If value is , the return value is startIndex. - /// The to seek. - /// The search starting position. - /// startIndex is less than zero or specifies a position not within this instance. - /// value is null. - public int LastIndexOf(string value, int startIndex) { return _raw.LastIndexOf(value, startIndex); } - - /// Reports the index of the last occurrence of a specified string within the current object. A parameter specifies the type of search to use for the specified string. - /// The index position of the value parameter if that string is found, or -1 if it is not. - /// One of the values. - /// The object to seek. - /// comparisonType is not a valid value. - /// value is null. - public int LastIndexOf(string value, StringComparison comparisonType) { return _raw.LastIndexOf(value, comparisonType); } - - /// Reports the index position of the last occurrence of the specified Unicode character in a substring within this instance. The search starts at a specified character position and examines a specified number of character positions. - /// The index position of value if that character is found, or -1 if it is not. - /// The number of character positions to examine. - /// A Unicode character to seek. - /// The starting position of a substring within this instance. - /// value is null. - /// startIndex or count is less than zero or greater than the length of this instance. - public int LastIndexOf(char value, int startIndex, int count) { return _raw.LastIndexOf(value, startIndex, count); } - /// Reports the index position of the last occurrence of a specified within this instance. The search starts at a specified character position and examines a specified number of character positions. - /// The index position of value if that string is found, or -1 if it is not. If value is , the return value is startIndex. - /// The number of character positions to examine. - /// The to seek. - /// The search starting position. - /// count or startIndex is negative.-or- count plus startIndex specify a position not within this instance. - /// value is null. - public int LastIndexOf(string value, int startIndex, int count) { return _raw.LastIndexOf(value, startIndex, count); } - - /// Reports the index of the last occurrence of a specified string within the current object. Parameters specify the starting search position in the current string, and type of search to use for the specified string. - /// The index position of the value parameter if that string is found, or -1 if it is not. - /// One of the values. - /// The object to seek. - /// The search starting position. - /// startIndex is less than zero or specifies a position that is not within this instance. - /// comparisonType is not a valid value. - /// value is null. - public int LastIndexOf(string value, int startIndex, StringComparison comparisonType) { return _raw.LastIndexOf(value, startIndex, comparisonType); } - - /// Reports the index position of the last occurrence of a specified object within this instance. Parameters specify the starting search position in the current string, the number of characters in the current string to search, and the type of search to use for the specified string. - /// The index position of the value parameter if that string is found, or -1 if it is not. - /// The number of character positions to examine. - /// One of the values. - /// The object to seek. - /// The search starting position. - /// count or startIndex is negative.-or- count plus startIndex specify a position that is not within this instance. - /// comparisonType is not a valid value. - /// value is null. - public int LastIndexOf(string value, int startIndex, int count, StringComparison comparisonType) { return _raw.LastIndexOf(value, startIndex, count, comparisonType); } - - /// Reports the index position of the last occurrence in this instance of one or more characters specified in a Unicode array. - /// The index position of the last occurrence in this instance where any character in anyOf was found; otherwise, -1 if no character in anyOf was found. - /// A Unicode character array containing one or more characters to seek. - /// anyOf is null. - public int LastIndexOfAny(char[] anyOf) { return _raw.LastIndexOfAny(anyOf); } - - /// Reports the index position of the last occurrence in this instance of one or more characters specified in a Unicode array. The search starts at a specified character position. - /// The index position of the last occurrence in this instance where any character in anyOf was found; otherwise, -1 if no character in anyOf was found. - /// A Unicode character array containing one or more characters to seek. - /// The search starting position. - /// startIndex specifies a position not within this instance. - /// anyOf is null. - public int LastIndexOfAny(char[] anyOf, int startIndex) { return _raw.LastIndexOfAny(anyOf, startIndex); } - - /// Reports the index position of the last occurrence in this instance of one or more characters specified in a Unicode array. The search starts at a specified character position and examines a specified number of character positions. - /// The index position of the last occurrence in this instance where any character in anyOf was found; otherwise, -1 if no character in anyOf was found. - /// The number of character positions to examine. - /// A Unicode character array containing one or more characters to seek. - /// The search starting position. - /// count or startIndex is negative.-or- count plus startIndex specify a position not within this instance. - /// anyOf is null. - public int LastIndexOfAny(char[] anyOf, int startIndex, int count) { return _raw.LastIndexOfAny(anyOf, startIndex, count); } - - /// Returns a new string whose textual value is the same as this string, but whose binary representation is in Unicode normalization form C. - /// A new, normalized string whose textual value is the same as this string, but whose binary representation is in normalization form C. - public string Normalize() { return _raw.Normalize(); } - - /// Returns a new string whose textual value is the same as this string, but whose binary representation is in the specified Unicode normalization form. - /// A new string whose textual value is the same as this string, but whose binary representation is in the normalization form specified by the normalizationForm parameter. - /// A Unicode normalization form. - public string Normalize(NormalizationForm normalizationForm) { return _raw.Normalize(normalizationForm); } - - /// Right-aligns the characters in this instance, padding with spaces on the left for a specified total length. - /// A new that is equivalent to this instance, but right-aligned and padded on the left with as many spaces as needed to create a length of totalWidth. Or, if totalWidth is less than the length of this instance, a new object that is identical to this instance. - /// The number of characters in the resulting string, equal to the number of original characters plus any additional padding characters. - /// totalWidth is less than zero. - public string PadLeft(int totalWidth) { return _raw.PadLeft(totalWidth); } - - /// Right-aligns the characters in this instance, padding on the left with a specified Unicode character for a specified total length. - /// A new that is equivalent to this instance, but right-aligned and padded on the left with as many paddingChar characters as needed to create a length of totalWidth. Or, if totalWidth is less than the length of this instance, a new that is identical to this instance. - /// A Unicode padding character. - /// The number of characters in the resulting string, equal to the number of original characters plus any additional padding characters. - /// totalWidth is less than zero. - public string PadLeft(int totalWidth, char paddingChar) { return _raw.PadLeft(totalWidth, paddingChar); } - - /// Left-aligns the characters in this string, padding with spaces on the right, for a specified total length. - /// A new that is equivalent to this instance, but left-aligned and padded on the right with as many spaces as needed to create a length of totalWidth. Or, if totalWidth is less than the length of this instance, a new that is identical to this instance. - /// The number of characters in the resulting string, equal to the number of original characters plus any additional padding characters. - /// totalWidth is less than zero. - public string PadRight(int totalWidth) { return _raw.PadRight(totalWidth); } - - /// Left-aligns the characters in this string, padding on the right with a specified Unicode character, for a specified total length. - /// A new that is equivalent to this instance, but left-aligned and padded on the right with as many paddingChar characters as needed to create a length of totalWidth. Or, if totalWidth is less than the length of this instance, a new that is identical to this instance. - /// A Unicode padding character. - /// The number of characters in the resulting string, equal to the number of original characters plus any additional padding characters. - /// totalWidth is less than zero. - public string PadRight(int totalWidth, char paddingChar) { return _raw.PadRight(totalWidth, paddingChar); } - - /// Deletes all the characters from this string beginning at a specified position and continuing through the last position. - /// A new object that is equivalent to this string less the removed characters. - /// The position to begin deleting characters. - /// startIndex is less than zero.-or- startIndex specifies a position that is not within this string. - public string Remove(int startIndex) { return _raw.Remove(startIndex); } - - /// Deletes a specified number of characters from this instance beginning at a specified position. - /// A new that is equivalent to this instance less count number of characters. - /// The number of characters to delete. - /// The position to begin deleting characters. - /// Either startIndex or count is less than zero.-or- startIndex plus count specify a position outside this instance. - public string Remove(int startIndex, int count) { return _raw.Remove(startIndex, count); } - - /// Replaces all occurrences of a specified Unicode character in this instance with another specified Unicode character. - /// A equivalent to this instance but with all instances of oldChar replaced with newChar. - /// A Unicode character to replace all occurrences of oldChar. - /// A Unicode character to be replaced. - public string Replace(char oldChar, char newChar) { return _raw.Replace(oldChar, newChar); } - - /// Replaces all occurrences of a specified in this instance, with another specified . - /// A equivalent to this instance but with all instances of oldValue replaced with newValue. - /// A to be replaced. - /// A to replace all occurrences of oldValue. - /// oldValue is null. - /// oldValue is the empty string (""). - public string Replace(string oldValue, string newValue) { return _raw.Replace(oldValue, newValue); } - - /// Returns a array containing the substrings in this instance that are delimited by elements of a specified array. - /// An array whose elements contain the substrings in this instance that are delimited by one or more characters in separator. For more information, see the Remarks section. - /// An array of Unicode characters that delimit the substrings in this instance, an empty array containing no delimiters, or null. - public string[] Split(params char[] separator) { return _raw.Split(separator); } - - /// Returns a array containing the substrings in this instance that are delimited by elements of a specified array. A parameter specifies the maximum number of substrings to return. - /// An array whose elements contain the substrings in this instance that are delimited by one or more characters in separator. For more information, see the Remarks section. - /// The maximum number of substrings to return. - /// An array of Unicode characters that delimit the substrings in this instance, an empty array containing no delimiters, or null. - /// count is negative. - public string[] Split(char[] separator, int count) { return _raw.Split(separator, count); } - - /// Returns a array containing the substrings in this string that are delimited by elements of a specified array. A parameter specifies whether to return empty array elements. - /// An array whose elements contain the substrings in this string that are delimited by one or more characters in separator. For more information, see the Remarks section. - /// Specify to omit empty array elements from the array returned, or to include empty array elements in the array returned. - /// An array of Unicode characters that delimit the substrings in this string, an empty array containing no delimiters, or null. - /// options is not one of the values. - public string[] Split(char[] separator, StringSplitOptions options) { return _raw.Split(separator, options); } - - /// Returns a array containing the substrings in this string that are delimited by elements of a specified array. A parameter specifies whether to return empty array elements. - /// An array whose elements contain the substrings in this string that are delimited by one or more strings in separator. For more information, see the Remarks section. - /// Specify to omit empty array elements from the array returned, or to include empty array elements in the array returned. - /// An array of strings that delimit the substrings in this string, an empty array containing no delimiters, or null. - /// options is not one of the values. - public string[] Split(string[] separator, StringSplitOptions options) { return _raw.Split(separator, options); } - - /// Returns a array containing the substrings in this string that are delimited by elements of a specified array. Parameters specify the maximum number of substrings to return and whether to return empty array elements. - /// An array whose elements contain the substrings in this stringthat are delimited by one or more characters in separator. For more information, see the Remarks section. - /// The maximum number of substrings to return. - /// Specify to omit empty array elements from the array returned, or to include empty array elements in the array returned. - /// An array of Unicode characters that delimit the substrings in this string, an empty array containing no delimiters, or null. - /// options is not one of the values. - /// count is negative. - public string[] Split(char[] separator, int count, StringSplitOptions options) { return _raw.Split(separator, count, options); } - - /// Returns a array containing the substrings in this string that are delimited by elements of a specified array. Parameters specify the maximum number of substrings to return and whether to return empty array elements. - /// An array whose elements contain the substrings in this string that are delimited by one or more strings in separator. For more information, see the Remarks section. - /// The maximum number of substrings to return. - /// Specify to omit empty array elements from the array returned, or to include empty array elements in the array returned. - /// An array of strings that delimit the substrings in this string, an empty array containing no delimiters, or null. - /// options is not one of the values. - /// count is negative. - public string[] Split(string[] separator, int count, StringSplitOptions options) { return _raw.Split(separator, count, options); } - - /// Determines whether the beginning of this instance matches the specified string. - /// true if value matches the beginning of this string; otherwise, false. - /// The to compare. - /// value is null. - public bool StartsWith(string value) { return _raw.StartsWith(value); } - - /// Determines whether the beginning of this string matches the specified string when compared using the specified comparison option. - /// true if the value parameter matches the beginning of this string; otherwise, false. - /// One of the values that determines how this string and value are compared. - /// A object to compare to. - /// comparisonType is not a value. - /// value is null. - public bool StartsWith(string value, StringComparison comparisonType) { return _raw.StartsWith(value, comparisonType); } - - /// Determines whether the beginning of this string matches the specified string when compared using the specified culture. - /// true if the value parameter matches the beginning of this string; otherwise, false. - /// Cultural information that determines how this string and value are compared. If culture is null, the current culture is used. - /// true to ignore case when comparing this string and value; otherwise, false. - /// The object to compare. - /// value is null. - public bool StartsWith(string value, bool ignoreCase, System.Globalization.CultureInfo culture) { return _raw.StartsWith(value, ignoreCase, culture); } - - /// Retrieves a substring from this instance. The substring starts at a specified character position. - /// A object equivalent to the substring that begins at startIndex in this instance, or if startIndex is equal to the length of this instance. - /// The starting character position of a substring in this instance. - /// startIndex is less than zero or greater than the length of this instance. - public string Substring(int startIndex) { return _raw.Substring(startIndex); } - - /// Retrieves a substring from this instance. The substring starts at a specified character position and has a specified length. - /// A equivalent to the substring of length length that begins at startIndex in this instance, or if startIndex is equal to the length of this instance and length is zero. - /// The index of the start of the substring. - /// The number of characters in the substring. - /// startIndex plus length indicates a position not within this instance.-or- startIndex or length is less than zero. - public string Substring(int startIndex, int length) { return _raw.Substring(startIndex, length); } - - IEnumerator IEnumerable.GetEnumerator() { return ((IEnumerable)_raw).GetEnumerator(); } - - IEnumerator IEnumerable.GetEnumerator() { return ((IEnumerable)_raw).GetEnumerator(); } - - /// Copies the characters in this instance to a Unicode character array. - /// A Unicode character array whose elements are the individual characters of this instance. If this instance is an empty string, the returned array is empty and has a zero length. - public char[] ToCharArray() { return _raw.ToCharArray(); } - - /// Copies the characters in a specified substring in this instance to a Unicode character array. - /// A Unicode character array whose elements are the length number of characters in this instance starting from character position startIndex. - /// The starting position of a substring in this instance. - /// The length of the substring in this instance. - /// startIndex or length is less than zero.-or- startIndex plus length is greater than the length of this instance. - public char[] ToCharArray(int startIndex, int length) { return _raw.ToCharArray(startIndex, length); } - - /// Returns a copy of this converted to lowercase, using the casing rules of the current culture. - /// A in lowercase. - public string ToLower() { return _raw.ToLower(); } - - /// Returns a copy of this converted to lowercase, using the casing rules of the specified culture. - /// A in lowercase. - /// A object that supplies culture-specific casing rules. - /// culture is null. - public string ToLower(System.Globalization.CultureInfo culture) { return _raw.ToLower(culture); } - - /// Returns a copy of this object converted to lowercase using the casing rules of the invariant culture. - /// A object in lowercase. - public string ToLowerInvariant() { return _raw.ToLowerInvariant(); } - - /// Returns a copy of this converted to uppercase, using the casing rules of the current culture. - /// A in uppercase. - public string ToUpper() { return _raw.ToUpper(); } - - /// Returns a copy of this converted to uppercase, using the casing rules of the specified culture. - /// A in uppercase. - /// A object that supplies culture-specific casing rules. - /// culture is null. - public string ToUpper(System.Globalization.CultureInfo culture) { return _raw.ToUpper(culture); } - - /// Returns a copy of this object converted to uppercase using the casing rules of the invariant culture. - /// A object in uppercase. - public string ToUpperInvariant() { return _raw.ToUpperInvariant(); } - - /// Removes all occurrences of white space characters from the beginning and end of this instance. - /// A new equivalent to this instance after white space characters are removed from the beginning and end. - public string Trim() { return _raw.Trim(); } - - /// Removes all occurrences of a set of characters specified in an array from the beginning and end of this instance. - /// The that remains after all occurrences of the characters in trimChars are removed from the beginning and end of this instance. If trimChars is null, white space characters are removed instead. - /// An array of Unicode characters to be removed or null. - public string Trim(params char[] trimChars) { return _raw.Trim(trimChars); } - - /// Removes all occurrences of a set of characters specified in an array from the end of this instance. - /// The that remains after all occurrences of the characters in trimChars are removed from the end. If trimChars is null, white space characters are removed instead. - /// An array of Unicode characters to be removed or null. - public string TrimEnd(params char[] trimChars) { return _raw.TrimEnd(trimChars); } - - /// Removes all occurrences of a set of characters specified in an array from the beginning of this instance. - /// The that remains after all occurrences of characters in trimChars are removed from the beginning. If trimChars is null, white space characters are removed instead. - /// An array of Unicode characters to be removed or null. - public string TrimStart(params char[] trimChars) { return _raw.TrimStart(trimChars); } - - /// Gets the character at a specified character position in this instance. - /// A Unicode character. - /// A character position in this instance. - /// index is greater than or equal to the length of this object or less than zero. - public char this[int index] { get { return _raw[index]; } } - - /// Gets the number of characters in this instance. - /// The number of characters in this instance. - public int Length { get { return _raw.Length; } } - #endregion - - #region IConvertible Members - - TypeCode IConvertible.GetTypeCode() - { - return TypeCode.Object; - } - - bool IConvertible.ToBoolean(IFormatProvider provider) - { - return this; - } - - byte IConvertible.ToByte(IFormatProvider provider) - { - return this; - } - - char IConvertible.ToChar(IFormatProvider provider) - { - return this; - } - - DateTime IConvertible.ToDateTime(IFormatProvider provider) - { - return this; - } - - decimal IConvertible.ToDecimal(IFormatProvider provider) - { - return this; - } - - double IConvertible.ToDouble(IFormatProvider provider) - { - return this; - } - - short IConvertible.ToInt16(IFormatProvider provider) - { - return this; - } - - int IConvertible.ToInt32(IFormatProvider provider) - { - return this; - } - - long IConvertible.ToInt64(IFormatProvider provider) - { - return this; - } - - sbyte IConvertible.ToSByte(IFormatProvider provider) - { - return this; - } - - float IConvertible.ToSingle(IFormatProvider provider) - { - return this; - } - - string IConvertible.ToString(IFormatProvider provider) - { - return this; - } - - ushort IConvertible.ToUInt16(IFormatProvider provider) - { - return this; - } - - uint IConvertible.ToUInt32(IFormatProvider provider) - { - return this; - } - - ulong IConvertible.ToUInt64(IFormatProvider provider) - { - return this; - } - - object IConvertible.ToType(Type conversionType, IFormatProvider provider) - { - if (conversionType == typeof(Var)) - { - return (Var)this; - } - else if (conversionType == typeof(string)) - { - return (string)this; - } - else if (conversionType == typeof(double)) - { - return (double)this; - } - else if (conversionType == typeof(float)) - { - return (float)this; - } - else if (conversionType == typeof(int)) - { - return (int)this; - } - else if (conversionType == typeof(uint)) - { - return (uint)this; - } - else if (conversionType == typeof(long)) - { - return (long)this; - } - else if (conversionType == typeof(ulong)) - { - return (ulong)this; - } - else if (conversionType == typeof(short)) - { - return (short)this; - } - else if (conversionType == typeof(ushort)) - { - return (ushort)this; - } - else if (conversionType == typeof(byte)) - { - return (byte)this; - } - else if (conversionType == typeof(sbyte)) - { - return (sbyte)this; - } - else if (conversionType == typeof(char)) - { - return (char)this; - } - else if (conversionType == typeof(decimal)) - { - return (decimal)this; - } - else if (conversionType == typeof(Guid)) - { - return (Guid)this; - } - else if (conversionType == typeof(DateTime)) - { - return (DateTime)this; - } - return this; - } - - #endregion - } - #endregion - - #region NoPreamble - /// - /// An encoding based on another encoding but with no preamble (BOM). - /// - public class NoPreambleEncoding : System.Text.Encoding - { - private readonly Encoding _baseEncoding; - private static readonly byte[] _preamble = new byte[0]; - - private readonly Encoding _utf16 = new NoPreambleEncoding(Encoding.Unicode); - - /// - /// A UTF16 encoding with no preamble. - /// - public Encoding UTF16 - { - get - { - return _utf16; - } - } - - // backing field for BigEndianUTF16 - private readonly Encoding _bigEndianUtf16 = new NoPreambleEncoding(Encoding.BigEndianUnicode); - - /// - /// A big-endian UTF16 encoding with no preamble. - /// - public Encoding BigEndianUTF16 - { - get - { - return _bigEndianUtf16; - } - } - - /// - /// Create a new encoding based on the specified encoding but with no preamble. - /// - /// the encoding to base this one on - /// The baseEncoding was null. - public NoPreambleEncoding(Encoding baseEncoding) - { - Contracts.CheckValue(baseEncoding, nameof(baseEncoding)); - _baseEncoding = baseEncoding; - } - - /// - /// Returns a sequence of bytes that specifies the encoding used (empty, in this case). - /// - /// - /// A byte array of length zero. - /// - public override byte[] GetPreamble() - { - return _preamble; - } - /// - /// Gets a name for the current encoding that can be used with mail agent body tags. - /// - /// A name for the current that can be used with mail agent body tags.-or- An empty string (""), if the current cannot be used. - public override string BodyName - { - get - { - return _baseEncoding.BodyName; - } - } - /// - /// Creates a shallow copy of the current object. - /// - /// - /// A copy of the current object. - /// - public override object Clone() - { - return new NoPreambleEncoding((Encoding)_baseEncoding.Clone()); - } - /// - /// Gets the code page identifier of the current . - /// - /// The code page identifier of the current . - public override int CodePage - { - get - { - return _baseEncoding.CodePage; - } - } - /// - /// Gets the human-readable description of the current encoding. - /// - /// The human-readable description of the current . - public override string EncodingName - { - get - { - return _baseEncoding.EncodingName; - } - } - /// - /// Calculates the number of bytes produced by encoding all the characters in the specified character array. - /// - /// The character array containing the characters to encode. - /// - /// The number of bytes produced by encoding all the characters in the specified character array. - /// - /// chars is null. - public override int GetByteCount(char[] chars) - { - return _baseEncoding.GetByteCount(chars); - } - /// - /// Calculates the number of bytes produced by encoding the characters in the specified . - /// - /// The containing the set of characters to encode. - /// - /// The number of bytes produced by encoding the specified characters. - /// - /// s is null. - public override int GetByteCount(string s) - { - return _baseEncoding.GetByteCount(s); - } - /// - /// Encodes all the characters in the specified character array into a sequence of bytes. - /// - /// The character array containing the characters to encode. - /// - /// A byte array containing the results of encoding the specified set of characters. - /// - /// chars is null. - public override byte[] GetBytes(char[] chars) - { - return _baseEncoding.GetBytes(chars); - } - /// - /// Encodes a set of characters from the specified character array into a sequence of bytes. - /// - /// The character array containing the set of characters to encode. - /// The index of the first character to encode. - /// The number of characters to encode. - /// - /// A byte array containing the results of encoding the specified set of characters. - /// - /// index or count is less than zero.-or- index and count do not denote a valid range in chars. - /// chars is null. - public override byte[] GetBytes(char[] chars, int index, int count) - { - return _baseEncoding.GetBytes(chars, index, count); - } - /// - /// Encodes all the characters in the specified into a sequence of bytes. - /// - /// The containing the characters to encode. - /// - /// A byte array containing the results of encoding the specified set of characters. - /// - /// s is null. - public override byte[] GetBytes(string s) - { - return _baseEncoding.GetBytes(s); - } - /// - /// Encodes a set of characters from the specified into the specified byte array. - /// - /// The containing the set of characters to encode. - /// The index of the first character to encode. - /// The number of characters to encode. - /// The byte array to contain the resulting sequence of bytes. - /// The index at which to start writing the resulting sequence of bytes. - /// - /// The actual number of bytes written into bytes. - /// - /// bytes does not have enough capacity from byteIndex to the end of the array to accommodate the resulting bytes. - /// charIndex or charCount or byteIndex is less than zero.-or- charIndex and charCount do not denote a valid range in chars.-or- byteIndex is not a valid index in bytes. - /// s is null.-or- bytes is null. - public override int GetBytes(string s, int charIndex, int charCount, byte[] bytes, int byteIndex) - { - return _baseEncoding.GetBytes(s, charIndex, charCount, bytes, byteIndex); - } - /// - /// Calculates the number of characters produced by decoding all the bytes in the specified byte array. - /// - /// The byte array containing the sequence of bytes to decode. - /// - /// The number of characters produced by decoding the specified sequence of bytes. - /// - /// bytes is null. - public override int GetCharCount(byte[] bytes) - { - return _baseEncoding.GetCharCount(bytes); - } - /// - /// Decodes all the bytes in the specified byte array into a set of characters. - /// - /// The byte array containing the sequence of bytes to decode. - /// - /// A character array containing the results of decoding the specified sequence of bytes. - /// - /// bytes is null. - public override char[] GetChars(byte[] bytes) - { - return _baseEncoding.GetChars(bytes); - } - /// - /// Decodes a sequence of bytes from the specified byte array into a set of characters. - /// - /// The byte array containing the sequence of bytes to decode. - /// The index of the first byte to decode. - /// The number of bytes to decode. - /// - /// A character array containing the results of decoding the specified sequence of bytes. - /// - /// bytes is null. - /// index or count is less than zero.-or- index and count do not denote a valid range in bytes. - public override char[] GetChars(byte[] bytes, int index, int count) - { - return _baseEncoding.GetChars(bytes, index, count); - } - /// - /// Obtains a decoder that converts an encoded sequence of bytes into a sequence of characters. - /// - /// - /// A that converts an encoded sequence of bytes into a sequence of characters. - /// - public override Decoder GetDecoder() - { - return _baseEncoding.GetDecoder(); - } - /// - /// Obtains an encoder that converts a sequence of Unicode characters into an encoded sequence of bytes. - /// - /// - /// An that converts a sequence of Unicode characters into an encoded sequence of bytes. - /// - public override Encoder GetEncoder() - { - return _baseEncoding.GetEncoder(); - } - /// - /// Decodes all the bytes in the specified byte array into a string. - /// - /// The byte array containing the sequence of bytes to decode. - /// - /// A containing the results of decoding the specified sequence of bytes. - /// - /// bytes is null. - public override string GetString(byte[] bytes) - { - return _baseEncoding.GetString(bytes); - } - /// - /// Decodes a sequence of bytes from the specified byte array into a string. - /// - /// The byte array containing the sequence of bytes to decode. - /// The index of the first byte to decode. - /// The number of bytes to decode. - /// - /// A containing the results of decoding the specified sequence of bytes. - /// - /// bytes is null. - /// index or count is less than zero.-or- index and count do not denote a valid range in bytes. - public override string GetString(byte[] bytes, int index, int count) - { - return _baseEncoding.GetString(bytes, index, count); - } - /// - /// Gets a name for the current encoding that can be used with mail agent header tags. - /// - /// A name for the current that can be used with mail agent header tags.-or- An empty string (""), if the current cannot be used. - public override string HeaderName - { - get - { - return _baseEncoding.HeaderName; - } - } - /// - /// Gets a value indicating whether the current encoding is always normalized, using the specified normalization form. - /// - /// One of the values. - /// - /// true if the current object is always normalized using the specified value; otherwise, false. The default is false. - /// - public override bool IsAlwaysNormalized(NormalizationForm form) - { - return _baseEncoding.IsAlwaysNormalized(form); - } - /// - /// Gets a value indicating whether the current encoding can be used by browser clients for displaying content. - /// - /// - /// true if the current can be used by browser clients for displaying content; otherwise, false. - public override bool IsBrowserDisplay - { - get - { - return _baseEncoding.IsBrowserDisplay; - } - } - /// - /// Gets a value indicating whether the current encoding can be used by browser clients for saving content. - /// - /// - /// true if the current can be used by browser clients for saving content; otherwise, false. - public override bool IsBrowserSave - { - get - { - return _baseEncoding.IsBrowserSave; - } - } - /// - /// Gets a value indicating whether the current encoding can be used by mail and news clients for displaying content. - /// - /// - /// true if the current can be used by mail and news clients for displaying content; otherwise, false. - public override bool IsMailNewsDisplay - { - get - { - return _baseEncoding.IsMailNewsDisplay; - } - } - /// - /// Gets a value indicating whether the current encoding can be used by mail and news clients for saving content. - /// - /// - /// true if the current can be used by mail and news clients for saving content; otherwise, false. - public override bool IsMailNewsSave - { - get - { - return _baseEncoding.IsMailNewsSave; - } - } - /// - /// Gets a value indicating whether the current encoding uses single-byte code points. - /// - /// - /// true if the current uses single-byte code points; otherwise, false. - public override bool IsSingleByte - { - get - { - return _baseEncoding.IsSingleByte; - } - } - /// - /// Returns a that represents the current . - /// - /// - /// A that represents the current . - /// - public override string ToString() - { - return _baseEncoding.ToString(); - } - /// - /// Gets the name registered with the Internet Assigned Numbers Authority (IANA) for the current encoding. - /// - /// - /// The IANA name for the current . - public override string WebName - { - get - { - return _baseEncoding.WebName; - } - } - /// - /// Gets the Windows operating system code page that most closely corresponds to the current encoding. - /// - /// - /// The Windows operating system code page that most closely corresponds to the current . - public override int WindowsCodePage - { - get - { - return _baseEncoding.WindowsCodePage; - } - } - /// - /// Determines whether the specified is equal to the current instance. - /// - /// The to compare with the current instance. - /// - /// true if value is an instance of and is equal to the current instance; otherwise, false. - /// - public override bool Equals(object value) - { - return value is NoPreambleEncoding && ((NoPreambleEncoding)value)._baseEncoding.Equals(_baseEncoding); - } - /// - /// Returns the hash code for the current instance. - /// - /// The hash code for the current instance. - public override int GetHashCode() - { - return _baseEncoding.GetHashCode(); - } - /// - /// Calculates the number of bytes produced by encoding a set of characters from the specified character array. - /// - /// The character array containing the set of characters to encode. - /// The index of the first character to encode. - /// The number of characters to encode. - /// - /// The number of bytes produced by encoding the specified characters. - /// - /// index or count is less than zero.-or- index and count do not denote a valid range in chars. - /// chars is null. - public override int GetByteCount(char[] chars, int index, int count) - { - return _baseEncoding.GetByteCount(chars, index, count); - } - - /// - /// Encodes a set of characters from the specified character array into the specified byte array. - /// - /// The character array containing the set of characters to encode. - /// The index of the first character to encode. - /// The number of characters to encode. - /// The byte array to contain the resulting sequence of bytes. - /// The index at which to start writing the resulting sequence of bytes. - /// - /// The actual number of bytes written into bytes. - /// - /// chars is null.-or- bytes is null. - /// bytes does not have enough capacity from byteIndex to the end of the array to accommodate the resulting bytes. - /// charIndex or charCount or byteIndex is less than zero.-or- charIndex and charCount do not denote a valid range in chars.-or- byteIndex is not a valid index in bytes. - public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) - { - return _baseEncoding.GetBytes(chars, charIndex, charCount, bytes, byteIndex); - } - - /// - /// Calculates the number of characters produced by decoding a sequence of bytes from the specified byte array. - /// - /// The byte array containing the sequence of bytes to decode. - /// The index of the first byte to decode. - /// The number of bytes to decode. - /// - /// The number of characters produced by decoding the specified sequence of bytes. - /// - /// bytes is null. - /// index or count is less than zero.-or- index and count do not denote a valid range in bytes. - public override int GetCharCount(byte[] bytes, int index, int count) - { - return _baseEncoding.GetCharCount(bytes, index, count); - } - - /// - /// Decodes a sequence of bytes from the specified byte array into the specified character array. - /// - /// The byte array containing the sequence of bytes to decode. - /// The index of the first byte to decode. - /// The number of bytes to decode. - /// The character array to contain the resulting set of characters. - /// The index at which to start writing the resulting set of characters. - /// - /// The actual number of characters written into chars. - /// - /// byteIndex or byteCount or charIndex is less than zero.-or- byteindex and byteCount do not denote a valid range in bytes.-or- charIndex is not a valid index in chars. - /// bytes is null.-or- chars is null. - /// chars does not have enough capacity from charIndex to the end of the array to accommodate the resulting characters. - public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex) - { - return _baseEncoding.GetChars(bytes, byteIndex, byteCount, chars, charIndex); - } - - /// - /// calculates the maximum number of bytes produced by encoding the specified number of characters. - /// - /// The number of characters to encode. - /// - /// The maximum number of bytes produced by encoding the specified number of characters. - /// - /// charCount is less than zero. - public override int GetMaxByteCount(int charCount) - { - return _baseEncoding.GetMaxByteCount(charCount); - } - - /// - /// calculates the maximum number of characters produced by decoding the specified number of bytes. - /// - /// The number of bytes to decode. - /// - /// The maximum number of characters produced by decoding the specified number of bytes. - /// - /// byteCount is less than zero. - public override int GetMaxCharCount(int byteCount) - { - return _baseEncoding.GetMaxCharCount(byteCount); - } - } - #endregion - - #region Parallel Enumeration -#if ENABLE_PARALLEL_ENUMERATION - /// - /// - /// - public class BackgroundEnumerable : IEnumerable - { - IEnumerable baseEnumerable; - - /// - /// - /// - /// - public BackgroundEnumerable(IEnumerable baseEnumerable) - { - this.baseEnumerable = baseEnumerable; - } - -#if EXPLICIT - public IEnumerator GetEnumerator() - { - return new BackgroundEnumerator(baseEnumerable.GetEnumerator()); - } - - private class BackgroundEnumerator : IEnumerator, IDisposable - { - IEnumerator baseEnumerator; - Thread thread; - AutoResetEvent fillMutex; - AutoResetEvent drainMutex; - object current; - object next; - - public BackgroundEnumerator(IEnumerator baseEnumerator) - { - this.baseEnumerator = baseEnumerator; - fillMutex = new AutoResetEvent(false); - drainMutex = new AutoResetEvent(true); - thread = new Thread(new ThreadStart(Enumerate)); - thread.Start(); - } - - private void Enumerate() - { - } - - public object Current - { - get - { - return current; - } - } - - public bool MoveNext() - { - fillMutex.WaitOne(); - current = next; - drainMutex.Set(); - } - - public void Reset() - { - if (thread != null) - { - try - { - thread.Abort(); - } - catch - { - } - thread = null; - } - baseEnumerator.Reset(); - fillMutex = new AutoResetEvent(false); - drainMutex = new AutoResetEvent(true); - thread = new Thread(new ThreadStart(Enumerate)); - thread.Start(); - } - - void IDisposable.Dispose() - { - if (thread != null) - { - try - { - thread.Abort(); - } - catch - { - } - thread = null; - } - } - } -#else - /// - /// - /// - /// - public IEnumerator GetEnumerator() - { - try - { - fillMutex = new AutoResetEvent(false); - drainMutex = new AutoResetEvent(true); - thread = new Thread(new ThreadStart(Enumerate)); - thread.Start(); - - while (true) - { - fillMutex.WaitOne(); - current = next; - if (current == null) yield break; - drainMutex.Set(); - yield return current; - } - } - finally - { - if (thread != null) - { - try - { - thread.Abort(); - } - catch - { - } - thread = null; - } - } - } - - Thread thread; - AutoResetEvent fillMutex; - AutoResetEvent drainMutex; - object current; - object next; - - private void Enumerate() - { - foreach (object obj in baseEnumerable) - { - drainMutex.WaitOne(); - next = obj; - fillMutex.Set(); - } - drainMutex.WaitOne(); - next = null; - fillMutex.Set(); - } -#endif - } - - /// - /// - /// - public class BackgroundEnumerable : IEnumerable - { - IEnumerable baseEnumerable; - - /// - /// - /// - /// - public BackgroundEnumerable(IEnumerable baseEnumerable) - { - this.baseEnumerable = baseEnumerable; - } - -#if EXPLICIT - public IEnumerator GetEnumerator() - { - return new BackgroundEnumerator(baseEnumerable.GetEnumerator()); - } - - private class BackgroundEnumerator : IEnumerator, IDisposable - { - IEnumerator baseEnumerator; - Thread thread; - AutoResetEvent fillMutex; - AutoResetEvent drainMutex; - object current; - object next; - - public BackgroundEnumerator(IEnumerator baseEnumerator) - { - this.baseEnumerator = baseEnumerator; - fillMutex = new AutoResetEvent(false); - drainMutex = new AutoResetEvent(true); - thread = new Thread(new ThreadStart(Enumerate)); - thread.Start(); - } - - private void Enumerate() - { - } - - public object Current - { - get - { - return current; - } - } - - public bool MoveNext() - { - fillMutex.WaitOne(); - current = next; - drainMutex.Set(); - } - - public void Reset() - { - if (thread != null) - { - try - { - thread.Abort(); - } - catch - { - } - thread = null; - } - baseEnumerator.Reset(); - fillMutex = new AutoResetEvent(false); - drainMutex = new AutoResetEvent(true); - thread = new Thread(new ThreadStart(Enumerate)); - thread.Start(); - } - - void IDisposable.Dispose() - { - if (thread != null) - { - try - { - thread.Abort(); - } - catch - { - } - thread = null; - } - } - } -#else - /// - /// - /// - /// - public IEnumerator GetEnumerator() - { - try - { - next = new T[3][]; - for (int i = 0; i < next.Length; i++) next[i] = new T[4]; - T[] current = new T[next[0].Length]; - fillMutex = new AutoResetEvent[next.Length]; - drainMutex = new AutoResetEvent[next.Length]; - for (int i = 0; i < fillMutex.Length; i++) - { - fillMutex[i] = new AutoResetEvent(false); - drainMutex[i] = new AutoResetEvent(true); - } - thread = new Thread(new ThreadStart(Enumerate)); - thread.IsBackground = true; - thread.Start(); - - int currentIndex = 0; - while (true) - { - fillMutex[currentIndex].WaitOne(); - T[] tmp = current; - current = next[currentIndex]; - next[currentIndex] = tmp; - drainMutex[currentIndex].Set(); - if (current == null) yield break; - for (int i = 0; i < current.Length; i++) - { - yield return current[i]; - } - currentIndex = (currentIndex + 1) % next.Length; - } - } - finally - { - if (thread != null) - { - try - { - thread.Abort(); - } - catch - { - } - thread = null; - } - } - } - - Thread thread; - AutoResetEvent[] fillMutex; - AutoResetEvent[] drainMutex; - T[][] next; - - private void Enumerate() - { - int nextIndex = 0; - int index = 0; - foreach (T obj in baseEnumerable) - { - next[nextIndex][index] = obj; - index++; - if (index == next[nextIndex].Length) - { - fillMutex[nextIndex].Set(); - nextIndex = (nextIndex + 1) % next.Length; - drainMutex[nextIndex].WaitOne(); - index = 0; - } - } - if (index != 0) - { - T[] old = next[nextIndex]; - next[nextIndex] = new T[index]; - Array.Copy(old, next[nextIndex], next[nextIndex].Length); - fillMutex[nextIndex].Set(); - nextIndex = (nextIndex + 1) % next.Length; - drainMutex[nextIndex].WaitOne(); - index = 0; - } - next[nextIndex] = null; - fillMutex[nextIndex].Set(); - thread = null; - } -#endif - - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - } -#endif - #endregion - -#if NDOC - /// - /// - /// The Microsoft.ML.Runtime.Internal namespace includes classes and functionality developed - /// by the Text Mining, Search, and Navigation group in Microsoft Research. - /// All functionality is for internal use only and is copyright - /// 2007 Microsoft Corporation. - /// - /// - /// Microsoft.ML.Runtime.Internal.IO contains functionality to help with input and output operations. - /// Using the ZStream classes to open files, support is automatically provided for - /// compression and decompression, HTTP downloading, Cosmos stream fetching, console - /// interaction, named streams, SQL table reading, lists of files, and so on. - /// The concept of filename is extended to allow this to - /// be simple and automatic. Other classes provide memory-mapped files, unbuffered - /// I/O, table reading, and other useful functionality. - /// - /// - /// The most essential feature is the ability to open the expanded class of stream names, - /// easily and efficiently: - /// - /// - /// Stream streamIn = ZStreamIn.Open("filename"); - /// Stream streamOut = ZStreamOut.Open("filename"); - /// StreamReader reader = ZStreamReader.Open("filename"); - /// StreamWriter writer = ZStreamWriter.Open("filename"); - /// - /// - /// See the Open() documentation to see the allowed names for file, console, - /// null, URL, Cosmos, and compressed input and output streams. - /// - /// - /// Compression support relies on executable utilities to be in the path. - /// See for 7z.exe and 7za.exe (for many formats - - /// .7z, .gz, .zip, .rar, .bz2, .cab, .arj), for gzip.exe - /// (for .gz), or for unrar.exe (for .rar). Gzip support built-in to .NET - /// 2.0 can be used, but it has extreme deficiencies in terms of speed, size, and flexibility. - /// - /// - /// The IOUtil static methods are useful functions for file and stream operations. Many duplicate - /// existing Framework methods, but they are extended to handle all streamnames. - /// - /// - /// BinaryReaderEx and BinaryWriterEx allow for easy and efficient serialization of many data types. - /// They also make using .NET binary serialization as simple as using Write and Read on serializable objects. - /// - /// - public class NamespaceDoc - { - // this only exists for NDoc - } -#endif -} diff --git a/src/Microsoft.ML.InternalStreams/LowFragmentationStream.cs b/src/Microsoft.ML.InternalStreams/LowFragmentationStream.cs deleted file mode 100644 index 673d54bc70..0000000000 --- a/src/Microsoft.ML.InternalStreams/LowFragmentationStream.cs +++ /dev/null @@ -1,722 +0,0 @@ -// owner: rragno - -#define LZMA_PLAIN -#define UNBUFFERED - -using System; -using System.IO; -using System.Collections; -using System.Collections.Specialized; -using System.Text; -using System.Threading; -using System.Runtime.InteropServices; -//using System.Xml; -//using System.Data; -//using System.Data.OleDb; - -namespace Microsoft.ML.Runtime.Internal.IO -{ - // Should this start out very small for tiny files? - - /// - /// This is a writable FileStream that reduces the fragmentation by - /// extending the output in chunks. - /// - public class LowFragmentationStream : FileStream - { - #region Static Settings - - private static long _minExtension = 50L * 1024L * 1024L; - private static long _maxExtension = 2L * 1024L * 1024L * 1024L; - private static float _extensionFactor = 0.5F; - - /// - /// Get or set the minimum size increment by which a file will be extended, in bytes. - /// Defaults to 50MB. - /// - public static long MinExtension - { - get { return _minExtension; } - set - { - if (value <= 0) - value = 50L * 1024L * 1024L; - _minExtension = value; - } - } - - /// - /// Get or set the maximum size increment by which a file will be extended, in bytes. - /// Defaults to 2GB. - /// - public static long MaxExtension - { - get { return _maxExtension; } - set - { - if (value <= 0) - value = 2L * 1024L * 1024L * 1024L; - _maxExtension = value; - } - } - - /// - /// Get or set the factor by which a file wll be extended when needed. - /// Defaults to 0.5, and must be greater than 0.0 and less than or equal to 1.0. - /// - public static float ExtensionFactor - { - get { return _extensionFactor; } - set - { - if (value <= 0.0F) - value = 0.5F; - if (value > 1.0F) - value = 1.0F; - _extensionFactor = value; - } - } - - #endregion - - /// - /// This is really equal to base.Length while writing, but base.Length - /// is less efficient. - /// - private long _extendedLength; - /// - /// If positive, this represents the true length of the file. - /// Otherwise, that is the Position value. - /// - private long _length; - /// - /// Hopefully, this is always positive, and is the cached Position for efficiency. - /// - private long _position; - private bool _closed = false; - - /// - /// Construct a writable Stream outputting to the given file. - /// - /// the name of the file to write to - /// - /// An I/O error occurs - /// The caller does not have the required permission. - /// The specified path is invalid, such as being on an unmapped drive. - /// The access needed is not permitted by the operating system for the specified . - /// The specified path, file name, or both exceed the system-defined maximum length. - public LowFragmentationStream(string fileName) - : this(fileName, false) - { - } - /// - /// Construct a writable Stream outputting to the given file. - /// - /// the name of the file to write to - /// the size of the buffer to use, in bytes - /// - /// is negative or zero. - /// An I/O error occurs - /// The caller does not have the required permission. - /// The specified path is invalid, such as being on an unmapped drive. - /// The access needed is not permitted by the operating system for the specified . - /// The specified path, file name, or both exceed the system-defined maximum length. - public LowFragmentationStream(string fileName, int bufferSize) - : this(fileName, false, bufferSize) - { - } - /// - /// Construct a writable Stream outputting to the given file. - /// - /// the name of the file to write to - /// if true, append to the file; otherwise, overwrite - /// - /// Append is specified, and the file cannot be found. - /// An I/O error occurs - /// The caller does not have the required permission. - /// The specified path is invalid, such as being on an unmapped drive. - /// The access needed is not permitted by the operating system for the specified . - /// The specified path, file name, or both exceed the system-defined maximum length. - public LowFragmentationStream(string fileName, bool append) - : this(fileName, append, 64 * 1024) - { - } - /// - /// Construct a writable Stream outputting to the given file. - /// - /// the name of the file to write to - /// if true, append to the file; otherwise, overwrite - /// the size of the buffer to use, in bytes - /// - /// is negative or zero. - /// Append is specified, and the file cannot be found. - /// An I/O error occurs - /// The caller does not have the required permission. - /// The specified path is invalid, such as being on an unmapped drive. - /// The access needed is not permitted by the operating system for the specified . - /// The specified path, file name, or both exceed the system-defined maximum length. - public LowFragmentationStream(string fileName, bool append, int bufferSize) - : base(fileName, append ? FileMode.Append : FileMode.Create, FileAccess.Write, FileShare.Read, bufferSize) - { - if (append) - { - _length = base.Length; - _position = _length; - _extendedLength = _length; - Extend(); - } - else - { - _length = -1; - _position = 0; - _extendedLength = 0; - Extend(); - } - } - - #region Allocation - - /// - /// Set the desired file allocation, preferably without changing the end of file. - /// - /// the size to set, in bytes - /// if true, use the low-level undocumented NT API to allocate, without setting the length - private void SetAllocation(long length, bool useNT) - { - // using NT calls seems unreliable, beyond being undocumented, unsupported, and - // likely platform dependant. - // - // Sometimes, it works perfectly. Other times, it generates roughly the same - // number of fragments as not doing anything (but can vary). Other times, it - // generates 10 times as many fragments as not doing anything. - // - // It does, at least, leave the file without extra allocated space at all times. - -#if !NT_ALLOCATE - if (!useNT) - { - base.SetLength(length); - } - else - { -#endif - //FILE_ALLOCATION_INFORMATION allocInfo = new FILE_ALLOCATION_INFORMATION(length); - long allocInfo = length; - IOUtil.Win32.IO_STATUS_BLOCK status = new IOUtil.Win32.IO_STATUS_BLOCK(); // = IO_STATUS_BLOCK.NullBlock; - IOUtil.Win32.NtSetInformationFile( - base.SafeFileHandle.DangerousGetHandle(), - ref status, - ref allocInfo, - 8, //sizeof(FILE_ALLOCATION_INFORMATION), - IOUtil.Win32.FILE_INFORMATION_CLASS.FileAllocationInformation); - // FILE_INFORMATION_CLASS.FileEndOfFileInformation); -#if !NT_ALLOCATE - } -#endif - } - #endregion - - /// - /// Remove allocated space that was not written to. - /// - /// the file to trim - /// true if the file is resized; false otherwise - /// - /// This is not normally needed. The LowFragmentationStream trims itself upon finalization. - /// However, if the runtime is terminated abruptly, it is possible for a file to be left - /// with unused space. In that case, this method will remove the unused space. - /// - /// Append is specified, and the file cannot be found. - /// An I/O error occurs - /// The caller does not have the required permission. - /// The specified path is invalid, such as being on an unmapped drive. - /// The access needed is not permitted by the operating system for the specified . - /// The specified path, file name, or both exceed the system-defined maximum length. - public static bool TrimOverextension(string fileName) - { - long length = -1; - using (FileStream s = new FileStream(fileName, FileMode.Open, FileAccess.Read)) - { - byte[] buffer = new byte[1024 * 1024]; - s.Seek(-1, SeekOrigin.End); - int lastByte = s.ReadByte(); - if (lastByte == 0) - { - long end = s.Length; - while (length < 0 && end > 0) - { - end -= buffer.Length; - if (end < 0) - end = 0; - s.Seek(end, SeekOrigin.Begin); - int count = s.Read(buffer, 0, buffer.Length); - for (int c = count - 1; c >= 0; c--) - { - if (buffer[c] != 0) - { - length = end + c; - break; - } - } - } - } - } - if (length < 0) - return false; - IOUtil.ResizeFile(fileName, length); - return true; - } - - private void Extend() - { - // really, using NtSetInformationFile might be better, because it will - // automatically truncate, but that is ugly and not really supported... *** - // Also, it seems to cause random fragmentation, unpredictably. - long pos = Position; - long ext = (long)(pos * _extensionFactor); - if (ext < _minExtension) - { - ext = _minExtension; - } - else if (ext > _maxExtension) - { - ext = _maxExtension; - } - // this calls out to the system: - //long len = base.Length; - long len = _extendedLength; - _extendedLength = pos + ext; - if (_extendedLength > len) - { - try - { - // attempt to keep the over-allocation down, at the beginning - bool useNT = len < _minExtension; - SetAllocation(_extendedLength, useNT); - } - catch - { - // ignore? This could mean there is not enough space on disk! - // If we leave the extendedLength set high, this instance will - // harmlessly believe it has preallocated and just allow the writes - // to extend normally... (of course, we could retry at a smaller - // increment...) - try - { - ext = pos + _minExtension; - base.SetLength(ext); - _extendedLength = ext; - } - catch - { - // give up - } - } - } - else - { - - } - } - - private void Truncate() - { - long len = _length; - if (len <= 0) - { - len = Position; - } - if (len != base.Length) - { - base.SetLength(len); - } - _extendedLength = len; - } - - /// - /// Clean up the stream - truncate the file length as needed. - /// - ~LowFragmentationStream() - { - try - { - Truncate(); - //Flush(true); - } - catch - { - // ignore - } - } - - /// - /// Close the stream. - /// - public override void Close() - { - if (_closed) - return; - try - { - Truncate(); - } - catch - { - // ignore - } - base.Close(); - _closed = true; - GC.SuppressFinalize(this); - } - - /// - /// Release the resources used by this instance. - /// - /// true if disposing - protected override void Dispose(bool disposing) - { - //Truncate(); - Flush(true); - base.Dispose(disposing); - } - - /// - /// Write any pending data, without truncating. - /// - /// - /// Note that this does not truncate the file! - /// - public override void Flush() - { - // should this truncate? *** - Flush(false); - } - /// - /// Write any pending data and optionally truncate. - /// - /// if true, also truncate the file; if false, do not. - public new void Flush(bool truncate) - { - try - { - base.Flush(); - if (truncate) - { - Truncate(); - base.Flush(); - } - } - catch - { - // ignore?? - } - } - - /// - /// Extend the capacity to be at least a certain number of bytes. - /// - /// the number of bytes to allocate - /// - /// This is useful when the approximate or exact size of the output is known. - /// It will not affect the final length of the file, but it can help with efficiency - /// and fragmentation. - /// - public void Reserve(long length) - { - if (length > _extendedLength) - { - base.SetLength(length); - _extendedLength = length; - } - } - - #region Reading - // /// - // /// Read data into the buffer. - // /// - // /// the buffer to place the data in - // /// the starting index in buffer - // /// the maximum number of bytes to read - // /// the number of bytes read - // public override int Read(byte[] buffer, int offset, int count) - // { - // return base.Read(buffer, offset, count); - // } - // - // /// - // /// Read a single byte. - // /// - // /// the byte read, or a negative number if end of file - // public override int ReadByte() - // { - // return base.ReadByte(); - // } - // - // /// - // /// Read data into the buffer. - // /// - // /// the buffer to place the data in - // /// the starting index in buffer - // /// the maximum number of bytes to read - // /// the callback to use - // /// the state to use for the callback - // /// the number of bytes read - // /// Read is positioned out of bounds. - // public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback callback, object state) - // { - // return base.BeginRead(buffer, offset, count, callback, state); - // } - // - // /// - // /// End an asynchronous read. - // /// - // /// the result - // /// number of bytes read - // public override int EndRead(IAsyncResult asyncResult) - // { - // return base.EndRead(asyncResult); - // } - #endregion - - /// - /// Sets the current position of this stream to the given value. - /// - /// The byte number to seek to. - /// - /// The new position in the stream. - /// - /// An I/O error occurs. - /// Attempted seeking before the beginning of the stream. - /// Methods were called after the stream was closed. - public long Seek(long offset) - { - return Seek(offset, SeekOrigin.Begin); - } - /// - /// Sets the current position of this stream to the given value. - /// - /// The point relative to to seek to, in bytes. - /// Specifies the beginning, the end, or the current position as a reference point for , using a value of type . - /// - /// The new position in the stream. - /// - /// An I/O error occurs. - /// Attempted seeking before the beginning of the stream. - /// Methods were called after the stream was closed. - public override long Seek(long offset, SeekOrigin origin) - { - switch (origin) - { - case SeekOrigin.Begin: - break; - case SeekOrigin.Current: - offset = Position + offset; - break; - case SeekOrigin.End: - offset = (_length > 0 ? _length : Position) + offset; - break; - } - - if (offset != Position) - { - Flush(true); - if (_length < 0 && offset < Position) - { - // now need to record the current length, since it is not Position: - _length = Position; - } - } - _position = base.Seek(offset, SeekOrigin.Begin); - return _position; - } - - /// - /// Set the file length - avoid using this manually except for truncation, - /// since the size is normally automatically extended. - /// - /// the length to set it to, in bytes - /// - /// To simply ensure a predicted length, use instead. - /// - public override void SetLength(long value) - { - base.SetLength(value); - if (Position < value) - { - // must remember the true length... - _length = value; - } - else - { - // length now moves with position... - _length = -1; - _position = base.Position; - } - _extendedLength = value; - } - - #region Writing - - /// - /// Write data from the buffer. - /// - /// the buffer to read the data from - /// the starting index in buffer - /// the maximum number of bytes to write - public override void Write(byte[] buffer, int offset, int count) - { - long next = Position + count; - if (_length > 0 && next >= _length) - { - _length = -1; - } - if (next >= _extendedLength) - { - Extend(); - } - _position = -1; - base.Write(buffer, offset, count); - _position = next; - } - - /// - /// Write a single byte. - /// - /// the byte to write - public override void WriteByte(byte value) - { - long next = Position + 1; - if (_length > 0 && next >= _length) - { - _length = -1; - } - if (next >= _extendedLength) - { - Extend(); - } - _position = -1; - base.WriteByte(value); - _position = next; - } - - /// - /// Begins an asynchronous write. - /// - /// The buffer to write data to. - /// The zero based byte offset at which to begin writing. - /// The maximum number of bytes to write. - /// The method to be called when the asynchronous write operation is completed. - /// A user-provided object that distinguishes this particular asynchronous write request from other requests. - /// - /// An that references the asynchronous write. - /// - /// The stream does not support writing. - /// The stream is closed. - /// An I/O error occurs. - public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback callback, object stateObject) - { - // we could wrap the state object... - // just call this +1, in case less is written? - long next = Position + 1; - if (_length > 0 && next >= _length) - { - // should we do this, without EndWrite being called? - _length = -1; - } - if (Position + count >= _extendedLength) - { - Extend(); - } - // we just don't know how much will really be read... - _position = -1; - return base.BeginWrite(buffer, offset, count, callback, stateObject); - } - - /// - /// Ends an asynchronous write, blocking until the I/O operation has completed. - /// - /// The pending asynchronous I/O request. - /// is . - /// This object was not created by calling on this class. - /// is called multiple times. - public override void EndWrite(IAsyncResult asyncResult) - { - // should we sync the position here? - _position = Position; - if (_length > 0) - { - if (Position >= _length) - { - _length = -1; - } - } - base.EndWrite(asyncResult); - } - - #endregion - - /// - /// Get whether the stream can read. - /// - public override bool CanRead - { - get - { - // just pass through? - return base.CanRead; - } - } - - /// - /// Get whether the stream can seek. - /// - public override bool CanSeek - { - get - { - return base.CanSeek; - } - } - - /// - /// Get whether the stream can write. - /// - public override bool CanWrite - { - get - { - return base.CanWrite; - } - } - - /// - /// Get the length of the file. - /// - public override long Length - { - get - { - if (_length > 0) - return _length; - return Position; - } - } - - /// - /// Get or set the position in the file. - /// - public override long Position - { - get - { - if (_position < 0) - _position = base.Position; - return _position; - } - set - { - Seek(value); - } - } - } -} diff --git a/src/Microsoft.ML.InternalStreams/Microsoft.ML.InternalStreams.csproj b/src/Microsoft.ML.InternalStreams/Microsoft.ML.InternalStreams.csproj deleted file mode 100644 index 31074af4f9..0000000000 --- a/src/Microsoft.ML.InternalStreams/Microsoft.ML.InternalStreams.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - netstandard2.0 - Microsoft.ML - CORECLR - TRACE;USE_FASTTREENATIVE;NO_STORE;CORECLR;DEBUG;NETSTANDARD2_0 - true - 1591,1572,1573 - Microsoft.ML.InternalStreams - - - - - - - diff --git a/src/Microsoft.ML.InternalStreams/TableIO.cs b/src/Microsoft.ML.InternalStreams/TableIO.cs deleted file mode 100644 index c9d8207dc3..0000000000 --- a/src/Microsoft.ML.InternalStreams/TableIO.cs +++ /dev/null @@ -1,3414 +0,0 @@ -// owner: rragno - -using System; -using System.IO; -using System.Collections; -using System.Collections.Specialized; -using System.Text; -#if !ENABLE_BARTOK -using System.Xml; -#endif -#if ALLOW_DB -using System.Data; -using System.Data.OleDb; -#endif - -namespace Microsoft.ML.Runtime.Internal.IO -{ - //// TODO: consider unifying all formats, syncing with other data access models - //// TODO: simple XML support - //// TODO: consider generalizing header support; changing from "row,item" to "item,field"; - //// making a base TableReader abstract class. - - /// - /// Process tabular data. - /// - public interface ITableProcessor - { - /// - /// Gets or sets whether to trim whitespace from each field. - /// - bool TrimWhitespace { get; set; } - - /// - /// Gets or sets whether to ignore case when matching header names. - /// - bool IgnoreHeaderCase { get; set; } - - /// - /// Gets or sets the header names. - /// - string[] Headers { get; set; } - - /// - /// Check for end of file. - /// - /// true if at end of file, false otherwise - bool Eof(); - - /// - /// Advance to the next row. - /// - void NextRow(); - - /// - /// Close the table. - /// - void Close(); - } - - /// - /// - /// - public interface ITableRow - { - /// - /// Get the field at the column index. - /// - string this[int index] { get; } - - /// - /// Get the field at the column with the given header. - /// - string this[string header] { get; } - } - - /// - /// Read tabular data. - /// - public interface ITableReader : ITableProcessor, IEnumerable, ITableRow - { - /// - /// Gets or sets whether to return "", not null, when the end of a row is reached, - /// until the row is advanced: - /// - bool FillBlankColumns { get; set; } - - /// - /// Check for end of row. - /// - /// true if at end of row, false otherwise - bool RowEnd(); - - /// - /// Get the next field and advance the reader. - /// - /// the field at the next column - string ReadItem(); - - /// - /// Get the next field and advance the reader, filling with empty fields at the end of the row. - /// - /// the field at the next column - string ReadItemLinear(); - - /// - /// Get the number of fields in the current row. - /// - /// the number of fields in the current row - int RowLength(); - - /// - /// Read an entire row and advance the reader. - /// - /// The current row as an array of fields - string[] ReadRow(); - - /// - /// Read an entire row and advance the reader. - /// - /// The length of the row to read, truncating or filling with empty fields as needed - /// The current row as an array of fields - string[] ReadRow(int len); - - /// - /// Reset the reader to the beginning. - /// - void Reset(); - } - - /// - /// Write tabular data. - /// - public interface ITableWriter : ITableProcessor - { - /// - /// Write the next field and advance the writer. - /// - /// the field to write - void WriteItem(string item); - - /// - /// Write an entire row and advance the writer. - /// - /// The row to write as an array of fields - void WriteRow(string[] items); - - /// - /// Write an entire row and advance the writer. - /// - /// The length of the row to write, truncating or filling with empty fields as needed - /// The row to write as an array of fields - void WriteRow(string[] items, int len); - - /// - /// Set the field at the column index. - /// - string this[int index] { set; } - - /// - /// Set the field at the column with the given header. - /// - string this[string header] { set; } - } - - /// - /// Enumerator to read through the rows in a table. - /// - public class TableEnumerator : IEnumerator - { - ITableReader _reader; - - #region IEnumerator Members - /// - /// Create a new enumerator to read through the table rows - /// - /// the table to read lines from - public TableEnumerator(ITableReader reader) - { - _reader = reader; - } - /// - /// Return the enumerator to the initial state. - /// - public void Reset() - { - _reader.Reset(); - } - /// - /// Get the current row of the table. - /// - public ITableRow Current - { - get - { - return (ITableRow)_reader; - } - } - - /// - /// Get the current row of the table. - /// - object IEnumerator.Current - { - get - { - return ((TableEnumerator)this).Current; - } - } - - /// - /// Move the enumerator to the next row. - /// - /// true if the next row exists, or false if at the end of the table - public bool MoveNext() - { - _reader.NextRow(); - return !_reader.Eof(); - } - - #endregion - } - - ////////////////////////////////////////////// - ////// CSV support - ////////////////////////////////////////////// - - /// - /// Read TSV formatted data. - /// - public class TsvReader : CsvReader - { - private void Configure() - { - Delimiter = "\t"; - DelimiterSet = false; - ParseQuotes = false; - ReadHeaders = true; - SkipBlankColumnsLines = true; - SkipBlankLines = true; - // should this one be set? *** - IgnoreHeaderCase = true; - } - - /// - /// Create a TsvReader based on the TextReader, - /// - /// the TextReader to read the table from - public TsvReader(TextReader tr) - : base(tr) - { - Configure(); - } - - /// - /// Create a TsvReader based on the specified file, - /// - /// the name of the file to read the table from - public TsvReader(string fname) - : base(fname) - { - Configure(); - } - - /// - /// Create a TsvReader based on the Stream, - /// - /// the Stream to read the table from - public TsvReader(Stream fstream) - : base(fstream) - { - Configure(); - } - - /// - /// Create a TsvReader based on the specified file, - /// - /// the name of the file to read the table from - /// the encoding to use to interpet the file - public TsvReader(string fname, Encoding encoding) - : base(fname, encoding) - { - Configure(); - } - - /// - /// Create a TsvReader based on the Stream, - /// - /// the Stream to read the table from - /// the encoding to use to interpet the Stream - public TsvReader(Stream fstream, Encoding encoding) - : base(fstream, encoding) - { - Configure(); - } - } - - /// - /// Tab-Separated Value writer. - /// - public class TsvWriter : CsvWriter - { - private void Configure() - { - Delimiter = "\t"; - ParseQuotes = false; - SkipBlankLines = true; - EndInNewline = true; - // should this one be set? *** - IgnoreHeaderCase = true; - } - - /// - /// Create a TsvWriter based on the TextWriter, - /// - /// the TextWriter to write the table to - public TsvWriter(TextWriter tr) - : base(tr) - { - Configure(); - } - - /// - /// Create a TsvWriter based on the specified file, - /// - /// the name of the file to write the table to - public TsvWriter(string fname) - : base(fname) - { - Configure(); - } - - /// - /// Create a TsvWriter based on the Stream, - /// - /// the Stream to write the table to - public TsvWriter(Stream fstream) - : base(fstream) - { - Configure(); - } - - /// - /// Create a TsvWriter based on the specified file, - /// - /// the name of the file to write the table to - /// the encoding to use - public TsvWriter(string fname, Encoding encoding) - : base(fname, encoding) - { - Configure(); - } - - /// - /// Create a TsvWriter based on the Stream, - /// - /// the Stream to write the table to - /// the encoding to use - public TsvWriter(Stream fstream, Encoding encoding) - : base(fstream, encoding) - { - Configure(); - } - } - - /// - /// Read CSV formatted data. - /// - public class CsvReader : ITableReader, IDisposable - { - // trim whitespace from each entry: - private bool _trimWhitespace = true; - // skip lines that are only whitespace: - private bool _skipBlankLines = true; - // skip lines that have delimeters but only whitespace otherwise: - private bool _skipBlankColumnsLines = true; - // return "", not null, when the end of a row is reached, until the row is advanced: - private bool _fillBlankColumns = true; - // treat repeated delimiters as a single delimiter: - private bool _collapseDelimiters = false; - // treat delimiter string as a set of delimiter characters: - private bool _delimiterSet = false; - // read the first line as header names for the columns - private bool _readHeaders = false; - // the names of the headers - private string[] _headers = null; - private string[] _headersNormalized = null; - private bool _ignoreHeaderCase = true; - private bool _initialized = false; - // use the quoteChar to determine quoted sections: - private bool _parseQuotes = true; - private string _quoteChar = "\""; - private string _delimiter = ","; - - private TextReader _file; - private string _curLine; - private long _rowNumber = 0; // gives 1-based rows, like excel... - - // support 1-row read-ahead: - private StringCollection _curRow; - private string[] _curRowArray; - private int _curCol; - - /// - /// Create a CsvReader based on the TextReader, - /// - /// the TextReader to read the table from - public CsvReader(TextReader tr) - { - _file = tr; - } - - /// - /// Create a CsvReader based on the specified file, - /// - /// the name of the file to read the table from - public CsvReader(string fname) - : this(ZStreamReader.Open(fname)) - { - } - - /// - /// Create a CsvReader based on the Stream, - /// - /// the Stream to read the table from - public CsvReader(Stream fstream) - : this(new StreamReader(fstream)) - { - } - - /// - /// Create a CsvReader based on the specified file, - /// - /// the name of the file to read the table from - /// the encoding to use to interpet the file - public CsvReader(string fname, Encoding encoding) - : this(new StreamReader(fname, encoding, true)) - { - } - - /// - /// Create a CsvReader based on the Stream, - /// - /// the Stream to read the table from - /// the encoding to use to interpet the Stream - public CsvReader(Stream fstream, Encoding encoding) - : this(new StreamReader(fstream, encoding, true)) - { - } - - /// - /// Gets or sets whether to trim whitespace from each field. - /// - public bool TrimWhitespace - { - get { return _trimWhitespace; } - set { _trimWhitespace = value; } - } - /// - /// Get or set whether to skip blank lines. - /// - public bool SkipBlankLines - { - get { return _skipBlankLines; } - set { _skipBlankLines = value; } - } - /// - /// Get or set whether to skip lines with all fields empty. - /// - public bool SkipBlankColumnsLines - { - get { return _skipBlankColumnsLines; } - set { _skipBlankColumnsLines = value; } - } - /// - /// Gets or sets whether to return "", not null, when the end of a row is reached, - /// until the row is advanced: - /// - public bool FillBlankColumns - { - get { return _fillBlankColumns; } - set { _fillBlankColumns = value; } - } - /// - /// Get or set whether to respect quotes when parsing - /// - public bool ParseQuotes - { - get { return _quoteChar != null && _quoteChar.Length > 0 && _parseQuotes; } - set { _parseQuotes = value; } - } - /// - /// Get or set the string to use for a quote symbol - /// - public string QuoteChar - { - get { return _quoteChar; } - set { _quoteChar = value; } - } - /// - /// Get or set the column delimiter string. - /// - public string Delimiter - { - get { return _delimiter; } - set { _delimiter = value; } - } - /// - /// Get or set whether to collapse consecutive delimiters. - /// - public bool CollapseDelimiters - { - get { return _collapseDelimiters; } - set { _collapseDelimiters = value; } - } - /// - /// Get or set whether to treate the delimiter string as a set of characters. - /// - public bool DelimiterSet - { - get { return _delimiterSet; } - set { _delimiterSet = value; } - } - /// - /// Get or set whether to read the headers from the first line of the input. - /// - public bool ReadHeaders - { - get { return _readHeaders; } - set { _readHeaders = value; } - } - /// - /// Gets or sets whether to ignore case when matching header names. - /// - public bool IgnoreHeaderCase - { - get { return _ignoreHeaderCase; } - set - { - if (_ignoreHeaderCase != value) - { - _ignoreHeaderCase = value; - FixupNormalizedHeaders(); - } - } - } - /// - /// Gets or sets the header names. - /// - public string[] Headers - { - get - { - Initialize(); - return _headers; - } - set - { - _headers = value; - FixupNormalizedHeaders(); - } - } - /// - /// Get the number of the current row. - /// - public long RowNumber - { - get { return _rowNumber; } - } - - /// - /// Check for end of file. - /// - /// true if at end of file, false otherwise - public bool Eof() - { - // a little tricky when finishing the last row with remaining whitespace... - // won't show as EOF until after nextRow is called... - if (_file == null) - return true; - if (_file.Peek() != -1) - return false; - if (!_initialized) - Initialize(); - return RowEnd(); - } - - /// - /// Check for end of row. - /// - /// true if at end of row, false otherwise - public bool RowEnd() - { - return ((_curRow == null || _curCol >= _curRow.Count) && - (_curRowArray == null || _curCol >= _curRowArray.Length)); - } - - private void Initialize() - { - if (_initialized) - return; - _initialized = true; - NextRow(); - // read and store the column header names: - if (ReadHeaders) - { - // always skip blanks when reading headers - bool givenSkipBlankLines = SkipBlankLines; - bool givenSkipBlankColumnsLines = SkipBlankColumnsLines; - SkipBlankLines = true; - SkipBlankColumnsLines = true; - Headers = ReadRow(); - SkipBlankLines = givenSkipBlankLines; - SkipBlankColumnsLines = givenSkipBlankColumnsLines; - } - } - - private void FixupNormalizedHeaders() - { - if (_headers == null) - { - _headersNormalized = null; - return; - } - _headersNormalized = new string[_headers.Length]; - for (int i = 0; i < _headers.Length; i++) - { - string header = _headers[i]; - if (header == null) - continue; - header = header.Trim(); - if (IgnoreHeaderCase) - header = header.ToLower(); - _headersNormalized[i] = header; - } - } - - /// - /// Get the field at the column index. - /// - public string this[int index] - { - get - { - if (!_initialized) - Initialize(); - if (_curRow == null) - { - // look in the string[], instead... - if (_curRowArray == null) - return null; - if (index < 0 || index >= _curRowArray.Length) - { - if (_fillBlankColumns) - return ""; - return null; - } - return _curRowArray[index]; - } - if (index < 0 || index >= _curRow.Count) - { - if (_fillBlankColumns) - return ""; - return null; - } - return _curRow[index]; - } - } - - /// - /// Get the field at the column with the given header. - /// - public string this[string header] - { - get - { - if (!_initialized) - Initialize(); - if (header == null || _headers == null) - return null; - header = header.Trim(); - if (IgnoreHeaderCase) - header = header.ToLower(); - for (int i = 0; i < _headers.Length; i++) - { - if (header == _headersNormalized[i]) - { - return this[i]; - } - } - return null; - } - } - - /// - /// Get the index of the column which has the given header. - /// - public int GetColumnIndex(string header) - { - if (!_initialized) - Initialize(); - if (header == null || _headers == null) - return -1; - header = header.Trim(); - if (IgnoreHeaderCase) - header = header.ToLower(); - for (int i = 0; i < _headers.Length; i++) - { - if (header == _headersNormalized[i]) - { - return i; - } - } - return -1; - } - - /// - /// Get the next field and advance the reader. - /// - /// the field at the next column - public string ReadItem() - { - // check if we need to setup the reading: - if (!_initialized) - Initialize(); - - //// Should actually return a blank entry for the *last* blank entry and null beyond *** - if (RowEnd()) - { - if (_fillBlankColumns) - return ""; - else - return null; - } - - string res = _curRow != null ? _curRow[_curCol] : _curRowArray[_curCol]; - _curCol++; - return res; - } - - // should optimize for 1-character quotes and delimiters... - private string ReadItemFromLine(ref int curPos) - { - if (curPos >= _curLine.Length) - return null; - - string resStr; - curPos++; - // check for empty case: - bool empty = (curPos >= _curLine.Length); - if (!empty) - { - // optimized check for empty in middle of line - if (_delimiterSet) // look for *any* of the characters - { - if (_curLine.Length - curPos >= 1 && - _delimiter.IndexOf(_curLine[curPos]) >= 0) - { - // leave cursor on the delimiter - empty = true; - } - } - else - { - if (_curLine.Length - curPos >= _delimiter.Length && - string.CompareOrdinal(_curLine, curPos, _delimiter, 0, _delimiter.Length) == 0) - { - // leave cursor on end of the delimiter: - curPos += _delimiter.Length - 1; - empty = true; - } - } - } - if (empty) - { - resStr = ""; - } - else - { - // check for unquoted case: - int nextDelimiterIndex; - if (_delimiterSet) - { - // should optimize this: - nextDelimiterIndex = _curLine.IndexOfAny(_delimiter.ToCharArray(), curPos); - } - else - { - nextDelimiterIndex = _curLine.IndexOf(_delimiter, curPos); - } - if (nextDelimiterIndex < 0) - { - nextDelimiterIndex = _curLine.Length; - } - if (!_parseQuotes || _curLine.IndexOf(_quoteChar, curPos, nextDelimiterIndex - curPos) < 0) - { - // simple case! no quote characters... - resStr = _curLine.Substring(curPos, nextDelimiterIndex - curPos); - curPos = nextDelimiterIndex; - } - else - { - StringBuilder res = new StringBuilder(); - for (bool inQuoted = false; true; curPos++) - { - // check for end of line: - if (curPos >= _curLine.Length) - { - if (!inQuoted) - { - // done with item! - break; - } - else - { - // uh oh... a quoted newline... - if (_file == null) - break; - if (_file.Peek() == -1) - break; - // read a new row... - _curLine = _file.ReadLine(); - curPos = -1; - res.Append('\n'); - continue; - } - } - - // check for delimiters: - if (!inQuoted) - { - if (_delimiterSet) // look for *any* of the characters - { - if (_curLine.Length - curPos >= 1 && - _delimiter.IndexOf(_curLine[curPos]) >= 0) - { - // leave cursor on the delimiter - break; - } - } - else - { - if (_curLine.Length - curPos >= _delimiter.Length && - string.CompareOrdinal(_curLine, curPos, _delimiter, 0, _delimiter.Length) == 0) - { - // leave cursor on end of the delimiter: - curPos += _delimiter.Length - 1; - break; - } - } - } - // hard to say how to best handle quotes... Want to be lenient. - if (_parseQuotes && - (_curLine.Length - curPos >= _quoteChar.Length && - string.CompareOrdinal(_curLine, curPos, _quoteChar, 0, _quoteChar.Length) == 0)) - { - // double quote characters mean a literal quote character - // - but only when in quoted mode! - if (inQuoted && - _curLine.Length - curPos >= 2 * _quoteChar.Length && - string.CompareOrdinal(_curLine, curPos + _quoteChar.Length, _quoteChar, 0, _quoteChar.Length) == 0) - { - res.Append(_quoteChar); - curPos += 2 * _quoteChar.Length - 1; - } - else - { - inQuoted = !inQuoted; - curPos += _quoteChar.Length - 1; - } - continue; - } - res.Append(_curLine[curPos]); - } - resStr = res.ToString(); - } - if (_trimWhitespace) - { - // not perfect! *** - if (resStr.Length > 0 && (resStr[0] == ' ' || resStr[resStr.Length - 1] == ' ')) - { - resStr = resStr.Trim(); - } - } - } - // if we are collapsing repeated delimiters, we can do this by skipping blank elements... - if (_collapseDelimiters && resStr.Length == 0) - { - return ReadItemFromLine(ref curPos); - } - return resStr; - } - - /// - /// Get the next field and advance the reader, filling with empty fields at the end of the row. - /// - /// the field at the next column - public string ReadItemLinear() - { - if (!_initialized) - Initialize(); - - if (RowEnd()) - NextRow(); - return ReadItem(); - } - - /// - /// Get the number of fields in the current row. - /// - /// the number of fields in the current row - public int RowLength() - { - if (!_initialized) - Initialize(); - - if (_curRow == null) - { - if (_curRowArray == null) - return 0; - return _curRowArray.Length; - } - return _curRow.Count; - } - - // test if all entries in current row are blank - private bool RowBlank() - { - if (!_initialized) - Initialize(); - - if (_curRow == null) - { - if (_curRowArray == null) - return true; - for (int i = 0; i < _curRowArray.Length; i++) - { - string field = _curRowArray[i]; - if (field.Length != 0) - { - if (_trimWhitespace && - (field[0] == ' ' || field[field.Length - 1] == ' ')) - { - if (field.Trim().Length != 0) - return false; - } - else - { - return false; - } - } - } - return true; - } - for (int i = 0; i < _curRow.Count; i++) - { - string field = _curRow[i]; - if (field.Length != 0) - { - if (_trimWhitespace && - (field[0] == ' ' || field[field.Length - 1] == ' ')) - { - if (field.Trim().Length != 0) - return false; - } - else - { - return false; - } - } - } - return true; - } - - /// - /// Read an entire row and advance the reader. - /// - /// The current row as an array of fields - public string[] ReadRow() - { - return ReadRow(-1); - } - - /// - /// Read an entire row and advance the reader. - /// - /// The length of the row to read, truncating or filling with empty fields as needed - /// The current row as an array of fields - public string[] ReadRow(int len) - { - // check if we need to setup the reading: - if (!_initialized) - { - Initialize(); - } - if (Eof()) - return null; - if (_curRow == null && _curRowArray == null) - { - NextRow(); - } - if (_curRow == null && _curRowArray == null) - return null; - - if (!_parseQuotes && (_delimiterSet || _delimiter.Length == 1) && _curRow == null) - { - _readRowResult = _curRowArray; - NextRow(); - return _readRowResult; - } - - if (_curRow == null) - { - if (len < 0 || len == _curRowArray.Length) - { - _readRowResult = _curRowArray; - } - else - { - if (_readRowResult.Length != len) - _readRowResult = new string[len]; - if (len < _curRowArray.Length) - { -#if ENABLE_BARTOK - for (int i = 0; i < readRowResult.Length; i++) - { - readRowResult[i] = curRowArray[i]; - } -#else - Array.Copy(_curRowArray, _readRowResult, len); -#endif - } - else - { - _curRowArray.CopyTo(_readRowResult, 0); - for (int i = _curRowArray.Length; i < _readRowResult.Length; i++) - { - _readRowResult[i] = ""; - } - } - } - NextRow(); - return _readRowResult; - } - // always make a new array, for now... - _readRowResult = new string[len < 0 ? _curRow.Count : len]; - if (len < 0 || len == _curRow.Count) - { - if (_readRowResult.Length != _curRow.Count) - _readRowResult = new string[_curRow.Count]; - _curRow.CopyTo(_readRowResult, 0); - } - else - { - if (_readRowResult.Length != len) - _readRowResult = new string[len]; - if (len < _curRow.Count) - { - for (int i = 0; i < _readRowResult.Length; i++) - { - _readRowResult[i] = _curRow[i]; - } - } - else - { - _curRow.CopyTo(_readRowResult, 0); - for (int i = _curRow.Count; i < _readRowResult.Length; i++) - { - _readRowResult[i] = ""; - } - } - } - NextRow(); - return _readRowResult; - } - string[] _readRowResult = new string[0]; - - // only reads the *remaining* items... - // also advances the row! - private void ReadRowFromLine() - { - _curRow = null; - _curRowArray = null; - if (_curLine == null) - return; - if (_file == null) - return; - - if (_parseQuotes || !(_delimiterSet || _delimiter.Length == 1)) - { - _curRow = new StringCollection(); - int linePos = -1; - while (linePos < _curLine.Length - 1) - { - string field = ReadItemFromLine(ref linePos); - if (field != null) - { - _curRow.Add(field); - } - } - } - else - { - // we just split, now... - if (_delimiter.Length == 1) - { - _curRowArray = _curLine.Split(_delimiter[0]); - } - else - { - _curRowArray = _curLine.Split(_delimiter.ToCharArray()); - } - } - _curLine = null; - _curCol = 0; - } - - /// - /// Advance to the next row. - /// - public void NextRow() - { - if (_file == null) - return; - if (!_initialized) - { - Initialize(); - return; - } - _curRow = null; - _curRowArray = null; - _curCol = 0; - while (true) - { - _rowNumber++; - _curLine = _file.ReadLine(); - if (_curLine == null) - { - return; - } - if (_skipBlankLines && - (_curLine.Length == 0 || - (_curLine[0] == ' ' && _curLine.Trim().Length == 0))) - { - continue; - } - ReadRowFromLine(); - if (_skipBlankColumnsLines && RowBlank()) - { - _curRow = null; - _curRowArray = null; - continue; - } - break; - } - } - - /// - /// Close the table. - /// - public void Close() - { - if (_file == null) - return; - _file.Close(); - _curCol = -1; - _curLine = null; - _curRow = null; - _curRowArray = null; - _file = null; - } - - /// - /// Reset the reader to the beginning. - /// - /// The reader is not based on a Stream. - public void Reset() - { - if (_file == null) - return; - if (!(_file is StreamReader)) - { - throw new InvalidOperationException("Cannot reset CsvReader not based on a Stream."); - } - ((StreamReader)_file).BaseStream.Seek(0, SeekOrigin.Begin); - ((StreamReader)_file).DiscardBufferedData(); - _initialized = false; - _rowNumber = 0; - _curCol = -1; - _curLine = null; - _curRow = null; - _curRowArray = null; - } - - /// - /// Allows for random access into the file. The user is responsible - /// to make sure that position is at the beginning of a row. - /// - /// new position - /// relative to what - /// The reader is not based on a Stream. - public void Seek(long position, SeekOrigin origin) - { - if (_file == null) - return; - if (!(_file is StreamReader)) - { - throw new InvalidOperationException("Cannot seek reader not based on a Stream."); - } - - ((StreamReader)_file).BaseStream.Seek(position, origin); - if (position == 0) - { - _initialized = false; - } - _curCol = -1; - _curLine = null; - _curRow = null; - _curRowArray = null; - } - - /// - /// Returns position of the cursor in the file - /// - /// byte offset of the current position - /// The reader is not based on a Stream. - public long Position() - { - if (_file == null) - return 0; - if (!(_file is StreamReader)) - { - throw new InvalidOperationException("Cannot tell position for reader not based on a Stream."); - } - return ((StreamReader)_file).BaseStream.Position; - } - - #region IDisposable Members - - /// - /// Dispose. - /// - public void Dispose() - { - Close(); - } - - #endregion - /// - /// Return the enumerator - /// - /// an enumerator for the rows in this table - public IEnumerator GetEnumerator() - { - return new TableEnumerator(this); - } - } - - //// TODO: support reorder list - //// TODO: support filtering (ignoring) columns - /// - /// Write CSV formatted data. - /// - public class CsvWriter : ITableWriter, IDisposable - { - // trim whitespace from each entry: - private bool _trimWhitespace = true; - // skip lines that are only whitespace: - private bool _skipBlankLines = true; - // use the quoteChar to determine quoted sections: - private bool _parseQuotes = true; - // always end in a newline: - private bool _endInNewline = true; - // transform tab, carriage return, and newline into space: - private bool _normalizeWhitespace = false; - // characters to use if normalizing whitespace: - private char[] _whitespaceChars = new char[] { '\r', '\n', '\t' }; - - // support for addressing by column headers: - private string[] _headers = null; - private string[] _headersNormalized = null; - private bool _ignoreHeaderCase = true; - private bool _initialized = false; - private StringCollection _curRow = null; - private bool _writeHeaders = true; - - private string _quoteChar = "\""; - private string _delimiter = ","; - private char[] _delimiterOrQuoteOrNewline = new char[] { ',', '"', '\r', '\n' }; - - private bool _lineStart = true; - private long _rowNumber; - - private TextWriter _file; - - /// - /// Create a CsvWriter based on the TextWriter, - /// - /// the TextWriter to write the table to - public CsvWriter(TextWriter tr) - { - _file = tr; - _rowNumber = 1; - } - - /// - /// Create a CsvWriter based on the specified file, - /// - /// the name of the file to write the table to - public CsvWriter(string fname) - : this(ZStreamWriter.Open(fname)) - { - } - - /// - /// Create a CsvWriter based on the Stream, - /// - /// the Stream to write the table to - public CsvWriter(Stream fstream) - : this(new StreamWriter(fstream)) - { - } - - /// - /// Create a CsvWriter based on the specified file, - /// - /// the name of the file to write the table to - /// the encoding to use - public CsvWriter(string fname, Encoding encoding) - : this(new StreamWriter(fname, false, encoding)) - { - } - - /// - /// Create a CsvWriter based on the Stream, - /// - /// the Stream to write the table to - /// the encoding to use - public CsvWriter(Stream fstream, Encoding encoding) - : this(new StreamWriter(fstream, encoding)) - { - } - - /// - /// Get or set whether to convert all whitespace into space characters. - /// - public bool NormalizeWhitespace - { - get { return _normalizeWhitespace; } - set { _normalizeWhitespace = value; } - } - /// - /// Get or set the characters to consider as whitespace. - /// - public char[] WhitespaceChars - { - get { return _whitespaceChars; } - set - { - if (value != null) - { - _whitespaceChars = value; - } - else - { - _whitespaceChars = new char[] { '\r', '\n', '\t', ' ' }; - } - } - } - - /// - /// Get or set whether to write the headers as the first row. - /// - public bool WriteHeaders - { - get { return _writeHeaders; } - set { _writeHeaders = value; } - } - - /// - /// Gets or sets whether to trim whitespace from each field. - /// - public bool TrimWhitespace - { - get { return _trimWhitespace; } - set { _trimWhitespace = value; } - } - - /// - /// Get or set whether to skip blank lines. - /// - public bool SkipBlankLines - { - get { return _skipBlankLines; } - set { _skipBlankLines = value; } - } - - /// - /// Get or set whether to interpet quote characters when parsing. - /// - public bool ParseQuotes - { - get { return _parseQuotes; } - set { _parseQuotes = value; } - } - - /// - /// Get or set the string to use as a quote symbol. - /// - public string QuoteChar - { - get { return _quoteChar; } - set - { - _quoteChar = value; - FixupDelimiterOrQuoteOrNewline(); - } - } - - /// - /// Get or set the string to use to delimit columns. - /// - public string Delimiter - { - get { return _delimiter; } - set - { - _delimiter = value; - FixupDelimiterOrQuoteOrNewline(); - } - } - private void FixupDelimiterOrQuoteOrNewline() - { - string dqn = (_delimiter == null ? "" : _delimiter) + (_quoteChar == null ? "" : _quoteChar) + "\r\n"; - _delimiterOrQuoteOrNewline = dqn.ToCharArray(); - } - - /// - /// Get or set whether to end the file in a newline. - /// - public bool EndInNewline - { - get { return _endInNewline; } - set { _endInNewline = value; } - } - - /// - /// Gets or sets whether to ignore case when matching header names. - /// - public bool IgnoreHeaderCase - { - get { return _ignoreHeaderCase; } - set - { - if (_ignoreHeaderCase != value) - { - _ignoreHeaderCase = value; - FixupNormalizedHeaders(); - } - } - } - /// - /// Gets or sets the header names. - /// - public string[] Headers - { - get - { - return _headers; - } - set - { - _headers = value; - FixupNormalizedHeaders(); - } - } - private void FixupNormalizedHeaders() - { - if (_headers == null) - { - _headersNormalized = null; - return; - } - _headersNormalized = new string[_headers.Length]; - for (int i = 0; i < _headers.Length; i++) - { - string header = _headers[i]; - if (header == null) - continue; - header = header.Trim(); - if (IgnoreHeaderCase) - header = header.ToLower(); - _headersNormalized[i] = header; - } - } - - /// - /// Add a new header to the header list. - /// - /// the header to add - public void AddHeader(string header) - { - if (_headers == null) - _headers = new string[0]; - string[] newHeaders = new string[_headers.Length + 1]; - _headers.CopyTo(newHeaders, 0); - newHeaders[newHeaders.Length - 1] = header; - Headers = newHeaders; - } - - /// - /// Check for end of file. - /// - /// true if at end of file, false otherwise - public bool Eof() - { - return (_file == null); - } - - private void Initialize() - { - if (!_initialized) - { - _initialized = true; - // check if headers were needed and not written: - if (_writeHeaders && _headers != null) - { - WriteRow(_headers); - } - } - } - - private string Quotify(string item) - { - if (_trimWhitespace) - { - if (item.Length != 0 && - (item[0] == ' ' || item[item.Length - 1] == ' ')) - { - item = item.Trim(); - } - } - if (!_parseQuotes) - { - return item; - } - - if (item.IndexOfAny(_delimiterOrQuoteOrNewline) < 0) - { - // no delimiters, newlines, or quotes in string - return item; - } - StringBuilder sb = new StringBuilder(item); - sb.Replace(_quoteChar, _quoteChar + _quoteChar); - sb.Insert(0, _quoteChar); - sb.Append(_quoteChar); - return sb.ToString(); - } - - /// - /// Write the next field and advance the writer. - /// - /// the field to write - public void WriteItem(string item) - { - Initialize(); - WriteStoredRow(); - if (Eof()) - return; - - if (_lineStart) - { - _lineStart = false; - } - else - { - _file.Write(_delimiter); - } - - if (_normalizeWhitespace) - { - item = NormalizeWS(item); - } - item = Quotify(item); - _file.Write(item); - } - - private string NormalizeWS(string orig) - { - if (orig.IndexOfAny(_whitespaceChars) < 0) - return orig; - orig = orig.Replace("\r\n", " "); - orig = orig.Replace('\r', ' '); - orig = orig.Replace('\n', ' '); - orig = orig.Replace('\t', ' '); - return orig; - } - - /// - /// Write an entire row and advance the writer. - /// - /// The row to write as an array of fields - public void WriteRow(string[] items) - { - WriteRow(items, -1); - } - - /// - /// Write an entire row and advance the writer. - /// - /// The length of the row to write, truncating or filling with empty fields as needed - /// The row to write as an array of fields - public void WriteRow(string[] items, int len) - { - Initialize(); - WriteStoredRow(); - if (Eof()) - return; - - if (!_parseQuotes) - { - if (_normalizeWhitespace) - { - // this modifies the input parameter! - for (int i = 0; i < items.Length; i++) - { - items[i] = NormalizeWS(items[i]); - } - } - - if (_skipBlankLines && _lineStart && items.Length == 0) - return; - - if (items.Length == 0) - { - if (len > 0) - { - for (int i = 1; i < len; i++) - { - _file.Write(_delimiter); - } - } - } - else - { - _file.Write(items[0]); - if (len < 0 || len >= items.Length) - { - for (int i = 1; i < items.Length; i++) - { - _file.Write(_delimiter); - if (items[i].Length != 0) - _file.Write(items[i]); - } - for (int i = items.Length; i < len; i++) - { - _file.Write(_delimiter); - } - } - else - { - for (int i = 1; i < len; i++) - { - _file.Write(_delimiter); - if (items[i].Length != 0) - _file.Write(items[i]); - } - } - } - _file.WriteLine(); - - _lineStart = true; - _rowNumber++; - return; - } - - for (int i = 0; i < items.Length && (len < 0 || i < len); i++) - { - WriteItem(items[i]); - } - // pad with empty strings: - if (len > 0) - { - for (int i = items.Length; i < len; i++) - { - WriteItem(""); - } - } - NextRow(); - } - - /// - /// Write an entire row and advance the writer. - /// - /// The row to write as a collection of fields - public void WriteRow(StringCollection items) - { - WriteRow(items, -1); - } - - /// - /// Write an entire row and advance the writer. - /// - /// The length of the row to write, truncating or filling with empty fields as needed - /// The row to write as a collection of fields - public void WriteRow(StringCollection items, int len) - { - Initialize(); - WriteStoredRow(); - if (Eof()) - return; - - if (!_parseQuotes) - { - if (_normalizeWhitespace) - { - for (int i = 0; i < items.Count; i++) - { - items[i] = NormalizeWS(items[i]); - } - } - - if (_skipBlankLines && _lineStart && items.Count == 0) - return; - - if (items.Count == 0) - { - if (len > 0) - { - for (int i = 1; i < len; i++) - { - _file.Write(_delimiter); - } - } - } - else - { - _file.Write(items[0]); - if (len < 0 || len >= items.Count) - { - for (int i = 1; i < items.Count; i++) - { - _file.Write(_delimiter); - if (items[i].Length != 0) - _file.Write(items[i]); - } - for (int i = items.Count; i < len; i++) - { - _file.Write(_delimiter); - } - } - else - { - for (int i = 1; i < len; i++) - { - _file.Write(_delimiter); - if (items[i].Length != 0) - _file.Write(items[i]); - } - } - } - _file.WriteLine(); - - _lineStart = true; - _rowNumber++; - return; - } - - for (int i = 0; i < items.Count && (len < 0 || i < len); i++) - { - WriteItem(items[i]); - } - // pad with empty strings: - if (len > 0) - { - for (int i = items.Count; i < len; i++) - { - WriteItem(""); - } - } - NextRow(); - } - - /// - /// Set the field at the column index. - /// - public string this[int index] - { - set - { - if (index < 0) - return; - if (_curRow == null) - _curRow = new StringCollection(); - if (_curRow.Count <= index) // extend it... - { - for (int i = _curRow.Count; i <= index; i++) - { - _curRow.Add(""); - } - } - _curRow[index] = value; - } - } - - /// - /// Set the field at the column with the given header. - /// - public string this[string header] - { - set - { - if (header == null || _headers == null) - return; - header = header.Trim(); - if (IgnoreHeaderCase) - header = header.ToLower(); - for (int i = 0; i < _headersNormalized.Length; i++) - { - if (header == _headersNormalized[i]) - { - this[i] = value; - return; - } - } - // do nothing! - return; - } - } - - private void WriteStoredRow() - { - if (_curRow == null) - return; - WriteRow(_curRow); - _curRow = null; - } - - /// - /// Advance to the next row. - /// - public void NextRow() - { - if (Eof()) - return; - if (_curRow != null) - { - WriteStoredRow(); - return; - } - if (_lineStart && _skipBlankLines) - return; - _file.WriteLine(); - _lineStart = true; - _rowNumber++; - } - - /// - /// Get the number of the current row. - /// - public long RowNumber - { - get { return _rowNumber; } - } - - /// - /// Close the table. - /// - public void Close() - { - if (Eof()) - return; - if (_curRow != null) - { - WriteStoredRow(); - return; - } - if (!_lineStart && _endInNewline) - NextRow(); - _file.Flush(); - _file.Close(); - _lineStart = true; - _file = null; - } - #region IDisposable Members - - /// - /// Dispose. - /// - public void Dispose() - { - Close(); - } - - #endregion - } - -#if !ENABLE_BARTOK - -#if !DISABLE_XML - ////////////////////////////////////////////// - ////// XML support - ////////////////////////////////////////////// - - //// TODO: Decide how to handle attributes vs. child elements - //// TODO: Allow for categories (pushing and popping levels) - //// TODO: Allow override of given element names by header order? - - /// - /// Read XML formatted data. - /// - public class XmlTableReader : ITableReader - { - private bool _trimWhitespace = true; - private bool _fillBlankColumns = true; - private bool _ignoreHeaderCase = true; - private bool _addUnknownHeaders = true; - private bool _headerOrdered = true; - - private XmlTextReader _file; - private Hashtable _currentRow; - private ArrayList _currentRowSequence; - private string _currentName; - private int _currentCol; - //private bool initialized = false; - private string _tableName; - private string[] _headers; - private bool _initialized = false; - - private int _level = 0; - - /// - /// Create a new XmlTableReader - /// - /// the source to base the XmlTableReader on - public XmlTableReader(XmlTextReader tr) - { - tr.DtdProcessing = DtdProcessing.Prohibit; - _headers = new string[0]; - _file = tr; - if (_file != null) - { - _file.WhitespaceHandling = WhitespaceHandling.None; - _file.MoveToContent(); - // read in the "Table" element - _tableName = ""; - if (_file.IsStartElement()) - { - _tableName = _file.Name; - _file.Read(); - } - } - } - - /// - /// Create a new XmlTableReader - /// - /// the name of the file to base the XmlTableReader on - public XmlTableReader(string fname) - : this(new XmlTextReader(ZStreamReader.Open(fname)) { DtdProcessing = DtdProcessing.Prohibit }) - { - } - - /// - /// Create a new XmlTableReader - /// - /// the source to base the XmlTableReader on - public XmlTableReader(Stream tr) - : this(new XmlTextReader(tr) { DtdProcessing = DtdProcessing.Prohibit }) - { - } - - /// - /// Create a new XmlTableReader - /// - /// the source to base the XmlTableReader on - public XmlTableReader(TextReader tr) - : this(new XmlTextReader(tr) { DtdProcessing = DtdProcessing.Prohibit }) - { - } - - /// - /// Get the current in the hierarchy. - /// - public int Level - { - get { return _level; } - } - - /// - /// Gets or sets the header names. - /// - public string[] Headers - { - get { return _headers; } - set { _headers = value; } - } - - /// - /// Get or set whether to add new headers to the header list as they are encountered. - /// - public bool AddUnknownHeaders - { - get { return _addUnknownHeaders; } - set { _addUnknownHeaders = value; } - } - - /// - /// Get or set whether to sort the headers. - /// - public bool HeaderOrdered - { - get { return _headerOrdered; } - set { _headerOrdered = value; } - } - - /// - /// Get the name of the table. - /// - public string TableName - { - get { return _tableName; } - } - - /// - /// Check for end of file. - /// - /// true if at end of file, false otherwise - public bool Eof() - { - if (!_initialized) - Initialize(); - return _currentRow == null && (_file.ReadState == ReadState.Closed || _file.EOF); - } - - /// - /// Check for end of row. - /// - /// true if at end of row, false otherwise - public bool RowEnd() - { - if (!_initialized) - Initialize(); - return Eof() || _currentRowSequence == null || - _currentCol >= _currentRowSequence.Count; - } - - /// - /// Get the next field and advance the reader. - /// - /// the field at the next column - public string ReadItem() - { - if (!_initialized) - Initialize(); - if (Eof()) - return null; - if (RowEnd()) - { - // should we fill even this?? - if (FillBlankColumns) - return ""; - return null; - } - string res = (string)_currentRowSequence[_currentCol]; - _currentCol++; - return res; - } - - /// - /// Get the next field and advance the reader, filling with empty fields at the end of the row. - /// - /// the field at the next column - public string ReadItemLinear() - { - if (!_initialized) - Initialize(); - if (Eof()) - return null; - if (RowEnd()) - NextRow(); - return ReadItem(); - } - - /// - /// Get the number of fields in the current row. - /// - /// the number of fields in the current row - public int RowLength() - { - if (!_initialized) - Initialize(); - if (_currentRowSequence == null) - return 0; - return _currentRowSequence.Count; - } - - /// - /// Read an entire row and advance the reader. - /// - /// The current row as an array of fields - public string[] ReadRow() - { - if (!_initialized) - Initialize(); - if (_currentRowSequence == null) - return null; - string[] res = (string[])_currentRowSequence.ToArray(typeof(string)); - NextRow(); - return res; - } - - /// - /// Read an entire row and advance the reader. - /// - /// The length of the row to read, truncating or filling with empty fields as needed - /// The current row as an array of fields - public string[] ReadRow(int len) - { - if (!_initialized) - Initialize(); - if (len < 0) - return ReadRow(); - if (_currentRowSequence == null) - return null; - string[] res = new string[len]; - for (int i = 0; i < res.Length; i++) - { - if (i < _currentRowSequence.Count) - { - res[i] = (string)_currentRowSequence[i]; - } - else - { - res[i] = ""; - } - } - NextRow(); - return res; - } - - private void Initialize() - { - if (_initialized) - return; - _initialized = true; - NextRow(); - // can't read and store the column header names: - } - - /// - /// Advance to the next row. - /// - public void NextRow() - { - if (!_initialized) - { - Initialize(); - return; - } - _currentRow = null; - _currentRowSequence = null; - _currentName = null; - _currentCol = -1; - if (Eof()) - { - return; - } - - if (!_file.IsStartElement()) - { - // what to do here? It's wrong. - _file.Read(); - NextRow(); - return; - } - - _currentName = _file.Name; - _currentRow = new Hashtable(); - // assumes headers not null - _currentRowSequence = new ArrayList(_headers.Length); - _currentCol = 0; - if (!_file.IsEmptyElement) - { - string givenHeader, name, val; - - if (_file.HasAttributes) - { - for (int a = 0; a < _file.AttributeCount; a++) - { - _file.MoveToAttribute(a); - givenHeader = _file.Name; - name = givenHeader.Trim(); - if (IgnoreHeaderCase) - name = name.ToLower(); - val = _file.Value; - if (TrimWhitespace) - val = val.Trim(); - - int index = -1; - for (int i = 0; i < _headers.Length; i++) - { - string match = _headers[i].Trim(); - if (IgnoreHeaderCase) - match = match.ToLower(); - if (name == match) - { - index = i; - break; - } - } - if (index < 0) - { - if (_addUnknownHeaders) - { - string[] oldHeaders = _headers; - _headers = new string[oldHeaders.Length + 1]; - oldHeaders.CopyTo(_headers, 0); - index = _headers.Length - 1; - _headers[index] = givenHeader; - } - } - _currentRow[name] = val; - if (!_headerOrdered) - { - // skip adding to the sequence for attributes! - //currentRowSequence.Add(val); - } - else - { - if (val.Length != 0 && index >= 0 && index < _headers.Length) - { - for (int b = _currentRowSequence.Count; b <= index; b++) - { - _currentRowSequence.Add(""); - } - _currentRowSequence[index] = val; - } - } - } - _file.MoveToElement(); // Moves the reader back to the element node. - } - - _file.Read(); // pass start element - while (!(_file.NodeType == XmlNodeType.EndElement)) - { - if (_file.IsStartElement()) - { - givenHeader = _file.Name; - name = givenHeader.Trim(); - if (IgnoreHeaderCase) - name = name.ToLower(); - val = ""; - if (!_file.IsEmptyElement) - { - _file.Read(); - val = _file.ReadString(); - if (TrimWhitespace) - val = val.Trim(); - } - - int index = -1; - for (int i = 0; i < _headers.Length; i++) - { - string match = _headers[i].Trim(); - if (IgnoreHeaderCase) - match = match.ToLower(); - if (name == match) - { - index = i; - break; - } - } - if (index < 0) - { - if (_addUnknownHeaders) - { - string[] oldHeaders = _headers; - _headers = new string[oldHeaders.Length + 1]; - oldHeaders.CopyTo(_headers, 0); - index = _headers.Length - 1; - _headers[index] = givenHeader; - } - } - _currentRow[name] = val; - if (!_headerOrdered) - { - // skip adding to the sequence for attributes! - } - else - { - if (val.Length != 0 && index >= 0 && index < _headers.Length) - { - for (int b = _currentRowSequence.Count; b <= index; b++) - { - _currentRowSequence.Add(""); - } - _currentRowSequence[index] = val; - } - } - _file.Read(); // skip empty element OR end element... - } - else - { - // text not within a child element?? - val = _file.ReadString(); - if (TrimWhitespace) - val = val.Trim(); - - _currentRow[""] = val; - } - } - } - _file.Read(); - // try to skip the document closing tag? (XmlReader is disgusting, really) - while (_file.NodeType == XmlNodeType.EndElement) - { - _file.Read(); - _level--; - } - if (_level < 0) - _level = 0; - } - - /// - /// Get the field at the column index. - /// - public string this[int index] - { - get - { - if (!_initialized) - Initialize(); - if (_currentRowSequence == null || index < 0 || - index >= _currentRowSequence.Count) - { - if (FillBlankColumns) - return ""; - return null; - } - return (string)_currentRowSequence[index]; - } - } - - /// - /// Get the field at the column with the given header. - /// - public string this[string header] - { - get - { - if (!_initialized) - Initialize(); - header = header.Trim(); - if (IgnoreHeaderCase) - header = header.ToLower(); - string res = (string)_currentRow[header]; - if (res == null && FillBlankColumns) - res = ""; - return res; - } - } - - /// - /// Close the table. - /// - public void Close() - { - _file.Close(); - } - - /// - /// Reset the reader to the beginning. - /// - /// Always thrown, currently. - public void Reset() - { - throw new InvalidOperationException("Cannot reset XmlTableReader."); - } - - /// - /// Gets or sets whether to trim whitespace from each field. - /// - public bool TrimWhitespace - { - get - { - return _trimWhitespace; - } - set - { - _trimWhitespace = value; - } - } - - /// - /// Gets or sets whether to return "", not null, when the end of a row is reached, - /// until the row is advanced: - /// - public bool FillBlankColumns - { - get - { - return _fillBlankColumns; - } - set - { - _fillBlankColumns = value; - } - } - - /// - /// Gets or sets whether to ignore case when matching header names. - /// - public bool IgnoreHeaderCase - { - get - { - return _ignoreHeaderCase; - } - set - { - _ignoreHeaderCase = value; - } - } - - /// - /// Get a row enumerator. - /// - /// and enumerator for the row in this table - public IEnumerator GetEnumerator() - { - return new TableEnumerator(this); - } - } - - /// - /// Write XML formatted data. - /// - public class XmlTableWriter : ITableWriter - { - private bool _trimWhitespace = true; - private bool _ignoreHeaderCase = true; - private bool _addUnknownHeaders = true; - private bool _skipEmptyElements = true; - - private XmlTextWriter _file; - //private Hashtable currentRow; - private ArrayList _currentRowSequence; - private string _tableName = "Table"; - private string _currentName = "Item"; - private int _currentCol; - private bool _initialized; - private string[] _headers; - private bool _fieldsAsAttributes; - private bool _elementEndPending; - private int _openElementCount; - - /// - /// Create a new XmlTableWriter. - /// - /// the destination to write the table to - public XmlTableWriter(XmlTextWriter tr) - { - _file = tr; - _currentCol = 0; - if (_file != null) - { - _file.Formatting = Formatting.Indented; - _file.Indentation = 4; - //Write the XML delcaration - _file.WriteStartDocument(); - } - } - - /// - /// Create a new XmlTableWriter. - /// - /// the filename of the destination to write the table to - public XmlTableWriter(string fname) - : this(new XmlTextWriter(ZStreamWriter.Open(fname))) - { - } - - /// - /// Create a new XmlTableWriter. - /// - /// the destination to write the table to - public XmlTableWriter(Stream tr) - : this(new XmlTextWriter(tr, null)) - { - } - - /// - /// Create a new XmlTableWriter. - /// - /// the destination to write the table to - public XmlTableWriter(TextWriter tr) - : this(new XmlTextWriter(tr)) - { - } - - /// - /// Gets or sets the header names. - /// - public string[] Headers - { - get - { - return _headers; - } - set - { - _headers = value; - if (_headers != null) - { - for (int i = 0; i < _headers.Length; i++) - { - if (_headers[i] == null || _headers[i].Length == 0) - { - _headers[i] = "C" + i; - } - else - { - _headers[i] = _headers[i].Replace(' ', '_'); - _headers[i] = XmlConvert.EncodeName(_headers[i]); - } - } - } - } - } - - /// - /// Get or set the name to use for each item element, - /// - public string ItemName - { - get - { - return _currentName; - } - set - { - _currentName = value; - } - } - - /// - /// Get or set the name to use for the table element. - /// - public string TableName - { - get - { - return _tableName; - } - set - { - _tableName = value; - } - } - - /// - /// Get or set whether to represent the fields as attributes, instead of children. - /// - public bool FieldsAsAttributes - { - get { return _fieldsAsAttributes; } - set { _fieldsAsAttributes = value; } - } - - /// - /// Get or set whether to skip all empty elements. - /// - public bool SkipEmptyElements - { - get { return _skipEmptyElements; } - set { _skipEmptyElements = value; } - } - - /// - /// Check for end of file. - /// - /// true if at end of file, false otherwise - public bool Eof() - { - return _file.WriteState == WriteState.Closed; - } - - /// - /// Write the next field and advance the writer. - /// - /// the field to write - public void WriteItem(string item) - { - if (Eof()) - return; - this[_currentCol] = item; - _currentCol++; - } - - /// - /// Write an entire row and advance the writer. - /// - /// The row to write as an array of fields - public void WriteRow(string[] items) - { - WriteRow(items, -1); - } - - /// - /// Write an entire row and advance the writer. - /// - /// The length of the row to write, truncating or filling with empty fields as needed - /// The row to write as an array of fields - public void WriteRow(string[] items, int len) - { - // only handles sequence - not named items!!! *** - if (Eof()) - return; - if (items == null) - return; - if (!_initialized) - { - _initialized = true; - _file.WriteStartElement(TableName); - } - - if (_elementEndPending) - { - if (_openElementCount > 0) - { - if (TrimWhitespace) - { - _file.WriteEndElement(); - } - else - { - _file.WriteFullEndElement(); - } - _openElementCount--; - } - _elementEndPending = false; - } - - if (len < 0) - len = items.Length; - _file.WriteStartElement(ItemName); - for (int i = 0; i < len; i++) - { - string name; - if (_headers != null && i < _headers.Length) - { - name = _headers[i]; - } - else - { - name = "C" + i; - } - if (FieldsAsAttributes) - { - string val = (i < items.Length) ? (string)items[i] : ""; - if (val == null) - val = ""; - if (TrimWhitespace) - val = val.Trim(); - if (SkipEmptyElements && val.Length == 0) - { - // just skip it... - } - else - { - _file.WriteAttributeString(name, val); - } - } - else - { - string val = (i < items.Length) ? (string)items[i] : ""; - if (val == null) - val = ""; - if (TrimWhitespace) - val = val.Trim(); - - if (SkipEmptyElements && val.Length == 0) - { - // just skip it... - } - else - { - _file.WriteStartElement(name); - if (val.Length == 0) - { - _file.WriteEndElement(); - } - else - { - _file.WriteString(val); - _file.WriteFullEndElement(); - } - } - } - } - - _elementEndPending = true; - _openElementCount++; - } - - /// - /// Advance to the next row. - /// - public void NextRow() - { - if (_currentRowSequence != null) - { - // only handles sequence - not named items!!! *** - WriteRow((string[])_currentRowSequence.ToArray(typeof(string))); - _currentRowSequence = null; - } - - _currentRowSequence = null; - _currentCol = 0; - } - - /// - /// Set the field at the column index. - /// - public string this[int index] - { - // checking headers can be done here or at write time. Not much difference. - set - { - if (index < 0) - return; - if (_currentRowSequence == null) - _currentRowSequence = new ArrayList(); - if (_currentRowSequence.Count <= index) // extend it... - { - for (int i = _currentRowSequence.Count; i <= index; i++) - { - _currentRowSequence.Add(""); - } - } - _currentRowSequence[index] = value; - } - } - - /// - /// Set the field at the column with the given header. - /// - public string this[string header] - { - // checking headers can be done here or at write time. Not much difference. - set - { - if (header == null) - return; - - string givenHeader = header; - header = header.Trim(); - if (IgnoreHeaderCase) - header = header.ToLower(); - // don't use hash table: - //currentRow[header] = value; - // find in headers: - for (int i = 0; i < _headers.Length; i++) - { - string match = _headers[i].Trim(); - if (IgnoreHeaderCase) - match = match.ToLower(); - if (header == match) - { - this[i] = value; - return; - } - } - // add new header if required: - if (!AddUnknownHeaders) - return; - string[] oldHeaders = _headers; - _headers = new string[oldHeaders.Length + 1]; - oldHeaders.CopyTo(_headers, 0); - _headers[_headers.Length - 1] = givenHeader; - this[_headers.Length - 1] = value; - } - } - - /// - /// Close the table. - /// - public void Close() - { - if (_file == null || Eof()) - return; - // we don't really want this NextRow unless needed... *** - NextRow(); - - for (; _openElementCount > 0; _openElementCount--) - { - if (TrimWhitespace) - { - _file.WriteEndElement(); - } - else - { - _file.WriteFullEndElement(); - } - } - _elementEndPending = false; - - _file.WriteFullEndElement(); - _file.WriteEndDocument(); - _file.Flush(); - _file.Close(); - } - - /// - /// Increase the hierarchy depth. - /// - public void LevelIn() - { - _elementEndPending = false; - } - - /// - /// Decrease the hierarchy depth. - /// - public void LevelOut() - { - if (_openElementCount > 1 || (!_elementEndPending && _openElementCount > 0)) - { - if (TrimWhitespace) - { - _file.WriteEndElement(); - } - else - { - _file.WriteFullEndElement(); - } - - _openElementCount--; - } - } - - /// - /// Gets or sets whether to trim whitespace from each field. - /// - public bool TrimWhitespace - { - get { return _trimWhitespace; } - set { _trimWhitespace = value; } - } - - /// - /// Get or set whether to add new headers to the header list as they are encountered. - /// - public bool AddUnknownHeaders - { - get { return _addUnknownHeaders; } - set { _addUnknownHeaders = value; } - } - - /// - /// Gets or sets whether to ignore case when matching header names. - /// - public bool IgnoreHeaderCase - { - get { return _ignoreHeaderCase; } - set { _ignoreHeaderCase = value; } - } - } - -#endif - -#endif - -#if ALLOW_DB - ////////////////////////////////////////////// - ////// Excel support - ////////////////////////////////////////////// - - //// TODO: ODBC driver support? - //// Bare CsvReader support / merging? - //// Decent exception handling - //// Add in enumerator - - /// - /// Read spreadsheet data in various formats. - /// - public class SpreadsheetReader - { - private string filename; - private bool xlsFormat; - private OleDbConnection oleConn; - private OleDbDataReader oleReader; - private bool isEof; - private string[] cols; - - // trim whitespace from each entry: - private bool trimWhitespace = true; - // skip lines that are empty: - private bool skipBlankLines = true; - // skip lines that have fields but only whitespace in them: - private bool skipBlankColumnsLines = true; - private long rowNumber = 2; // gives 1-base row numbers... - - /// - /// Gets or sets whether to trim whitespace from each field. - /// - public bool TrimWhitespace - { - get { return trimWhitespace; } - set { trimWhitespace = value; } - } - /// - /// Get or set whether to skip blank lines. - /// - public bool SkipBlankLines - { - get { return skipBlankLines; } - set { skipBlankLines = value; } - } - /// - /// Get or set whether to skip lines with all columns blank. - /// - public bool SkipBlankColumnsLines - { - get { return skipBlankColumnsLines; } - set { skipBlankColumnsLines = value; } - } - /// - /// Get the number of the current row. - /// - public long RowNumber - { - get { return rowNumber; } - } - - /// - /// Create a new SpreadsheetReader. - /// - /// the source of the spreadsheet - public SpreadsheetReader(string fname) - { - filename = fname; - // determine format: - xlsFormat = false; - if (Path.GetExtension(filename).ToLower() == ".xls") - { - xlsFormat = true; - } - // construct connection string: - string connString = "Provider=Microsoft.Jet.OLEDB.4.0;" + - "Data Source="; - if (xlsFormat) // Excel Format - { - connString += "\"" + Path.GetFullPath(filename) + "\"" + ";" + - "Extended Properties=\"Excel 8.0;HDR=Yes\""; - } - else // assume CSV - { - connString += Path.GetDirectoryName(Path.GetFullPath(filename)) + ";" + - "Extended Properties=\"text;HDR=Yes;FMT=Delimited\""; - } - string selectString = "SELECT * FROM "; - if (xlsFormat) // Excel Format - { - selectString += "[Sheet1$]"; // hard-coded first sheet? *** - } - else // assume CSV - { - selectString += Path.GetFileName(filename); - } - - oleConn = new OleDbConnection(connString); - oleConn.Open(); - OleDbCommand openCmd = oleConn.CreateCommand(); - openCmd.CommandText = selectString; - oleReader = openCmd.ExecuteReader(); - - // Initialize the row reading: - isEof = !Next(); - // Initialize the column names: - SetupColumnNames(); - } - - /// - /// Test the connection to the spreadsheet. - /// - public void TestConnection() - { - if (oleReader == null) - { - Console.WriteLine("OleDbDataReader is not initialized!"); - return; - } - // display column names - for (int c = 0; c < oleReader.FieldCount; c++) - { - Console.Write(oleReader.GetName(c) + " \t"); - } - Console.WriteLine(""); - Console.WriteLine("--------------------------------------------------"); - // display data - while (oleReader.Read()) - { - object[] fields = new Object[oleReader.FieldCount]; - oleReader.GetValues(fields); - for (int i = 0; i < fields.Length; i++) - { - Console.Write("" + fields[i] + " \t"); - } - Console.WriteLine(""); - } - } - - /// - /// Read an entire row and advance the reader. - /// - /// The current row as an array of fields - public object[] ReadRowObjects() - { - if (Eof()) - return new Object[0]; // should it be a null? - object[] fields = new Object[oleReader.FieldCount]; - oleReader.GetValues(fields); - for (int i = 0; i < fields.Length; i++) - { - if (fields[i] == DBNull.Value) - { - fields[i] = null; - } - } - Next(); - return fields; - } - - /// - /// Read an entire row and advance the reader. - /// - /// The current row as an array of fields - public string[] ReadRow() - { - if (Eof()) - return new string[0]; // should it be a null? - object[] fields = new Object[oleReader.FieldCount]; - string[] fieldsStr = new string[oleReader.FieldCount]; - oleReader.GetValues(fields); - for (int i = 0; i < fields.Length; i++) - { - if (fields[i] == DBNull.Value) - { - fieldsStr[i] = ""; - } - else - { - fieldsStr[i] = fields[i].ToString(); - } - } - Next(); - return fieldsStr; - } - - //// Not certain if the FieldCount is stable. *** - private void SetupColumnNames() - { - if (oleReader.FieldCount <= 0) - { - cols = new string[0]; - return; - } - cols = new string[oleReader.FieldCount]; - for (int c = 0; c < oleReader.FieldCount; c++) - { - cols[c] = oleReader.GetName(c); - if (cols[c] == null) - { - cols[c] = ""; - } - } - } - - /// - /// Get the names of the columns. - /// - public string[] ColumnNames - { - get - { - return cols; - } - } - - /// - /// Get the field at the column index. - /// - public object this[int i] - { - get - { - if (Eof()) - return null; - object item = oleReader[i]; - if (item == DBNull.Value) - return null; - if (TrimWhitespace) - { - if (item.GetType() == typeof(String)) - { - item = ((string)item).Trim(); - } - } - return item; - } - } - - /// - /// Set the field at the column with the given header. - /// - public object this[string col] - { - get - { - if (Eof()) - return null; - object item = oleReader[col]; - if (item == DBNull.Value) - return null; - if (TrimWhitespace) - { - if (item.GetType() == typeof(String)) - { - item = ((string)item).Trim(); - } - } - return item; - } - } - - /// - /// Get the field at the column index. - /// - public string this[int i, bool b] - { - get - { - object item = this[i]; - if (item == null) - return ""; - string res = item.ToString(); - if (TrimWhitespace) - { - res = res.Trim(); - } - return res; - } - } - - /// - /// Set the field at the column with the given header. - /// - public string this[string col, bool b] - { - get - { - object item = this[col]; - if (item == null) - return ""; - string res = item.ToString(); - if (TrimWhitespace) - { - res = res.Trim(); - } - return res; - } - } - - /// - /// Advance to the next row. - /// - /// true if there are more rows, false if at end of table - public bool Next() - { - if (Eof() || oleReader == null) - return false; - while (true) - { - rowNumber++; - isEof = !oleReader.Read(); - if (isEof) - break; - if (!SkipBlankLines && !SkipBlankColumnsLines) - break; - if (oleReader.FieldCount == 0) - continue; // does this happen? - int col; - for (col = 0; col < RowLength(); col++) - { - if (SkipBlankColumnsLines) - { - if (this[col, true] != "") - break; - } - else - { - if (this[col] != null) - break; - } - } - if (col < RowLength()) - break; - } - return !isEof; - } - - /// - /// Get the number of fields in the current row. - /// - /// the number of fields in the current row - public int RowLength() - { - if (Eof()) - return 0; - return oleReader.FieldCount; - } - - /// - /// Check for end of file. - /// - /// true if at end of file, false otherwise - public bool Eof() - { - return isEof; - } - - /// - /// Close the table. - /// - public void Close() - { - if (oleReader != null) - oleReader.Close(); - oleReader = null; - if (oleConn != null) - oleConn.Close(); - oleConn = null; - isEof = true; - } - } - - //// TODO: ODBC driver support? - //// Bare CsvReader support / merging? - //// Decent exception handling - /// - /// Write spreadsheet data in various formats. - /// - public class SpreadsheetWriter - { - private string filename; - private bool xlsFormat; - private OleDbConnection oleConn; - - // trim whitespace from each entry: - private bool trimWhitespace = true; - - /// - /// Gets or sets whether to trim whitespace from each field. - /// - public bool TrimWhitespace - { - get { return trimWhitespace; } - set { trimWhitespace = value; } - } - - /// - /// Create a new SpreadsheetWriter. - /// - /// the source of the spreadsheet - public SpreadsheetWriter(string fname) - { - filename = fname; - - // determine format: - xlsFormat = false; - if (Path.GetExtension(filename).ToLower() == ".xls") - { - xlsFormat = true; - } - - // construct connection string: - string connString = "Provider=Microsoft.Jet.OLEDB.4.0;" + - "Data Source="; - if (xlsFormat) // Excel Format - { - connString += filename + ";" + - "Extended Properties=\"Excel 8.0;HDR=No\""; - } - else // assume CSV - { - connString += Path.GetDirectoryName(Path.GetFullPath(filename)) + ";" + - "Extended Properties=\"text;HDR=Yes;FMT=Delimited\""; - } - - string selectString = "SELECT * FROM "; - if (xlsFormat) // Excel Format - { - selectString += "[Sheet1$]"; // hard-coded first sheet? *** - } - else // assume CSV - { - selectString += Path.GetFileName(filename); - } - - //// open the connection: - oleConn = new OleDbConnection(connString); - oleConn.Open(); - - } - - private string sqlString(string item) - { - return "'" + item.Replace("'", "''") + "'"; - } - - private string columnNameTrue(int col) - { - int aInt = (int)'A'; - int zInt = (int)'Z'; - int range = (zInt - aInt + 1); - - string res = ""; - int high = col / range; - if (high > 0) - { - col = col % range; - res += (char)(aInt + high - 1); - } - res += (char)(aInt + col); - return res; - } - - private string columnName(int col) - { - string res = "F"; - res += (col + 1); - return res; - } - - /// - /// Write an entire row and advance the writer. - /// - /// The row to write as an array of fields - public void WriteRow(object[] fields) - { - OleDbCommand oleCmd = new OleDbCommand(); - oleCmd.Connection = oleConn; - string cmd = "INSERT INTO " + "[Sheet1$] "; // + "(FirstName, LastName) "; - - // give column names: - cmd += "("; - for (int i = 0; i < fields.Length; i++) - { - if (i > 0) - { - cmd += ", "; - } - cmd += columnName(i); - } - cmd += ") "; - - cmd += "VALUES ("; - for (int i = 0; i < fields.Length; i++) - { - if (i > 0) - { - cmd += ", "; - } - cmd += sqlString(fields[i].ToString()); - } - //'Bill', 'Brown' - cmd += ")"; - oleCmd.CommandText = cmd; - try - { - oleCmd.ExecuteNonQuery(); - } - catch (Exception) - { - throw new Exception("Excel insert error."); - } - } - - /// - /// Get the number of fields in the current row. - /// - /// the number of fields in the current row - public int RowLength() - { - return 0; - //return dataTable.Columns.Count; - } - - /// - /// Close the table. - /// - public void Close() - { - if (oleConn != null) - { - oleConn.Close(); - oleConn = null; - } - } - } -#endif -} diff --git a/src/Microsoft.ML.InternalStreams/UnbufferedStream.cs b/src/Microsoft.ML.InternalStreams/UnbufferedStream.cs deleted file mode 100644 index 8f005e9817..0000000000 --- a/src/Microsoft.ML.InternalStreams/UnbufferedStream.cs +++ /dev/null @@ -1,1626 +0,0 @@ -// owner: rragno - -//#define MONITOR -using System; -using System.IO; -using System.Security; -using System.Threading; -using Microsoft.ML.Runtime; -using Microsoft.ML.Runtime.Internal.Utilities; -//using System.Xml; -//using System.Data; -//using System.Data.OleDb; - -namespace Microsoft.ML.Runtime.Internal.IO -{ - // * Unbuffered writer - // * Robust stream - // * Improve seek? - // * custom StreamReader - // * custom BinaryReader / BinaryReaderEx - - /// - /// Stream using unbuffered I/O for efficient reading from fast disk arrays. - /// - public sealed class UnbufferedStream : FileStream, IDisposable - { - #region Static Configuration - - /// - /// Get or Set whether to read in the background in a separate thread. - /// - /// - ///

- /// This does not affect existing instances. - ///

- ///

- /// Normally, it is faster to perform a multithreaded read, even on a - /// single-processor machine. However, the thread creation cost does make - /// it somewhat more expensive to create an instance. - ///

- /// - public static bool ParallelRead { get; set; } = true; - - private static int _initialBlockSize = 4 * 1024 * 1024; - - /// - /// Get or Set the memory used for reading, in bytes. - /// - /// - ///

- /// If the memory cannot be allocated, a backoff strategy will be used. - ///

- ///

- /// The default is 8 MB. - ///

- /// - public static int BufferSize - { - get - { - return _initialBlockSize * 2; - } - set - { - int newSize = value / 2; - if (newSize < 0) - newSize = 1024 * 1024; - // allow 0 for a minimal (sector-sized) buffer - if (newSize != _initialBlockSize) - _initialBlockSize = newSize; - } - } - #endregion - - #region Instance Fields - - private long _length; - private string _fileName; - - private readonly uint _sectorSize; - private IntPtr[] _buffer; - private IntPtr[] _alignedBuffer; - private readonly IntPtr _handle; - private int _blockSize; - private bool _parallel; - private bool _released = true; - - #endregion - - #region Creation and Cleanup - - /// - /// Exception that represents a failure in the internal memory allocation. - /// - public class VirtualAllocException : IOException - { - /// - /// Create a new exception. - /// - /// the message associated with this exception - public VirtualAllocException(string msg) - : base(msg) - { - } - } - - /// - /// Open a file for reading without NTFS caching. - /// The stream should be accessed sequentially - seeking can be slow - and it will not - /// support writing. - /// - /// name of file to open - /// Unbuffered file stream - /// fileName is null. - /// fileName is invalid. - /// fileName cannot be found. - /// An I/O error has occurred. - // /// This may throw a different exception than FileNotFound if the file does - // /// not exist. - // is relying on the order of evaluation bad?? *** - public UnbufferedStream(string fileName) - : this(fileName, - new FileAlignmentInfo(fileName), - IOUtil.Win32.CreateFile(fileName, IOUtil.Win32.FileAccess.GENERIC_READ, IOUtil.Win32.FileShare.FILE_SHARE_READ, IntPtr.Zero, IOUtil.Win32.CreationDisposition.OPEN_EXISTING, - IOUtil.Win32.FileFlagsAndAttributes.FILE_FLAG_NO_BUFFERING | IOUtil.Win32.FileFlagsAndAttributes.FILE_FLAG_SEQUENTIAL_SCAN, IntPtr.Zero)) - { - } - /// - /// Open a file for reading without NTFS caching. - /// The stream should be accessed sequentially - seeking can be slow - and it will not - /// support writing. - /// - /// name of file to open - /// the length, sector size, and bytes from the end that are not sector-aligned - /// the handle of the specified file - /// Unbuffered file stream - /// A problem occurred alocating memory at a low level. -#if !UNBUFFERED_AS_STREAM - private UnbufferedStream(string fileName, FileAlignmentInfo alignInfo, IntPtr handle) - : base(new Microsoft.Win32.SafeHandles.SafeFileHandle(handle, true), FileAccess.Read, 1, false) -#else - private UnbufferedStream(string fileName, long length, IntPtr handle) -#endif - { - //Console.WriteLine("new unbuffered: " + fileName); -#if MONITOR - Console.WriteLine(new string('-', (":: " + Path.GetFileName(fileName) + " :: " + "Open").Length)); - Console.WriteLine(":: " + Path.GetFileName(fileName) + " :: " + "Open"); -#endif - _fileName = fileName; - //this.length = base.Length; - _length = alignInfo.Length; - _unalignedTail = alignInfo.UnalignedTail; - _sectorSize = alignInfo.SectorSize; - //this.handle = (IntPtr)typeof(FileStream).GetProperty("Handle").GetValue(this, null); - //this.handle = base.Handle; - _handle = handle; - - // open for first time: - Reopen(); - } - - /// - /// Release resources. - /// - ~UnbufferedStream() - { - Dispose(true); - } - - /// - /// Release the resources used for the unbuffered file. - /// - /// true if disposing, false otherwise - protected override void Dispose(bool disposing) - { - try - { - base.Dispose(disposing); - } - catch - { - // ignore - } - try - { - ReleaseBuffer(); - } - catch - { - // ignore - } - try - { - IOUtil.Win32.CloseHandle(_handle); - } - catch - { - // ignore - } -#if MONITOR - Console.WriteLine(":: " + Path.GetFileName(fileName) + " :: " + "Close"); - Console.WriteLine(new string('-', (":: " + Path.GetFileName(fileName) + " :: " + "Close").Length)); -#endif - _closed = true; - } - - //static int allocCount = 0; - - private void ReleaseBuffer() - { - if (!_released) - { - if (_buffer != null) - { - try - { - for (int j = 0; j < _buffer.Length; j++) - { - if (_buffer[j] != IntPtr.Zero) - { - try - { -#if REUSE_BUFFERS - if (buffer[j] != IntPtr.Zero) - { - if (blockSize == initialBlockSize) - { - // save the buffer for later... - lock (standingBuffers) - { - if (standingBufferCount < standingBuffers.Length) - { - standingBuffers[standingBufferCount++] = buffer[j]; - buffer[j] = IntPtr.Zero; - } - } - } - } -#endif - - if (_buffer[j] != IntPtr.Zero) - { - bool ret = IOUtil.Win32.VirtualFree(_buffer[j], IntPtr.Zero, IOUtil.Win32.FreeType.MEM_RELEASE); - _buffer[j] = IntPtr.Zero; - } - } - catch - { - } - } - } - } - catch - { - } - } - _buffer = null; - _released = true; - } - if (_fillThread != null) - { - _fillThread = null; - } - } - - private bool _closed; - - /// - /// Close the stream. - /// - public override void Close() - { - if (_closed) - return; - _closed = true; - base.Close(); - Dispose(true); - } - -#if REUSE_BUFFERS - private static IntPtr[] standingBuffers = new IntPtr[32]; - private static int standingBufferCount = 0; -#endif - - private void Reopen() - { - Reopen(0, true); - } - private void Reopen(long startPosition, bool allocate) - { - if (_fillThread != null) - { - try - { - _fillThread.Abort(); - } - catch - { - } - _fillThread = null; - } - long startRemainder = (startPosition % _sectorSize); - try - { - startPosition -= startRemainder; - long res; - IOUtil.Win32.SetFilePointerEx(_handle, startPosition, out res, IOUtil.Win32.SeekOrigin.FILE_BEGIN); - } - catch - { - // ignore? - } - - try - { - _released = false; - - _totalRead = startPosition; - _totalGet = startPosition; - _getCount = 0; - _bufferFillIndex = 0; - _bufferGetIndex = 0; - _fillCount = 0; - _currentBuffer = IntPtr.Zero; - _currentBufferLimit = 0; - _currentBufferBottom = 0; - _gotDone = false; - - if (allocate) - { - _parallel = ParallelRead; - int parallelLevel = _parallel ? 2 : 1; - _blockSize = _initialBlockSize; - // force non-parallel for small files... - if (_parallel && (_length < 4 * _sectorSize || _length < 2 * _blockSize)) - { - _parallel = false; - parallelLevel = 1; - } - _blockSize = _blockSize / parallelLevel; - if (_blockSize % _sectorSize != 0) - _blockSize -= (int)(_blockSize % _sectorSize); - if (_blockSize < _sectorSize) - _blockSize = (int)_sectorSize; - - _readMutex = new AutoResetEvent[parallelLevel]; - _pullMutex = new AutoResetEvent[parallelLevel]; - _readSize = new int[parallelLevel]; - _done = new bool[parallelLevel]; - - _buffer = new IntPtr[parallelLevel]; - _alignedBuffer = new IntPtr[parallelLevel]; - bool completed = false; - while (!completed) - { - completed = true; - for (int i = 0; i < _buffer.Length; i++) - { - _buffer[i] = IntPtr.Zero; - -#if REUSE_BUFFERS - if (blockSize == initialBlockSize) - { - lock (standingBuffers) - { - if (standingBufferCount > 0) - { - standingBufferCount--; - buffer[i] = standingBuffers[standingBufferCount]; - } - } - } -#endif - - if (_buffer[i] == IntPtr.Zero) - { - _buffer[i] = IOUtil.Win32.VirtualAlloc(IntPtr.Zero, new IntPtr(_blockSize + 8), IOUtil.Win32.AllocationType.MEM_RESERVE | IOUtil.Win32.AllocationType.MEM_COMMIT, IOUtil.Win32.Protect.PAGE_READWRITE); - } - - if (_buffer[i] == IntPtr.Zero || _buffer[i].ToInt64() < 0) - { - for (int j = 0; j < i; j++) - { - if (_buffer[j] != IntPtr.Zero) - { - try - { -#if REUSE_BUFFERS - if (buffer[j] != IntPtr.Zero) - { - if (blockSize == initialBlockSize) - { - // save the buffer for later... - lock (standingBuffers) - { - if (standingBufferCount < standingBuffers.Length) - { - standingBuffers[standingBufferCount++] = buffer[j]; - buffer[j] = IntPtr.Zero; - } - } - } - } -#endif - - if (_buffer[j] != IntPtr.Zero) - { - bool ret = IOUtil.Win32.VirtualFree(_buffer[j], IntPtr.Zero, IOUtil.Win32.FreeType.MEM_RELEASE); - _buffer[j] = IntPtr.Zero; - } - } - catch - { - } - } - } - if (_blockSize <= 256 * 1024) - { - throw Contracts.Process(new VirtualAllocException("Could not allocate buffer for UnbufferedStream.")); - } - // try reducing the buffer size: - _blockSize = _blockSize / 2; - GC.WaitForPendingFinalizers(); - GC.Collect(); - // try again: - completed = false; - break; - } - _alignedBuffer[i] = new IntPtr((_buffer[i].ToInt64() + 7) & ~7); - _readMutex[i] = new AutoResetEvent(false); - _pullMutex[i] = new AutoResetEvent(true); - } - } - } - else - { - // clear these, in any case: - for (int i = 0; i < _readMutex.Length; i++) - { - _readSize[i] = 0; - _done[i] = false; - _readMutex[i].Reset(); - _pullMutex[i].Set(); - } - } - - _alignedLimit = _length - (_unalignedTail == null ? 0 : _unalignedTail.Length); - if (_parallel) - { - _fillThread = Utils.CreateBackgroundThread(FillBuffer); - // start now? - // skip if small? *** - _fillThread.Start(); - } - - if (startRemainder != 0) - { - Skip(startRemainder); - } - } - catch - { - for (int j = 0; j < _buffer.Length; j++) - { - if (_buffer[j] != IntPtr.Zero) - { - try - { - bool ret = IOUtil.Win32.VirtualFree(_buffer[j], IntPtr.Zero, IOUtil.Win32.FreeType.MEM_RELEASE); - _buffer[j] = IntPtr.Zero; - } - catch - { - } - } - } - if (_fillThread != null) - { - try - { - _fillThread.Abort(); - _fillThread = null; - } - catch - { - } - } - try - { - base.Dispose(true); - } - catch - { - } - throw; - } - } - - #endregion - - #region Properties - - /// - /// Get the Length of this file, in bytes. - /// - public override long Length - { - get - { - // return base.Length; - // use the simple, recorded length: - return _length; - } - } - - /// - /// Get whether the stream supports reading - always true. - /// - public override bool CanRead - { - get - { - return true; - //return base.CanRead; - } - } - - /// - /// Get whether the stream supports writing - always false. - /// - public override bool CanWrite - { - get - { - return false; - //return base.CanWrite; - } - } - - #endregion - - #region Seeking - - /// - /// Get or Set the position in the file, in bytes. - /// - public override long Position - { - get - { - //return base.Position; - return _totalGet - _currentBufferLimit + _currentBufferBottom; - } - set - { - //throw new NotSupportedException("UnbufferedStream cannot seek."); - Seek(value); - } - } - - /// - /// Move forward by reading and discarding bytes. - /// - /// the number of bytes to skip - private void Skip(long count) - { - if (count == 0) - return; - //if (forWriting) throw new NotSupportedException(); - byte[] dump = new byte[Math.Min(count, 128 * 1024)]; - if (dump.Length != count) - { - int rem = (int)(count % dump.Length); - if (Read(dump, 0, rem) < rem) - return; - count -= rem; - } - while (count > 0) - { - int read = Read(dump, 0, dump.Length); - if (read != dump.Length) - { - System.Diagnostics.Debug.WriteLine("Skip failed! Read " + read + " / " + dump.Length + " for chunk."); - return; - } - count -= dump.Length; - } - } - - /// - /// Seek to a new position in the file, in bytes. - /// - /// the offset in bytes - /// the new position - public long Seek(long offset) - { - return Seek(offset, SeekOrigin.Begin); - } - /// - /// Seek to a new position in the file, in bytes. - /// - /// the offset in bytes - /// the SeekOrigin to take the offset from - /// the new position - public override long Seek(long offset, SeekOrigin origin) - { -#if SLOW_SEEK - long cur = Position; - switch (origin) - { - case SeekOrigin.Begin: - break; - case SeekOrigin.Current: - offset += cur; - break; - case SeekOrigin.End: - offset = Length - offset; - break; - } - if (offset < 0) offset = 0; - if (offset == cur) return cur; - - if (offset > cur) - { - Skip(offset - cur); - return Position; - } - else - { - Reopen(); - Skip(offset); - return Position; - } -#else - long cur = Position; - switch (origin) - { - case SeekOrigin.Begin: - break; - case SeekOrigin.Current: - offset += cur; - break; - case SeekOrigin.End: - offset = Length - offset; - break; - } - if (offset < 0) - offset = 0; - if (offset == cur) - return cur; - - if (offset > cur && (offset - cur) < 4000000) - { - Skip(offset - cur); - return Position; - } - else - { - Reopen(offset, false); - return Position; - } -#endif - } - - /// - /// Get whether the stream supports seeking. true, although performance - /// might not be optimal. - /// - public override bool CanSeek - { - get - { - return true; - } - } - - #endregion - - #region Reading - - #region Private Read Helpers - - private long _totalRead; - private int _bufferFillIndex; - private int _bufferGetIndex; - private int _fillCount; - private Thread _fillThread; - private AutoResetEvent[] _readMutex; - private AutoResetEvent[] _pullMutex; - private int[] _readSize; - private byte[] _unalignedTail; - - /// - /// Data about the file size and alignment. - /// - private struct FileAlignmentInfo - { - public readonly long Length; - public readonly uint SectorSize; - public readonly byte[] UnalignedTail; - - /// - /// Hack to read the unaligned tail first, instead of at the end. This helps with - /// keeping the file handles open, which enables seeking later. - /// - /// the file to read the tail of - /// the bytes of the unaligned tail, or null if the file is aligned - /// The file cannot be found. - /// The tail could not be read. - public FileAlignmentInfo(string fileName) - { - Length = (new FileInfo(fileName)).Length; - SectorSize = FindSectorSize(fileName); - int remainder = (int)(Length % SectorSize); - if (remainder == 0) - { - UnalignedTail = null; - } - else - { - UnalignedTail = new byte[remainder]; - int overCount; - for (int i = 0; i < 3; i++) - { - try - { - using (FileStream overRead = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - { - overRead.Seek(-UnalignedTail.Length, SeekOrigin.End); - overCount = overRead.Read(UnalignedTail, 0, UnalignedTail.Length); - } - if (overCount != remainder) - { - throw Contracts.ExceptIO("UnbufferedStream could not read tail of file: " + - "expected " + remainder + ", read " + overCount); - } - } - catch (FileNotFoundException) - { - throw; - } - catch - { - if (i == 2) - throw; - if (i == 0) - Thread.Sleep(10); - if (i == 1) - Thread.Sleep(100); - } - } - } - } - - /// - /// Return the sector size of the drive of the given path. - /// - /// path name for the drive, file, or directory - /// drive sector size in bytes - private static uint FindSectorSize(string path) - { - uint size = 512; - uint ignore; - IOUtil.Win32.GetDiskFreeSpace(Path.GetPathRoot(path), out ignore, out size, out ignore, out ignore); - return size; - } - } - - private long _alignedLimit; - - private void FillBuffer() - { - // This loop is the only one making reads - they are sequential and on only this - // thread - but the buffers it reads into are read in parallel by other threads. - // The idea is just to keep passing this thread buckets to let it pump data. - try - { - while (FillBufferPass()) - { - // keep reading... - } - } - catch (Exception) // ex) - { - // ignore all exceptions on the filling thread... - // may be needed, but will hide potential problems. - } - } - - private bool FillBufferPass() - { - // read the aligned amount only, specified by 'alignedLimit': - if (_totalRead < _alignedLimit) - { - int curBlock = (int)Math.Min(_blockSize, _alignedLimit - _totalRead); - - // wait for pull of last read: -#if MONITOR - Console.WriteLine(":: " + Path.GetFileName(fileName) + " :: " + "".PadLeft(4) + - (" fill wait: " + fillCount + " pullMutex[" + bufferFillIndex + "]").PadRight(56) + - " bytes: " + totalRead + " / " + length + " (" + alignedLimit + ")"); -#endif - _pullMutex[_bufferFillIndex].WaitOne(); -#if MONITOR - Console.WriteLine(":: " + Path.GetFileName(fileName) + " :: " + "".PadLeft(4) + - (" fill got: " + fillCount + " pullMutex[" + bufferFillIndex + "]").PadRight(56) + - " bytes: " + totalRead + " / " + length + " (" + alignedLimit + ")"); -#endif - try - { - // read into buffer: - int readBytes; - bool readres = IOUtil.Win32.Raw.ReadFile(_handle, _alignedBuffer[_bufferFillIndex], - (int)Math.Min(_blockSize, _alignedLimit - _totalRead), out readBytes); - if (!readres) - { - // try again?? *** - System.Diagnostics.Debug.WriteLine(""); - System.Diagnostics.Debug.WriteLine("ReadFile failed! at: " + _totalRead); - Thread.Sleep(10); - readres = IOUtil.Win32.Raw.ReadFile(_handle, _alignedBuffer[_bufferFillIndex], - (int)Math.Min(_blockSize, _alignedLimit - _totalRead), out readBytes); - if (!readres) - { - System.Diagnostics.Debug.WriteLine(""); - System.Diagnostics.Debug.WriteLine("ReadFile failed!! at: " + _totalRead); - - Thread.Sleep(50); - readres = IOUtil.Win32.Raw.ReadFile(_handle, _alignedBuffer[_bufferFillIndex], - (int)Math.Min(_blockSize, _alignedLimit - _totalRead), out readBytes); - - if (!readres) - { - System.Diagnostics.Debug.WriteLine(""); - System.Diagnostics.Debug.WriteLine("ReadFile failed!!! at: " + _totalRead); - readBytes = 0; - } - } - } - - // set read amounts: - _readSize[_bufferFillIndex] = readBytes; - _totalRead += readBytes; - - // signal completed read: - _fillCount++; - -#if MONITOR - Console.WriteLine(":: " + Path.GetFileName(fileName) + " :: " + "".PadLeft(4) + - (" fill set: " + fillCount + " readMutex[" + bufferFillIndex + "]").PadRight(56) + - " bytes: " + totalRead + " / " + length + " (" + alignedLimit + ")"); -#endif - if (readBytes == 0) - { - if (_totalRead < _alignedLimit) - { - throw Contracts.ExceptIO("Could not read complete buffer"); - } - } - - if (_totalRead < _alignedLimit || (_unalignedTail != null && _unalignedTail.Length != 0)) - { - _readMutex[_bufferFillIndex].Set(); - _bufferFillIndex = (_bufferFillIndex + 1) % _readMutex.Length; - return true; - } - else - { - _done[_bufferFillIndex] = true; - _readMutex[_bufferFillIndex].Set(); - _bufferFillIndex = (_bufferFillIndex + 1) % _readMutex.Length; - return false; - } - } - catch - { - _pullMutex[_bufferFillIndex].Set(); - throw; - } - } - else if (_alignedLimit == _totalRead && _unalignedTail != null && _unalignedTail.Length != 0) - { - _pullMutex[_bufferFillIndex].WaitOne(); - - try - { - // use the tail we read at the beginning: - // unalignedTail must not be null, since we are not aligned - _totalRead += _unalignedTail.Length; - CopyBuffer(_unalignedTail, _alignedBuffer[_bufferFillIndex], _unalignedTail.Length); - _readSize[_bufferFillIndex] = _unalignedTail.Length; - _done[_bufferFillIndex] = true; - } - catch (Exception ex) - { - _pullMutex[_bufferFillIndex].Set(); - throw ex; - } - _readMutex[_bufferFillIndex].Set(); - _bufferFillIndex = (_bufferFillIndex + 1) % _readMutex.Length; - - return false; - } - else - { - _readSize[_bufferFillIndex] = 0; - _done[_bufferFillIndex] = true; - _readMutex[_bufferFillIndex].Set(); - _bufferFillIndex = (_bufferFillIndex + 1) % _readMutex.Length; - return false; - } - } - - private bool[] _done; - private bool _gotDone; - private long _totalGet; - private int _getCount; - - private void GetBuffer() - { - if (_gotDone) - { - _currentBufferBottom = 0; - _currentBufferLimit = 0; - _currentBuffer = _alignedBuffer[0]; - return; - } - IntPtr res = _alignedBuffer[_bufferGetIndex]; - - if (_getCount != 0) - { - _pullMutex[(_bufferGetIndex + _readMutex.Length - 1) % _readMutex.Length].Set(); -#if MONITOR - Console.WriteLine(":: " + Path.GetFileName(fileName) + " :: " + "".PadLeft(32) + - (" drain set: " + getCount + " pullMutex[" + ((bufferGetIndex + readMutex.Length - 1) % readMutex.Length) + "]").PadRight(28) + - " bytes: " + totalGet + " / " + length); -#endif - } - -#if MONITOR - Console.WriteLine(":: " + Path.GetFileName(fileName) + " :: " + "".PadLeft(32) + - (" drain wait: " + getCount + " readMutex[" + bufferGetIndex + "]").PadRight(28) + - " bytes: " + totalGet + " / " + length); -#endif - // if needed, do it ourselves... - if (!_parallel) - { - FillBufferPass(); - } - - _readMutex[_bufferGetIndex].WaitOne(); - _getCount++; - - _currentBufferBottom = 0; - _currentBufferLimit = _readSize[_bufferGetIndex]; - if (_done[_bufferGetIndex] || _currentBufferLimit == 0) - { - for (int i = 0; i < _readSize.Length; i++) - { - _readSize[i] = 0; - } - _gotDone = true; - } - _totalGet += _currentBufferLimit; - _currentBuffer = res; -#if MONITOR - Console.WriteLine(":: " + Path.GetFileName(fileName) + " :: " + "".PadLeft(32) + - (" drain got: " + getCount + " readMutex[" + bufferGetIndex + "]").PadRight(28) + - " bytes: " + totalGet + " / " + length); -#endif - - _bufferGetIndex = (_bufferGetIndex + 1) % _readMutex.Length; - - } - - private void CopyBuffer(byte[] source, IntPtr dest, int count) - { - if (count <= 0) - return; - - #region copy variations - unsafe - { - byte* d = (byte*)dest.ToPointer(); - for (int i = 0; i < count; i++) - { - *d = source[i]; - d++; - } - } - #endregion - - } - private void CopyBuffer(IntPtr source, int sourceStart, byte[] dest, int destStart, int count) - { - if (count <= 0) - return; - - #region copy variations - unsafe - { - // pointer to array copy: - byte* s = ((byte*)source.ToPointer()) + sourceStart; - count += destStart; - for (; destStart < count; destStart++) - { - dest[destStart] = *(s++); - } - } - #endregion - - } - - private IntPtr _currentBuffer; - private int _currentBufferLimit; - private int _currentBufferBottom; - - private byte[] _sharedBuffer; - - #endregion - - #region Public Read Functionality - - /// - /// Reads a block of bytes from the stream and writes the data into a buffer. - /// The buffer is automatically allocated, but it may be shared across calls to this method. - /// - /// the array in which the values are replaced by the bytes read - /// - /// The total number of bytes read into the buffer. This will be 0 if the end - /// of the stream has been reached, and is guaranteed to be less than buffer.Length only if - /// fewer than buffer.Length bytes remain (and it will then equal the remainder of the bytes). - /// - public int Read(out byte[] buffer) - { - if (_sharedBuffer == null) - { - // could use a weak reference... - _sharedBuffer = new byte[_blockSize]; - } - buffer = _sharedBuffer; - int res = Read(buffer); - // should we resize the array here? it might be convenient... - if (res == 0) - { - _sharedBuffer = null; - buffer = new byte[0]; - } - return res; - } - /// - /// Reads a block of bytes from the stream and writes the data in a given buffer. - /// - /// the array in which the values are replaced by the bytes read - /// - /// - /// The total number of bytes read into the buffer. This will be 0 if the end - /// of the stream has been reached, and is guaranteed to be less than buffer.Length only if - /// fewer than buffer.Length bytes remain (and it will then equal the remainder of the bytes). - /// - public int Read(byte[] buffer) - { - return Read(buffer, 0, buffer.Length); - } - - /// - /// Reads a block of bytes from the stream and writes the data in a given buffer. - /// - /// the array in which the values between offset and (offset + count - 1) are replaced by the bytes read - /// The byte offset in array at which to begin reading. - /// The maximum number of bytes to read. - /// - /// The total number of bytes read into the buffer. This will be 0 if the end - /// of the stream has been reached, and is guaranteed to be less than count only if - /// fewer than count bytes remain (and it will then equal the remainder of the bytes). - /// - /// The counts are out of range. - /// The buffer is null - public override int Read(byte[] buffer, int offset, int count) - { -#if !OLD_DIRECT_READ - - Contracts.CheckValue(buffer, nameof(buffer)); - Contracts.CheckParam(0 <= offset && offset <= buffer.Length, nameof(offset)); - Contracts.CheckParam(offset <= offset + count && offset + count <= buffer.Length, nameof(count)); - - int read = 0; - while (count > 0) - { - if (_currentBufferLimit == _currentBufferBottom) - { - GetBuffer(); - if (_currentBufferLimit == 0) - break; - } - if (_currentBufferLimit - _currentBufferBottom >= count) - { - CopyBuffer(_currentBuffer, _currentBufferBottom, buffer, offset, count); - _currentBufferBottom += count; - read += count; - count = 0; - break; - } - else - { - int chunk = _currentBufferLimit - _currentBufferBottom; - CopyBuffer(_currentBuffer, _currentBufferBottom, buffer, offset, chunk); - read += chunk; - count -= chunk; - offset += chunk; - _currentBufferBottom = _currentBufferLimit; - } - } - return read; - -#else - if (overRead != null) return overRead.Read(buffer, offset, count); - if (Position + count <= length) - { - -#if SYNCHRONOUS - int readBytes; - bool readres; - readres = ReadFile(handle, alignedBuffer, blockSize, out readBytes, IntPtr.Zero); - if (!readres) return 0; - - fixed (byte* amBuffer = mBuffer) - { - int* abIn = (int*)alignedBuffer.ToPointer(); - int* abOut = (int*)amBuffer; - int* abOutEnd = (int*)(abOut + ((readBytes + 7) >> 2)); - while (abOut != abOutEnd) - { - *abOut = *abIn; - abOut++; - abIn++; - } - } - return readBytes; -#else - -#if ASYNC_BAD - int readBytes; - - if (overlappedIndex < 0) - { - overlappedIndex = 0; - overlappeds[overlappedIndex].Offset = (uint)(totalRead & 0xFFFFFFFF); - overlappeds[overlappedIndex].OffsetHigh = (uint)(totalRead >> 32); - overlappeds[overlappedIndex].hEvent = IntPtr.Zero; - ReadFile(handle, alignedBuffer[overlappedIndex], blockSize, - out readBytes, - ref overlappeds[overlappedIndex]); - totalRead += blockSize; - } - - bool result = GetOverlappedResult(handle, - alignedBuffer[overlappedIndex], - ref overlappeds[overlappedIndex], out readBytes, true); - - Console.WriteLine("read: " + readBytes); - if (readBytes == 0) return 0; - - IntPtr inBuffer = alignedBuffer[overlappedIndex]; - - overlappedIndex = (overlappedIndex + 1) % 2; - overlappeds[overlappedIndex].Offset = (uint)(totalRead & 0xFFFFFFFF); - overlappeds[overlappedIndex].OffsetHigh = (uint)(totalRead >> 32); - overlappeds[overlappedIndex].hEvent = IntPtr.Zero; - ReadFile(handle, alignedBuffer[overlappedIndex], blockSize, - out readBytes, - ref overlappeds[overlappedIndex]); - totalRead += blockSize; - - fixed (byte* amBuffer = mBuffer) - { - int* abIn = (int*)inBuffer.ToPointer(); - int* abOut = (int*)amBuffer; - int* abOutEnd = (int*)(abOut + ((readBytes + 7) >> 2)); - while (abOut != abOutEnd) - { - *abOut = *abIn; - abOut++; - abIn++; - } - } - return readBytes; -#else - IntPtr inBuffer = GetBuffer(); - - fixed (byte* amBuffer = mBuffer) - { - int* abIn = (int*)inBuffer.ToPointer(); - int* abOut = (int*)amBuffer; - int* abOutEnd = (int*)(abOut + ((blockSize + 7) >> 2)); - while (abOut != abOutEnd) - { - *abOut = *abIn; - abOut++; - abIn++; - } - } - return blockSize; -#endif -#endif - } - if (Position >= length) - { - return 0; - } - // Unbuffered reads *must* be aligned to the sector size (assumed as 512, here). - // This will use a padded file to read the correct amount, but otherwise will - // have to truncate the read. - if (count == 0) return 0; - int remaining = (int)(length - Position); - int paddedRemaining = (int)(base.Length - Position); - int res = paddedRemaining; - if (res == 0) return 0; - //bool resized = false; - if (res % sectorSize != 0) - { - - //// total hack: - int first = (int)((res / sectorSize) * sectorSize); - if (first != 0) - { - Read(buffer, offset, first); - offset += first; - count -= first; - } - long pos = Position; - //Console.WriteLine(" pos: " + pos); - Close(); - overRead = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - overRead.Seek(pos, SeekOrigin.Begin); - return first + Read(buffer, offset, count); - } - return Math.Min(res, remaining); -#endif - } - - /// - /// Read a block of bytes from the stream and advance the position. - /// - /// returns a pointer to a (pinned) buffer of bytes read from the stream - /// - /// the number of bytes read - this will be positive unless the end of - /// stream has been reached, in which case it will be 0 - /// - /// The buffer returned here may be shared across calls to this method. - public unsafe int Read(out byte* buffer) - { - if (_currentBufferLimit == _currentBufferBottom) - { - GetBuffer(); - } - buffer = ((byte*)_currentBuffer.ToPointer()) + _currentBufferBottom; - int count = _currentBufferLimit - _currentBufferBottom; - _currentBufferBottom = _currentBufferLimit; - return count; - } - - /// - /// Retrieve the next byte in the stream and advance the position. - /// - /// the next byte, or -1 if at end of stream - /// This is not as efficient as block reading, because of overhead issues. - public override int ReadByte() - { - if (_currentBufferLimit == _currentBufferBottom) - { - GetBuffer(); - } - if (_currentBufferLimit == 0) - return -1; - // note the (post)increment of currentBufferBottom: - unsafe - { - return *(((byte*)_currentBuffer.ToPointer()) + _currentBufferBottom++); - } - } - - /// - /// Retrieve the next byte in the stream, without advancing the position. - /// - /// the next byte, or -1 if at end of stream - /// This is not as efficient as block reading, because of overhead issues. - public int Peek() - { - if (_currentBufferLimit == _currentBufferBottom) - { - GetBuffer(); - } - if (_currentBufferLimit == 0) - return -1; - unsafe - { - return *(((byte*)_currentBuffer.ToPointer()) + _currentBufferBottom); - } - } - - /// - /// Check if the end of file has been reached. - /// - /// true if no more bytes remain; false otherwise - public bool Eof() - { - if (_currentBufferBottom < _currentBufferLimit) - return false; - // is this good enough? We want to be cheap... - return (Position < Length); - } - - #endregion - - #endregion - - #region Unsupported Members - -#if !UNBUFFERED_AS_STREAM - /// - /// Get whether the stream was opened asynchronously. Always false, but this does not - /// matter for managed code. - /// - public override bool IsAsync - { - get - { - return false; - } - } -#endif - - /// - /// Write all pending data. This method does nothing. - /// - public override void Flush() - { - } - - /// - /// Set the length of the file - not supported. - /// - /// the length that the file will not be set to - /// Always thrown. - public override void SetLength(long value) - { - throw Contracts.ExceptNotSupp("UnbufferedStream cannot set the file length."); - } - - /// - /// Write a section of buffer to the stream - not supported. - /// - /// the buffer that will not be written - /// the offset in buffer at which to not start writing - /// the number of bytes to not write - /// Always thrown. - public override void Write(byte[] buffer, int offset, int count) - { - throw Contracts.ExceptNotSupp("UnbufferedStream cannot write."); - } - - #endregion - } - -#if UNBUFFEREDREADER - /// - /// StreamReader using unbuffered I/O for efficient reading from fast disk arrays. - /// - public sealed class UnbufferedStreamReader : StreamReader - { - private readonly UnbufferedStream stream; - private Encoding encoding; - private Decoder decoder; - private byte* byteBuffer; - private int byteCount = 0; - private int bytePos = 0; - private char[] charBuffer; - private int charLen = 0; - private int charPos = 0; - - public UnbufferedStreamReader(string fileName) - : this(new UnbufferedStream(fileName)) - { - } - - public UnbufferedStreamReader(UnbufferedStream stream) - : base(stream) - { - this.stream = stream; - Peek(); - encoding = CurrentEncoding; - buffer = null; - bufferCount = 0; - Init(); - } - - private void Init() - { - this.decoder = encoding.GetDecoder(); - byteBuffer = null; - byteCount = 0; - charBuffer = null; - this.byteLen = 0; - this.bytePos = 0; - //this._isBlocked = false; - //this._closable = true; - } - - /// Reads the next character from the input stream and advances the character position by one character. - /// The next character from the input stream represented as an object, or -1 if no more characters are available. - /// An I/O error occurs. - public override int Read() - { - if (stream == null) - throw new IOException("Reader is closed."); - if ((this.charPos == this.charLen) && (this.ReadBuffer() == 0)) - { - return -1; - } - int num1 = this.charBuffer[this.charPos]; - this.charPos++; - return num1; - } - - /// Returns the next available character but does not consume it. - /// The next character to be read, or -1 if no more characters are available or the stream does not support seeking. - /// An I/O error occurs. - public override int Peek() - { - if (stream == null) - throw new IOException("Reader is closed."); - if (charPos < charLen) - { - return charBuffer[charPos]; - } - if ((this.charPos != this.charLen) || (!this._isBlocked && (this.ReadBuffer() != 0))) - { - return this.charBuffer[this.charPos]; - } - return -1; - } - - /// Reads a maximum of count characters from the current stream into buffer, beginning at index. - /// The number of characters that have been read, or 0 if at the end of the stream and no data was read. The number will be less than or equal to the count parameter, depending on whether the data is available within the stream. - /// The maximum number of characters to read. - /// When this method returns, contains the specified character array with the values between index and (index + count - 1) replaced by the characters read from the current source. - /// The index of buffer at which to begin writing. - /// buffer is null. - /// index or count is negative. - /// The buffer length minus index is less than count. - /// An I/O error occurs, such as the stream is closed. - public override int Read([In, Out] char[] buffer, int index, int count) - { - if (stream == null) - throw new IOException("Reader is closed."); - if (buffer == null) - { - throw new ArgumentNullException("buffer", Environment.GetResourceString("ArgumentNull_Buffer")); - } - if ((index < 0) || (count < 0)) - { - throw new ArgumentOutOfRangeException((index < 0) ? "index" : "count", Environment.GetResourceString("ArgumentOutOfRange_NeedNonNegNum")); - } - if ((buffer.Length - index) < count) - { - throw new ArgumentException(Environment.GetResourceString("Argument_InvalidOffLen")); - } - int num1 = 0; - bool flag1 = false; - while (count > 0) - { - int num2 = this.charLen - this.charPos; - if (num2 == 0) - { - num2 = this.ReadBuffer(buffer, index + num1, count, out flag1); - } - if (num2 == 0) - { - return num1; - } - if (num2 > count) - { - num2 = count; - } - if (!flag1) - { - Buffer.InternalBlockCopy(this.charBuffer, this.charPos * 2, buffer, (index + num1) * 2, num2 * 2); - this.charPos += num2; - } - num1 += num2; - count -= num2; - if (this._isBlocked) - { - return num1; - } - } - return num1; - } - - /// Reads a maximum of count characters from the current stream and writes the data to buffer, beginning at index. - /// The number of characters that have been read. The number will be less than or equal to count, depending on whether all input characters have been read. - /// The maximum number of characters to read. - /// When this method returns, this parameter contains the specified character array with the values between index and (index + count -1) replaced by the characters read from the current source. - /// The place in buffer at which to begin writing. - /// An I/O error occurs. - /// index or count is negative. - /// The buffer length minus index is less than count. - /// buffer is null. - /// The is closed. - public virtual int ReadBlock([In, Out] char[] buffer, int index, int count) - { - int num1; - int num2 = 0; - do - { - num2 += num1 = this.Read(buffer, index + num2, count - num2); - } - while ((num1 > 0) && (num2 < count)); - return num2; - } - - /// Reads a line of characters from the current stream and returns the data as a string. - /// The next line from the input stream, or null if the end of the input stream is reached. - /// An I/O error occurs. - /// There is insufficient memory to allocate a buffer for the returned string. - public override string ReadLine() - { - if (stream == null) - throw new IOException("Reader is closed."); - if ((this.charPos == this.charLen) && (this.ReadBuffer() == 0)) - { - return null; - } - StringBuilder builder1 = null; - while (true) - { - int num1 = this.charPos; - do - { - char ch1 = this.charBuffer[num1]; - if ((ch1 == '\r') || (ch1 == '\n')) - { - string text1; - if (builder1 != null) - { - builder1.Append(this.charBuffer, this.charPos, num1 - this.charPos); - text1 = builder1.ToString(); - } - else - { - text1 = new string(this.charBuffer, this.charPos, num1 - this.charPos); - } - this.charPos = num1 + 1; - if (((ch1 == '\r') && ((this.charPos < this.charLen) || (this.ReadBuffer() > 0))) && (this.charBuffer[this.charPos] == '\n')) - { - this.charPos++; - } - return text1; - } - num1++; - } - while (num1 < this.charLen); - num1 = this.charLen - this.charPos; - if (builder1 == null) - { - builder1 = new StringBuilder(num1 + 80); - } - builder1.Append(this.charBuffer, this.charPos, num1); - if (this.ReadBuffer() <= 0) - { - return builder1.ToString(); - } - } - } - - /// Reads the stream from the current position to the end of the stream. - /// The rest of the stream as a string, from the current position to the end. If the current position is at the end of the stream, returns the empty string(""). - /// An I/O error occurs. - /// There is insufficient memory to allocate a buffer for the returned string. - public override string ReadToEnd() - { - if (stream == null) - throw new IOException("Reader is closed."); - StringBuilder builder1 = new StringBuilder(this.charLen - this.charPos); - do - { - builder1.Append(this.charBuffer, this.charPos, this.charLen - this.charPos); - this.charPos = this.charLen; - this.ReadBuffer(); - } - while (this.charLen > 0); - return builder1.ToString(); - } - - private int ReadBuffer() - { - byteCount = stream.Read(out byteBuffer); - int maxCharsPerBuffer = encoding.GetMaxCharCount(byteCount); - if (charBuffer == null || charBuffer.Length < maxCharsPerBuffer) - { - charBuffer = new char[maxCharsPerBuffer]; - } - - charLen = 0; - charPos = 0; - } - - /// Closes the underlying stream, releases the unmanaged resources used by the , and optionally releases the managed resources. - /// true to release both managed and unmanaged resources; false to release only unmanaged resources. - protected override void Dispose(bool disposing) - { - try - { - base.Dispose(disposing); - } - finally - { - this.stream = null; - this.encoding = null; - } - } - } -#endif -} diff --git a/src/Microsoft.ML.Sweeper/Microsoft.ML.Sweeper.csproj b/src/Microsoft.ML.Sweeper/Microsoft.ML.Sweeper.csproj index 7e022fd5ba..265809dd62 100644 --- a/src/Microsoft.ML.Sweeper/Microsoft.ML.Sweeper.csproj +++ b/src/Microsoft.ML.Sweeper/Microsoft.ML.Sweeper.csproj @@ -12,7 +12,6 @@ - From 1b1e5bbef3d6303156a88ef8a49dcebd9c405e38 Mon Sep 17 00:00:00 2001 From: Daniel Drews Date: Wed, 11 Jul 2018 19:05:13 -0500 Subject: [PATCH 4/6] Address notes from @Ivanidzo4ka --- src/Microsoft.ML.CpuMath/Thunk.cs | 244 +++++++++--------- .../Application/LogLossApplication.cs | 2 +- .../FactorizationMachineInterface.cs | 6 +- 3 files changed, 126 insertions(+), 126 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/Thunk.cs b/src/Microsoft.ML.CpuMath/Thunk.cs index b2df7760f6..bc23963bbe 100644 --- a/src/Microsoft.ML.CpuMath/Thunk.cs +++ b/src/Microsoft.ML.CpuMath/Thunk.cs @@ -17,155 +17,155 @@ internal unsafe static class Thunk public static extern bool ChkAvx(); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulA(bool add, float* pmat, float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulA(bool add, /*const*/ float* pmat, /*const*/ float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulX(bool add, float* pmat, float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulX(bool add, /*const*/ float* pmat, /*const*/ float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulPA(bool add, float* pmat, int* pposSrc, float* psrc, + public static extern void MatMulPA(bool add, /*const*/ float* pmat, /*const*/ int* pposSrc, /*const*/ float* psrc, int posMin, int iposMin, int iposLim, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulPX(bool add, float* pmat, int* pposSrc, float* psrc, + public static extern void MatMulPX(bool add, /*const*/ float* pmat, /*const*/ int* pposSrc, /*const*/ float* psrc, int posMin, int iposMin, int iposLim, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulRU(bool add, int* pstarts, int* pindices, float* pcoefs, - float* psrc, float* pdst, int crow); + public static extern void MatMulRU(bool add, /*const*/ int* pstarts, /*const*/ int* pindices, /*const*/ float* pcoefs, + /*const*/ float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulRX(bool add, int* pstarts, int* pindices, float* pcoefs, - float* psrc, float* pdst, int crow); + public static extern void MatMulRX(bool add, /*const*/ int* pstarts, /*const*/ int* pindices, /*const*/ float* pcoefs, + /*const*/ float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulCU(bool add, int* pmprowiv, int* pmprowcol, - int* pruns, float* pcoefs, float* psrc, float* pdst, int crow); + public static extern void MatMulCU(bool add, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, + /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulDU(bool add, int* pmprowiv, int* pmprowcol, int* pmprowrun, - int* pruns, float* pcoefs, float* psrc, float* pdst, int crow); + public static extern void MatMulDU(bool add, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, /*const*/ int* pmprowrun, + /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulCX(bool add, int* pmprowiv, int* pmprowcol, - int* pruns, float* pcoefs, float* psrc, float* pdst, int crow); + public static extern void MatMulCX(bool add, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, + /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulDX(bool add, int* pmprowiv, int* pmprowcol, int* pmprowrun, - int* pruns, float* pcoefs, float* psrc, float* pdst, int crow); + public static extern void MatMulDX(bool add, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, /*const*/ int* pmprowrun, + /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MeanU(bool add, int* pmprowcol, int* pmprowindices, int* pindices, - float* psrc, float* pdst, int crow); + public static extern void MeanU(bool add, /*const*/ int* pmprowcol, /*const*/ int* pmprowindices, /*const*/ int* pindices, + /*const*/ float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MaxU(bool add, int* pmprowcol, int* pmprowindices, int* pindices, - float* psrc, float* pdst, int crow); + public static extern void MaxU(bool add, /*const*/ int* pmprowcol, /*const*/ int* pmprowindices, /*const*/ int* pindices, + /*const*/ float* psrc, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void RespNormU(bool add, float alpha, float beta, bool avgOverFullKernel, float offset, - int* pmprowcol, int* pmprowindices, int* pindices, - float* psrc, float* pdst, int crow); + /*const*/ int* pmprowcol, /*const*/ int* pmprowindices, /*const*/ int* pindices, + /*const*/ float* psrc, float* pdst, int crow); // These treat pmat as if it is stored in column-major order. Thus, crow and ccol are the numbers of rows // and columns from that perspective. Alternatively, crow is the number of rows in the transpose of pmat // (thought of as row-major order). [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranA(bool add, float* pmat, float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranA(bool add, /*const*/ float* pmat, /*const*/ float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranX(bool add, float* pmat, float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranX(bool add, /*const*/ float* pmat, /*const*/ float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranPA(bool add, float* pmat, int* pposSrc, float* psrc, + public static extern void MatMulTranPA(bool add, /*const*/ float* pmat, /*const*/ int* pposSrc, /*const*/ float* psrc, int posMin, int iposMin, int iposLim, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranPX(bool add, float* pmat, int* pposSrc, float* psrc, + public static extern void MatMulTranPX(bool add, /*const*/ float* pmat, /*const*/ int* pposSrc, /*const*/ float* psrc, int posMin, int iposMin, int iposLim, float* pdst, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranRU(bool add, int* pstarts, int* pindices, float* pcoefs, - float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranRU(bool add, /*const*/ int* pstarts, /*const*/ int* pindices, /*const*/ float* pcoefs, + /*const*/ float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranRX(bool add, int* pstarts, int* pindices, float* pcoefs, - float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranRX(bool add, /*const*/ int* pstarts, /*const*/ int* pindices, /*const*/ float* pcoefs, + /*const*/ float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranCU(bool add, int* pmpcoliv, int* pmpcolrow, - int* pruns, float* pcoefs, float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranCU(bool add, /*const*/ int* pmpcoliv, /*const*/ int* pmpcolrow, + /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranDU(bool add, int* pmpcoliv, int* pmpcolrow, int* pmpcolrun, - int* pruns, float* pcoefs, float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranDU(bool add, /*const*/ int* pmpcoliv, /*const*/ int* pmpcolrow, /*const*/ int* pmpcolrun, + /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranCX(bool add, int* pmpcoliv, int* pmpcolrow, - int* pruns, float* pcoefs, float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranCX(bool add, /*const*/ int* pmpcoliv, /*const*/ int* pmpcolrow, + /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MatMulTranDX(bool add, int* pmpcoliv, int* pmpcolrow, int* pmpcolrun, - int* pruns, float* pcoefs, float* psrc, float* pdst, int crow, int ccol); + public static extern void MatMulTranDX(bool add, /*const*/ int* pmpcoliv, /*const*/ int* pmpcolrow, /*const*/ int* pmpcolrun, + /*const*/ int* pruns, /*const*/ float* pcoefs, /*const*/ float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MeanBackU(bool add, int* pmpcolrow, int* pmpcolindices, int* pindices, - float* psrc, float* pdst, int crow, int ccol); + public static extern void MeanBackU(bool add, /*const*/ int* pmpcolrow, /*const*/ int* pmpcolindices, /*const*/ int* pindices, + /*const*/ float* psrc, float* pdst, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MaxBackU(bool add, int* pmpcolrow, int* pmpcolindices, int* pindices, - float* psrc, float* pdst, float* pval, int crow, int ccol); + public static extern void MaxBackU(bool add, /*const*/ int* pmpcolrow, /*const*/ int* pmpcolindices, /*const*/ int* pindices, + /*const*/ float* psrc, float* pdst, /*const*/ float* pval, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void RespNormBackU(bool add, float alpha, float beta, bool avgOverFullKernel, float offset, - int* pmpcolrow, int* pmpcolindices, int* pindices, - float* perrors, float* perrorsPrev, float* pvaluesPrev, int crow, int ccol); + /*const*/ int* pmpcolrow, /*const*/ int* pmpcolindices, /*const*/ int* pindices, + /*const*/ float* perrors, float* perrorsPrev, /*const*/ float* pvaluesPrev, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranA(float a, float* px, float* py, float* pmat, int crow, int ccol, float decay); + public static extern void AddXYTranA(float a, /*const*/ float* px, /*const*/ float* py, float* pmat, int crow, int ccol, float decay); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranX(float a, float* px, float* py, float* pmat, int crow, int ccol, float decay); + public static extern void AddXYTranX(float a, /*const*/ float* px, /*const*/ float* py, float* pmat, int crow, int ccol, float decay); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranPA(float a, float* px, int* pposY, float* pvaluesY, + public static extern void AddXYTranPA(float a, /*const*/ float* px, /*const*/ int* pposY, /*const*/ float* pvaluesY, int posMinY, int iposMinY, int iposLimY, float* pmat, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranPX(float a, float* px, int* pposY, float* pvaluesY, + public static extern void AddXYTranPX(float a, /*const*/ float* px, /*const*/ int* pposY, /*const*/ float* pvaluesY, int posMinY, int iposMinY, int iposLimY, float* pmat, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranRU(float a, float* px, float* py, - int* pstarts, int* pindices, float* pcoefs, int crow, float decay); + public static extern void AddXYTranRU(float a, /*const*/ float* px, /*const*/ float* py, + /*const*/ int* pstarts, /*const*/ int* pindices, float* pcoefs, int crow, float decay); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranRX(float a, float* px, float* py, - int* pstarts, int* pindices, float* pcoefs, int crow, float decay); + public static extern void AddXYTranRX(float a, /*const*/ float* px, /*const*/ float* py, + /*const*/ int* pstarts, /*const*/ int* pindices, float* pcoefs, int crow, float decay); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranCU(float a, float* px, float* py, int* pmprowiv, int* pmprowcol, - int* pruns, float* pcoefs, int crow); + public static extern void AddXYTranCU(float a, /*const*/ float* px, /*const*/ float* py, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, + /*const*/ int* pruns, float* pcoefs, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranDU(float a, float* px, float* py, int* pmprowiv, int* pmprowcol, - int* pmprowrun, int* pruns, float* pcoefs, int crow); + public static extern void AddXYTranDU(float a, /*const*/ float* px, /*const*/ float* py, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, + /*const*/ int* pmprowrun, /*const*/ int* pruns, float* pcoefs, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranCX(float a, float* px, float* py, int* pmprowiv, int* pmprowcol, - int* pruns, float* pcoefs, int crow); + public static extern void AddXYTranCX(float a, /*const*/ float* px, /*const*/ float* py, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, + /*const*/ int* pruns, float* pcoefs, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranDX(float a, float* px, float* py, int* pmprowiv, int* pmprowcol, - int* pmprowrun, int* pruns, float* pcoefs, int crow); + public static extern void AddXYTranDX(float a, /*const*/ float* px, /*const*/ float* py, /*const*/ int* pmprowiv, /*const*/ int* pmprowcol, + /*const*/ int* pmprowrun, /*const*/ int* pruns, float* pcoefs, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranMomA(float a, float* px, float* py, float* pmat, float momentum, float* pdel, int crow, int ccol); + public static extern void AddXYTranMomA(float a, /*const*/ float* px, /*const*/ float* py, float* pmat, float momentum, float* pdel, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranMomX(float a, float* px, float* py, float* pmat, float momentum, float* pdel, int crow, int ccol); + public static extern void AddXYTranMomX(float a, /*const*/ float* px, /*const*/ float* py, float* pmat, float momentum, float* pdel, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranGradA(float* px, float* py, float* pmat, float* paccGrads, float* paccUpdates, + public static extern void AddXYTranGradA(/*const*/ float* px, /*const*/ float* py, float* pmat, float* paccGrads, float* paccUpdates, float decay, float cond, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranGradX(float* px, float* py, float* pmat, float* paccGrads, float* paccUpdates, + public static extern void AddXYTranGradX(/*const*/ float* px, /*const*/ float* py, float* pmat, float* paccGrads, float* paccUpdates, float decay, float cond, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranGradRU(float* px, float* py, int* pstarts, int* pindices, + public static extern void AddXYTranGradRU(/*const*/ float* px, /*const*/ float* py, /*const*/ int* pstarts, /*const*/ int* pindices, float* pcoefs, float* paccGrads, float* paccUpdates, float decay, float cond, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranGradRX(float* px, float* py, int* pstarts, int* pindices, + public static extern void AddXYTranGradRX(/*const*/ float* px, /*const*/ float* py, /*const*/ int* pstarts, /*const*/ int* pindices, float* pcoefs, float* paccGrads, float* paccUpdates, float decay, float cond, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranGradPA(float* px, int* pposY, float* pvaluesY, + public static extern void AddXYTranGradPA(/*const*/ float* px, /*const*/ int* pposY, /*const*/ float* pvaluesY, int posMinY, int iposMinY, int iposLimY, float* pmat, float* paccGrads, float* paccUpdates, float decay, float cond, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddXYTranGradPX(float* px, int* pposY, float* pvaluesY, + public static extern void AddXYTranGradPX(/*const*/ float* px, /*const*/ int* pposY, /*const*/ float* pvaluesY, int posMinY, int iposMinY, int iposLimY, float* pmat, float* paccGrads, float* paccUpdates, float decay, float cond, int crow, int ccol); @@ -176,7 +176,7 @@ public static extern void AddXYTranGradPX(float* px, int* pposY, float* pvaluesY [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void ScaleX(float a, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ScaleSrcU(float a, float* ps, float* pd, int c); + public static extern void ScaleSrcU(float a, /*const*/ float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void ScaleAddU(float a, float b, float* pd, int c); @@ -187,85 +187,85 @@ public static extern void AddXYTranGradPX(float* px, int* pposY, float* pvaluesY [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void ScaleMaxNormTranU(float maxNorm, float* pmat, int crow, int ccol); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ScaleMaxNormRU(float maxNorm, int* pstarts, float* pmat, int crow); + public static extern void ScaleMaxNormRU(float maxNorm, /*const*/ int* pstarts, float* pmat, int crow); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void ScaleMaxNormCU(float maxNorm, int kernCount, int kernSize, float* pmat); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleA(float a, float* ps, float* pd, int c); + public static extern void AddScaleA(float a, /*const*/ float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleU(float a, float* ps, float* pd, int c); + public static extern void AddScaleU(float a, /*const*/ float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleX(float a, float* ps, float* pd, int c); + public static extern void AddScaleX(float a, /*const*/ float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleSU(float a, float* ps, int* pi, float* pd, int c); + public static extern void AddScaleSU(float a, /*const*/ float* ps, /*const*/ int* pi, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleCopyU(float a, float* ps, float* pd, float* pr, int c); + public static extern void AddScaleCopyU(float a, /*const*/ float* ps, /*const*/ float* pd, float* pr, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleMomA(float a, float* ps, float* pd, float momentum, float* pdel, int c); + public static extern void AddScaleMomA(float a, /*const*/ float* ps, float* pd, float momentum, float* pdel, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleMomX(float a, float* ps, float* pd, float momentum, float* pdel, int c); + public static extern void AddScaleMomX(float a, /*const*/ float* ps, float* pd, float momentum, float* pdel, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleGradA(float* ps, float* pd, float* paccGrads, float* paccUpdates, + public static extern void AddScaleGradA(/*const*/ float* ps, float* pd, float* paccGrads, float* paccUpdates, float decay, float cond, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleGradX(float* ps, float* pd, float* paccGrads, float* paccUpdates, + public static extern void AddScaleGradX(/*const*/ float* ps, float* pd, float* paccGrads, float* paccUpdates, float decay, float cond, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddScaleMultiA(int count, float* ps, float* pd, float* paccGrads, + public static extern void AddScaleMultiA(int count, /*const*/ float* ps, float* pd, float* paccGrads, float* paccUpdates, float decay, float cond, int size); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void AddScalarU(float a, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddU(float* ps, float* pd, int c); + public static extern void AddU(/*const*/ float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddA(float* ps, float* pd, int c); + public static extern void AddA(/*const*/ float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddX(float* ps, float* pd, int c); + public static extern void AddX(/*const*/ float* ps, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void AddSU(float* ps, int* pi, float* pd, int c); + public static extern void AddSU(/*const*/ float* ps, /*const*/ int* pi, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumA(float* ps, int c); + public static extern float SumA(/*const*/ float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumU(float* ps, int c); + public static extern float SumU(/*const*/ float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumX(float* ps, int c); + public static extern float SumX(/*const*/ float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumSqU(float* ps, int c); + public static extern float SumSqU(/*const*/ float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumSqDiffU(float mean, float* ps, int c); + public static extern float SumSqDiffU(float mean, /*const*/ float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumAbsU(float* ps, int c); + public static extern float SumAbsU(/*const*/ float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float SumAbsDiffU(float mean, float* ps, int c); + public static extern float SumAbsDiffU(float mean, /*const*/ float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float MulElementWiseU(float* ps1, float* ps2, float* pd, int c); + public static extern float MulElementWiseU(/*const*/ float* ps1, /*const*/float* ps2, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float MulElementWiseSU(float* ps1, float* ps2, int* pi, float* pd, int c); + public static extern float MulElementWiseSU(/*const*/ float* ps1, /*const*/float* ps2, /*const*/ int* pi, float* pd, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float MaxAbsU(float* ps, int c); + public static extern float MaxAbsU(/*const*/ float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float MaxAbsDiffU(float mean, float* ps, int c); + public static extern float MaxAbsDiffU(float mean, /*const*/ float* ps, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float DotU(float* pa, float* pb, int c); + public static extern float DotU(/*const*/ float* pa, /*const*/ float* pb, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float DotSU(float* pa, float* pb, int* pi, int c); + public static extern float DotSU(/*const*/ float* pa, /*const*/ float* pb, /*const*/ int* pi, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern float Dist2(float* px, float* py, int c); + public static extern float Dist2(/*const*/ float* px, /*const*/ float* py, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplySigmoidA(float* ps, float* pd, int c); + public static extern void ApplySigmoidA(/*const*/ float* ps, float* pd, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplySigmoidX(float* ps, float* pd, int c) + public static void ApplySigmoidX(/*const*/ float* ps, float* pd, int c) { ApplySigmoidA(ps, pd, c); } @@ -345,84 +345,84 @@ public static void ApplyBoundedRectifiedLinearX(float* ps, float* pd, int c) } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplySigmoidDerivativeA(float* pv, float* pg, int c); + public static extern void ApplySigmoidDerivativeA(/*const*/ float* pv, float* pg, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplySigmoidDerivativeX(float* pv, float* pg, int c) + public static void ApplySigmoidDerivativeX(/*const*/ float* pv, float* pg, int c) { ApplySigmoidDerivativeA(pv, pg, c); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplyRectifiedLinearDerivativeA(float* pv, float* pg, int c); + public static extern void ApplyRectifiedLinearDerivativeA(/*const*/ float* pv, float* pg, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplyRectifiedLinearDerivativeX(float* pv, float* pg, int c) + public static void ApplyRectifiedLinearDerivativeX(/*const*/ float* pv, float* pg, int c) { ApplyRectifiedLinearDerivativeA(pv, pg, c); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplySquareDerivativeA(float* px, float* py, float* pg, int c, bool drop); + public static extern void ApplySquareDerivativeA(/*const*/ float* px, /*const*/ float* py, float* pg, int c, bool drop); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplySquareDerivativeX(float* px, float* py, float* pg, int c, bool drop) + public static void ApplySquareDerivativeX(/*const*/ float* px, /*const*/ float* py, float* pg, int c, bool drop) { ApplySquareDerivativeA(px, py, pg, c, drop); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplySqrtDerivativeA(float* pv, float* pg, int c); + public static extern void ApplySqrtDerivativeA(/*const*/ float* pv, float* pg, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplySqrtDerivativeX(float* pv, float* pg, int c) + public static void ApplySqrtDerivativeX(/*const*/ float* pv, float* pg, int c) { ApplySqrtDerivativeA(pv, pg, c); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplySoftRectifiedLinearDerivativeU(float* px, float* py, float* pg, int c); + public static extern void ApplySoftRectifiedLinearDerivativeU(/*const*/ float* px, /*const*/ float* py, float* pg, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplySoftRectifiedLinearDerivativeA(float* px, float* py, float* pg, int c) + public static void ApplySoftRectifiedLinearDerivativeA(/*const*/ float* px, /*const*/ float* py, float* pg, int c) { ApplySoftRectifiedLinearDerivativeU(px, py, pg, c); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplySoftRectifiedLinearDerivativeX(float* px, float* py, float* pg, int c) + public static void ApplySoftRectifiedLinearDerivativeX(/*const*/ float* px, /*const*/ float* py, float* pg, int c) { ApplySoftRectifiedLinearDerivativeU(px, py, pg, c); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplyAbsDerivativeA(float* px, float* py, float* pg, int c, bool drop); + public static extern void ApplyAbsDerivativeA(/*const*/ float* px, /*const*/ float* py, float* pg, int c, bool drop); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplyAbsDerivativeX(float* px, float* py, float* pg, int c, bool drop) + public static void ApplyAbsDerivativeX(/*const*/ float* px, /*const*/ float* py, float* pg, int c, bool drop) { ApplyAbsDerivativeA(px, py, pg, c, drop); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplyTanhDerivativeA(float* pv, float* pg, int c); + public static extern void ApplyTanhDerivativeA(/*const*/ float* pv, float* pg, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplyTanhDerivativeX(float* pv, float* pg, int c) + public static void ApplyTanhDerivativeX(/*const*/ float* pv, float* pg, int c) { ApplyTanhDerivativeA(pv, pg, c); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ApplyBoundedRectifiedLinearDerivativeA(float* pv, float* pg, int c); + public static extern void ApplyBoundedRectifiedLinearDerivativeA(/*const*/ float* pv, float* pg, int c); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void ApplyBoundedRectifiedLinearDerivativeX(float* pv, float* pg, int c) + public static void ApplyBoundedRectifiedLinearDerivativeX(/*const*/ float* pv, float* pg, int c) { ApplyBoundedRectifiedLinearDerivativeA(pv, pg, c); } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ZeroItemsU(float* pd, int c, int* pindices, int cindices); + public static extern void ZeroItemsU(float* pd, int c, /*const*/ int* pindices, int cindices); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void ZeroMatrixItemsCore(float* pd, int c, int ccol, int cfltRow, int* pindices, int cindices); + public static extern void ZeroMatrixItemsCore(float* pd, int c, int ccol, int cfltRow, /*const*/ int* pindices, int cindices); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void SdcaL1UpdateU(float primalUpdate, float* ps, float threshold, float* pd1, float* pd2, int c); + public static extern void SdcaL1UpdateU(float primalUpdate, /*const*/ float* ps, float threshold, float* pd1, float* pd2, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void SdcaL1UpdateSU(float primalUpdate, float* ps, int* pi, float threshold, float* pd1, float* pd2, int c); + public static extern void SdcaL1UpdateSU(float primalUpdate, /*const*/ float* ps, /*const*/ int* pi, float threshold, float* pd1, float* pd2, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void ScaleAdadeltaU(float* mat, float* accGrads, float* accUpdates, float decay, float cond, float* grads, int size); @@ -435,7 +435,7 @@ public static void ApplyBoundedRectifiedLinearDerivativeX(float* pv, float* pg, // In CoreCLR we use Buffer.MemoryCopy directly instead of // plumbing our own version. [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void MemCpy(void* dst, void* src, long count); + public static extern void MemCpy(void* dst, /*const*/ void* src, long count); #endif } } diff --git a/src/Microsoft.ML.FastTree/Application/LogLossApplication.cs b/src/Microsoft.ML.FastTree/Application/LogLossApplication.cs index 2c2a254ad8..15d449ea72 100644 --- a/src/Microsoft.ML.FastTree/Application/LogLossApplication.cs +++ b/src/Microsoft.ML.FastTree/Application/LogLossApplication.cs @@ -106,7 +106,7 @@ protected override void GetGradientInOneQuery(int query, int threadIndex) double delta = (_coef * labelDiff) / (1.0 + Math.Exp(_coef * labelDiff * (_scores[d1] - _scores[d2]))); _gradient[d1] += delta; - _gradient[d2] -= delta; + _gradient[d2] -= delta; } } } diff --git a/src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachineInterface.cs b/src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachineInterface.cs index eae140f2d2..b5fdbd0262 100644 --- a/src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachineInterface.cs +++ b/src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachineInterface.cs @@ -31,12 +31,12 @@ private static bool Compat(AlignedArray a) } [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void CalculateIntermediateVariablesNative(int fieldCount, int latentDim, int count, int* fieldIndices, int* featureIndices, - float* featureValues, float* linearWeights, float* latentWeights, float* latentSum, float* response); + public static extern void CalculateIntermediateVariablesNative(int fieldCount, int latentDim, int count, int* /*const*/ fieldIndices, int* /*const*/ featureIndices, + float* /*const*/ featureValues, float* /*const*/ linearWeights, float* /*const*/ latentWeights, float* latentSum, float* response); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void CalculateGradientAndUpdateNative(float lambdaLinear, float lambdaLatent, float learningRate, int fieldCount, int latentDim, float weight, - int count, int* fieldIndices, int* featureIndices, float* featureValues, float* latentSum, float slope, + int count, int* /*const*/ fieldIndices, int* /*const*/ featureIndices, float* /*const*/ featureValues, float* /*const*/ latentSum, float slope, float* linearWeights, float* latentWeights, float* linearAccumulatedSquaredGrads, float* latentAccumulatedSquaredGrads); public static void CalculateIntermediateVariables(int fieldCount, int latentDim, int count, int[] fieldIndices, int[] featureIndices, float[] featureValues, From 7bcd0a6fd0812f9934862fca99b58f30a3ca5bbb Mon Sep 17 00:00:00 2001 From: Daniel Drews Date: Wed, 11 Jul 2018 19:06:40 -0500 Subject: [PATCH 5/6] Remove TreeOrderedCandidatesSearch --- .../Algorithms/SmacSweeper.cs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/Microsoft.ML.Sweeper/Algorithms/SmacSweeper.cs b/src/Microsoft.ML.Sweeper/Algorithms/SmacSweeper.cs index 927ffc6b5d..795c06084b 100644 --- a/src/Microsoft.ML.Sweeper/Algorithms/SmacSweeper.cs +++ b/src/Microsoft.ML.Sweeper/Algorithms/SmacSweeper.cs @@ -182,24 +182,6 @@ private ParameterSet[] GenerateCandidateConfigurations(int numOfCandidates, IEnu return configs; } - private ParameterSet[] TreeOrderedCandidatesSearch(FastForestRegressionPredictor forest, int numOfCandidates, IEnumerable previousRuns) - { - // Step 1: Get ordered list of all leaf values. - SortedList> leafValueList = new SortedList>(Comparer.Create((x, y) => y.CompareTo(x))); - for (int i = 0; i < forest.TrainedEnsemble.NumTrees; i++) - { - RegressionTree t = forest.TrainedEnsemble.GetTreeAt(i); - for (int j = 0; j < t.NumLeaves; j++) - { - double val = t.LeafValue(j); - while (leafValueList.ContainsKey(val)) - val += Double.Epsilon; - leafValueList.Add(val, Tuple.Create(i, j)); - } - } - return null; - } - /// /// Does a mix of greedy local search around best performing parameter sets, while throwing random parameter sets into the mix. /// From d41d11691a1bfbb012e1827ca034f5cf99f3ca85 Mon Sep 17 00:00:00 2001 From: Daniel Drews Date: Thu, 12 Jul 2018 20:33:39 -0500 Subject: [PATCH 6/6] Remove whitespace and reinstate commented out tests --- src/Microsoft.ML.Core/Utilities/MathUtils.cs | 2 +- .../UnitTests/TestEntryPoints.cs | 110 +++++++++++++++++- 2 files changed, 109 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ML.Core/Utilities/MathUtils.cs b/src/Microsoft.ML.Core/Utilities/MathUtils.cs index 65090e1b85..7fd0829708 100644 --- a/src/Microsoft.ML.Core/Utilities/MathUtils.cs +++ b/src/Microsoft.ML.Core/Utilities/MathUtils.cs @@ -257,7 +257,7 @@ public static Float SoftMax(Float[] inputs, int count) if (count == 1) return max; - + double intermediate = 0.0; Float cutoff = max - LogTolerance; diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index 30a2df8337..1fe05f239c 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -201,6 +201,55 @@ public void EntryPointCaching() Done(); } + //[Fact] + //public void EntryPointSchemaManipulation() + //{ + // var dv1_data = new[] + // { + // new Dv1 { Col1 = 11, Col2 = 21, Col3 = 31, Col4 = 41 }, + // new Dv1 { Col1 = 12, Col2 = 22, Col3 = 32, Col4 = 42 }, + // new Dv1 { Col1 = 13, Col2 = 23, Col3 = 33, Col4 = 43 }, + // }; + // var dv1 = Env.CreateDataView(dv1_data); + + // var concatOut = SchemaManipulation.ConcatColumns(Env, + // new ConcatTransform.Arguments { Column = new[] { ConcatTransform.Column.Parse("ColA:Col1,Col2") }, Data = dv1 }); + + // var postConcatDv = Env.CreateTransform("Concat{col=ColA:Col1,Col2}", dv1); + // CheckSameValues(concatOut.OutputData, postConcatDv); + + // var copyOut = SchemaManipulation.CopyColumns(Env, + // new CopyColumnsTransform.Arguments + // { + // Column = new[] { CopyColumnsTransform.Column.Parse("ColB:Col3"), CopyColumnsTransform.Column.Parse("ColC:Col4") }, + // Data = concatOut.OutputData, + // }); + + // var postCopyDv = Env.CreateTransform("Copy{col=ColB:Col3 col=ColC:Col4}", postConcatDv); + // CheckSameValues(copyOut.OutputData, postCopyDv); + + // var dropOut = SchemaManipulation.DropColumns(Env, + // new DropColumnsTransform.Arguments { Column = new[] { "Col1", "Col2", "Col3", "Col4" }, Data = copyOut.OutputData }); + + // var postDropDv = Env.CreateTransform("Drop{col=Col1 col=Col2 col=Col3 col=Col4}", postCopyDv); + // CheckSameValues(dropOut.OutputData, postDropDv); + + // var selectOut = SchemaManipulation.SelectColumns(Env, + // new DropColumnsTransform.KeepArguments { Column = new[] { "ColA", "ColB" }, Data = dropOut.OutputData }); + + // var postSelectDv = Env.CreateTransform("Keep{col=ColA col=ColB}", postDropDv); + + // CheckSameValues(selectOut.OutputData, postSelectDv); + + // var combinedModel = ModelOperations.CombineTransformModels(Env, + // new ModelOperations.CombineTransformModelsInput + // { + // Models = new[] { concatOut.Model, copyOut.Model, dropOut.Model, selectOut.Model } + // }).OutputModel; + // CheckSameValues(selectOut.OutputData, combinedModel.Apply(Env, dv1)); + // Done(); + //} + /// /// Helper function to get the type of build being used. /// @@ -551,6 +600,63 @@ public void EntryPointExecGraphCommand() cmd.Run(); } + //[Fact] + //public void EntryPointArrayOfVariables() + //{ + // string inputGraph = @" + // { + // ""Nodes"": [ + // { + // ""Name"": ""SchemaManipulation.ConcatColumns"", + // ""Inputs"": { + // ""Data"": ""$data1"", + // ""Column"": [{""Name"":""ColA"", ""Source"":[""Col1"", ""Col2""]}] + // }, + // ""Outputs"": { + // ""Model"": ""$model1"", + // ""OutputData"": ""$data2"" + // } + // }, + // { + // ""Name"": ""SchemaManipulation.CopyColumns"", + // ""Inputs"": { + // ""Data"": ""$data2"", + // ""Column"": [{""Name"":""ColB"", ""Source"":""Col3""}, {""Name"":""ColC"", ""Source"":""Col4""}] + // }, + // ""Outputs"": { + // ""Model"": ""$model2"", + // ""OutputData"": ""$data3"" + // } + // }, + // { + // ""Name"": ""ModelOperations.CombineTransformModels"", + // ""Inputs"": { + // Models: [""$model1"", ""$model2""] + // }, + // ""Outputs"": { + // OutputModel: ""$model3"" + // } + // } + // ] + // }"; + + // JObject graph = JObject.Parse(inputGraph); + // var catalog = ModuleCatalog.CreateInstance(Env); + // var runner = new GraphRunner(Env, catalog, graph[FieldNames.Nodes] as JArray); + + // var dv1_data = new[] + // { + // new Dv1 { Col1 = 11, Col2 = 21, Col3 = 31, Col4 = 41 }, + // new Dv1 { Col1 = 12, Col2 = 22, Col3 = 32, Col4 = 42 }, + // new Dv1 { Col1 = 13, Col2 = 23, Col3 = 33, Col4 = 43 }, + // }; + // var dv1 = Env.CreateDataView(dv1_data); + // runner.SetInput("data1", dv1); + // runner.RunAll(); + // var model = runner.GetOutput("model3"); + // Assert.NotNull(model); + //} + [Fact] public void EntryPointCalibrate() { @@ -687,7 +793,7 @@ public void EntryPointPipelineEnsemble() Data = splitOutput.TestData[nModels], PredictorModel = regressionEnsembleModel }).ScoredData; - + var anomalyEnsembleModel = EntryPoints.EnsembleCreator.CreateAnomalyPipelineEnsemble(Env, new EntryPoints.EnsembleCreator.PipelineAnomalyInput() { @@ -3318,7 +3424,7 @@ public void EntryPointLinearPredictorSummary() { var dataPath = GetDataPath("breast-cancer-withheader.txt"); var inputFile = new SimpleFileHandle(Env, dataPath, false, false); - + var dataView = ImportTextData.TextLoader(Env, new ImportTextData.LoaderInput() { Arguments =