diff --git a/src/Microsoft.Data.Analysis/DataFrame.IO.cs b/src/Microsoft.Data.Analysis/DataFrame.IO.cs
index eff4c3a90b..1d39d08258 100644
--- a/src/Microsoft.Data.Analysis/DataFrame.IO.cs
+++ b/src/Microsoft.Data.Analysis/DataFrame.IO.cs
@@ -172,7 +172,7 @@ private static DataFrameColumn CreateColumn(Type kind, string[] columnNames, int
return ret;
}
- private static DataFrame ReadCsvLinesIntoDataFrame(IEnumerable<string> lines,
+ private static DataFrame ReadCsvLinesIntoDataFrame(WrappedStreamReaderOrStringReader wrappedReader,
char separator = ',', bool header = true,
string[] columnNames = null, Type[] dataTypes = null,
long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false
@@ -183,140 +183,139 @@ private static DataFrame ReadCsvLinesIntoDataFrame(IEnumerable<string> lines,
throw new ArgumentException(string.Format(Strings.ExpectedEitherGuessRowsOrDataTypes, nameof(guessRows), nameof(dataTypes)));
}
- var linesForGuessType = new List<string[]>();
- long rowline = 0;
- int numberOfColumns = dataTypes?.Length ?? 0;
-
- if (header == true && numberOfRowsToRead != -1)
+ List<DataFrameColumn> columns;
+ string[] fields;
+ using (var textReader = wrappedReader.GetTextReader())
{
- numberOfRowsToRead++;
- }
+ TextFieldParser parser = new TextFieldParser(textReader);
+ parser.SetDelimiters(separator.ToString());
- List<DataFrameColumn> columns;
- // First pass: schema and number of rows.
- string line = null;
+ var linesForGuessType = new List<string[]>();
+ long rowline = 0;
+ int numberOfColumns = dataTypes?.Length ?? 0;
- var enumerator = lines.GetEnumerator();
- while (enumerator.MoveNext())
- {
- line = enumerator.Current;
- if ((numberOfRowsToRead == -1) || rowline < numberOfRowsToRead)
+ if (header == true && numberOfRowsToRead != -1)
+ {
+ numberOfRowsToRead++;
+ }
+
+ // First pass: schema and number of rows.
+ while ((fields = parser.ReadFields()) != null)
{
- if (linesForGuessType.Count < guessRows || (header && rowline == 0))
+ if ((numberOfRowsToRead == -1) || rowline < numberOfRowsToRead)
{
- var spl = line.Split(separator);
- if (header && rowline == 0)
+ if (linesForGuessType.Count < guessRows || (header && rowline == 0))
{
- if (columnNames == null)
+ if (header && rowline == 0)
{
- columnNames = spl;
+ if (columnNames == null)
+ {
+ columnNames = fields;
+ }
+ }
+ else
+ {
+ linesForGuessType.Add(fields);
+ numberOfColumns = Math.Max(numberOfColumns, fields.Length);
}
}
- else
- {
- linesForGuessType.Add(spl);
- numberOfColumns = Math.Max(numberOfColumns, spl.Length);
- }
+ }
+ ++rowline;
+ if (rowline == guessRows || guessRows == 0)
+ {
+ break;
}
}
- ++rowline;
- if (rowline == guessRows || guessRows == 0)
+
+ if (rowline == 0)
{
- break;
+ throw new FormatException(Strings.EmptyFile);
}
- }
- if (rowline == 0)
- {
- throw new FormatException(Strings.EmptyFile);
- }
-
- columns = new List<DataFrameColumn>(numberOfColumns);
- // Guesses types or looks up dataTypes and adds columns.
- for (int i = 0; i < numberOfColumns; ++i)
- {
- Type kind = dataTypes == null ? GuessKind(i, linesForGuessType) : dataTypes[i];
- columns.Add(CreateColumn(kind, columnNames, i));
+ columns = new List<DataFrameColumn>(numberOfColumns);
+ // Guesses types or looks up dataTypes and adds columns.
+ for (int i = 0; i < numberOfColumns; ++i)
+ {
+ Type kind = dataTypes == null ? GuessKind(i, linesForGuessType) : dataTypes[i];
+ columns.Add(CreateColumn(kind, columnNames, i));
+ }
}
DataFrame ret = new DataFrame(columns);
- line = null;
// Fill values.
- enumerator.Reset();
- rowline = 0;
- while (enumerator.MoveNext() && (numberOfRowsToRead == -1 || rowline < numberOfRowsToRead))
+ using (var textReader = wrappedReader.GetTextReader())
{
- line = enumerator.Current;
- var spl = line.Split(separator);
- if (header && rowline == 0)
- {
- // Skips.
- }
- else
+ TextFieldParser parser = new TextFieldParser(textReader);
+ parser.SetDelimiters(separator.ToString());
+
+ long rowline = 0;
+ while ((fields = parser.ReadFields()) != null && (numberOfRowsToRead == -1 || rowline < numberOfRowsToRead))
{
- ret.Append(spl, inPlace: true);
+ if (header && rowline == 0)
+ {
+ // Skips.
+ }
+ else
+ {
+ ret.Append(fields, inPlace: true);
+ }
+ ++rowline;
}
- ++rowline;
- }
- if (addIndexColumn)
- {
- PrimitiveDataFrameColumn<int> indexColumn = new PrimitiveDataFrameColumn<int>("IndexColumn", columns[0].Length);
- for (int i = 0; i < columns[0].Length; i++)
+ if (addIndexColumn)
{
- indexColumn[i] = i;
+ PrimitiveDataFrameColumn<int> indexColumn = new PrimitiveDataFrameColumn<int>("IndexColumn", columns[0].Length);
+ for (int i = 0; i < columns[0].Length; i++)
+ {
+ indexColumn[i] = i;
+ }
+ columns.Insert(0, indexColumn);
}
- columns.Insert(0, indexColumn);
- }
- return ret;
- }
- private class CsvLines : IEnumerable<string>
- {
- private CsvLineEnumerator enumerator;
- public CsvLines(CsvLineEnumerator csvLineEnumerator)
- {
- enumerator = csvLineEnumerator;
}
- public IEnumerator<string> GetEnumerator() => enumerator;
-
- IEnumerator IEnumerable.GetEnumerator() => enumerator;
+ return ret;
}
- private class CsvLineEnumerator : IEnumerator<string>
+ private class WrappedStreamReaderOrStringReader
{
- private StreamReader streamReader;
- private string currentLine;
- private long streamStartPosition;
- public CsvLineEnumerator(StreamReader csvStream)
- {
- streamStartPosition = csvStream.BaseStream.Position;
- streamReader = csvStream;
- currentLine = null;
- }
-
- public string Current => currentLine;
-
- object IEnumerator.Current => currentLine;
+ private Stream _stream;
+ private long _initialPosition;
+ private Encoding _encoding;
+ private string _csvString;
- public void Dispose()
+ public WrappedStreamReaderOrStringReader(Stream stream, Encoding encoding)
{
- throw new NotImplementedException();
+ _stream = stream;
+ _initialPosition = stream.Position;
+ _encoding = encoding;
+ _csvString = null;
}
- public bool MoveNext()
+ public WrappedStreamReaderOrStringReader(string csvString)
{
- currentLine = streamReader.ReadLine();
- return currentLine != null;
+ _csvString = csvString;
+ _initialPosition = 0;
+ _encoding = null;
+ _stream = null;
}
- public void Reset()
+ // Returns a new TextReader. If the wrapped object is a stream, the stream is reset to its initial position.
+ public TextReader GetTextReader()
{
- streamReader.DiscardBufferedData();
- streamReader.BaseStream.Seek(streamStartPosition, SeekOrigin.Begin);
+ if (_stream != null)
+ {
+ _stream.Seek(_initialPosition, SeekOrigin.Begin);
+ return new StreamReader(_stream, _encoding, detectEncodingFromByteOrderMarks: true, DefaultStreamReaderBufferSize, leaveOpen: true);
+ }
+ else
+ {
+ return new StringReader(_csvString);
+ }
+
}
+
}
/// <summary>
@@ -336,8 +335,8 @@ public static DataFrame LoadCsvFromString(string csvString,
string[] columnNames = null, Type[] dataTypes = null,
long numberOfRowsToRead = -1, int guessRows = 10, bool addIndexColumn = false)
{
- string[] lines = csvString.Split(new[] { Environment.NewLine }, StringSplitOptions.None);
- return ReadCsvLinesIntoDataFrame(lines, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn);
+ WrappedStreamReaderOrStringReader wrappedStreamReaderOrStringReader = new WrappedStreamReaderOrStringReader(csvString);
+ return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn);
}
/// <summary>
@@ -369,12 +368,8 @@ public static DataFrame LoadCsv(Stream csvStream,
throw new ArgumentException(string.Format(Strings.ExpectedEitherGuessRowsOrDataTypes, nameof(guessRows), nameof(dataTypes)));
}
- using (var streamReader = new StreamReader(csvStream, encoding ?? Encoding.UTF8, detectEncodingFromByteOrderMarks: true, DefaultStreamReaderBufferSize, leaveOpen: true))
- {
- CsvLineEnumerator linesEnumerator = new CsvLineEnumerator(streamReader);
- IEnumerable<string> lines = new CsvLines(linesEnumerator);
- return ReadCsvLinesIntoDataFrame(lines, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn);
- }
+ WrappedStreamReaderOrStringReader wrappedStreamReaderOrStringReader = new WrappedStreamReaderOrStringReader(csvStream, encoding ?? Encoding.UTF8);
+ return ReadCsvLinesIntoDataFrame(wrappedStreamReaderOrStringReader, separator, header, columnNames, dataTypes, numberOfRowsToRead, guessRows, addIndexColumn);
}
/// <summary>
diff --git a/src/Microsoft.Data.Analysis/strings.Designer.cs b/src/Microsoft.Data.Analysis/Strings.Designer.cs
similarity index 68%
rename from src/Microsoft.Data.Analysis/strings.Designer.cs
rename to src/Microsoft.Data.Analysis/Strings.Designer.cs
index fc64940869..ff3cd6cadd 100644
--- a/src/Microsoft.Data.Analysis/strings.Designer.cs
+++ b/src/Microsoft.Data.Analysis/Strings.Designer.cs
@@ -69,6 +69,24 @@ internal static string BadColumnCast {
}
}
+ /// <summary>
+ /// Looks up a localized string similar to Line {0} cannot be parsed with the current Delimiters..
+ /// </summary>
+ internal static string CannotParseWithDelimiters {
+ get {
+ return ResourceManager.GetString("CannotParseWithDelimiters", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to Line {0} cannot be parsed with the current FieldWidths..
+ /// </summary>
+ internal static string CannotParseWithFieldWidths {
+ get {
+ return ResourceManager.GetString("CannotParseWithFieldWidths", resourceCulture);
+ }
+ }
+
/// <summary>
/// Looks up a localized string similar to Cannot resize down.
/// </summary>
@@ -87,6 +105,15 @@ internal static string ColumnIndexOutOfRange {
}
}
+ /// <summary>
+ /// Looks up a localized string similar to Comment token cannot contain whitespace.
+ /// </summary>
+ internal static string CommentTokenCannotContainWhitespace {
+ get {
+ return ResourceManager.GetString("CommentTokenCannotContainWhitespace", resourceCulture);
+ }
+ }
+
/// <summary>
/// Looks up a localized string similar to DataType.
/// </summary>
@@ -96,6 +123,15 @@ internal static string DataType {
}
}
+ /// <summary>
+ /// Looks up a localized string similar to Delimiter cannot be new line characters.
+ /// </summary>
+ internal static string DelimiterCannotBeNewlineChar {
+ get {
+ return ResourceManager.GetString("DelimiterCannotBeNewlineChar", resourceCulture);
+ }
+ }
+
/// <summary>
/// Looks up a localized string similar to Length (excluding null values).
/// </summary>
@@ -114,6 +150,24 @@ internal static string DuplicateColumnName {
}
}
+ /// <summary>
+ /// Looks up a localized string similar to Delimiters is empty..
+ /// </summary>
+ internal static string EmptyDelimiters {
+ get {
+ return ResourceManager.GetString("EmptyDelimiters", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to FieldWidths is empty..
+ /// </summary>
+ internal static string EmptyFieldWidths {
+ get {
+ return ResourceManager.GetString("EmptyFieldWidths", resourceCulture);
+ }
+ }
+
/// <summary>
/// Looks up a localized string similar to Empty file.
/// </summary>
@@ -123,6 +177,15 @@ internal static string EmptyFile {
}
}
+ /// <summary>
+ /// Looks up a localized string similar to Exceeded maximum buffer size..
+ /// </summary>
+ internal static string ExceededMaxBufferSize {
+ get {
+ return ResourceManager.GetString("ExceededMaxBufferSize", resourceCulture);
+ }
+ }
+
/// <summary>
/// Looks up a localized string similar to Parameter.Count exceeds the number of columns({0}) in the DataFrame .
/// </summary>
@@ -150,6 +213,24 @@ internal static string ExpectedEitherGuessRowsOrDataTypes {
}
}
+ /// <summary>
+ /// Looks up a localized string similar to {0} not found..
+ /// </summary>
+ internal static string FileNotFound {
+ get {
+ return ResourceManager.GetString("FileNotFound", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to A double quote is not a legal delimiter when HasFieldsEnclosedInQuotes is set to True..
+ /// </summary>
+ internal static string IllegalQuoteDelimiter {
+ get {
+ return ResourceManager.GetString("IllegalQuoteDelimiter", resourceCulture);
+ }
+ }
+
/// <summary>
/// Looks up a localized string similar to Column is immutable.
/// </summary>
@@ -186,6 +267,15 @@ internal static string InvalidColumnName {
}
}
+ /// <summary>
+ /// Looks up a localized string similar to All field widths, except the last element, must be greater than zero. A field width less than or equal to zero in the last element indicates the last field is of variable length..
+ /// </summary>
+ internal static string InvalidFieldWidths {
+ get {
+ return ResourceManager.GetString("InvalidFieldWidths", resourceCulture);
+ }
+ }
+
/// <summary>
/// Looks up a localized string similar to Line {0} has less columns than expected.
/// </summary>
@@ -195,6 +285,15 @@ internal static string LessColumnsThatExpected {
}
}
+ /// <summary>
+ /// Looks up a localized string similar to Line {0} cannot be read because it exceeds the max line size..
+ /// </summary>
+ internal static string LineExceedsMaxLineSize {
+ get {
+ return ResourceManager.GetString("LineExceedsMaxLineSize", resourceCulture);
+ }
+ }
+
/// <summary>
/// Looks up a localized string similar to MapIndices exceeds column length.
/// </summary>
@@ -266,6 +365,24 @@ internal static string NotSupportedColumnType {
return ResourceManager.GetString("NotSupportedColumnType", resourceCulture);
}
}
+
+ /// <summary>
+ /// Looks up a localized string similar to Delimiters is null..
+ /// </summary>
+ internal static string NullDelimiters {
+ get {
+ return ResourceManager.GetString("NullDelimiters", resourceCulture);
+ }
+ }
+
+ /// <summary>
+ /// Looks up a localized string similar to FieldWidths is null..
+ /// </summary>
+ internal static string NullFieldWidths {
+ get {
+ return ResourceManager.GetString("NullFieldWidths", resourceCulture);
+ }
+ }
/// <summary>
/// Looks up a localized string similar to numeric column.
@@ -276,6 +393,15 @@ internal static string NumericColumnType {
}
}
+ /// <summary>
+ /// Looks up a localized string similar to {0} must be greater than 0.
+ /// </summary>
+ internal static string PositiveNumberOfCharacters {
+ get {
+ return ResourceManager.GetString("PositiveNumberOfCharacters", resourceCulture);
+ }
+ }
+
/// <summary>
/// Looks up a localized string similar to Cannot span multiple buffers.
/// </summary>
@@ -284,5 +410,14 @@ internal static string SpansMultipleBuffers {
return ResourceManager.GetString("SpansMultipleBuffers", resourceCulture);
}
}
+
+ /// <summary>
+ /// Looks up a localized string similar to Stream doesn't support reading.
+ /// </summary>
+ internal static string StreamDoesntSupportReading {
+ get {
+ return ResourceManager.GetString("StreamDoesntSupportReading", resourceCulture);
+ }
+ }
}
}
diff --git a/src/Microsoft.Data.Analysis/strings.resx b/src/Microsoft.Data.Analysis/Strings.resx
similarity index 81%
rename from src/Microsoft.Data.Analysis/strings.resx
rename to src/Microsoft.Data.Analysis/Strings.resx
index ad9f114050..de91078cec 100644
--- a/src/Microsoft.Data.Analysis/strings.resx
+++ b/src/Microsoft.Data.Analysis/Strings.resx
@@ -120,24 +120,45 @@
<data name="BadColumnCast" xml:space="preserve">
<value>Cannot cast column holding {0} values to type {1}</value>
</data>
+ <data name="CannotParseWithDelimiters" xml:space="preserve">
+ <value>Line {0} cannot be parsed with the current Delimiters.</value>
+ </data>
+ <data name="CannotParseWithFieldWidths" xml:space="preserve">
+ <value>Line {0} cannot be parsed with the current FieldWidths.</value>
+ </data>
<data name="CannotResizeDown" xml:space="preserve">
<value>Cannot resize down</value>
</data>
<data name="ColumnIndexOutOfRange" xml:space="preserve">
<value>Index cannot be greater than the Column's Length</value>
</data>
+ <data name="CommentTokenCannotContainWhitespace" xml:space="preserve">
+ <value>Comment token cannot contain whitespace</value>
+ </data>
<data name="DataType" xml:space="preserve">
<value>DataType</value>
</data>
+ <data name="DelimiterCannotBeNewlineChar" xml:space="preserve">
+ <value>Delimiter cannot be new line characters</value>
+ </data>
<data name="DescriptionMethodLength" xml:space="preserve">
<value>Length (excluding null values)</value>
</data>
<data name="DuplicateColumnName" xml:space="preserve">
<value>DataFrame already contains a column called {0}</value>
</data>
+ <data name="EmptyDelimiters" xml:space="preserve">
+ <value>Delimiters is empty.</value>
+ </data>
+ <data name="EmptyFieldWidths" xml:space="preserve">
+ <value>FieldWidths is empty.</value>
+ </data>
<data name="EmptyFile" xml:space="preserve">
<value>Empty file</value>
</data>
+ <data name="ExceededMaxBufferSize" xml:space="preserve">
+ <value>Exceeded maximum buffer size.</value>
+ </data>
<data name="ExceedsNumberOfColumns" xml:space="preserve">
<value>Parameter.Count exceeds the number of columns({0}) in the DataFrame </value>
</data>
@@ -147,6 +168,12 @@
<data name="ExpectedEitherGuessRowsOrDataTypes" xml:space="preserve">
<value>Expected either {0} or {1} to be provided</value>
</data>
+ <data name="FileNotFound" xml:space="preserve">
+ <value>{0} not found.</value>
+ </data>
+ <data name="IllegalQuoteDelimiter" xml:space="preserve">
+ <value>A double quote is not a legal delimiter when HasFieldsEnclosedInQuotes is set to True.</value>
+ </data>
<data name="ImmutableColumn" xml:space="preserve">
<value>Column is immutable</value>
</data>
@@ -159,9 +186,15 @@
<data name="InvalidColumnName" xml:space="preserve">
<value>Column does not exist</value>
</data>
+ <data name="InvalidFieldWidths" xml:space="preserve">
+ <value>All field widths, except the last element, must be greater than zero. A field width less than or equal to zero in the last element indicates the last field is of variable length.</value>
+ </data>
<data name="LessColumnsThatExpected" xml:space="preserve">
<value>Line {0} has less columns than expected</value>
</data>
+ <data name="LineExceedsMaxLineSize" xml:space="preserve">
+ <value>Line {0} cannot be read because it exceeds the max line size.</value>
+ </data>
<data name="MapIndicesExceedsColumnLenth" xml:space="preserve">
<value>MapIndices exceeds column length</value>
</data>
@@ -186,10 +219,22 @@
<data name="NotSupportedColumnType" xml:space="preserve">
<value>{0} is not a supported column type.</value>
</data>
+ <data name="NullDelimiters" xml:space="preserve">
+ <value>Delimiters is null.</value>
+ </data>
+ <data name="NullFieldWidths" xml:space="preserve">
+ <value>FieldWidths is null.</value>
+ </data>
<data name="NumericColumnType" xml:space="preserve">
<value>numeric column</value>
</data>
+ <data name="PositiveNumberOfCharacters" xml:space="preserve">
+ <value>{0} must be greater than 0</value>
+ </data>
<data name="SpansMultipleBuffers" xml:space="preserve">
<value>Cannot span multiple buffers</value>
</data>
+ <data name="StreamDoesntSupportReading" xml:space="preserve">
+ <value>Stream doesn't support reading</value>
+ </data>
</root>
\ No newline at end of file
diff --git a/src/Microsoft.Data.Analysis/TextFieldParser.cs b/src/Microsoft.Data.Analysis/TextFieldParser.cs
index 6cdada1aff..e5c279db1d 100644
--- a/src/Microsoft.Data.Analysis/TextFieldParser.cs
+++ b/src/Microsoft.Data.Analysis/TextFieldParser.cs
@@ -6,6 +6,7 @@
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
+using System.Globalization;
using System.IO;
using System.Runtime.CompilerServices;
using System.Text;
@@ -19,6 +20,102 @@ internal enum FieldType
FixedWidth
}
+ internal class QuoteDelimitedFieldBuilder
+ {
+ private StringBuilder _field;
+ private bool _fieldFinished;
+ private int _index;
+ private int _delimiterLength;
+ private Regex _delimiterRegex;
+ private string _spaceChars;
+ private bool _malformedLine;
+
+ public QuoteDelimitedFieldBuilder(Regex delimiterRegex, string spaceChars)
+ {
+ _delimiterRegex = delimiterRegex;
+ _spaceChars = spaceChars;
+ _field = new StringBuilder();
+ }
+
+ public bool FieldFinished => _fieldFinished;
+
+ public string Field => _field.ToString();
+
+ public int Index => _index;
+
+ public int DelimiterLength => _delimiterLength;
+
+ public bool MalformedLine => _malformedLine;
+
+ public void BuildField(string line, int startAt)
+ {
+ _index = startAt;
+ int length = line.Length;
+
+ while (_index < length)
+ {
+ if (line[_index] == '"')
+ {
+ // Are we at the end of a file?
+ if (_index + 1 == length)
+ {
+ // We've found the end of the field
+ _fieldFinished = true;
+ _delimiterLength = 1;
+
+ // Move index past end of line
+ _index++;
+ return;
+ }
+ // Check to see if this is an escaped quote
+ if (_index + 1 < line.Length && line[_index + 1] == '"')
+ {
+ _field.Append('"');
+ _index += 2;
+ continue;
+ }
+
+ // Find the next delimiter and make sure everything between the quote and delimiter is ignorable
+ int Limit;
+ Match delimiterMatch = _delimiterRegex.Match(line, _index + 1);
+ if (!delimiterMatch.Success)
+ {
+ Limit = length - 1;
+ }
+ else
+ {
+ Limit = delimiterMatch.Index - 1;
+ }
+
+ for (int i = _index + 1; i < Limit; i++)
+ {
+ if (_spaceChars.IndexOf(line[i]) < 0)
+ {
+ _malformedLine = true;
+ return;
+ }
+ }
+
+ // The length of the delimiter is the length of the closing quote (1) + any spaces + the length of the delimiter we matched if any
+ _delimiterLength = 1 + Limit - _index;
+ if (delimiterMatch.Success)
+ {
+ _delimiterLength += delimiterMatch.Length;
+ }
+
+ _fieldFinished = true;
+ return;
+ }
+ else
+ {
+ _field.Append(line[_index]);
+ _index += 1;
+ }
+ }
+ }
+ }
+
+
internal class TextFieldParser : IDisposable
{
private delegate int ChangeBufferFunction();
@@ -47,7 +144,13 @@ internal class TextFieldParser : IDisposable
private string[] _delimitersCopy;
- private Regex _whiteSpaceRegEx = new Regex("\\s", RegexOptions.CultureInvariant);
+ private Regex _delimiterRegex;
+
+ private Regex _delimiterWithEndCharsRegex;
+
+ private int[] _whitespaceCodes = new int[] { '\u0009', '\u000B', '\u000C', '\u0020', '\u0085', '\u00A0', '\u1680', '\u2000', '\u2001', '\u2002', '\u2003', '\u2004', '\u2005', '\u2006', '\u2007', '\u2008', '\u2009', '\u200A', '\u200B', '\u2028', '\u2029', '\u3000', '\uFEFF' };
+
+ private Regex _beginQuotesRegex;
private bool _trimWhiteSpace = true;
@@ -57,12 +160,20 @@ internal class TextFieldParser : IDisposable
private int _charsRead;
+ private bool _needPropertyCheck = true;
+
private const int DEFAULT_BUFFER_LENGTH = 4096;
private char[] _buffer = new char[DEFAULT_BUFFER_LENGTH];
private bool _hasFieldsEnclosedInQuotes = true;
+ private int _lineLength;
+
+ private string _spaceChars;
+
+ private int _maxLineSize = 10000000;
+
private int _maxBufferSize = 10000000;
private bool _leaveOpen;
@@ -76,6 +187,7 @@ public string[] CommentTokens
{
CheckCommentTokensForWhitespace(value);
_commentTokens = value;
+ _needPropertyCheck = true;
}
}
@@ -120,17 +232,18 @@ public long LineNumber
public FieldType TextFieldType
{
- get =>_textFieldType;
+ get => _textFieldType;
set
{
ValidateFieldTypeEnumValue(value, "value");
_textFieldType = value;
+ _needPropertyCheck = true;
}
}
public int[] FieldWidths
{
- get =>_fieldWidths;
+ get => _fieldWidths;
private set
{
if (value != null)
@@ -143,6 +256,7 @@ private set
_fieldWidthsCopy = null;
}
_fieldWidths = value;
+ _needPropertyCheck = true;
}
}
@@ -161,12 +275,14 @@ private set
_delimitersCopy = null;
}
_delimiters = value;
+ _needPropertyCheck = true;
+ _beginQuotesRegex = null;
}
}
public bool TrimWhiteSpace
{
- get =>_trimWhiteSpace;
+ get => _trimWhiteSpace;
set
{
_trimWhiteSpace = value;
@@ -175,13 +291,65 @@ public bool TrimWhiteSpace
public bool HasFieldsEnclosedInQuotes
{
- get =>_hasFieldsEnclosedInQuotes;
+ get => _hasFieldsEnclosedInQuotes;
set
{
_hasFieldsEnclosedInQuotes = value;
}
}
+ private Regex BeginQuotesRegex
+ {
+ get
+ {
+ if (_beginQuotesRegex == null)
+ {
+ string pattern = string.Format(CultureInfo.InvariantCulture, "\\G[{0}]*\"", WhitespacePattern);
+ _beginQuotesRegex = new Regex(pattern, RegexOptions.CultureInvariant);
+ }
+ return _beginQuotesRegex;
+ }
+ }
+
+ private string EndQuotePattern => string.Format(CultureInfo.InvariantCulture, "\"[{0}]*", WhitespacePattern);
+
+ private string WhitespaceCharacters
+ {
+ get
+ {
+ StringBuilder builder = new StringBuilder();
+ int[] whitespaceCodes = _whitespaceCodes;
+ foreach (int code in whitespaceCodes)
+ {
+ char spaceChar = (char)code;
+ if (!CharacterIsInDelimiter(spaceChar))
+ {
+ builder.Append(spaceChar);
+ }
+ }
+ return builder.ToString();
+ }
+ }
+
+ private string WhitespacePattern
+ {
+ get
+ {
+ StringBuilder builder = new StringBuilder();
+ int[] whitespaceCodes = _whitespaceCodes;
+ for (int i = 0; i < whitespaceCodes.Length; i++)
+ {
+ int code = whitespaceCodes[i];
+ char spaceChar = (char)code;
+ if (!CharacterIsInDelimiter(spaceChar))
+ {
+ builder.Append("\\u" + code.ToString("X4", CultureInfo.InvariantCulture));
+ }
+ }
+ return builder.ToString();
+ }
+ }
+
public TextFieldParser(string path)
{
InitializeFromPath(path, Encoding.ASCII, detectEncoding: true);
@@ -254,6 +422,25 @@ public string ReadLine()
return line.TrimEnd(newLineChars);
}
+ public string[] ReadFields()
+ {
+ if ((_reader == null) | (_buffer == null))
+ {
+ return null;
+ }
+ ValidateReadyToRead();
+ switch (_textFieldType)
+ {
+ case FieldType.FixedWidth:
+ return ParseFixedWidthLine();
+ case FieldType.Delimited:
+ return ParseDelimitedLine();
+ default:
+ Debug.Fail("The TextFieldType is not supported");
+ return null;
+ }
+ }
+
///<summary>
/// Peek at <paramref name="numberOfChars"/> characters of the next data line without reading the line
///</summary>
@@ -263,7 +450,7 @@ public string PeekChars(int numberOfChars)
{
if (numberOfChars <= 0)
{
- throw new ArgumentException($"{nameof(numberOfChars)} must be greater than 0");
+ throw new ArgumentException(string.Format(Strings.PositiveNumberOfCharacters, nameof(numberOfChars)));
}
if ((_reader == null) | (_buffer == null))
@@ -355,6 +542,8 @@ private void FinishReading()
_lineNumber = -1L;
_endOfData = true;
_buffer = null;
+ _delimiterRegex = null;
+ _beginQuotesRegex = null;
}
private void InitializeFromPath(string path, Encoding defaultEncoding, bool detectEncoding)
@@ -381,7 +570,7 @@ private void InitializeFromStream(Stream stream, Encoding defaultEncoding, bool
}
if (!stream.CanRead)
{
- throw new ArgumentException("stream can't read");
+ throw new ArgumentException(Strings.StreamDoesntSupportReading);
}
if (defaultEncoding == null)
{
@@ -395,7 +584,7 @@ private string ValidatePath(string path)
{
if (!File.Exists(path))
{
- throw new FileNotFoundException($"{path} not found.");
+ throw new FileNotFoundException(Strings.FileNotFound);
}
return path;
}
@@ -476,7 +665,7 @@ private int IncreaseBufferSize()
int bufferSize = _buffer.Length + DEFAULT_BUFFER_LENGTH;
if (bufferSize > _maxBufferSize)
{
- throw new Exception("Exceeded maximum buffer size");
+ throw new Exception(Strings.ExceededMaxBufferSize);
}
char[] tempArray = new char[bufferSize];
Array.Copy(_buffer, tempArray, _buffer.Length);
@@ -487,6 +676,23 @@ private int IncreaseBufferSize()
return charsRead;
}
+ private string ReadNextDataLine()
+ {
+ ChangeBufferFunction BufferFunction = ReadToBuffer;
+ string line;
+ do
+ {
+ line = ReadNextLine(ref _position, BufferFunction);
+ _lineNumber++;
+ }
+ while (IgnoreLine(line));
+ if (line == null)
+ {
+ CloseReader();
+ }
+ return line;
+ }
+
private string PeekNextDataLine()
{
ChangeBufferFunction BufferFunction = IncreaseBufferSize;
@@ -562,6 +768,205 @@ private string ReadNextLine(ref int cursor, ChangeBufferFunction changeBuffer)
return Builder.ToString();
}
+ private string[] ParseDelimitedLine()
+ {
+ string line = ReadNextDataLine();
+ if (line == null)
+ {
+ return null;
+ }
+ long currentLineNumber = _lineNumber - 1;
+ int index = 0;
+ List<string> Fields = new List<string>();
+ int lineEndIndex = GetEndOfLineIndex(line);
+ while (index <= lineEndIndex)
+ {
+ Match matchResult = null;
+ bool quoteDelimited = false;
+ if (HasFieldsEnclosedInQuotes)
+ {
+ matchResult = BeginQuotesRegex.Match(line, index);
+ quoteDelimited = matchResult.Success;
+ }
+ string field;
+ if (quoteDelimited)
+ {
+ // Move the Index beyond quote
+ index = matchResult.Index + matchResult.Length;
+
+ // Looking for the closing quote
+ QuoteDelimitedFieldBuilder endHelper = new QuoteDelimitedFieldBuilder(_delimiterWithEndCharsRegex, _spaceChars);
+ endHelper.BuildField(line, index);
+ if (endHelper.MalformedLine)
+ {
+ _errorLine = line.TrimEnd(newLineChars);
+ _errorLineNumber = currentLineNumber;
+ throw new Exception(string.Format(Strings.CannotParseWithDelimiters, currentLineNumber));
+ }
+ if (endHelper.FieldFinished)
+ {
+ field = endHelper.Field;
+ index = endHelper.Index + endHelper.DelimiterLength;
+ }
+ else
+ {
+ // We may have an embedded line end character, so grab next line
+ do
+ {
+ int endOfLine = line.Length;
+ string newLine = ReadNextDataLine();
+ if (newLine == null)
+ {
+ _errorLine = line.TrimEnd(newLineChars);
+ _errorLineNumber = currentLineNumber;
+ throw new Exception(string.Format(Strings.CannotParseWithDelimiters, currentLineNumber));
+ }
+ if (line.Length + newLine.Length > _maxLineSize)
+ {
+ _errorLine = line.TrimEnd(newLineChars);
+ _errorLineNumber = currentLineNumber;
+ throw new Exception(string.Format(Strings.LineExceedsMaxLineSize, currentLineNumber));
+ }
+ line += newLine;
+ lineEndIndex = GetEndOfLineIndex(line);
+ endHelper.BuildField(line, endOfLine);
+ if (endHelper.MalformedLine)
+ {
+ _errorLine = line.TrimEnd(newLineChars);
+ _errorLineNumber = currentLineNumber;
+ throw new Exception(string.Format(Strings.CannotParseWithDelimiters, currentLineNumber));
+ }
+ }
+ while (!endHelper.FieldFinished);
+ field = endHelper.Field;
+ index = endHelper.Index + endHelper.DelimiterLength;
+ }
+ if (_trimWhiteSpace)
+ {
+ field = field.Trim();
+ }
+ Fields.Add(field);
+ continue;
+ }
+
+ // Find the next delimiter
+ Match delimiterMatch = _delimiterRegex.Match(line, index);
+ if (delimiterMatch.Success)
+ {
+ field = line.Substring(index, delimiterMatch.Index - index);
+ if (_trimWhiteSpace)
+ {
+ field = field.Trim();
+ }
+ Fields.Add(field);
+ index = delimiterMatch.Index + delimiterMatch.Length;
+ continue;
+ }
+ field = line.Substring(index).TrimEnd(newLineChars);
+ if (_trimWhiteSpace)
+ {
+ field = field.Trim();
+ }
+ Fields.Add(field);
+ break;
+ }
+ return Fields.ToArray();
+ }
+
+ private string[] ParseFixedWidthLine()
+ {
+ Debug.Assert(_fieldWidths != null, "No field widths");
+ string line = ReadNextDataLine();
+ if (line == null)
+ {
+ return null;
+ }
+ line = line.TrimEnd(newLineChars);
+ StringInfo lineInfo = new StringInfo(line);
+ ValidateFixedWidthLine(lineInfo, _lineNumber - 1);
+ int index = 0;
+ int length = _fieldWidths.Length;
+ string[] Fields = new string[length];
+ for (int i = 0; i < length; i++)
+ {
+ Fields[i] = GetFixedWidthField(lineInfo, index, _fieldWidths[i]);
+ index += _fieldWidths[i];
+ }
+ return Fields;
+ }
+
+ private string GetFixedWidthField(StringInfo line, int index, int fieldLength)
+ {
+ string field = (fieldLength > 0) ? line.SubstringByTextElements(index, fieldLength) : ((index < line.LengthInTextElements) ? line.SubstringByTextElements(index).TrimEnd(newLineChars) : string.Empty);
+ if (_trimWhiteSpace)
+ {
+ return field.Trim();
+ }
+ return field;
+ }
+
+ private int GetEndOfLineIndex(string line)
+ {
+ Debug.Assert(line != null, "We are parsing null");
+ int length = line.Length;
+ Debug.Assert(length > 0, "A blank line shouldn't be parsed");
+ if (length == 1)
+ {
+ Debug.Assert(!line[0].Equals('\r') & !line[0].Equals('\n'), "A blank line shouldn't be parsed");
+ return length;
+ }
+ checked
+ {
+ if (line[length - 2].Equals('\r') | line[length - 2].Equals('\n'))
+ {
+ return length - 2;
+ }
+ if (line[length - 1].Equals('\r') | line[length - 1].Equals('\n'))
+ {
+ return length - 1;
+ }
+ return length;
+ }
+ }
+
+ private void ValidateFixedWidthLine(StringInfo line, long lineNumber)
+ {
+ Debug.Assert(line != null, "No Line sent");
+ if (line.LengthInTextElements < _lineLength)
+ {
+ _errorLine = line.String;
+ _errorLineNumber = checked(_lineNumber - 1);
+ throw new Exception(string.Format(Strings.CannotParseWithFieldWidths, lineNumber));
+ }
+ }
+
+ private void ValidateFieldWidths()
+ {
+ if (_fieldWidths == null)
+ {
+ throw new InvalidOperationException(Strings.NullFieldWidths);
+ }
+ if (_fieldWidths.Length == 0)
+ {
+ throw new InvalidOperationException(Strings.EmptyFieldWidths);
+ }
+ checked
+ {
+ int widthBound = _fieldWidths.Length - 1;
+ _lineLength = 0;
+ int num = widthBound - 1;
+ for (int i = 0; i <= num; i++)
+ {
+ Debug.Assert(_fieldWidths[i] > 0, "Bad field width, this should have been caught on input");
+ _lineLength += _fieldWidths[i];
+ }
+ if (_fieldWidths[widthBound] > 0)
+ {
+ _lineLength += _fieldWidths[widthBound];
+ }
+ }
+ }
+
private void ValidateFieldWidthsOnInput(int[] widths)
{
Debug.Assert(widths != null, "There are no field widths");
@@ -570,11 +975,81 @@ private void ValidateFieldWidthsOnInput(int[] widths)
{
if (widths[i] < 1)
{
- throw new ArgumentException("All field widths, except the last element, must be greater than zero. A field width less than or equal to zero in the last element indicates the last field is of variable length.");
+ throw new ArgumentException(Strings.InvalidFieldWidths);
}
}
}
+ private void ValidateAndEscapeDelimiters()
+ {
+ if (_delimiters == null)
+ {
+ throw new Exception(Strings.NullDelimiters);
+ }
+ if (_delimiters.Length == 0)
+ {
+ throw new Exception(Strings.EmptyDelimiters);
+ }
+ int length = _delimiters.Length;
+ StringBuilder builder = new StringBuilder();
+ StringBuilder quoteBuilder = new StringBuilder();
+ quoteBuilder.Append(EndQuotePattern + "(");
+ for (int i = 0; i <= length - 1; i++)
+ {
+ if (_delimiters[i] != null)
+ {
+ if (_hasFieldsEnclosedInQuotes && _delimiters[i].IndexOf('"') > -1)
+ {
+ throw new Exception(Strings.IllegalQuoteDelimiter);
+ }
+ string escapedDelimiter = Regex.Escape(_delimiters[i]);
+ builder.Append(escapedDelimiter + "|");
+ quoteBuilder.Append(escapedDelimiter + "|");
+ }
+ else
+ {
+ Debug.Fail("Delimiter element is empty. This should have been caught on input");
+ }
+ }
+ _spaceChars = WhitespaceCharacters;
+ _delimiterRegex = new Regex(builder.ToString(0, builder.Length - 1), (RegexOptions)512);
+ builder.Append("\r|\n");
+ _delimiterWithEndCharsRegex = new Regex(builder.ToString(), (RegexOptions)512);
+ quoteBuilder.Append("\r|\n)|\"$");
+ }
+
+ private void ValidateReadyToRead()
+ {
+ if (!(_needPropertyCheck | ArrayHasChanged()))
+ {
+ return;
+ }
+ switch (_textFieldType)
+ {
+ case FieldType.Delimited:
+ ValidateAndEscapeDelimiters();
+ break;
+ case FieldType.FixedWidth:
+ ValidateFieldWidths();
+ break;
+ default:
+ Debug.Fail("Unknown TextFieldType");
+ break;
+ }
+ if (_commentTokens != null)
+ {
+ string[] commentTokens = _commentTokens;
+ foreach (string token in commentTokens)
+ {
+ if (token != string.Empty && (_hasFieldsEnclosedInQuotes & (_textFieldType == FieldType.Delimited)) && string.Compare(token.Trim(), "\"", StringComparison.Ordinal) == 0)
+ {
+ throw new Exception(Strings.IllegalQuoteDelimiter);
+ }
+ }
+ }
+ _needPropertyCheck = false;
+ }
+
private void ValidateDelimiters(string[] delimiterArray)
{
if (delimiterArray == null)
@@ -585,15 +1060,68 @@ private void ValidateDelimiters(string[] delimiterArray)
{
if (delimiter == string.Empty)
{
- throw new Exception("Delimiter cannot be empty");
+ throw new Exception(Strings.EmptyDelimiters);
}
if (delimiter.IndexOfAny(newLineChars) > -1)
{
- throw new Exception("Delimiter cannot be new line characters");
+ throw new Exception(Strings.DelimiterCannotBeNewlineChar);
}
}
}
+ private bool ArrayHasChanged()
+ {
+ int lowerBound = 0;
+ int upperBound = 0;
+ switch (_textFieldType)
+ {
+ case FieldType.Delimited:
+ {
+ Debug.Assert(((_delimitersCopy == null) & (_delimiters == null)) | ((_delimitersCopy != null) & (_delimiters != null)), "Delimiters and copy are not both Nothing or both not Nothing");
+ if (_delimiters == null)
+ {
+ return false;
+ }
+ lowerBound = _delimitersCopy.GetLowerBound(0);
+ upperBound = _delimitersCopy.GetUpperBound(0);
+ int num3 = lowerBound;
+ int num4 = upperBound;
+ for (int i = num3; i <= num4; i++)
+ {
+ if (_delimiters[i] != _delimitersCopy[i])
+ {
+ return true;
+ }
+ }
+ break;
+ }
+ case FieldType.FixedWidth:
+ {
+ Debug.Assert(((_fieldWidthsCopy == null) & (_fieldWidths == null)) | ((_fieldWidthsCopy != null) & (_fieldWidths != null)), "FieldWidths and copy are not both Nothing or both not Nothing");
+ if (_fieldWidths == null)
+ {
+ return false;
+ }
+ lowerBound = _fieldWidthsCopy.GetLowerBound(0);
+ upperBound = _fieldWidthsCopy.GetUpperBound(0);
+ int num = lowerBound;
+ int num2 = upperBound;
+ for (int j = num; j <= num2; j++)
+ {
+ if (_fieldWidths[j] != _fieldWidthsCopy[j])
+ {
+ return true;
+ }
+ }
+ break;
+ }
+ default:
+ Debug.Fail("Unknown TextFieldType");
+ break;
+ }
+ return false;
+ }
+
private void CheckCommentTokensForWhitespace(string[] tokens)
{
if (tokens == null)
@@ -604,9 +1132,23 @@ private void CheckCommentTokensForWhitespace(string[] tokens)
{
if (token.Length == 1 && char.IsWhiteSpace(token[0]))
{
- throw new Exception("Comment token cannot contain whitespace");
+ throw new Exception(Strings.CommentTokenCannotContainWhitespace);
}
}
}
+
+ private bool CharacterIsInDelimiter(char testCharacter)
+ {
+ Debug.Assert(_delimiters != null, "No delimiters set!");
+ string[] delimiters = _delimiters;
+ foreach (string delimiter in delimiters)
+ {
+ if (delimiter.IndexOf(testCharacter) > -1)
+ {
+ return true;
+ }
+ }
+ return false;
+ }
}
}
diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs
index 9e8ccb6903..93bf2dad54 100644
--- a/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs
+++ b/test/Microsoft.Data.Analysis.Tests/DataFrame.IOTests.cs
@@ -101,24 +101,29 @@ internal static void VerifyColumnTypes(DataFrame df, bool testArrowStringColumn
}
}
- [Fact]
- public void TestReadCsvWithHeader()
+ private static Stream GetStream(string streamData)
{
- string data = @"vendor_id,rate_code,passenger_count,trip_time_in_secs,trip_distance,payment_type,fare_amount
-CMT,1,1,1271,3.8,CRD,17.5
-CMT,1,1,474,1.5,CRD,8
-CMT,1,1,637,1.4,CRD,8.5
-CMT,1,1,181,0.6,CSH,4.5";
+ return new MemoryStream(Encoding.Default.GetBytes(streamData));
+ }
+
+ [Theory]
+ [InlineData(false)]
+ [InlineData(true)]
+ public void TestReadCsvWithHeader(bool useQuotes)
+ {
+ string CMT = useQuotes ? @"""C,MT""" : "CMT";
+ string verifyCMT = useQuotes ? "C,MT" : "CMT";
+ string data = @$"vendor_id,rate_code,passenger_count,trip_time_in_secs,trip_distance,payment_type,fare_amount
+{CMT},1,1,1271,3.8,CRD,17.5
+{CMT},1,1,474,1.5,CRD,8
+{CMT},1,1,637,1.4,CRD,8.5
+{CMT},1,1,181,0.6,CSH,4.5";
- Stream GetStream(string streamData)
- {
- return new MemoryStream(Encoding.Default.GetBytes(streamData));
- }
void RegularTest(DataFrame df)
{
Assert.Equal(4, df.Rows.Count);
Assert.Equal(7, df.Columns.Count);
- Assert.Equal("CMT", df.Columns["vendor_id"][3]);
+ Assert.Equal(verifyCMT, df.Columns["vendor_id"][3]);
VerifyColumnTypes(df);
}
DataFrame df = DataFrame.LoadCsv(GetStream(data));
@@ -130,7 +135,7 @@ void ReducedRowsTest(DataFrame reducedRows)
{
Assert.Equal(3, reducedRows.Rows.Count);
Assert.Equal(7, reducedRows.Columns.Count);
- Assert.Equal("CMT", reducedRows.Columns["vendor_id"][2]);
+ Assert.Equal(verifyCMT, reducedRows.Columns["vendor_id"][2]);
VerifyColumnTypes(df);
}
DataFrame reducedRows = DataFrame.LoadCsv(GetStream(data), numberOfRowsToRead: 3);
@@ -140,22 +145,61 @@ void ReducedRowsTest(DataFrame reducedRows)
}
[Fact]
- public void TestReadCsvNoHeader()
+ public void TestReadCsvSplitAcrossMultipleLines()
{
- string data = @"CMT,1,1,1271,3.8,CRD,17.5
-CMT,1,1,474,1.5,CRD,8
-CMT,1,1,637,1.4,CRD,8.5
-CMT,1,1,181,0.6,CSH,4.5";
+ string CMT = @"""C
+MT""";
+ string verifyCMT = @"C
+MT";
+ string data = @$"{CMT},1,1,1271,3.8,CRD,17.5
+{CMT},1,1,474,1.5,CRD,8
+{CMT},1,1,637,1.4,CRD,8.5
+{CMT},1,1,181,0.6,CSH,4.5";
- Stream GetStream(string streamData)
+ void RegularTest(DataFrame df)
{
- return new MemoryStream(Encoding.Default.GetBytes(streamData));
+ Assert.Equal(4, df.Rows.Count);
+ Assert.Equal(7, df.Columns.Count);
+ Assert.Equal(verifyCMT, df.Columns["Column0"][3]);
+ VerifyColumnTypes(df);
}
+
+ DataFrame df = DataFrame.LoadCsv(GetStream(data), header: false);
+ RegularTest(df);
+ DataFrame csvDf = DataFrame.LoadCsvFromString(data, header: false);
+ RegularTest(csvDf);
+
+ void ReducedRowsTest(DataFrame reducedRows)
+ {
+ Assert.Equal(3, reducedRows.Rows.Count);
+ Assert.Equal(7, reducedRows.Columns.Count);
+ Assert.Equal(verifyCMT, reducedRows.Columns["Column0"][2]);
+ VerifyColumnTypes(df);
+ }
+
+ DataFrame reducedRows = DataFrame.LoadCsv(GetStream(data), header: false, numberOfRowsToRead: 3);
+ ReducedRowsTest(reducedRows);
+ csvDf = DataFrame.LoadCsvFromString(data, header: false, numberOfRowsToRead: 3);
+ ReducedRowsTest(csvDf);
+ }
+
+ [Theory]
+ [InlineData(false)]
+ [InlineData(true)]
+ public void TestReadCsvNoHeader(bool useQuotes)
+ {
+ string CMT = useQuotes ? @"""C,MT""" : "CMT";
+ string verifyCMT = useQuotes ? "C,MT" : "CMT";
+ string data = @$"{CMT},1,1,1271,3.8,CRD,17.5
+{CMT},1,1,474,1.5,CRD,8
+{CMT},1,1,637,1.4,CRD,8.5
+{CMT},1,1,181,0.6,CSH,4.5";
+
void RegularTest(DataFrame df)
{
Assert.Equal(4, df.Rows.Count);
Assert.Equal(7, df.Columns.Count);
- Assert.Equal("CMT", df.Columns["Column0"][3]);
+ Assert.Equal(verifyCMT, df.Columns["Column0"][3]);
VerifyColumnTypes(df);
}
@@ -168,7 +212,7 @@ void ReducedRowsTest(DataFrame reducedRows)
{
Assert.Equal(3, reducedRows.Rows.Count);
Assert.Equal(7, reducedRows.Columns.Count);
- Assert.Equal("CMT", reducedRows.Columns["Column0"][2]);
+ Assert.Equal(verifyCMT, reducedRows.Columns["Column0"][2]);
VerifyColumnTypes(df);
}
@@ -241,10 +285,6 @@ False 10 Null
CMT,1,1,637,1.4,CRD,8.5
CMT,1,1,181,0.6,CSH,4.5";
- Stream GetStream(string streamData)
- {
- return new MemoryStream(Encoding.Default.GetBytes(streamData));
- }
string data = header ? headerLine + dataLines : dataLines;
DataFrame df = DataFrame.LoadCsv(GetStream(data),
@@ -295,10 +335,6 @@ public void TestReadCsvWithTypes()
,,,,,,
CMT,1,1,181,0.6,CSH,4.5";
- Stream GetStream(string streamData)
- {
- return new MemoryStream(Encoding.Default.GetBytes(streamData));
- }
void Verify(DataFrame df)
{
Assert.Equal(5, df.Rows.Count);
@@ -358,10 +394,6 @@ public void TestReadCsvWithPipeSeparator()
||||||
CMT|1|1|181|0.6|CSH|4.5";
- Stream GetStream(string streamData)
- {
- return new MemoryStream(Encoding.Default.GetBytes(streamData));
- }
void Verify(DataFrame df)
{
Assert.Equal(5, df.Rows.Count);
@@ -401,10 +433,6 @@ public void TestReadCsvWithSemicolonSeparator()
;;;;;;
CMT;1;1;181;0.6;CSH;4.5";
- Stream GetStream(string streamData)
- {
- return new MemoryStream(Encoding.Default.GetBytes(streamData));
- }
void Verify(DataFrame df)
{
Assert.Equal(5, df.Rows.Count);
@@ -443,10 +471,6 @@ public void TestReadCsvWithExtraColumnInHeader()
CMT,1,1,637,1.4,CRD,8.5
CMT,1,1,181,0.6,CSH,4.5";
- Stream GetStream(string streamData)
- {
- return new MemoryStream(Encoding.Default.GetBytes(streamData));
- }
void Verify(DataFrame df)
{
Assert.Equal(4, df.Rows.Count);
@@ -476,11 +500,6 @@ public void TestReadCsvWithExtraColumnInRow()
CMT,1,1,637,1.4,CRD,8.5,0
CMT,1,1,181,0.6,CSH,4.5,0";
- Stream GetStream(string streamData)
- {
- return new MemoryStream(Encoding.Default.GetBytes(streamData));
- }
-
Assert.Throws<IndexOutOfRangeException>(() => DataFrame.LoadCsv(GetStream(data)));
Assert.Throws<IndexOutOfRangeException>(() => DataFrame.LoadCsvFromString(data));
}
@@ -494,11 +513,6 @@ public void TestReadCsvWithLessColumnsInRow()
CMT,1,1,637,1.4,CRD
CMT,1,1,181,0.6,CSH";
- Stream GetStream(string streamData)
- {
- return new MemoryStream(Encoding.Default.GetBytes(streamData));
- }
-
void Verify(DataFrame df)
{
Assert.Equal(4, df.Rows.Count);
@@ -530,11 +544,6 @@ public void TestReadCsvWithAllNulls()
null,null,null,null
null,null,null,null";
- Stream GetStream(string streamData)
- {
- return new MemoryStream(Encoding.Default.GetBytes(streamData));
- }
-
void Verify(DataFrame df)
{
Assert.Equal(6, df.Rows.Count);
@@ -578,11 +587,6 @@ public void TestReadCsvWithNullsAndDataTypes()
,,,
CMT,1,1,null";
- Stream GetStream(string streamData)
- {
- return new MemoryStream(Encoding.Default.GetBytes(streamData));
- }
-
void Verify(DataFrame df)
{
Assert.Equal(6, df.Rows.Count);
@@ -645,11 +649,6 @@ public void TestReadCsvWithNulls()
,,,
CMT,1,1,null";
- Stream GetStream(string streamData)
- {
- return new MemoryStream(Encoding.Default.GetBytes(streamData));
- }
-
void Verify(DataFrame df)
{
Assert.Equal(6, df.Rows.Count);
@@ -867,10 +866,6 @@ public void TestMixedDataTypesInCsv()
,
CMT,";
- Stream GetStream(string streamData)
- {
- return new MemoryStream(Encoding.Default.GetBytes(streamData));
- }
DataFrame df = DataFrame.LoadCsv(GetStream(data));
Assert.Equal(6, df.Rows.Count);
Assert.Equal(2, df.Columns.Count);
diff --git a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj
index 2b6f43253f..8032812924 100644
--- a/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj
+++ b/test/Microsoft.Data.Analysis.Tests/Microsoft.Data.Analysis.Tests.csproj
@@ -28,4 +28,10 @@
<DependentUpon>DataFrameColumn.BinaryOperationTests.tt</DependentUpon>
</Compile>
</ItemGroup>
+
+ <ItemGroup>
+ <Compile Include="../../src/Microsoft.Data.Analysis/Strings.Designer.cs" />
+ <EmbeddedResource Include="../../src/Microsoft.Data.Analysis/Strings.resx" LogicalName="Microsoft.Data.Analysis.Strings.resources" />
+ </ItemGroup>
+
</Project>
diff --git a/test/Microsoft.Data.Analysis.Tests/TextFieldParserTests.cs b/test/Microsoft.Data.Analysis.Tests/TextFieldParserTests.cs
index 96ac2d4787..c481d13b28 100644
--- a/test/Microsoft.Data.Analysis.Tests/TextFieldParserTests.cs
+++ b/test/Microsoft.Data.Analysis.Tests/TextFieldParserTests.cs
@@ -164,5 +164,256 @@ public void ReadLine_ReadToEnd()
Assert.True(parser.EndOfData);
}
}
+
+ [Fact]
+ public void ErrorLine()
+ {
+ string data = @"abc 123
+def 45
+ghi 789";
+
+ using (var parser = new TextFieldParser(GetStream(data)))
+ {
+ parser.TextFieldType = FieldType.FixedWidth;
+ parser.SetFieldWidths(new[] { 3, 4 });
+
+ Assert.Equal(-1, parser.ErrorLineNumber);
+ Assert.Equal("", parser.ErrorLine);
+
+ Assert.Equal(new[] { "abc", "123" }, parser.ReadFields());
+ Assert.Equal(-1, parser.ErrorLineNumber);
+ Assert.Equal("", parser.ErrorLine);
+
+ Assert.Throws<Exception>(() => parser.ReadFields());
+ Assert.Equal(2, parser.ErrorLineNumber);
+ Assert.Equal("def 45", parser.ErrorLine);
+
+ Assert.Equal(new[] { "ghi", "789" }, parser.ReadFields());
+ Assert.Equal(2, parser.ErrorLineNumber);
+ Assert.Equal("def 45", parser.ErrorLine);
+ }
+ }
+
+ [Fact]
+ public void HasFieldsEnclosedInQuotes_TrimWhiteSpace()
+ {
+ string data = @""""", "" "" ,""abc"", "" 123 "" ,";
+
+ using (var parser = new TextFieldParser(GetStream(data)))
+ {
+ parser.SetDelimiters(new[] { "," });
+ Assert.Equal(new[] { "", "", "abc", "123", "" }, parser.ReadFields());
+ }
+
+ using (var parser = new TextFieldParser(GetStream(data)))
+ {
+ parser.TrimWhiteSpace = false;
+ parser.SetDelimiters(new[] { "," });
+ Assert.Equal(new[] { "", " ", "abc", " 123 ", "" }, parser.ReadFields());
+ }
+
+ using (var parser = new TextFieldParser(GetStream(data)))
+ {
+ parser.HasFieldsEnclosedInQuotes = false;
+ parser.SetDelimiters(new[] { "," });
+ Assert.Equal(new[] { @"""""", @""" """, @"""abc""", @""" 123 """, "" }, parser.ReadFields());
+ }
+
+ using (var parser = new TextFieldParser(GetStream(data)))
+ {
+ parser.TrimWhiteSpace = false;
+ parser.HasFieldsEnclosedInQuotes = false;
+ parser.SetDelimiters(new[] { "," });
+ Assert.Equal(new[] { @"""""", @" "" "" ", @"""abc""", @" "" 123 "" ", "" }, parser.ReadFields());
+ }
+
+ data = @""","", "", """;
+ using (var parser = new TextFieldParser(GetStream(data)))
+ {
+ parser.TrimWhiteSpace = false;
+ parser.SetDelimiters(new[] { "," });
+ Assert.Equal(new[] { ",", ", " }, parser.ReadFields());
+ }
+ }
+
+ [Fact]
+ public void PeekChars()
+ {
+ string data = @"abc,123
+def,456
+ghi,789";
+
+ using (var parser = new TextFieldParser(GetStream(data)))
+ {
+ Assert.Throws<ArgumentException>(() => parser.PeekChars(0));
+
+ Assert.Equal("a", parser.PeekChars(1));
+ Assert.Equal("abc,123", parser.PeekChars(10));
+
+ Assert.Equal("abc,123", parser.ReadLine());
+
+ parser.TextFieldType = FieldType.FixedWidth;
+ parser.SetFieldWidths(new[] { 3, -1 });
+
+ Assert.Equal("d", parser.PeekChars(1));
+ Assert.Equal("def,456", parser.PeekChars(10));
+ Assert.Equal(new[] { "def", ",456" }, parser.ReadFields());
+
+ parser.TextFieldType = FieldType.Delimited;
+ parser.SetDelimiters(new[] { "," });
+
+ Assert.Equal("g", parser.PeekChars(1));
+ Assert.Equal("ghi,789", parser.PeekChars(10));
+ Assert.Equal(new[] { "ghi", "789" }, parser.ReadFields());
+
+ Assert.Null(parser.PeekChars(1));
+ Assert.Null(parser.PeekChars(10));
+ }
+ }
+
+ [Fact]
+ public void ReadFields_FieldWidths()
+ {
+ string data = @"abc,123
+def,456
+ghi,789";
+
+ using (var parser = new TextFieldParser(GetStream(data)))
+ {
+ parser.TextFieldType = FieldType.FixedWidth;
+
+ Assert.Throws<InvalidOperationException>(() => parser.ReadFields());
+
+ parser.SetFieldWidths(new[] { -1 });
+ Assert.Equal(new[] { "abc,123" }, parser.ReadFields());
+
+ parser.SetFieldWidths(new[] { 3, -1 });
+ Assert.Equal(new[] { "def", ",456" }, parser.ReadFields());
+
+ parser.SetFieldWidths(new[] { 3, 2 });
+ Assert.Equal(new[] { "ghi", ",7" }, parser.ReadFields());
+
+ parser.SetFieldWidths(new[] { 3, 2 });
+ Assert.Null(parser.ReadFields());
+ }
+ }
+
+ [Fact]
+ public void ReadFields_Delimiters_LineNumber()
+ {
+ string data = @"abc,123
+def,456
+ghi,789";
+
+ using (var parser = new TextFieldParser(GetStream(data)))
+ {
+ Assert.Equal(1, parser.LineNumber);
+
+ Assert.Throws<Exception>(() => parser.ReadFields());
+ Assert.Equal(1, parser.LineNumber);
+
+ parser.SetDelimiters(new[] { "," });
+ Assert.Equal(new[] { "abc","123" }, parser.ReadFields());
+ Assert.Equal(2, parser.LineNumber);
+
+ parser.SetDelimiters(new[] { ";", "," });
+ Assert.Equal(new[] { "def", "456" }, parser.ReadFields());
+ Assert.Equal(3, parser.LineNumber);
+
+ parser.SetDelimiters(new[] { "g", "9" });
+ Assert.Equal(new[] { "", "hi,78", "" }, parser.ReadFields());
+ Assert.Equal(-1, parser.LineNumber);
+ }
+
+ data = @",,
+
+,
+";
+
+ using (var parser = new TextFieldParser(GetStream(data)))
+ {
+ Assert.Equal(1, parser.LineNumber);
+
+ parser.SetDelimiters(new[] { "," });
+ Assert.Equal(new[] { "", "", "" }, parser.ReadFields());
+ Assert.Equal(2, parser.LineNumber);
+
+ // ReadFields should ignore the empty new line
+ Assert.Equal(new[] { "", "" }, parser.ReadFields());
+ Assert.Equal(-1, parser.LineNumber);
+
+ Assert.Null(parser.ReadFields());
+ Assert.Equal(-1, parser.LineNumber);
+
+ Assert.Null(parser.ReadFields());
+ Assert.Equal(-1, parser.LineNumber);
+ }
+ }
+
+ [Fact]
+ public void UnmatchedQuote_MalformedLineException()
+ {
+ string data = @""""", """;
+
+ using (var parser = new TextFieldParser(GetStream(data)))
+ {
+ parser.SetDelimiters(new[] { "," });
+ Assert.Throws<Exception>(() => parser.ReadFields());
+ }
+ }
+
+ [Fact]
+ public void ReadFields_QuoteOnNewLine()
+ {
+ string data = @"abc,""123
+123""
+def,456
+ghi,789";
+
+ using (var parser = new TextFieldParser(GetStream(data)))
+ {
+ Assert.Equal(1, parser.LineNumber);
+
+ Assert.Throws<Exception>(() => parser.ReadFields());
+ Assert.Equal(1, parser.LineNumber);
+
+ parser.SetDelimiters(new[] { "," });
+ Assert.Equal(new[] { "abc",@"123
+123" }, parser.ReadFields());
+ Assert.Equal(3, parser.LineNumber);
+
+ parser.SetDelimiters(new[] { ";", "," });
+ Assert.Equal(new[] { "def", "456" }, parser.ReadFields());
+ Assert.Equal(4, parser.LineNumber);
+
+ parser.SetDelimiters(new[] { "g", "9" });
+ Assert.Equal(new[] { "", "hi,78", "" }, parser.ReadFields());
+ Assert.Equal(-1, parser.LineNumber);
+ }
+
+ data = @",,
+
+,
+";
+
+ using (var parser = new TextFieldParser(GetStream(data)))
+ {
+ Assert.Equal(1, parser.LineNumber);
+
+ parser.SetDelimiters(new[] { "," });
+ Assert.Equal(new[] { "", "", "" }, parser.ReadFields());
+ Assert.Equal(2, parser.LineNumber);
+
+ // ReadFields should ignore the empty new line
+ Assert.Equal(new[] { "", "" }, parser.ReadFields());
+ Assert.Equal(-1, parser.LineNumber);
+
+ Assert.Null(parser.ReadFields());
+ Assert.Equal(-1, parser.LineNumber);
+
+ Assert.Null(parser.ReadFields());
+ Assert.Equal(-1, parser.LineNumber);
+ }
+ }
}
}