Append rows to a DataFrame (dotnet#2823)

Prashanth Govindarajan · msftbot[bot] · commit 82c315f088ac · 2020-01-28T05:39:13.000Z
* Append rows to a DataFrame

* Unit test

* Update unit tests and doc

* Need to perfrom a type check every time

* sq

* Update unit test

* Address comments
diff --git a/src/Microsoft.Data.Analysis/DataFrame.IO.cs b/src/Microsoft.Data.Analysis/DataFrame.IO.cs
@@ -271,7 +271,7 @@ public static DataFrame LoadCsv(Stream csvStream,
                     }
                     else
                     {
-                        ret.Append(spl);
+                        ret.Append(spl, inPlace: true);
                     }
                     ++rowline;
                     line = streamReader.ReadLine();
diff --git a/src/Microsoft.Data.Analysis/DataFrame.cs b/src/Microsoft.Data.Analysis/DataFrame.cs
@@ -424,56 +424,76 @@ private void ResizeByOneAndAppend(DataFrameColumn column, object value)
         }
 
         /// <summary> 
-        /// Appends a row inplace to the DataFrame 
+        /// Appends rows to the DataFrame 
+        /// </summary> 
+        /// <remarks>If an input column's value doesn't match a DataFrameColumn's data type, a conversion will be attempted</remarks> 
+        /// <remarks>If a <seealso cref="DataFrameRow"/> in <paramref name="rows"/> is null, a null value is appended to each column</remarks>
+        /// <param name="rows">The rows to be appended to this DataFrame </param> 
+        /// <param name="inPlace">If set, appends <paramref name="rows"/> in place. Otherwise, a new DataFrame is returned with the <paramref name="rows"/> appended</param>
+        public DataFrame Append(IEnumerable<DataFrameRow> rows, bool inPlace = false)
+        {
+            DataFrame ret = inPlace ? this : Clone();
+            foreach (DataFrameRow row in rows)
+            {
+                ret.Append(row, inPlace: true);
+            }
+            return ret;
+        }
+
+        /// <summary> 
+        /// Appends a row to the DataFrame 
         /// </summary> 
         /// <remarks>If a column's value doesn't match its column's data type, a conversion will be attempted</remarks> 
         /// <remarks>If <paramref name="row"/> is null, a null value is appended to each column</remarks>
         /// <param name="row"></param> 
-        public void Append(IEnumerable<object> row = null)
+        /// <param name="inPlace">If set, appends a <paramref name="row"/> in place. Otherwise, a new DataFrame is returned with an appended <paramref name="row"/> </param>
+        public DataFrame Append(IEnumerable<object> row = null, bool inPlace = false)
         {
-            IEnumerator<DataFrameColumn> columnEnumerator = Columns.GetEnumerator();
+            DataFrame ret = inPlace ? this : Clone();
+            IEnumerator<DataFrameColumn> columnEnumerator = ret.Columns.GetEnumerator();
             bool columnMoveNext = columnEnumerator.MoveNext();
             if (row != null)
             {
-                // Go through row first to make sure there are no data type incompatibilities
-                IEnumerator<object> rowEnumerator = row.GetEnumerator();
-                bool rowMoveNext = rowEnumerator.MoveNext();
-                List<object> cachedObjectConversions = new List<object>();
-                while (columnMoveNext && rowMoveNext)
-                {
-                    DataFrameColumn column = columnEnumerator.Current;
-                    object value = rowEnumerator.Current;
-                    // StringDataFrameColumn can accept empty strings. The other columns interpret empty values as nulls
-                    if (value is string stringValue && string.IsNullOrEmpty(stringValue) && column.DataType != typeof(string))
-                    {
-                        value = null;
-                    }
-                    if (value != null)
+                    // Go through row first to make sure there are no data type incompatibilities
+                    IEnumerator<object> rowEnumerator = row.GetEnumerator();
+                    bool rowMoveNext = rowEnumerator.MoveNext();
+                    List<object> cachedObjectConversions = new List<object>();
+                    while (columnMoveNext && rowMoveNext)
                     {
-                        value = Convert.ChangeType(value, column.DataType);
-                        if (value is null)
+                        DataFrameColumn column = columnEnumerator.Current;
+                        object value = rowEnumerator.Current;
+                        // StringDataFrameColumn can accept empty strings. The other columns interpret empty values as nulls
+                        if (value is string stringValue && string.IsNullOrEmpty(stringValue) && column.DataType != typeof(string))
                         {
-                            throw new ArgumentException(string.Format(Strings.MismatchedValueType, column.DataType), value.GetType().ToString());
+                            value = null;
                         }
+                        if (value != null)
+                        {
+                            value = Convert.ChangeType(value, column.DataType);
+                            if (value is null)
+                            {
+                                throw new ArgumentException(string.Format(Strings.MismatchedValueType, column.DataType), value.GetType().ToString());
+                            }
+                        }
+                        cachedObjectConversions.Add(value);
+                        columnMoveNext = columnEnumerator.MoveNext();
+                        rowMoveNext = rowEnumerator.MoveNext();
                     }
-                    cachedObjectConversions.Add(value);
-                    columnMoveNext = columnEnumerator.MoveNext();
-                    rowMoveNext = rowEnumerator.MoveNext();
-                }
-                if (rowMoveNext)
-                {
-                    throw new ArgumentException(string.Format(Strings.ExceedsNumberOfColumns, Columns.Count), nameof(row));
-                }
-                columnEnumerator.Reset();
+                    if (rowMoveNext)
+                    {
+                        throw new ArgumentException(string.Format(Strings.ExceedsNumberOfColumns, Columns.Count), nameof(row));
+                    }
+                // Reset the enumerators
+                columnEnumerator = ret.Columns.GetEnumerator();
                 columnMoveNext = columnEnumerator.MoveNext();
-                rowEnumerator.Reset();
+                rowEnumerator = row.GetEnumerator();
                 rowMoveNext = rowEnumerator.MoveNext();
                 int cacheIndex = 0;
                 while (columnMoveNext && rowMoveNext)
                 {
                     DataFrameColumn column = columnEnumerator.Current;
                     object value = cachedObjectConversions[cacheIndex];
-                    ResizeByOneAndAppend(column, value);
+                    ret.ResizeByOneAndAppend(column, value);
                     columnMoveNext = columnEnumerator.MoveNext();
                     rowMoveNext = rowEnumerator.MoveNext();
                     cacheIndex++;
@@ -483,19 +503,22 @@ public void Append(IEnumerable<object> row = null)
             {
                 // Fill the remaining columns with null
                 DataFrameColumn column = columnEnumerator.Current;
-                ResizeByOneAndAppend(column, null);
+                ret.ResizeByOneAndAppend(column, null);
                 columnMoveNext = columnEnumerator.MoveNext();
             }
-            Columns.RowCount++;
+            ret.Columns.RowCount++;
+            return ret;
         }
 
         /// <summary> 
-        /// Appends a row inplace by enumerating column names and values from <paramref name="row"/> 
+        /// Appends a row by enumerating column names and values from <paramref name="row"/> 
         /// </summary> 
         /// <remarks>If a column's value doesn't match its column's data type, a conversion will be attempted</remarks> 
-        /// <param name="row"></param> 
-        public void Append(IEnumerable<KeyValuePair<string, object>> row)
+        /// <param name="row">An enumeration of column name and value to be appended</param> 
+        /// <param name="inPlace">If set, appends <paramref name="row"/> in place. Otherwise, a new DataFrame is returned with an appended <paramref name="row"/> </param>
+        public DataFrame Append(IEnumerable<KeyValuePair<string, object>> row, bool inPlace = false)
         {
+            DataFrame ret = inPlace ? this : Clone();
             if (row == null)
             {
                 throw new ArgumentNullException(nameof(row));
@@ -505,13 +528,13 @@ public void Append(IEnumerable<KeyValuePair<string, object>> row)
             foreach (KeyValuePair<string, object> columnAndValue in row)
             {
                 string columnName = columnAndValue.Key;
-                int index = Columns.IndexOf(columnName);
+                int index = ret.Columns.IndexOf(columnName);
                 if (index == -1)
                 {
                     throw new ArgumentException(Strings.InvalidColumnName, nameof(columnName));
                 }
 
-                DataFrameColumn column = Columns[index];
+                DataFrameColumn column = ret.Columns[index];
                 object value = columnAndValue.Value;
                 if (value != null)
                 {
@@ -528,22 +551,23 @@ public void Append(IEnumerable<KeyValuePair<string, object>> row)
             foreach (KeyValuePair<string, object> columnAndValue in row)
             {
                 string columnName = columnAndValue.Key;
-                int index = Columns.IndexOf(columnName);
+                int index = ret.Columns.IndexOf(columnName);
 
-                DataFrameColumn column = Columns[index];
+                DataFrameColumn column = ret.Columns[index];
                 object value = cachedObjectConversions[cacheIndex];
-                ResizeByOneAndAppend(column, value);
+                ret.ResizeByOneAndAppend(column, value);
                 cacheIndex++;
             }
 
-            foreach (DataFrameColumn column in Columns)
+            foreach (DataFrameColumn column in ret.Columns)
             {
                 if (column.Length == Rows.Count)
                 {
-                    ResizeByOneAndAppend(column, null);
+                    ret.ResizeByOneAndAppend(column, null);
                 }
             }
-            Columns.RowCount++;
+            ret.Columns.RowCount++;
+            return ret;
         }
 
         /// <summary>
diff --git a/tests/Microsoft.Data.Analysis.Tests/DataFrameTests.cs b/tests/Microsoft.Data.Analysis.Tests/DataFrameTests.cs
@@ -1947,84 +1947,176 @@ public void TestMutationOnRows()
             }
         }
 
+        [Fact]
+        public void TestAppendRows()
+        {
+            DataFrame df = MakeDataFrame<float, bool>(10);
+            DataFrame df2 = MakeDataFrame<int, bool>(5);
+            Assert.Equal(10, df.Rows.Count);
+            Assert.Equal(1, df.Columns[0].NullCount);
+            Assert.Equal(1, df.Columns[1].NullCount);
+
+            DataFrame ret = df.Append(df2.Rows, inPlace: false);
+            Assert.Equal(10, df.Rows.Count);
+            Assert.Equal(1, df.Columns[0].NullCount);
+            Assert.Equal(1, df.Columns[1].NullCount);
+
+            Verify(ret, df, df2);
+
+            void Verify(DataFrame ret, DataFrame check1, DataFrame check2)
+            {
+                Assert.Equal(15, ret.Rows.Count);
+                Assert.Equal(2, ret.Columns[0].NullCount);
+                Assert.Equal(2, ret.Columns[1].NullCount);
+                for (long i = 0; i < ret.Rows.Count; i++)
+                {
+                    DataFrameRow row = ret.Rows[i];
+                    for (int j = 0; j < check1.Columns.Count; j++)
+                    {
+                        if (i < check1.Rows.Count)
+                        {
+                            Assert.Equal(row[j], check1.Rows[i][j]);
+                        }
+                        else
+                        {
+                            Assert.Equal(row[j]?.ToString(), (check2.Rows[i - check1.Rows.Count][j])?.ToString());
+                        }
+                    }
+                }
+            }
+
+            DataFrame dfClone = df.Clone();
+            df.Append(df2.Rows, inPlace: true);
+            Verify(df, dfClone, df2);
+        }
+
         [Fact]
         public void TestAppendRow()
         {
             DataFrame df = MakeDataFrame<int, bool>(10);
-            df.Append(new List<object> { 5, true });
+            df.Append(new List<object> { 5, true }, inPlace: true);
             Assert.Equal(11, df.Rows.Count);
             Assert.Equal(1, df.Columns[0].NullCount);
             Assert.Equal(1, df.Columns[1].NullCount);
 
-            df.Append(new List<object> { 100 });
+            DataFrame ret = df.Append(new List<object> { 5, true });
+            Assert.Equal(12, ret.Rows.Count);
+            Assert.Equal(1, ret.Columns[0].NullCount);
+            Assert.Equal(1, ret.Columns[1].NullCount);
+
+            df.Append(new List<object> { 100 }, inPlace: true);
             Assert.Equal(12, df.Rows.Count);
             Assert.Equal(1, df.Columns[0].NullCount);
             Assert.Equal(2, df.Columns[1].NullCount);
 
-            df.Append(new List<object> { null, null });
+            ret = df.Append(new List<object> { 100 }, inPlace: false);
+            Assert.Equal(13, ret.Rows.Count);
+            Assert.Equal(1, ret.Columns[0].NullCount);
+            Assert.Equal(3, ret.Columns[1].NullCount);
+
+            df.Append(new List<object> { null, null }, inPlace: true);
             Assert.Equal(13, df.Rows.Count);
             Assert.Equal(2, df.Columns[0].NullCount);
             Assert.Equal(3, df.Columns[1].NullCount);
+            ret = df.Append(new List<object> { null, null }, inPlace: false);
+            Assert.Equal(14, ret.Rows.Count);
+            Assert.Equal(3, ret.Columns[0].NullCount);
+            Assert.Equal(4, ret.Columns[1].NullCount);
 
-            df.Append(new Dictionary<string, object> { { "Column1", (object)5 } ,  { "Column2", false } });
+            df.Append(new Dictionary<string, object> { { "Column1", (object)5 }, { "Column2", false } }, inPlace: true);
             Assert.Equal(14, df.Rows.Count);
             Assert.Equal(2, df.Columns[0].NullCount);
             Assert.Equal(3, df.Columns[1].NullCount);
+            ret = df.Append(new Dictionary<string, object> { { "Column1", (object)5 }, { "Column2", false } }, inPlace: false);
+            Assert.Equal(15, ret.Rows.Count);
+            Assert.Equal(2, ret.Columns[0].NullCount);
+            Assert.Equal(3, ret.Columns[1].NullCount);
 
-            df.Append(new Dictionary<string, object> { { "Column1", 5 } });
+            df.Append(new Dictionary<string, object> { { "Column1", 5 } }, inPlace: true);
             Assert.Equal(15, df.Rows.Count);
 
             Assert.Equal(15, df["Column1"].Length);
             Assert.Equal(15, df["Column2"].Length);
             Assert.Equal(2, df.Columns[0].NullCount);
             Assert.Equal(4, df.Columns[1].NullCount);
+            ret = df.Append(new Dictionary<string, object> { { "Column1", 5 } }, inPlace: false);
+            Assert.Equal(16, ret.Rows.Count);
 
-            df.Append(new Dictionary<string, object> { { "Column2", false } });
+            Assert.Equal(16, ret["Column1"].Length);
+            Assert.Equal(16, ret["Column2"].Length);
+            Assert.Equal(2, ret.Columns[0].NullCount);
+            Assert.Equal(5, ret.Columns[1].NullCount);
+
+            df.Append(new Dictionary<string, object> { { "Column2", false } }, inPlace: true);
             Assert.Equal(16, df.Rows.Count);
             Assert.Equal(16, df["Column1"].Length);
             Assert.Equal(16, df["Column2"].Length);
             Assert.Equal(3, df.Columns[0].NullCount);
             Assert.Equal(4, df.Columns[1].NullCount);
-
-            df.Append((IEnumerable<object>)null);
+            ret = df.Append(new Dictionary<string, object> { { "Column2", false } }, inPlace: false);
+            Assert.Equal(17, ret.Rows.Count);
+            Assert.Equal(17, ret["Column1"].Length);
+            Assert.Equal(17, ret["Column2"].Length);
+            Assert.Equal(4, ret.Columns[0].NullCount);
+            Assert.Equal(4, ret.Columns[1].NullCount);
+
+            df.Append((IEnumerable<object>)null, inPlace: true);
             Assert.Equal(17, df.Rows.Count);
             Assert.Equal(17, df["Column1"].Length);
             Assert.Equal(17, df["Column2"].Length);
             Assert.Equal(4, df.Columns[0].NullCount);
             Assert.Equal(5, df.Columns[1].NullCount);
+            ret = df.Append((IEnumerable<object>)null, inPlace: false);
+            Assert.Equal(18, ret.Rows.Count);
+            Assert.Equal(18, ret["Column1"].Length);
+            Assert.Equal(18, ret["Column2"].Length);
+            Assert.Equal(5, ret.Columns[0].NullCount);
+            Assert.Equal(6, ret.Columns[1].NullCount);
 
             // DataFrame must remain usable even if Append throws
-            Assert.Throws<FormatException>(() => df.Append(new List<object> { 5, "str" }));
-            Assert.Throws<FormatException>(() => df.Append(new Dictionary<string, object> { { "Column2", "str" } }));
-            Assert.Throws<ArgumentException>(() => df.Append(new List<object> { 5, true, true }));
+            Assert.Throws<FormatException>(() => df.Append(new List<object> { 5, "str" }, inPlace: true));
+            Assert.Throws<FormatException>(() => df.Append(new Dictionary<string, object> { { "Column2", "str" } }, inPlace: true));
+            Assert.Throws<ArgumentException>(() => df.Append(new List<object> { 5, true, true }, inPlace: true));
+
+            df.Append(inPlace: true);
+            Assert.Equal(18, df.Rows.Count);
+            Assert.Equal(18, df["Column1"].Length);
+            Assert.Equal(18, df["Column2"].Length);
+            Assert.Equal(5, df.Columns[0].NullCount);
+            Assert.Equal(6, df.Columns[1].NullCount);
 
-            df.Append();
+            ret = df.Append(inPlace: false);
             Assert.Equal(18, df.Rows.Count);
             Assert.Equal(18, df["Column1"].Length);
             Assert.Equal(18, df["Column2"].Length);
             Assert.Equal(5, df.Columns[0].NullCount);
             Assert.Equal(6, df.Columns[1].NullCount);
+            Assert.Equal(19, ret.Rows.Count);
+            Assert.Equal(19, ret["Column1"].Length);
+            Assert.Equal(19, ret["Column2"].Length);
+            Assert.Equal(6, ret.Columns[0].NullCount);
+            Assert.Equal(7, ret.Columns[1].NullCount);
         }
 
         [Fact]
         public void TestAppendEmptyValue()
         {
             DataFrame df = MakeDataFrame<int, bool>(10);
-            df.Append(new List<object> { "", true });
+            df.Append(new List<object> { "", true }, inPlace: true);
             Assert.Equal(11, df.Rows.Count);
             Assert.Equal(2, df.Columns[0].NullCount);
             Assert.Equal(1, df.Columns[1].NullCount);
 
             StringDataFrameColumn column = new StringDataFrameColumn("Strings", Enumerable.Range(0, 11).Select(x => x.ToString()));
             df.Columns.Add(column);
 
-            df.Append(new List<object> { 1, true, "" });
+            df.Append(new List<object> { 1, true, "" }, inPlace: true);
             Assert.Equal(12, df.Rows.Count);
             Assert.Equal(2, df.Columns[0].NullCount);
             Assert.Equal(1, df.Columns[1].NullCount);
             Assert.Equal(0, df.Columns[2].NullCount);
 
-            df.Append(new List<object> { 1, true, null });
+            df.Append(new List<object> { 1, true, null }, inPlace: true);
             Assert.Equal(13, df.Rows.Count);
             Assert.Equal(1, df.Columns[2].NullCount);
         }

Original file line number	Diff line number	Diff line change
`@@ -271,7 +271,7 @@ public static DataFrame LoadCsv(Stream csvStream,`
`271`	`271`	`}`
`272`	`272`	`else`
`273`	`273`	`{`
`274`		`- ret.Append(spl);`
	`274`	`+ ret.Append(spl, inPlace: true);`
`275`	`275`	`}`
`276`	`276`	`++rowline;`
`277`	`277`	`line = streamReader.ReadLine();`