BUG: fix segfault introduced by prior #2981 fix

Author: wesm

pandas/io/tests/test_parsers.py

@@ -1548,26 +1548,6 @@ def test_int64_min_issues(self):
 
         tm.assert_frame_equal(result, expected)
 
-    def test_parse_ragged_csv(self):
-        data = """1,2,3
-1,2,3,4
-1,2,3,4,5
-1,2
-1,2,3,4"""
-
-        nice_data = """1,2,3,,
-1,2,3,4,
-1,2,3,4,5
-1,2,,,
-1,2,3,4,"""
-        result = self.read_csv(StringIO(data), header=None,
-                               names=['a', 'b', 'c', 'd', 'e'])
-
-        expected = self.read_csv(StringIO(nice_data), header=None,
-                                 names=['a', 'b', 'c', 'd', 'e'])
-
-        tm.assert_frame_equal(result, expected)
-
 
 class TestPythonParser(ParserTests, unittest.TestCase):
 
@@ -2078,6 +2058,36 @@ def test_na_trailing_columns(self):
         self.assertEquals(result['Date'][1], '2012-05-12')
         self.assertTrue(result['UnitPrice'].isnull().all())
 
+    def test_parse_ragged_csv(self):
+        data = """1,2,3
+1,2,3,4
+1,2,3,4,5
+1,2
+1,2,3,4"""
+
+        nice_data = """1,2,3,,
+1,2,3,4,
+1,2,3,4,5
+1,2,,,
+1,2,3,4,"""
+        result = self.read_csv(StringIO(data), header=None,
+                               names=['a', 'b', 'c', 'd', 'e'])
+
+        expected = self.read_csv(StringIO(nice_data), header=None,
+                                 names=['a', 'b', 'c', 'd', 'e'])
+
+        tm.assert_frame_equal(result, expected)
+
+        # too many columns, cause segfault if not careful
+        data = "1,2\n3,4,5"
+
+        result = self.read_csv(StringIO(data), header=None,
+                               names=range(50))
+        expected = self.read_csv(StringIO(data), header=None,
+                                 names=range(3)).reindex(columns=range(50))
+
+        tm.assert_frame_equal(result, expected)
+
 
 class TestParseSQL(unittest.TestCase):
 

pandas/src/parser/tokenizer.c

@@ -498,7 +498,14 @@ static int end_line(parser_t *self) {
     }
     else {
         /* missing trailing delimiters */
-        if (self->lines >= self->header + 1) {
+        if ((self->lines >= self->header + 1) && fields < ex_fields) {
+
+            /* Might overrun the buffer when closing fields */
+            if (make_stream_space(self, ex_fields - fields) < 0) {
+                self->error_msg = "out of memory";
+                return -1;
+            }
+
             while (fields < ex_fields){
                 end_field(self);
                 /* printf("Prior word: %s\n", self->words[self->words_len - 2]); */