Merge remote-tracking branch 'upstream/master' into remove-sparse

Author: TomAugspurger

MANIFEST.in

@@ -15,6 +15,7 @@ graft pandas
 global-exclude *.bz2
 global-exclude *.csv
 global-exclude *.dta
+global-exclude *.feather
 global-exclude *.gz
 global-exclude *.h5
 global-exclude *.html
@@ -24,7 +25,10 @@ global-exclude *.pickle
 global-exclude *.png
 global-exclude *.pyc
 global-exclude *.pyd
+global-exclude *.ods
+global-exclude *.odt
 global-exclude *.sas7bdat
+global-exclude *.sav
 global-exclude *.so
 global-exclude *.xls
 global-exclude *.xlsm

doc/source/user_guide/io.rst

@@ -4860,7 +4860,7 @@ The above example creates a partitioned dataset that may look like:
    from shutil import rmtree
    try:
        rmtree('test')
-   except Exception:
+   except OSError:
        pass
 
 .. _io.sql:

doc/source/whatsnew/v1.0.0.rst

@@ -37,6 +37,15 @@ Other enhancements
   (:issue:`28368`)
 -
 
+
+Build Changes
+^^^^^^^^^^^^^
+
+Pandas has added a `pyproject.toml <https://www.python.org/dev/peps/pep-0517/>`_ file and will no longer include
+cythonized files in the source distribution uploaded to PyPI (:issue:`28341`, :issue:`20775`). If you're installing
+a built distribution (wheel) or via conda, this shouldn't have any effect on you. If you're building pandas from
+source, you should no longer need to install Cython into your build environment before calling ``pip install pandas``.
+
 .. _whatsnew_1000.api_breaking:
 
 Backwards incompatible API changes
@@ -116,6 +125,7 @@ Performance improvements
 Bug fixes
 ~~~~~~~~~
 
+- Bug in :meth:`DataFrame.to_html` when using ``formatters=<list>`` and ``max_cols`` together. (:issue:`25955`)
 
 Categorical
 ^^^^^^^^^^^
@@ -180,6 +190,7 @@ Indexing
 ^^^^^^^^
 
 - Bug in assignment using a reverse slicer (:issue:`26939`)
+- Bug in :meth:`DataFrame.explode` would duplicate frame in the presence of duplicates in the index (:issue:`28010`)
 - Bug in reindexing a :meth:`PeriodIndex` with another type of index that contained a `Period` (:issue:`28323`) (:issue:`28337`)
 - Fix assignment of column via `.loc` with numpy non-ns datetime type (:issue:`27395`)
 
@@ -235,13 +246,6 @@ Sparse
 -
 -
 
-
-Build Changes
-^^^^^^^^^^^^^
-- Fixed pyqt development dependency issue because of different pyqt package name in conda and PyPI (:issue:`26838`)
-- Added a `pyproject.toml <https://www.python.org/dev/peps/pep-0517/>`_ file (:issue:`20775`)
-
-
 ExtensionArray
 ^^^^^^^^^^^^^^
 

pandas/core/frame.py

@@ -5253,7 +5253,6 @@ def reorder_levels(self, order, axis=0):
 
     def _combine_frame(self, other, func, fill_value=None, level=None):
         this, other = self.align(other, join="outer", level=level, copy=False)
-        new_index, new_columns = this.index, this.columns
 
         if fill_value is None:
             # since _arith_op may be called in a loop, avoid function call
@@ -5271,38 +5270,62 @@ def _arith_op(left, right):
 
         if ops.should_series_dispatch(this, other, func):
             # iterate over columns
-            return ops.dispatch_to_series(this, other, _arith_op)
+            new_data = ops.dispatch_to_series(this, other, _arith_op)
         else:
             with np.errstate(all="ignore"):
-                result = _arith_op(this.values, other.values)
-            result = dispatch_fill_zeros(func, this.values, other.values, result)
-            return self._constructor(
-                result, index=new_index, columns=new_columns, copy=False
-            )
+                res_values = _arith_op(this.values, other.values)
+            new_data = dispatch_fill_zeros(func, this.values, other.values, res_values)
+        return this._construct_result(other, new_data, _arith_op)
 
     def _combine_match_index(self, other, func, level=None):
         left, right = self.align(other, join="outer", axis=0, level=level, copy=False)
         # at this point we have `left.index.equals(right.index)`
 
         if left._is_mixed_type or right._is_mixed_type:
             # operate column-wise; avoid costly object-casting in `.values`
-            return ops.dispatch_to_series(left, right, func)
+            new_data = ops.dispatch_to_series(left, right, func)
         else:
             # fastpath --> operate directly on values
             with np.errstate(all="ignore"):
                 new_data = func(left.values.T, right.values).T
-            return self._constructor(
-                new_data, index=left.index, columns=self.columns, copy=False
-            )
+        return left._construct_result(other, new_data, func)
 
     def _combine_match_columns(self, other: Series, func, level=None):
         left, right = self.align(other, join="outer", axis=1, level=level, copy=False)
         # at this point we have `left.columns.equals(right.index)`
-        return ops.dispatch_to_series(left, right, func, axis="columns")
+        new_data = ops.dispatch_to_series(left, right, func, axis="columns")
+        return left._construct_result(right, new_data, func)
 
     def _combine_const(self, other, func):
         # scalar other or np.ndim(other) == 0
-        return ops.dispatch_to_series(self, other, func)
+        new_data = ops.dispatch_to_series(self, other, func)
+        return self._construct_result(other, new_data, func)
+
+    def _construct_result(self, other, result, func):
+        """
+        Wrap the result of an arithmetic, comparison, or logical operation.
+
+        Parameters
+        ----------
+        other : object
+        result : DataFrame
+        func : binary operator
+
+        Returns
+        -------
+        DataFrame
+
+        Notes
+        -----
+        `func` is included for compat with SparseDataFrame signature, is not
+        needed here.
+        """
+        out = self._constructor(result, index=self.index, copy=False)
+        # Pin columns instead of passing to constructor for compat with
+        #  non-unique columns case
+        out.columns = self.columns
+        return out
+        # TODO: finalize?  we do for SparseDataFrame
 
     def combine(self, other, func, fill_value=None, overwrite=True):
         """
@@ -6206,12 +6229,13 @@ def explode(self, column: Union[str, Tuple]) -> "DataFrame":
         if not self.columns.is_unique:
             raise ValueError("columns must be unique")
 
-        result = self[column].explode()
-        return (
-            self.drop([column], axis=1)
-            .join(result)
-            .reindex(columns=self.columns, copy=False)
-        )
+        df = self.reset_index(drop=True)
+        result = df[column].explode()
+        result = df.drop([column], axis=1).join(result)
+        result.index = self.index.take(result.index)
+        result = result.reindex(columns=self.columns, copy=False)
+
+        return result
 
     def unstack(self, level=-1, fill_value=None):
         """

pandas/core/generic.py

@@ -33,7 +33,6 @@
 from pandas.util._decorators import Appender, Substitution, rewrite_axis_style_signature
 from pandas.util._validators import validate_bool_kwarg, validate_fillna_kwargs
 
-from pandas.core.dtypes.cast import maybe_promote, maybe_upcast_putmask
 from pandas.core.dtypes.common import (
     ensure_int64,
     ensure_object,
@@ -968,15 +967,12 @@ def squeeze(self, axis=None):
         1
         """
         axis = self._AXIS_NAMES if axis is None else (self._get_axis_number(axis),)
-        try:
-            return self.iloc[
-                tuple(
-                    0 if i in axis and len(a) == 1 else slice(None)
-                    for i, a in enumerate(self.axes)
-                )
-            ]
-        except Exception:
-            return self
+        return self.iloc[
+            tuple(
+                0 if i in axis and len(a) == 1 else slice(None)
+                for i, a in enumerate(self.axes)
+            )
+        ]
 
     def swaplevel(self, i=-2, j=-1, axis=0):
         """
@@ -9042,22 +9038,9 @@ def _where(
                         # try to not change dtype at first (if try_quick)
                         if try_quick:
 
-                            try:
-                                new_other = com.values_from_object(self)
-                                new_other = new_other.copy()
-                                new_other[icond] = other
-                                other = new_other
-                            except Exception:
-                                try_quick = False
-
-                        # let's create a new (if we failed at the above
-                        # or not try_quick
-                        if not try_quick:
-
-                            dtype, fill_value = maybe_promote(other.dtype)
-                            new_other = np.empty(len(icond), dtype=dtype)
-                            new_other.fill(fill_value)
-                            maybe_upcast_putmask(new_other, icond, other)
+                            new_other = com.values_from_object(self)
+                            new_other = new_other.copy()
+                            new_other[icond] = other
                             other = new_other
 
                     else:

pandas/core/internals/blocks.py

@@ -600,41 +600,42 @@ def _astype(self, dtype, copy=False, errors="raise", **kwargs):
                 return self.copy()
             return self
 
-        try:
-            # force the copy here
-            if self.is_extension:
-                values = self.values.astype(dtype)
-            else:
-                if issubclass(dtype.type, str):
+        # force the copy here
+        if self.is_extension:
+            # TODO: Should we try/except this astype?
+            values = self.values.astype(dtype)
+        else:
+            if issubclass(dtype.type, str):
 
-                    # use native type formatting for datetime/tz/timedelta
-                    if self.is_datelike:
-                        values = self.to_native_types()
-
-                    # astype formatting
-                    else:
-                        values = self.get_values()
+                # use native type formatting for datetime/tz/timedelta
+                if self.is_datelike:
+                    values = self.to_native_types()
 
+                # astype formatting
                 else:
-                    values = self.get_values(dtype=dtype)
-
-                # _astype_nansafe works fine with 1-d only
-                vals1d = values.ravel()
-                values = astype_nansafe(vals1d, dtype, copy=True, **kwargs)
+                    values = self.get_values()
 
-            # TODO(extension)
-            # should we make this attribute?
-            if isinstance(values, np.ndarray):
-                values = values.reshape(self.shape)
+            else:
+                values = self.get_values(dtype=dtype)
 
-        except Exception:
-            # e.g. astype_nansafe can fail on object-dtype of strings
-            #  trying to convert to float
-            if errors == "raise":
-                raise
-            newb = self.copy() if copy else self
-        else:
-            newb = make_block(values, placement=self.mgr_locs, ndim=self.ndim)
+            # _astype_nansafe works fine with 1-d only
+            vals1d = values.ravel()
+            try:
+                values = astype_nansafe(vals1d, dtype, copy=True, **kwargs)
+            except (ValueError, TypeError):
+                # e.g. astype_nansafe can fail on object-dtype of strings
+                #  trying to convert to float
+                if errors == "raise":
+                    raise
+                newb = self.copy() if copy else self
+                return newb
+
+        # TODO(extension)
+        # should we make this attribute?
+        if isinstance(values, np.ndarray):
+            values = values.reshape(self.shape)
+
+        newb = make_block(values, placement=self.mgr_locs, ndim=self.ndim)
 
         if newb.is_numeric and self.is_numeric:
             if newb.shape != self.shape:

pandas/core/nanops.py

@@ -1300,10 +1300,12 @@ def _ensure_numeric(x):
     elif not (is_float(x) or is_integer(x) or is_complex(x)):
         try:
             x = float(x)
-        except Exception:
+        except ValueError:
+            # e.g. "1+1j" or "foo"
             try:
                 x = complex(x)
-            except Exception:
+            except ValueError:
+                # e.g. "foo"
                 raise TypeError(
                     "Could not convert {value!s} to numeric".format(value=x)
                 )

pandas/core/ops/__init__.py

@@ -512,12 +512,7 @@ def column_op(a, b):
         raise NotImplementedError(right)
 
     new_data = expressions.evaluate(column_op, str_rep, left, right)
-
-    result = left._constructor(new_data, index=left.index, copy=False)
-    # Pin columns instead of passing to constructor for compat with
-    # non-unique columns case
-    result.columns = left.columns
-    return result
+    return new_data
 
 
 def dispatch_to_extension_op(
@@ -1056,7 +1051,8 @@ def f(self, other, axis=default_axis, level=None):
             # Another DataFrame
             if not self._indexed_same(other):
                 self, other = self.align(other, "outer", level=level, copy=False)
-            return dispatch_to_series(self, other, na_op, str_rep)
+            new_data = dispatch_to_series(self, other, na_op, str_rep)
+            return self._construct_result(other, new_data, na_op)
 
         elif isinstance(other, ABCSeries):
             return _combine_series_frame(
@@ -1086,7 +1082,8 @@ def f(self, other):
                 raise ValueError(
                     "Can only compare identically-labeled DataFrame objects"
                 )
-            return dispatch_to_series(self, other, func, str_rep)
+            new_data = dispatch_to_series(self, other, func, str_rep)
+            return self._construct_result(other, new_data, func)
 
         elif isinstance(other, ABCSeries):
             return _combine_series_frame(

pandas/core/series.py

@@ -1277,11 +1277,6 @@ def _set_with(self, key, value):
 
             if is_scalar(key):
                 key = [key]
-            elif not isinstance(key, (list, Series, np.ndarray)):
-                try:
-                    key = list(key)
-                except Exception:
-                    key = [key]
 
             if isinstance(key, Index):
                 key_type = key.inferred_type

pandas/io/formats/format.py

@@ -657,6 +657,13 @@ def _chk_truncate(self) -> None:
                 frame = concat(
                     (frame.iloc[:, :col_num], frame.iloc[:, -col_num:]), axis=1
                 )
+                # truncate formatter
+                if isinstance(self.formatters, (list, tuple)):
+                    truncate_fmt = self.formatters
+                    self.formatters = [
+                        *truncate_fmt[:col_num],
+                        *truncate_fmt[-col_num:],
+                    ]
             self.tr_col_num = col_num
         if truncate_v:
             # cast here since if truncate_v is True, max_rows_adj is not None