Merge branch 'main' into issue-50977

Author: kostyafarber

.pre-commit-config.yaml

@@ -92,7 +92,7 @@ repos:
         args: [--disable=all, --enable=redefined-outer-name]
         stages: [manual]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.11.4
+    rev: 5.12.0
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade

LICENSE

@@ -3,7 +3,7 @@ BSD 3-Clause License
 Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team
 All rights reserved.
 
-Copyright (c) 2011-2022, Open source contributors.
+Copyright (c) 2011-2023, Open source contributors.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

ci/code_checks.sh

@@ -83,7 +83,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX04,GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,PR03,PR04,PR05,PR06,PR08,PR09,PR10,RT01,RT02,RT04,RT05,SA02,SA03,SA04,SS01,SS02,SS03,SS04,SS05,SS06
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    MSG='Partially validate docstrings (EX01)' ; echo $MSG
+    MSG='Partially validate docstrings (EX01)' ;  echo $MSG
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX01 --ignore_functions \
         pandas.Series.index \
         pandas.Series.dtype \
@@ -574,7 +574,73 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.DataFrame.sparse.to_coo \
         pandas.DataFrame.to_gbq \
         pandas.DataFrame.style \
-        pandas.DataFrame.__dataframe__ \
+        pandas.DataFrame.__dataframe__
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Partially validate docstrings (EX02)' ;  echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX02 --ignore_functions \
+        pandas.DataFrame.copy \
+        pandas.DataFrame.plot.line \
+        pandas.DataFrame.std \
+        pandas.DataFrame.var \
+        pandas.Index.factorize \
+        pandas.Period.strftime \
+        pandas.Series.copy \
+        pandas.Series.factorize \
+        pandas.Series.floordiv \
+        pandas.Series.plot.line \
+        pandas.Series.rfloordiv \
+        pandas.Series.sparse.density \
+        pandas.Series.sparse.npoints \
+        pandas.Series.sparse.sp_values \
+        pandas.Series.std \
+        pandas.Series.var \
+        pandas.Timestamp.fromtimestamp \
+        pandas.api.types.infer_dtype \
+        pandas.api.types.is_bool_dtype \
+        pandas.api.types.is_categorical_dtype \
+        pandas.api.types.is_complex_dtype \
+        pandas.api.types.is_datetime64_any_dtype \
+        pandas.api.types.is_datetime64_dtype \
+        pandas.api.types.is_datetime64_ns_dtype \
+        pandas.api.types.is_datetime64tz_dtype \
+        pandas.api.types.is_dict_like \
+        pandas.api.types.is_file_like \
+        pandas.api.types.is_float_dtype \
+        pandas.api.types.is_hashable \
+        pandas.api.types.is_int64_dtype \
+        pandas.api.types.is_integer_dtype \
+        pandas.api.types.is_interval_dtype \
+        pandas.api.types.is_iterator \
+        pandas.api.types.is_list_like \
+        pandas.api.types.is_named_tuple \
+        pandas.api.types.is_numeric_dtype \
+        pandas.api.types.is_object_dtype \
+        pandas.api.types.is_period_dtype \
+        pandas.api.types.is_re \
+        pandas.api.types.is_re_compilable \
+        pandas.api.types.is_signed_integer_dtype \
+        pandas.api.types.is_sparse \
+        pandas.api.types.is_string_dtype \
+        pandas.api.types.is_timedelta64_dtype \
+        pandas.api.types.is_timedelta64_ns_dtype \
+        pandas.api.types.is_unsigned_integer_dtype \
+        pandas.core.groupby.DataFrameGroupBy.take \
+        pandas.core.groupby.SeriesGroupBy.take \
+        pandas.factorize \
+        pandas.io.formats.style.Styler.concat \
+        pandas.io.formats.style.Styler.export \
+        pandas.io.formats.style.Styler.set_td_classes \
+        pandas.io.formats.style.Styler.use \
+        pandas.io.json.build_table_schema \
+        pandas.merge_ordered \
+        pandas.option_context \
+        pandas.plotting.andrews_curves \
+        pandas.plotting.autocorrelation_plot \
+        pandas.plotting.lag_plot \
+        pandas.plotting.parallel_coordinates \
+        pandas.plotting.radviz \
+        pandas.tseries.frequencies.to_offset
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

doc/source/reference/frame.rst

@@ -83,6 +83,7 @@ Binary operator functions
 .. autosummary::
    :toctree: api/
 
+   DataFrame.__add__
    DataFrame.add
    DataFrame.sub
    DataFrame.mul

doc/source/whatsnew/v2.0.0.rst

@@ -183,6 +183,7 @@ Other enhancements
 - Added ``copy`` parameter to :meth:`Series.infer_objects` and :meth:`DataFrame.infer_objects`, passing ``False`` will avoid making copies for series or columns that are already non-object or where no better dtype can be inferred (:issue:`50096`)
 - :meth:`DataFrame.plot.hist` now recognizes ``xlabel`` and ``ylabel`` arguments (:issue:`49793`)
 - :meth:`Series.drop_duplicates` has gained ``ignore_index`` keyword to reset index (:issue:`48304`)
+- :meth:`Series.dropna` and :meth:`DataFrame.dropna` has gained ``ignore_index`` keyword to reset index (:issue:`31725`)
 - Improved error message in :func:`to_datetime` for non-ISO8601 formats, informing users about the position of the first error (:issue:`50361`)
 - Improved error message when trying to align :class:`DataFrame` objects (for example, in :func:`DataFrame.compare`) to clarify that "identically labelled" refers to both index and columns (:issue:`50083`)
 - Added :meth:`DatetimeIndex.as_unit` and :meth:`TimedeltaIndex.as_unit` to convert to different resolutions; supported resolutions are "s", "ms", "us", and "ns" (:issue:`50616`)
@@ -619,10 +620,11 @@ Other API changes
   new DataFrame (shallow copy) instead of the original DataFrame, consistent with other
   methods to get a full slice (for example ``df.loc[:]`` or ``df[:]``) (:issue:`49469`)
 - Disallow computing ``cumprod`` for :class:`Timedelta` object; previously this returned incorrect values (:issue:`50246`)
+- Instantiating an :class:`Index` with an numeric numpy dtype with data containing :class:`NA` and/or :class:`NaT` now raises a ``ValueError``. Previously a ``TypeError`` was raised (:issue:`51050`)
 - Loading a JSON file with duplicate columns using ``read_json(orient='split')`` renames columns to avoid duplicates, as :func:`read_csv` and the other readers do (:issue:`50370`)
 - The levels of the index of the :class:`Series` returned from ``Series.sparse.from_coo`` now always have dtype ``int32``. Previously they had dtype ``int64`` (:issue:`50926`)
 - :func:`to_datetime` with ``unit`` of either "Y" or "M" will now raise if a sequence contains a non-round ``float`` value, matching the ``Timestamp`` behavior (:issue:`50301`)
--
+- The methods :meth:`Series.round`, :meth:`DataFrame.__invert__`, :meth:`Series.__invert__`, :meth:`DataFrame.swapaxes`, :meth:`DataFrame.first`, :meth:`DataFrame.last`, :meth:`Series.first`, :meth:`Series.last` and :meth:`DataFrame.align` will now always return new objects (:issue:`51032`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_200.deprecations:
@@ -639,6 +641,7 @@ Deprecations
 - :meth:`Index.is_categorical` has been deprecated. Use :func:`pandas.api.types.is_categorical_dtype` instead (:issue:`50042`)
 - :meth:`Index.is_object` has been deprecated. Use :func:`pandas.api.types.is_object_dtype` instead (:issue:`50042`)
 - :meth:`Index.is_interval` has been deprecated. Use :func:`pandas.api.types.is_intterval_dtype` instead (:issue:`50042`)
+- Deprecated ``all`` and ``any`` reductions with ``datetime64`` and :class:`DatetimeTZDtype` dtypes, use e.g. ``(obj != pd.Timestamp(0), tz=obj.tz).all()`` instead (:issue:`34479`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -758,7 +761,7 @@ Removal of prior version deprecations/changes
 - Disallow passing non-keyword arguments to :meth:`DataFrame.replace`, :meth:`Series.replace` except for ``to_replace`` and ``value`` (:issue:`47587`)
 - Disallow passing non-keyword arguments to :meth:`DataFrame.sort_values` except for ``by`` (:issue:`41505`)
 - Disallow passing non-keyword arguments to :meth:`Series.sort_values` (:issue:`41505`)
-- Disallow passing 2 non-keyword arguments to :meth:`DataFrame.reindex` (:issue:`17966`)
+- Disallow passing non-keyword arguments to :meth:`DataFrame.reindex` except for ``labels`` (:issue:`17966`)
 - Disallow :meth:`Index.reindex` with non-unique :class:`Index` objects (:issue:`42568`)
 - Disallowed constructing :class:`Categorical` with scalar ``data`` (:issue:`38433`)
 - Disallowed constructing :class:`CategoricalIndex` without passing ``data`` (:issue:`38944`)
@@ -1965,6 +1968,7 @@ Conversion
 - Bug in :class:`.arrays.ArrowExtensionArray` that would raise ``NotImplementedError`` when passed a sequence of strings or binary (:issue:`49172`)
 - Bug in :meth:`Series.astype` raising ``pyarrow.ArrowInvalid`` when converting from a non-pyarrow string dtype to a pyarrow numeric type (:issue:`50430`)
 - Bug in :meth:`Series.to_numpy` converting to NumPy array before applying ``na_value`` (:issue:`48951`)
+- Bug in :meth:`DataFrame.astype` not copying data when converting to pyarrow dtype (:issue:`50984`)
 - Bug in :func:`to_datetime` was not respecting ``exact`` argument when ``format`` was an ISO8601 format (:issue:`12649`)
 - Bug in :meth:`TimedeltaArray.astype` raising ``TypeError`` when converting to a pyarrow duration type (:issue:`49795`)
 -
@@ -2051,6 +2055,7 @@ Period
 - Bug in adding a :class:`Period` object to an array of :class:`DateOffset` objects incorrectly raising ``TypeError`` (:issue:`50162`)
 - Bug in :class:`Period` where passing a string with finer resolution than nanosecond would result in a ``KeyError`` instead of dropping the extra precision (:issue:`50417`)
 - Bug in parsing strings representing Week-periods e.g. "2017-01-23/2017-01-29" as minute-frequency instead of week-frequency (:issue:`50803`)
+- Bug in :meth:`GroupBy.sum`, :meth:`GroupBy.cumsum`, :meth:`GroupBy.prod`, :meth:`GroupBy.cumprod` with :class:`PeriodDtype` failing to raise ``TypeError`` (:issue:`51040`)
 -
 
 Plotting
@@ -2101,7 +2106,7 @@ Sparse
 ^^^^^^
 - Bug in :meth:`Series.astype` when converting a ``SparseDtype`` with ``datetime64[ns]`` subtype to ``int64`` dtype raising, inconsistent with the non-sparse behavior (:issue:`49631`,:issue:`50087`)
 - Bug in :meth:`Series.astype` when converting a from ``datetime64[ns]`` to ``Sparse[datetime64[ns]]`` incorrectly raising (:issue:`50082`)
--
+- Bug in :meth:`Series.sparse.to_coo` raising ``SystemError`` when :class:`MultiIndex` contains a ``ExtensionArray`` (:issue:`50996`)
 
 ExtensionArray
 ^^^^^^^^^^^^^^

pandas/_testing/__init__.py

@@ -100,7 +100,6 @@
     get_obj,
 )
 from pandas._testing.contexts import (
-    RNGContext,
     decompress_file,
     ensure_clean,
     ensure_safe_environment_variables,
@@ -1135,7 +1134,6 @@ def shares_memory(left, right) -> bool:
     "raise_assert_detail",
     "rands",
     "reset_display_options",
-    "RNGContext",
     "raises_chained_assignment_error",
     "round_trip_localpath",
     "round_trip_pathlib",

pandas/_testing/contexts.py

@@ -4,16 +4,13 @@
 import os
 from pathlib import Path
 import tempfile
-from types import TracebackType
 from typing import (
     IO,
     Any,
     Generator,
 )
 import uuid
 
-import numpy as np
-
 from pandas.compat import PYPY
 from pandas.errors import ChainedAssignmentError
 
@@ -198,40 +195,6 @@ def use_numexpr(use, min_elements=None) -> Generator[None, None, None]:
         set_option("compute.use_numexpr", olduse)
 
 
-class RNGContext:
-    """
-    Context manager to set the numpy random number generator speed. Returns
-    to the original value upon exiting the context manager.
-
-    Parameters
-    ----------
-    seed : int
-        Seed for numpy.random.seed
-
-    Examples
-    --------
-    with RNGContext(42):
-        np.random.randn()
-    """
-
-    def __init__(self, seed) -> None:
-        self.seed = seed
-
-    def __enter__(self) -> None:
-
-        self.start_state = np.random.get_state()
-        np.random.seed(self.seed)
-
-    def __exit__(
-        self,
-        exc_type: type[BaseException] | None,
-        exc_value: BaseException | None,
-        traceback: TracebackType | None,
-    ) -> None:
-
-        np.random.set_state(self.start_state)
-
-
 def raises_chained_assignment_error():
 
     if PYPY:

pandas/conftest.py

@@ -149,8 +149,6 @@ def pytest_collection_modifyitems(items, config) -> None:
     ignored_doctest_warnings = [
         # Docstring divides by zero to show behavior difference
         ("missing.mask_zero_div_zero", "divide by zero encountered"),
-        # Docstring demonstrates the call raises a warning
-        ("_validators.validate_axis_style_args", "Use named arguments"),
     ]
 
     for item in items:

pandas/core/algorithms.py

@@ -1197,8 +1197,10 @@ def is_valid_dtype_n_method(dtype: DtypeObj) -> bool:
         nsmallest/nlargest methods
         """
         return (
-            is_numeric_dtype(dtype) and not is_complex_dtype(dtype)
-        ) or needs_i8_conversion(dtype)
+            not is_complex_dtype(dtype)
+            if is_numeric_dtype(dtype)
+            else needs_i8_conversion(dtype)
+        )
 
 
 class SelectNSeries(SelectN):

pandas/core/arraylike.py

@@ -97,6 +97,92 @@ def _arith_method(self, other, op):
 
     @unpack_zerodim_and_defer("__add__")
     def __add__(self, other):
+        """
+        Get Addition of DataFrame and other, column-wise.
+
+        Equivalent to ``DataFrame.add(other)``.
+
+        Parameters
+        ----------
+        other : scalar, sequence, Series, dict or DataFrame
+            Object to be added to the DataFrame.
+
+        Returns
+        -------
+        DataFrame
+            The result of adding ``other`` to DataFrame.
+
+        See Also
+        --------
+        DataFrame.add : Add a DataFrame and another object, with option for index-
+            or column-oriented addition.
+
+        Examples
+        --------
+        >>> df = pd.DataFrame({'height': [1.5, 2.6], 'weight': [500, 800]},
+        ...                   index=['elk', 'moose'])
+        >>> df
+               height  weight
+        elk       1.5     500
+        moose     2.6     800
+
+        Adding a scalar affects all rows and columns.
+
+        >>> df[['height', 'weight']] + 1.5
+               height  weight
+        elk       3.0   501.5
+        moose     4.1   801.5
+
+        Each element of a list is added to a column of the DataFrame, in order.
+
+        >>> df[['height', 'weight']] + [0.5, 1.5]
+               height  weight
+        elk       2.0   501.5
+        moose     3.1   801.5
+
+        Keys of a dictionary are aligned to the DataFrame, based on column names;
+        each value in the dictionary is added to the corresponding column.
+
+        >>> df[['height', 'weight']] + {'height': 0.5, 'weight': 1.5}
+               height  weight
+        elk       2.0   501.5
+        moose     3.1   801.5
+
+        When `other` is a :class:`Series`, the index of `other` is aligned with the
+        columns of the DataFrame.
+
+        >>> s1 = pd.Series([0.5, 1.5], index=['weight', 'height'])
+        >>> df[['height', 'weight']] + s1
+               height  weight
+        elk       3.0   500.5
+        moose     4.1   800.5
+
+        Even when the index of `other` is the same as the index of the DataFrame,
+        the :class:`Series` will not be reoriented. If index-wise alignment is desired,
+        :meth:`DataFrame.add` should be used with `axis='index'`.
+
+        >>> s2 = pd.Series([0.5, 1.5], index=['elk', 'moose'])
+        >>> df[['height', 'weight']] + s2
+               elk  height  moose  weight
+        elk    NaN     NaN    NaN     NaN
+        moose  NaN     NaN    NaN     NaN
+
+        >>> df[['height', 'weight']].add(s2, axis='index')
+               height  weight
+        elk       2.0   500.5
+        moose     4.1   801.5
+
+        When `other` is a :class:`DataFrame`, both columns names and the
+        index are aligned.
+
+        >>> other = pd.DataFrame({'height': [0.2, 0.4, 0.6]},
+        ...                      index=['elk', 'moose', 'deer'])
+        >>> df[['height', 'weight']] + other
+               height  weight
+        deer      NaN     NaN
+        elk       1.7     NaN
+        moose     3.0     NaN
+        """
         return self._arith_method(other, operator.add)
 
     @unpack_zerodim_and_defer("__radd__")

pandas/core/arrays/arrow/array.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from copy import deepcopy
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -220,6 +221,9 @@ def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = Fal
         if isinstance(scalars, cls):
             scalars = scalars._data
         elif not isinstance(scalars, (pa.Array, pa.ChunkedArray)):
+            if copy and is_array_like(scalars):
+                # pa array should not get updated when numpy array is updated
+                scalars = deepcopy(scalars)
             try:
                 scalars = pa.array(scalars, type=pa_dtype, from_pandas=True)
             except pa.ArrowInvalid:

pandas/core/arrays/datetimelike.py

@@ -2034,11 +2034,12 @@ def ceil(
     # Reductions
 
     def any(self, *, axis: AxisInt | None = None, skipna: bool = True) -> bool:
-        # GH#34479 discussion of desired behavior long-term
+        # GH#34479 the nanops call will issue a FutureWarning for non-td64 dtype
         return nanops.nanany(self._ndarray, axis=axis, skipna=skipna, mask=self.isna())
 
     def all(self, *, axis: AxisInt | None = None, skipna: bool = True) -> bool:
-        # GH#34479 discussion of desired behavior long-term
+        # GH#34479 the nanops call will issue a FutureWarning for non-td64 dtype
+
         return nanops.nanall(self._ndarray, axis=axis, skipna=skipna, mask=self.isna())
 
     # --------------------------------------------------------------