CLN: assorted cleanups (#39856)

Author: jbrockmendel

Makefile

@@ -25,16 +25,3 @@ doc:
 	cd doc; \
 	python make.py clean; \
 	python make.py html
-
-check:
-	python3 scripts/validate_unwanted_patterns.py \
-		--validation-type="private_function_across_module" \
-		--included-file-extensions="py" \
-		--excluded-file-paths=pandas/tests,asv_bench/ \
-		pandas/
-
-	python3 scripts/validate_unwanted_patterns.py \
-		--validation-type="private_import_across_module" \
-		--included-file-extensions="py" \
-		--excluded-file-paths=pandas/tests,asv_bench/,doc/
-		pandas/

pandas/core/common.py

@@ -49,11 +49,7 @@
     ABCSeries,
 )
 from pandas.core.dtypes.inference import iterable_not_string
-from pandas.core.dtypes.missing import (  # noqa
-    isna,
-    isnull,
-    notnull,
-)
+from pandas.core.dtypes.missing import isna
 
 
 class SettingWithCopyError(ValueError):

pandas/core/frame.py

@@ -752,10 +752,11 @@ def _can_fast_transpose(self) -> bool:
         """
         if isinstance(self._mgr, ArrayManager):
             return False
-        if self._mgr.any_extension_types:
-            # TODO(EA2D) special case would be unnecessary with 2D EAs
+        blocks = self._mgr.blocks
+        if len(blocks) != 1:
             return False
-        return len(self._mgr.blocks) == 1
+
+        return not self._mgr.any_extension_types
 
     # ----------------------------------------------------------------------
     # Rendering Methods

pandas/core/indexes/interval.py

@@ -822,9 +822,7 @@ def _convert_list_indexer(self, keyarr):
     def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
         if not isinstance(dtype, IntervalDtype):
             return False
-        if self.closed != dtype.closed:
-            return False
-        common_subtype = find_common_type([self.dtype.subtype, dtype.subtype])
+        common_subtype = find_common_type([self.dtype, dtype])
         return not is_object_dtype(common_subtype)
 
     # --------------------------------------------------------------------

pandas/core/internals/blocks.py

@@ -1013,7 +1013,7 @@ def setitem(self, indexer, value):
             arr_value = value
         else:
             is_ea_value = False
-            arr_value = np.array(value)
+            arr_value = np.asarray(value)
 
         if transpose:
             values = values.T
@@ -1654,8 +1654,8 @@ def setitem(self, indexer, value):
         be a compatible shape.
         """
         if not self._can_hold_element(value):
-            # This is only relevant for DatetimeTZBlock, which has a
-            #  non-trivial `_can_hold_element`.
+            # This is only relevant for DatetimeTZBlock, ObjectValuesExtensionBlock,
+            #  which has a non-trivial `_can_hold_element`.
             # https://github.com/pandas-dev/pandas/issues/24020
             # Need a dedicated setitem until GH#24020 (type promotion in setitem
             #  for extension arrays) is designed and implemented.
@@ -2081,6 +2081,8 @@ def fillna(
 class DatetimeLikeBlockMixin(NDArrayBackedExtensionBlock):
     """Mixin class for DatetimeBlock, DatetimeTZBlock, and TimedeltaBlock."""
 
+    is_numeric = False
+    _can_hold_na = True
     _dtype: np.dtype
     _holder: Type[Union[DatetimeArray, TimedeltaArray]]
 
@@ -2133,10 +2135,6 @@ class DatetimeBlock(DatetimeLikeBlockMixin):
     _dtype = fill_value.dtype
     _holder = DatetimeArray
 
-    @property
-    def _can_hold_na(self):
-        return True
-
     def set_inplace(self, locs, values):
         """
         See Block.set.__doc__
@@ -2153,6 +2151,8 @@ class DatetimeTZBlock(ExtensionBlock, DatetimeBlock):
 
     __slots__ = ()
     is_extension = True
+    _can_hold_na = True
+    is_numeric = False
 
     _holder = DatetimeArray
 
@@ -2205,8 +2205,6 @@ def external_values(self):
 
 class TimeDeltaBlock(DatetimeLikeBlockMixin):
     __slots__ = ()
-    _can_hold_na = True
-    is_numeric = False
     _holder = TimedeltaArray
     fill_value = np.timedelta64("NaT", "ns")
     _dtype = fill_value.dtype

pandas/core/internals/concat.py

@@ -118,11 +118,8 @@ def concatenate_block_managers(
             else:
                 b = make_block(values, placement=placement, ndim=blk.ndim)
         else:
-            b = make_block(
-                _concatenate_join_units(join_units, concat_axis, copy=copy),
-                placement=placement,
-                ndim=len(axes),
-            )
+            new_values = _concatenate_join_units(join_units, concat_axis, copy=copy)
+            b = make_block(new_values, placement=placement, ndim=len(axes))
         blocks.append(b)
 
     return BlockManager(blocks, axes)

pandas/core/reshape/reshape.py

@@ -429,7 +429,7 @@ def unstack(obj, level, fill_value=None):
             return obj.T.stack(dropna=False)
     elif not isinstance(obj.index, MultiIndex):
         # GH 36113
-        # Give nicer error messages when unstack a  Series whose
+        # Give nicer error messages when unstack a Series whose
         # Index is not a MultiIndex.
         raise ValueError(
             f"index must be a MultiIndex to unstack, {type(obj.index)} was passed"
@@ -451,9 +451,10 @@ def _unstack_frame(obj, level, fill_value=None):
         mgr = obj._mgr.unstack(unstacker, fill_value=fill_value)
         return obj._constructor(mgr)
     else:
-        return _Unstacker(
-            obj.index, level=level, constructor=obj._constructor
-        ).get_result(obj._values, value_columns=obj.columns, fill_value=fill_value)
+        unstacker = _Unstacker(obj.index, level=level, constructor=obj._constructor)
+        return unstacker.get_result(
+            obj._values, value_columns=obj.columns, fill_value=fill_value
+        )
 
 
 def _unstack_extension_series(series, level, fill_value):

pandas/tests/arithmetic/test_timedelta64.py

@@ -2090,6 +2090,7 @@ def test_td64arr_div_numeric_array(
                 expected = [tdser[n] / vector[n] for n in range(len(tdser))]
             expected = pd.Index(expected)  # do dtype inference
             expected = tm.box_expected(expected, xbox)
+            assert tm.get_dtype(expected) == "m8[ns]"
 
             if using_array_manager and box_with_array is pd.DataFrame:
                 # TODO the behaviour is buggy here (third column with all-NaT

pandas/tests/indexing/test_iloc.py

@@ -921,7 +921,7 @@ def test_iloc_setitem_empty_frame_raises_with_3d_ndarray(self):
         with pytest.raises(ValueError, match=msg):
             obj.iloc[nd3] = 0
 
-    @pytest.mark.parametrize("indexer", [lambda x: x.loc, lambda x: x.iloc])
+    @pytest.mark.parametrize("indexer", [tm.loc, tm.iloc])
     def test_iloc_getitem_read_only_values(self, indexer):
         # GH#10043 this is fundamentally a test for iloc, but test loc while
         #  we're here

pandas/tests/series/indexing/test_setitem.py

@@ -242,6 +242,88 @@ def test_setitem_callable_other(self):
         tm.assert_series_equal(ser, expected)
 
 
+class TestSetitemWithExpansion:
+    def test_setitem_empty_series(self):
+        # GH#10193
+        key = Timestamp("2012-01-01")
+        series = Series(dtype=object)
+        series[key] = 47
+        expected = Series(47, [key])
+        tm.assert_series_equal(series, expected)
+
+    def test_setitem_empty_series_datetimeindex_preserves_freq(self):
+        # GH#33573 our index should retain its freq
+        series = Series([], DatetimeIndex([], freq="D"), dtype=object)
+        key = Timestamp("2012-01-01")
+        series[key] = 47
+        expected = Series(47, DatetimeIndex([key], freq="D"))
+        tm.assert_series_equal(series, expected)
+        assert series.index.freq == expected.index.freq
+
+    def test_setitem_empty_series_timestamp_preserves_dtype(self):
+        # GH 21881
+        timestamp = Timestamp(1412526600000000000)
+        series = Series([timestamp], index=["timestamp"], dtype=object)
+        expected = series["timestamp"]
+
+        series = Series([], dtype=object)
+        series["anything"] = 300.0
+        series["timestamp"] = timestamp
+        result = series["timestamp"]
+        assert result == expected
+
+    @pytest.mark.parametrize(
+        "td",
+        [
+            Timedelta("9 days"),
+            Timedelta("9 days").to_timedelta64(),
+            Timedelta("9 days").to_pytimedelta(),
+        ],
+    )
+    def test_append_timedelta_does_not_cast(self, td):
+        # GH#22717 inserting a Timedelta should _not_ cast to int64
+        expected = Series(["x", td], index=[0, "td"], dtype=object)
+
+        ser = Series(["x"])
+        ser["td"] = td
+        tm.assert_series_equal(ser, expected)
+        assert isinstance(ser["td"], Timedelta)
+
+        ser = Series(["x"])
+        ser.loc["td"] = Timedelta("9 days")
+        tm.assert_series_equal(ser, expected)
+        assert isinstance(ser["td"], Timedelta)
+
+
+def test_setitem_scalar_into_readonly_backing_data():
+    # GH#14359: test that you cannot mutate a read only buffer
+
+    array = np.zeros(5)
+    array.flags.writeable = False  # make the array immutable
+    series = Series(array)
+
+    for n in range(len(series)):
+        msg = "assignment destination is read-only"
+        with pytest.raises(ValueError, match=msg):
+            series[n] = 1
+
+        assert array[n] == 0
+
+
+def test_setitem_slice_into_readonly_backing_data():
+    # GH#14359: test that you cannot mutate a read only buffer
+
+    array = np.zeros(5)
+    array.flags.writeable = False  # make the array immutable
+    series = Series(array)
+
+    msg = "assignment destination is read-only"
+    with pytest.raises(ValueError, match=msg):
+        series[1:3] = 1
+
+    assert not array.any()
+
+
 class TestSetitemCasting:
     @pytest.mark.parametrize("unique", [True, False])
     @pytest.mark.parametrize("val", [3, 3.0, "3"], ids=type)
@@ -445,88 +527,6 @@ def val(self, request):
         return request.param
 
 
-class TestSetitemWithExpansion:
-    def test_setitem_empty_series(self):
-        # GH#10193
-        key = Timestamp("2012-01-01")
-        series = Series(dtype=object)
-        series[key] = 47
-        expected = Series(47, [key])
-        tm.assert_series_equal(series, expected)
-
-    def test_setitem_empty_series_datetimeindex_preserves_freq(self):
-        # GH#33573 our index should retain its freq
-        series = Series([], DatetimeIndex([], freq="D"), dtype=object)
-        key = Timestamp("2012-01-01")
-        series[key] = 47
-        expected = Series(47, DatetimeIndex([key], freq="D"))
-        tm.assert_series_equal(series, expected)
-        assert series.index.freq == expected.index.freq
-
-    def test_setitem_empty_series_timestamp_preserves_dtype(self):
-        # GH 21881
-        timestamp = Timestamp(1412526600000000000)
-        series = Series([timestamp], index=["timestamp"], dtype=object)
-        expected = series["timestamp"]
-
-        series = Series([], dtype=object)
-        series["anything"] = 300.0
-        series["timestamp"] = timestamp
-        result = series["timestamp"]
-        assert result == expected
-
-    @pytest.mark.parametrize(
-        "td",
-        [
-            Timedelta("9 days"),
-            Timedelta("9 days").to_timedelta64(),
-            Timedelta("9 days").to_pytimedelta(),
-        ],
-    )
-    def test_append_timedelta_does_not_cast(self, td):
-        # GH#22717 inserting a Timedelta should _not_ cast to int64
-        expected = Series(["x", td], index=[0, "td"], dtype=object)
-
-        ser = Series(["x"])
-        ser["td"] = td
-        tm.assert_series_equal(ser, expected)
-        assert isinstance(ser["td"], Timedelta)
-
-        ser = Series(["x"])
-        ser.loc["td"] = Timedelta("9 days")
-        tm.assert_series_equal(ser, expected)
-        assert isinstance(ser["td"], Timedelta)
-
-
-def test_setitem_scalar_into_readonly_backing_data():
-    # GH#14359: test that you cannot mutate a read only buffer
-
-    array = np.zeros(5)
-    array.flags.writeable = False  # make the array immutable
-    series = Series(array)
-
-    for n in range(len(series)):
-        msg = "assignment destination is read-only"
-        with pytest.raises(ValueError, match=msg):
-            series[n] = 1
-
-        assert array[n] == 0
-
-
-def test_setitem_slice_into_readonly_backing_data():
-    # GH#14359: test that you cannot mutate a read only buffer
-
-    array = np.zeros(5)
-    array.flags.writeable = False  # make the array immutable
-    series = Series(array)
-
-    msg = "assignment destination is read-only"
-    with pytest.raises(ValueError, match=msg):
-        series[1:3] = 1
-
-    assert not array.any()
-
-
 class TestSetitemTimedelta64IntoNumeric(SetitemCastingEquivalents):
     # timedelta64 should not be treated as integers when setting into
     #  numeric Series