Backport PR #42200 on branch 1.3.x (TYP: update for numpy-1.21.0) (#42205)

Co-authored-by: Simon Hawkins <[email protected]>

Author: meeseeksmachine

pandas/core/apply.py

@@ -1076,11 +1076,7 @@ def apply_standard(self) -> FrameOrSeriesUnion:
 
         with np.errstate(all="ignore"):
             if isinstance(f, np.ufunc):
-                # error: Argument 1 to "__call__" of "ufunc" has incompatible type
-                # "Series"; expected "Union[Union[int, float, complex, str, bytes,
-                # generic], Sequence[Union[int, float, complex, str, bytes, generic]],
-                # Sequence[Sequence[Any]], _SupportsArray]"
-                return f(obj)  # type: ignore[arg-type]
+                return f(obj)
 
             # row-wise access
             if is_extension_array_dtype(obj.dtype) and hasattr(obj._values, "map"):

pandas/core/arrays/masked.py

@@ -409,9 +409,7 @@ def isin(self, values) -> BooleanArray:  # type: ignore[override]
                 result += self._mask
             else:
                 result *= np.invert(self._mask)
-        # error: No overload variant of "zeros_like" matches argument types
-        # "BaseMaskedArray", "Type[bool]"
-        mask = np.zeros_like(self, dtype=bool)  # type: ignore[call-overload]
+        mask = np.zeros_like(self, dtype=bool)
         return BooleanArray(result, mask, copy=False)
 
     def copy(self: BaseMaskedArrayT) -> BaseMaskedArrayT:

pandas/core/arrays/period.py

@@ -341,7 +341,9 @@ def freq(self) -> BaseOffset:
     def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
         if dtype == "i8":
             return self.asi8
-        elif dtype == bool:
+        # error: Non-overlapping equality check (left operand type: "Optional[Union[str,
+        # dtype[Any]]]", right operand type: "Type[bool]")
+        elif dtype == bool:  # type: ignore[comparison-overlap]
             return ~self._isnan
 
         # This will raise TypeError for non-object dtypes

pandas/core/arrays/sparse/array.py

@@ -1463,11 +1463,7 @@ def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):
             return type(self)(result)
 
     def __abs__(self):
-        # error: Argument 1 to "__call__" of "ufunc" has incompatible type
-        # "SparseArray"; expected "Union[Union[int, float, complex, str, bytes,
-        # generic], Sequence[Union[int, float, complex, str, bytes, generic]],
-        # Sequence[Sequence[Any]], _SupportsArray]"
-        return np.abs(self)  # type: ignore[arg-type]
+        return np.abs(self)
 
     # ------------------------------------------------------------------------
     # Ops

pandas/core/common.py

@@ -410,7 +410,7 @@ def random_state(state=None):
 
     Returns
     -------
-    np.random.RandomState
+    np.random.RandomState or np.random if state is None
 
     """
     if (

pandas/core/computation/pytables.py

@@ -226,11 +226,7 @@ def stringify(value):
             if v not in metadata:
                 result = -1
             else:
-                # error: Incompatible types in assignment (expression has type
-                # "Union[Any, ndarray]", variable has type "int")
-                result = metadata.searchsorted(  # type: ignore[assignment]
-                    v, side="left"
-                )
+                result = metadata.searchsorted(v, side="left")
             return TermValue(result, result, "integer")
         elif kind == "integer":
             v = int(float(v))

pandas/core/dtypes/cast.py

@@ -15,6 +15,7 @@
     TYPE_CHECKING,
     Any,
     Sized,
+    TypeVar,
     cast,
     overload,
 )
@@ -107,6 +108,8 @@
 _int32_max = np.iinfo(np.int32).max
 _int64_max = np.iinfo(np.int64).max
 
+NumpyArrayT = TypeVar("NumpyArrayT", bound=np.ndarray)
+
 
 def maybe_convert_platform(
     values: list | tuple | range | np.ndarray | ExtensionArray,
@@ -659,7 +662,10 @@ def _ensure_dtype_type(value, dtype: np.dtype):
     object
     """
     # Start with exceptions in which we do _not_ cast to numpy types
-    if dtype == np.object_:
+
+    # error: Non-overlapping equality check (left operand type: "dtype[Any]", right
+    # operand type: "Type[object_]")
+    if dtype == np.object_:  # type: ignore[comparison-overlap]
         return value
 
     # Note: before we get here we have already excluded isna(value)
@@ -866,10 +872,10 @@ def maybe_infer_dtype_type(element):
 
 
 def maybe_upcast(
-    values: np.ndarray,
+    values: NumpyArrayT,
     fill_value: Scalar = np.nan,
     copy: bool = False,
-) -> tuple[np.ndarray, Scalar]:
+) -> tuple[NumpyArrayT, Scalar]:
     """
     Provide explicit type promotion and coercion.
 
@@ -1093,7 +1099,10 @@ def astype_nansafe(
         raise ValueError("dtype must be np.dtype or ExtensionDtype")
 
     if arr.dtype.kind in ["m", "M"] and (
-        issubclass(dtype.type, str) or dtype == object
+        issubclass(dtype.type, str)
+        # error: Non-overlapping equality check (left operand type: "dtype[Any]", right
+        # operand type: "Type[object]")
+        or dtype == object  # type: ignore[comparison-overlap]
     ):
         from pandas.core.construction import ensure_wrapped_if_datetimelike
 
@@ -1104,7 +1113,9 @@ def astype_nansafe(
         return lib.ensure_string_array(arr, skipna=skipna, convert_na_value=False)
 
     elif is_datetime64_dtype(arr):
-        if dtype == np.int64:
+        # Non-overlapping equality check (left operand type: "dtype[Any]", right
+        # operand type: "Type[signedinteger[Any]]")
+        if dtype == np.int64:  # type: ignore[comparison-overlap]
             warnings.warn(
                 f"casting {arr.dtype} values to int64 with .astype(...) "
                 "is deprecated and will raise in a future version. "
@@ -1124,7 +1135,9 @@ def astype_nansafe(
         raise TypeError(f"cannot astype a datetimelike from [{arr.dtype}] to [{dtype}]")
 
     elif is_timedelta64_dtype(arr):
-        if dtype == np.int64:
+        # error: Non-overlapping equality check (left operand type: "dtype[Any]", right
+        # operand type: "Type[signedinteger[Any]]")
+        if dtype == np.int64:  # type: ignore[comparison-overlap]
             warnings.warn(
                 f"casting {arr.dtype} values to int64 with .astype(...) "
                 "is deprecated and will raise in a future version. "
@@ -1398,10 +1411,9 @@ def convert_dtypes(
 
         if is_string_dtype(inferred_dtype):
             if not convert_string:
-                inferred_dtype = input_array.dtype
+                return input_array.dtype
             else:
-                inferred_dtype = pandas_dtype("string")
-            return inferred_dtype
+                return pandas_dtype("string")
 
         if convert_integer:
             target_int_dtype = pandas_dtype("Int64")
@@ -1454,7 +1466,9 @@ def convert_dtypes(
     else:
         return input_array.dtype
 
-    return inferred_dtype
+    # error: Incompatible return value type (got "Union[str, Union[dtype[Any],
+    # ExtensionDtype]]", expected "Union[dtype[Any], ExtensionDtype]")
+    return inferred_dtype  # type: ignore[return-value]
 
 
 def maybe_infer_to_datetimelike(
@@ -1831,7 +1845,9 @@ def construct_2d_arraylike_from_scalar(
 
     if dtype.kind in ["m", "M"]:
         value = maybe_unbox_datetimelike_tz_deprecation(value, dtype, stacklevel=4)
-    elif dtype == object:
+    # error: Non-overlapping equality check (left operand type: "dtype[Any]", right
+    # operand type: "Type[object]")
+    elif dtype == object:  # type: ignore[comparison-overlap]
         if isinstance(value, (np.timedelta64, np.datetime64)):
             # calling np.array below would cast to pytimedelta/pydatetime
             out = np.empty(shape, dtype=object)
@@ -2206,7 +2222,9 @@ def can_hold_element(arr: ArrayLike, element: Any) -> bool:
             return tipo.kind == "b"
         return lib.is_bool(element)
 
-    elif dtype == object:
+    # error: Non-overlapping equality check (left operand type: "dtype[Any]", right
+    # operand type: "Type[object]")
+    elif dtype == object:  # type: ignore[comparison-overlap]
         return True
 
     elif dtype.kind == "S":

pandas/core/frame.py

@@ -648,7 +648,7 @@ def __init__(
         elif isinstance(data, (np.ndarray, Series, Index)):
             if data.dtype.names:
                 # i.e. numpy structured array
-
+                data = cast(np.ndarray, data)
                 mgr = rec_array_to_mgr(
                     data,
                     index,
@@ -2280,9 +2280,7 @@ def to_records(
             if dtype_mapping is None:
                 formats.append(v.dtype)
             elif isinstance(dtype_mapping, (type, np.dtype, str)):
-                # error: Argument 1 to "append" of "list" has incompatible type
-                # "Union[type, dtype, str]"; expected "dtype"
-                formats.append(dtype_mapping)  # type: ignore[arg-type]
+                formats.append(dtype_mapping)
             else:
                 element = "row" if i < index_len else "column"
                 msg = f"Invalid dtype {dtype_mapping} specified for {element} {name}"

pandas/core/generic.py

@@ -9751,11 +9751,9 @@ def abs(self: FrameOrSeries) -> FrameOrSeries:
         2    6   30  -30
         3    7   40  -50
         """
-        # error: Argument 1 to "__call__" of "ufunc" has incompatible type
-        # "FrameOrSeries"; expected "Union[Union[int, float, complex, str, bytes,
-        # generic], Sequence[Union[int, float, complex, str, bytes, generic]],
-        # Sequence[Sequence[Any]], _SupportsArray]"
-        return np.abs(self)  # type: ignore[arg-type]
+        # error: Incompatible return value type (got "ndarray[Any, dtype[Any]]",
+        # expected "FrameOrSeries")
+        return np.abs(self)  # type: ignore[return-value]
 
     @final
     def describe(

pandas/core/groupby/ops.py

@@ -161,7 +161,9 @@ def _get_cython_function(
         f = getattr(libgroupby, ftype)
         if is_numeric:
             return f
-        elif dtype == object:
+        # error: Non-overlapping equality check (left operand type: "dtype[Any]", right
+        # operand type: "Literal['object']")
+        elif dtype == object:  # type: ignore[comparison-overlap]
             if "object" not in f.__signatures__:
                 # raise NotImplementedError here rather than TypeError later
                 raise NotImplementedError(

pandas/core/indexes/base.py

@@ -559,7 +559,9 @@ def _dtype_to_subclass(cls, dtype: DtypeObj):
 
             return Int64Index
 
-        elif dtype == object:
+        # error: Non-overlapping equality check (left operand type: "dtype[Any]", right
+        # operand type: "Type[object]")
+        elif dtype == object:  # type: ignore[comparison-overlap]
             # NB: assuming away MultiIndex
             return Index
 

pandas/core/indexes/category.py

@@ -193,17 +193,12 @@ def _can_hold_strings(self):
     def _engine_type(self):
         # self.codes can have dtype int8, int16, int32 or int64, so we need
         # to return the corresponding engine type (libindex.Int8Engine, etc.).
-
-        # error: Invalid index type "Type[generic]" for "Dict[Type[signedinteger[Any]],
-        # Any]"; expected type "Type[signedinteger[Any]]"
         return {
             np.int8: libindex.Int8Engine,
             np.int16: libindex.Int16Engine,
             np.int32: libindex.Int32Engine,
             np.int64: libindex.Int64Engine,
-        }[
-            self.codes.dtype.type  # type: ignore[index]
-        ]
+        }[self.codes.dtype.type]
 
     _attributes = ["name"]
 

pandas/core/indexes/extension.py

@@ -397,11 +397,13 @@ def astype(self, dtype, copy: bool = True) -> Index:
                 return self
             return self.copy()
 
+        # error: Non-overlapping equality check (left operand type: "dtype[Any]", right
+        # operand type: "Literal['M8[ns]']")
         if (
             isinstance(self.dtype, np.dtype)
             and isinstance(dtype, np.dtype)
             and dtype.kind == "M"
-            and dtype != "M8[ns]"
+            and dtype != "M8[ns]"  # type: ignore[comparison-overlap]
         ):
             # For now Datetime supports this by unwrapping ndarray, but DTI doesn't
             raise TypeError(f"Cannot cast {type(self).__name__} to dtype")

pandas/core/internals/blocks.py

@@ -1167,13 +1167,17 @@ def shift(self, periods: int, axis: int = 0, fill_value: Any = None) -> list[Blo
         # convert integer to float if necessary. need to do a lot more than
         # that, handle boolean etc also
 
-        # error: Argument 1 to "maybe_upcast" has incompatible type "Union[ndarray,
-        # ExtensionArray]"; expected "ndarray"
+        # error: Value of type variable "NumpyArrayT" of "maybe_upcast" cannot be
+        # "Union[ndarray[Any, Any], ExtensionArray]"
         new_values, fill_value = maybe_upcast(
-            self.values, fill_value  # type: ignore[arg-type]
+            self.values, fill_value  # type: ignore[type-var]
         )
 
-        new_values = shift(new_values, periods, axis, fill_value)
+        # error: Argument 1 to "shift" has incompatible type "Union[ndarray[Any, Any],
+        # ExtensionArray]"; expected "ndarray[Any, Any]"
+        new_values = shift(
+            new_values, periods, axis, fill_value  # type: ignore[arg-type]
+        )
 
         return [self.make_block(new_values)]
 
@@ -1894,9 +1898,7 @@ def get_block_type(values, dtype: Dtype | None = None):
         cls = ExtensionBlock
     elif isinstance(dtype, CategoricalDtype):
         cls = CategoricalBlock
-    # error: Non-overlapping identity check (left operand type: "Type[generic]",
-    # right operand type: "Type[Timestamp]")
-    elif vtype is Timestamp:  # type: ignore[comparison-overlap]
+    elif vtype is Timestamp:
         cls = DatetimeTZBlock
     elif isinstance(dtype, ExtensionDtype):
         # Note: need to be sure PandasArray is unwrapped before we get here

pandas/core/internals/construction.py

@@ -10,6 +10,7 @@
     Any,
     Hashable,
     Sequence,
+    cast,
 )
 import warnings
 
@@ -165,6 +166,9 @@ def rec_array_to_mgr(
 
     # fill if needed
     if isinstance(data, np.ma.MaskedArray):
+        # GH#42200 we only get here with MaskedRecords, but check for the
+        #  parent class MaskedArray to avoid the need to import MaskedRecords
+        data = cast("MaskedRecords", data)
         new_arrays = fill_masked_arrays(data, arr_columns)
     else:
         # error: Incompatible types in assignment (expression has type