Merge remote-tracking branch 'upstream/master' into excel-fixture-cleanup

Author: WillAyd

asv_bench/benchmarks/index_object.py

@@ -196,11 +196,20 @@ def setup(self, N):
         self.intv = IntervalIndex.from_arrays(left, right)
         self.intv._engine
 
+        self.left = IntervalIndex.from_breaks(np.arange(N))
+        self.right = IntervalIndex.from_breaks(np.arange(N - 3, 2 * N - 3))
+
     def time_monotonic_inc(self, N):
         self.intv.is_monotonic_increasing
 
     def time_is_unique(self, N):
         self.intv.is_unique
 
+    def time_intersection(self, N):
+        self.left.intersection(self.right)
+
+    def time_intersection_duplicate(self, N):
+        self.intv.intersection(self.right)
+
 
 from .pandas_vb_common import setup  # noqa: F401

doc/source/whatsnew/v0.25.0.rst

@@ -502,14 +502,15 @@ Performance Improvements
 - Improved performance of :meth:`Series.searchsorted`. The speedup is especially large when the dtype is
   int8/int16/int32 and the searched key is within the integer bounds for the dtype (:issue:`22034`)
 - Improved performance of :meth:`pandas.core.groupby.GroupBy.quantile` (:issue:`20405`)
-- Improved performance when slicing :class:`RangeIndex` (:issue:`26565`)
+- Improved performance of slicing and other selected operation on a :class:`RangeIndex` (:issue:`26565`, :issue:`26617`)
 - Improved performance of :meth:`read_csv` by faster tokenizing and faster parsing of small float numbers (:issue:`25784`)
 - Improved performance of :meth:`read_csv` by faster parsing of N/A and boolean values (:issue:`25804`)
 - Improved performance of :attr:`IntervalIndex.is_monotonic`, :attr:`IntervalIndex.is_monotonic_increasing` and :attr:`IntervalIndex.is_monotonic_decreasing` by removing conversion to :class:`MultiIndex` (:issue:`24813`)
 - Improved performance of :meth:`DataFrame.to_csv` when writing datetime dtypes (:issue:`25708`)
 - Improved performance of :meth:`read_csv` by much faster parsing of ``MM/YYYY`` and ``DD/MM/YYYY`` datetime formats (:issue:`25922`)
 - Improved performance of nanops for dtypes that cannot store NaNs. Speedup is particularly prominent for :meth:`Series.all` and :meth:`Series.any` (:issue:`25070`)
 - Improved performance of :meth:`Series.map` for dictionary mappers on categorical series by mapping the categories instead of mapping all values (:issue:`23785`)
+- Improved performance of :meth:`IntervalIndex.intersection` (:issue:`24813`)
 - Improved performance of :meth:`read_csv` by faster concatenating date columns without extra conversion to string for integer/float zero and float ``NaN``; by faster checking the string for the possibility of being a date (:issue:`25754`)
 - Improved performance of :attr:`IntervalIndex.is_unique` by removing conversion to ``MultiIndex`` (:issue:`24813`)
 
@@ -537,6 +538,7 @@ Datetimelike
 - Bug in :func:`to_datetime` which does not replace the invalid argument with ``NaT`` when error is set to coerce (:issue:`26122`)
 - Bug in adding :class:`DateOffset` with nonzero month to :class:`DatetimeIndex` would raise ``ValueError`` (:issue:`26258`)
 - Bug in :func:`to_datetime` which raises unhandled ``OverflowError`` when called with mix of invalid dates and ``NaN`` values with ``format='%Y%m%d'`` and ``error='coerce'`` (:issue:`25512`)
+- Bug in :func:`to_datetime` which raises ``TypeError`` for ``format='%Y%m%d'`` when called for invalid integer dates with length >= 6 digits with ``errors='ignore'``
 
 Timedelta
 ^^^^^^^^^

environment.yml

@@ -9,59 +9,72 @@ dependencies:
   - python-dateutil>=2.5.0
   - pytz
 
-  # development
+  # benchmarks
   - asv
+
+  # building
   - cython>=0.28.2
+
+  # code checks
+  - cpplint
   - flake8
-  - flake8-comprehensions
-  - flake8-rst>=0.6.0,<=0.7.0
-  - gitpython
-  - hypothesis>=3.82
-  - ipywidgets
-  - isort
-  - moto
+  - flake8-comprehensions  # used by flake8, linting of unnecessary comprehensions
+  - flake8-rst>=0.6.0,<=0.7.0  # linting of code blocks in rst files
+  - isort  # check that imports are in the right order
   - mypy
+  - pycodestyle  # used by flake8
+
+  # documentation
+  - gitpython  # obtain contributors from git for whatsnew
+  - sphinx
+  - numpydoc>=0.9.0
+
+  # documentation (jupyter notebooks)
   - nbconvert>=5.4.1
-  - nbformat
-  - notebook>=5.7.5
+  - nbsphinx
   - pandoc
-  - pycodestyle
-  - pyqt
-  - python-snappy
+
+  # testing
+  - boto3
+  - botocore>=1.11
+  - hypothesis>=3.82
+  - moto  # mock S3
   - pytest>=4.0.2
+  - pytest-cov
   - pytest-mock
-  - sphinx
-  - numpydoc>=0.9.0
+  - pytest-xdist
+  - seaborn
+  - statsmodels
+
+  # unused (required indirectly may be?)
+  - ipywidgets
+  - nbformat
+  - notebook>=5.7.5
   - pip
 
   # optional
-  - beautifulsoup4>=4.2.1
   - blosc
-  - botocore>=1.11
-  - boto3
   - bottleneck>=1.2.1
-  - fastparquet>=0.2.1
-  - html5lib
-  - ipython>=5.6.0
   - ipykernel
-  - jinja2
-  - lxml
-  - matplotlib>=2.2.2
-  - nbsphinx
+  - ipython>=5.6.0
+  - jinja2  # pandas.Styler
+  - matplotlib>=2.2.2  # pandas.plotting, Series.plot, DataFrame.plot
   - numexpr>=2.6.8
-  - openpyxl
-  - pyarrow>=0.9.0
-  - pytables>=3.4.2
-  - pytest-cov
-  - pytest-xdist
-  - s3fs
   - scipy>=1.1
-  - seaborn
-  - sqlalchemy
-  - statsmodels
-  - xarray
-  - xlrd
-  - xlsxwriter
-  - xlwt
-  - pip:
-    - cpplint
+
+  # optional for io
+  - beautifulsoup4>=4.2.1  # pandas.read_html
+  - fastparquet>=0.2.1  # pandas.read_parquet, DataFrame.to_parquet
+  - html5lib  # pandas.read_html
+  - lxml  # pandas.read_html
+  - openpyxl  # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile
+  - pyarrow>=0.9.0  # pandas.read_paquet, DataFrame.to_parquet, pandas.read_feather, DataFrame.to_feather
+  - pyqt  # pandas.read_clipbobard
+  - pytables>=3.4.2  # pandas.read_hdf, DataFrame.to_hdf
+  - python-snappy  # required by pyarrow
+  - s3fs  # pandas.read_csv... when using 's3://...' path
+  - sqlalchemy  # pandas.read_sql, DataFrame.to_sql
+  - xarray  # DataFrame.to_xarray
+  - xlrd  # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile
+  - xlsxwriter  # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile
+  - xlwt  # pandas.read_excel, DataFrame.to_excel, pandas.ExcelWriter, pandas.ExcelFile

pandas/_libs/tslibs/strptime.pyx

@@ -140,13 +140,13 @@ def array_strptime(object[:] values, object fmt,
                     iresult[i] = NPY_NAT
                     continue
                 raise ValueError("time data %r does not match "
-                                 "format %r (match)" % (values[i], fmt))
+                                 "format %r (match)" % (val, fmt))
             if len(val) != found.end():
                 if is_coerce:
                     iresult[i] = NPY_NAT
                     continue
                 raise ValueError("unconverted data remains: %s" %
-                                 values[i][found.end():])
+                                 val[found.end():])
 
         # search
         else:
@@ -156,7 +156,7 @@ def array_strptime(object[:] values, object fmt,
                     iresult[i] = NPY_NAT
                     continue
                 raise ValueError("time data %r does not match format "
-                                 "%r (search)" % (values[i], fmt))
+                                 "%r (search)" % (val, fmt))
 
         iso_year = -1
         year = 1900

pandas/conftest.py

@@ -378,12 +378,7 @@ def unique_nulls_fixture(request):
              FixedOffset(0), FixedOffset(-300), timezone.utc,
              timezone(timedelta(hours=1)),
              timezone(timedelta(hours=-1), name='foo')]
-TIMEZONE_IDS = ['None', 'UTC', 'US/Eastern', 'Asia/Tokyp',
-                'dateutil/US/Pacific', 'dateutil/Asia/Singapore',
-                'dateutil.tz.tzutz()', 'dateutil.tz.tzlocal()',
-                'pytz.FixedOffset(300)', 'pytz.FixedOffset(0)',
-                'pytz.FixedOffset(-300)', 'datetime.timezone.utc',
-                'datetime.timezone.+1', 'datetime.timezone.-1.named']
+TIMEZONE_IDS = [repr(i) for i in TIMEZONES]
 
 
 @td.parametrize_fixture_doc(str(TIMEZONE_IDS))

pandas/core/arrays/integer.py

@@ -78,17 +78,6 @@ def construct_array_type(cls):
         """
         return IntegerArray
 
-    @classmethod
-    def construct_from_string(cls, string):
-        """
-        Construction from a string, raise a TypeError if not
-        possible
-        """
-        if string == cls.name:
-            return cls()
-        raise TypeError("Cannot construct a '{}' from "
-                        "'{}'".format(cls, string))
-
 
 def integer_array(values, dtype=None, copy=False):
     """

pandas/core/dtypes/base.py

@@ -172,17 +172,27 @@ def construct_array_type(cls):
         raise NotImplementedError
 
     @classmethod
-    def construct_from_string(cls, string):
-        """
-        Attempt to construct this type from a string.
+    def construct_from_string(cls, string: str):
+        r"""
+        Construct this type from a string.
+
+        This is useful mainly for data types that accept parameters.
+        For example, a period dtype accepts a frequency parameter that
+        can be set as ``period[H]`` (where H means hourly frequency).
+
+        By default, in the abstract class, just the name of the type is
+        expected. But subclasses can overwrite this method to accept
+        parameters.
 
         Parameters
         ----------
         string : str
+            The name of the type, for example ``category``.
 
         Returns
         -------
-        self : instance of 'cls'
+        ExtensionDtype
+            Instance of the dtype.
 
         Raises
         ------
@@ -191,21 +201,26 @@ def construct_from_string(cls, string):
 
         Examples
         --------
-        If the extension dtype can be constructed without any arguments,
-        the following may be an adequate implementation.
+        For extension dtypes with arguments the following may be an
+        adequate implementation.
 
         >>> @classmethod
-        ... def construct_from_string(cls, string)
-        ...     if string == cls.name:
-        ...         return cls()
+        ... def construct_from_string(cls, string):
+        ...     pattern = re.compile(r"^my_type\[(?P<arg_name>.+)\]$")
+        ...     match = pattern.match(string)
+        ...     if match:
+        ...         return cls(**match.groupdict())
         ...     else:
         ...         raise TypeError("Cannot construct a '{}' from "
-        ...                         "'{}'".format(cls, string))
+        ...                         "'{}'".format(cls.__name__, string))
         """
-        raise AbstractMethodError(cls)
+        if string != cls.name:
+            raise TypeError("Cannot construct a '{}' from '{}'".format(
+                cls.__name__, string))
+        return cls()
 
     @classmethod
-    def is_dtype(cls, dtype):
+    def is_dtype(cls, dtype) -> bool:
         """Check if we match 'dtype'.
 
         Parameters

pandas/core/dtypes/common.py

@@ -1931,8 +1931,6 @@ def _is_dtype_type(arr_or_dtype, condition):
         if issubclass(arr_or_dtype, ExtensionDtype):
             arr_or_dtype = arr_or_dtype.type
         return condition(np.dtype(arr_or_dtype).type)
-    elif arr_or_dtype is None:
-        return condition(type(None))
 
     # if we have an array-like
     if hasattr(arr_or_dtype, 'dtype'):

pandas/core/dtypes/dtypes.py

@@ -440,19 +440,6 @@ def construct_array_type(cls):
         from pandas import Categorical
         return Categorical
 
-    @classmethod
-    def construct_from_string(cls, string):
-        """
-        attempt to construct this type from a string, raise a TypeError if
-        it's not possible """
-        try:
-            if string == 'category':
-                return cls()
-            else:
-                raise TypeError("cannot construct a CategoricalDtype")
-        except AttributeError:
-            pass
-
     @staticmethod
     def validate_ordered(ordered):
         """

pandas/core/indexes/base.py

@@ -2440,9 +2440,7 @@ def _union(self, other, sort):
     def _wrap_setop_result(self, other, result):
         return self._constructor(result, name=get_op_result_name(self, other))
 
-    # TODO: standardize return type of non-union setops type(self vs other)
-    def intersection(self, other, sort=False):
-        """
+    _index_shared_docs['intersection'] = """
         Form the intersection of two Index objects.
 
         This returns a new Index with elements common to the index and `other`.
@@ -2476,6 +2474,10 @@ def intersection(self, other, sort=False):
         >>> idx1.intersection(idx2)
         Int64Index([3, 4], dtype='int64')
         """
+
+    # TODO: standardize return type of non-union setops type(self vs other)
+    @Appender(_index_shared_docs['intersection'])
+    def intersection(self, other, sort=False):
         self._validate_sort_keyword(sort)
         self._assert_can_do_setop(other)
         other = ensure_index(other)
@@ -4013,11 +4015,7 @@ def __contains__(self, key):
 
     @Appender(_index_shared_docs['contains'] % _index_doc_kwargs)
     def contains(self, key):
-        hash(key)
-        try:
-            return key in self._engine
-        except (TypeError, ValueError):
-            return False
+        return key in self
 
     def __hash__(self):
         raise TypeError("unhashable type: %r" % type(self).__name__)