Merge remote-tracking branch 'upstream/master' into issue-pandas-dev#38781

Author: simonjayhawkins

.github/workflows/ci.yml

@@ -2,74 +2,81 @@ name: CI
 
 on:
   push:
-    branches: master
+    branches: [master]
   pull_request:
     branches:
       - master
       - 1.2.x
 
 env:
   ENV_FILE: environment.yml
+  PANDAS_CI: 1
 
 jobs:
   checks:
     name: Checks
     runs-on: ubuntu-latest
-    steps:
-
-    - name: Setting conda path
-      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
+    defaults:
+      run:
+        shell: bash -l {0}
 
+    steps:
     - name: Checkout
       uses: actions/checkout@v1
 
     - name: Looking for unwanted patterns
       run: ci/code_checks.sh patterns
       if: always()
 
-    - name: Setup environment and build pandas
-      run: ci/setup_env.sh
-      if: always()
+    - name: Cache conda
+      uses: actions/cache@v2
+      with:
+        path: ~/conda_pkgs_dir
+        key: ${{ runner.os }}-conda-${{ hashFiles('${{ env.ENV_FILE }}') }}
 
-    - name: Linting
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        activate-environment: pandas-dev
+        channel-priority: strict
+        environment-file: ${{ env.ENV_FILE }}
+        use-only-tar-bz2: true
+
+    - name: Environment Detail
       run: |
-        source activate pandas-dev
-        ci/code_checks.sh lint
+        conda info
+        conda list
+
+    - name: Build Pandas
+      run: |
+        python setup.py build_ext -j 2
+        python -m pip install -e . --no-build-isolation --no-use-pep517
+
+    - name: Linting
+      run: ci/code_checks.sh lint
       if: always()
 
     - name: Checks on imported code
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh code
+      run: ci/code_checks.sh code
       if: always()
 
     - name: Running doctests
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh doctests
+      run: ci/code_checks.sh doctests
       if: always()
 
     - name: Docstring validation
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh docstrings
+      run: ci/code_checks.sh docstrings
       if: always()
 
     - name: Typing validation
-      run: |
-        source activate pandas-dev
-        ci/code_checks.sh typing
+      run: ci/code_checks.sh typing
       if: always()
 
     - name: Testing docstring validation script
-      run: |
-        source activate pandas-dev
-        pytest --capture=no --strict-markers scripts
+      run: pytest --capture=no --strict-markers scripts
       if: always()
 
     - name: Running benchmarks
       run: |
-        source activate pandas-dev
         cd asv_bench
         asv check -E existing
         git remote add upstream https://github.com/pandas-dev/pandas.git
@@ -106,7 +113,6 @@ jobs:
       run: |
         source activate pandas-dev
         python web/pandas_web.py web/pandas --target-path=web/build
-
     - name: Build documentation
       run: |
         source activate pandas-dev
@@ -132,3 +138,22 @@ jobs:
     - name: Upload dev docs
       run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev
       if: github.event_name == 'push'
+
+  data_manager:
+    name: Test experimental data manager
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Setting conda path
+      run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH
+
+    - name: Checkout
+      uses: actions/checkout@v1
+
+    - name: Setup environment and build pandas
+      run: ci/setup_env.sh
+
+    - name: Run tests
+      run: |
+        source activate pandas-dev
+        pytest pandas/tests/frame/methods --array-manager

.pre-commit-config.yaml

@@ -145,11 +145,12 @@ repos:
         language: pygrep
         types_or: [python, cython]
     -   id: unwanted-typing
-        name: Check for use of comment-based annotation syntax and missing error codes
+        name: Check for outdated annotation syntax and missing error codes
         entry: |
             (?x)
             \#\ type:\ (?!ignore)|
-            \#\ type:\s?ignore(?!\[)
+            \#\ type:\s?ignore(?!\[)|
+            \)\ ->\ \"
         language: pygrep
         types: [python]
     -   id: np-bool

asv_bench/benchmarks/io/excel.py

@@ -43,6 +43,7 @@ class ReadExcel:
     params = ["xlrd", "openpyxl", "odf"]
     param_names = ["engine"]
     fname_excel = "spreadsheet.xlsx"
+    fname_excel_xls = "spreadsheet.xls"
     fname_odf = "spreadsheet.ods"
 
     def _create_odf(self):
@@ -63,10 +64,16 @@ def setup_cache(self):
         self.df = _generate_dataframe()
 
         self.df.to_excel(self.fname_excel, sheet_name="Sheet1")
+        self.df.to_excel(self.fname_excel_xls, sheet_name="Sheet1")
         self._create_odf()
 
     def time_read_excel(self, engine):
-        fname = self.fname_odf if engine == "odf" else self.fname_excel
+        if engine == "xlrd":
+            fname = self.fname_excel_xls
+        elif engine == "odf":
+            fname = self.fname_odf
+        else:
+            fname = self.fname_excel
         read_excel(fname, engine=engine)
 
 

doc/make.py

@@ -46,6 +46,7 @@ def __init__(
         warnings_are_errors=False,
     ):
         self.num_jobs = num_jobs
+        self.include_api = include_api
         self.verbosity = verbosity
         self.warnings_are_errors = warnings_are_errors
 
@@ -188,7 +189,14 @@ def _add_redirects(self):
                 if not row or row[0].strip().startswith("#"):
                     continue
 
-                path = os.path.join(BUILD_PATH, "html", *row[0].split("/")) + ".html"
+                html_path = os.path.join(BUILD_PATH, "html")
+                path = os.path.join(html_path, *row[0].split("/")) + ".html"
+
+                if not self.include_api and (
+                    os.path.join(html_path, "reference") in path
+                    or os.path.join(html_path, "generated") in path
+                ):
+                    continue
 
                 try:
                     title = self._get_page_title(row[1])
@@ -198,11 +206,6 @@ def _add_redirects(self):
                     # sphinx specific stuff
                     title = "this page"
 
-                if os.path.exists(path):
-                    raise RuntimeError(
-                        f"Redirection would overwrite an existing file: {path}"
-                    )
-
                 with open(path, "w") as moved_page_fd:
                     html = f"""\
 <html>

doc/source/conf.py

@@ -77,29 +77,32 @@
 try:
     import nbconvert
 except ImportError:
-    logger.warn("nbconvert not installed. Skipping notebooks.")
+    logger.warning("nbconvert not installed. Skipping notebooks.")
     exclude_patterns.append("**/*.ipynb")
 else:
     try:
         nbconvert.utils.pandoc.get_pandoc_version()
     except nbconvert.utils.pandoc.PandocMissing:
-        logger.warn("Pandoc not installed. Skipping notebooks.")
+        logger.warning("Pandoc not installed. Skipping notebooks.")
         exclude_patterns.append("**/*.ipynb")
 
 # sphinx_pattern can be '-api' to exclude the API pages,
 # the path to a file, or a Python object
 # (e.g. '10min.rst' or 'pandas.DataFrame.head')
 source_path = os.path.dirname(os.path.abspath(__file__))
 pattern = os.environ.get("SPHINX_PATTERN")
+single_doc = pattern is not None and pattern != "-api"
+include_api = pattern != "-api"
 if pattern:
     for dirname, dirs, fnames in os.walk(source_path):
+        reldir = os.path.relpath(dirname, source_path)
         for fname in fnames:
             if os.path.splitext(fname)[-1] in (".rst", ".ipynb"):
                 fname = os.path.relpath(os.path.join(dirname, fname), source_path)
 
                 if fname == "index.rst" and os.path.abspath(dirname) == source_path:
                     continue
-                elif pattern == "-api" and dirname == "reference":
+                elif pattern == "-api" and reldir.startswith("reference"):
                     exclude_patterns.append(fname)
                 elif pattern != "-api" and fname != pattern:
                     exclude_patterns.append(fname)
@@ -109,11 +112,11 @@
 with open(os.path.join(source_path, "index.rst"), "w") as f:
     f.write(
         t.render(
-            include_api=pattern is None,
-            single_doc=(pattern if pattern is not None and pattern != "-api" else None),
+            include_api=include_api,
+            single_doc=(pattern if single_doc else None),
         )
     )
-autosummary_generate = True if pattern is None else ["index"]
+autosummary_generate = True if include_api else ["index"]
 autodoc_typehints = "none"
 
 # numpydoc
@@ -315,7 +318,7 @@
         # ... and each of its public methods
         moved_api_pages.append((f"{old}.{method}", f"{new}.{method}"))
 
-if pattern is None:
+if include_api:
     html_additional_pages = {
         "generated/" + page[0]: "api_redirect.html" for page in moved_api_pages
     }
@@ -411,7 +414,7 @@
 # latex_use_modindex = True
 
 
-if pattern is None:
+if include_api:
     intersphinx_mapping = {
         "dateutil": ("https://dateutil.readthedocs.io/en/latest/", None),
         "matplotlib": ("https://matplotlib.org/", None),

doc/source/ecosystem.rst

@@ -85,6 +85,14 @@ Featuretools is a Python library for automated feature engineering built on top
 
 Compose is a machine learning tool for labeling data and prediction engineering. It allows you to structure the labeling process by parameterizing prediction problems and transforming time-driven relational data into target values with cutoff times that can be used for supervised learning.
 
+`STUMPY <https://github.com/TDAmeritrade/stumpy>`__
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+STUMPY is a powerful and scalable Python library for modern time series analysis.
+At its core, STUMPY efficiently computes something called a
+`matrix profile <https://stumpy.readthedocs.io/en/latest/Tutorial_The_Matrix_Profile.html>`__,
+which can be used for a wide variety of time series data mining tasks.
+
 .. _ecosystem.visualization:
 
 Visualization

doc/source/getting_started/comparison/comparison_with_spreadsheets.rst

@@ -35,7 +35,7 @@ General terminology translation
 ``DataFrame``
 ~~~~~~~~~~~~~
 
-A ``DataFrame`` in pandas is analogous to an Excel worksheet. While an Excel worksheet can contain
+A ``DataFrame`` in pandas is analogous to an Excel worksheet. While an Excel workbook can contain
 multiple worksheets, pandas ``DataFrame``\s exist independently.
 
 ``Series``

doc/source/getting_started/intro_tutorials/04_plotting.rst

@@ -151,7 +151,7 @@ I want each of the columns in a separate subplot.
 
 Separate subplots for each of the data columns are supported by the ``subplots`` argument
 of the ``plot`` functions. The builtin options available in each of the pandas plot
-functions that are worthwhile to have a look.
+functions are worth reviewing.
 
 .. raw:: html
 

doc/source/user_guide/enhancingperf.rst

@@ -247,7 +247,7 @@ We've gotten another big improvement. Let's check again where the time is spent:
 
 .. ipython:: python
 
-   %%prun -l 4 apply_integrate_f(df["a"].to_numpy(), df["b"].to_numpy(), df["N"].to_numpy())
+   %prun -l 4 apply_integrate_f(df["a"].to_numpy(), df["b"].to_numpy(), df["N"].to_numpy())
 
 As one might expect, the majority of the time is now spent in ``apply_integrate_f``,
 so if we wanted to make anymore efficiencies we must continue to concentrate our

doc/source/whatsnew/v0.8.0.rst

@@ -176,7 +176,7 @@ New plotting methods
 Vytautas Jancauskas, the 2012 GSOC participant, has added many new plot
 types. For example, ``'kde'`` is a new option:
 
-.. ipython:: python
+.. code-block:: python
 
    s = pd.Series(
        np.concatenate((np.random.randn(1000), np.random.randn(1000) * 0.5 + 3))

doc/source/whatsnew/v1.2.1.rst

@@ -14,18 +14,24 @@ including other versions of pandas.
 
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
-- The deprecated attributes ``_AXIS_NAMES`` and ``_AXIS_NUMBERS`` of :class:`DataFrame` and :class:`Series` will no longer show up in ``dir`` or ``inspect.getmembers`` calls (:issue:`38740`)
 - Fixed regression in :meth:`to_csv` that created corrupted zip files when there were more rows than ``chunksize`` (:issue:`38714`)
-- Fixed a regression in ``groupby().rolling()`` where :class:`MultiIndex` levels were dropped (:issue:`38523`)
 - Fixed regression in repr of float-like strings of an ``object`` dtype having trailing 0's truncated after the decimal (:issue:`38708`)
 - Fixed regression in :meth:`DataFrame.groupby()` with :class:`Categorical` grouping column not showing unused categories for ``grouped.indices`` (:issue:`38642`)
 - Fixed regression in :meth:`DataFrame.any` and :meth:`DataFrame.all` not returning a result for tz-aware ``datetime64`` columns (:issue:`38723`)
 - Fixed regression in :meth:`DataFrame.__setitem__` raising ``ValueError`` when expanding :class:`DataFrame` and new column is from type ``"0 - name"`` (:issue:`39010`)
 - Fixed regression in :meth:`.GroupBy.sem` where the presence of non-numeric columns would cause an error instead of being dropped (:issue:`38774`)
+- Fixed regression in :meth:`DataFrame.loc.__setitem__` raising ``ValueError`` when :class:`DataFrame` has unsorted :class:`MultiIndex` columns and indexer is a scalar (:issue:`38601`)
 - Fixed regression in :func:`read_excel` with non-rawbyte file handles (:issue:`38788`)
-- Bug in :meth:`read_csv` with ``float_precision="high"`` caused segfault or wrong parsing of long exponent strings. This resulted in a regression in some cases as the default for ``float_precision`` was changed in pandas 1.2.0 (:issue:`38753`)
 - Fixed regression in :meth:`Rolling.skew` and :meth:`Rolling.kurt` modifying the object inplace (:issue:`38908`)
 - Fixed regression in :meth:`read_csv` and other read functions were the encoding error policy (``errors``) did not default to ``"replace"`` when no encoding was specified (:issue:`38989`)
+- Fixed regression in :meth:`DataFrame.apply` with ``axis=1`` using str accessor in apply function (:issue:`38979`)
+- Fixed regression in :meth:`DataFrame.replace` raising ``ValueError`` when :class:`DataFrame` has dtype ``bytes`` (:issue:`38900`)
+- Fixed regression in :meth:`DataFrameGroupBy.diff` raising for ``int8`` and ``int16`` columns (:issue:`39050`)
+- Fixed regression in :meth:`Series.fillna` that raised ``RecursionError`` with ``datetime64[ns, UTC]`` dtype (:issue:`38851`)
+- Fixed regression that raised ``AttributeError`` with PyArrow versions [0.16.0, 1.0.0) (:issue:`38801`)
+- Fixed regression in :meth:`DataFrame.groupby` when aggregating an :class:`ExtensionDType` that could fail for non-numeric values (:issue:`38980`)
+- Fixed regression in :meth:`DataFrame.loc.__setitem__` raising ``KeyError`` with :class:`MultiIndex` and list-like columns indexer enlarging :class:`DataFrame` (:issue:`39147`)
+- Fixed regression in comparisons between ``NaT`` and ``datetime.date`` objects incorrectly returning ``True`` (:issue:`39151`)
 
 .. ---------------------------------------------------------------------------
 
@@ -34,14 +40,8 @@ Fixed regressions
 Bug fixes
 ~~~~~~~~~
 
-I/O
-^^^
-
-- Bumped minimum fastparquet version to 0.4.0 to avoid ``AttributeError`` from numba (:issue:`38344`)
-- Bumped minimum pymysql version to 0.8.1 to avoid test failures (:issue:`38344`)
-- Fixed ``AttributeError`` with PyArrow versions [0.16.0, 1.0.0) (:issue:`38801`)
-
--
+- Bug in :meth:`read_csv` with ``float_precision="high"`` caused segfault or wrong parsing of long exponent strings. This resulted in a regression in some cases as the default for ``float_precision`` was changed in pandas 1.2.0 (:issue:`38753`)
+- Bug in :func:`read_csv` not closing an opened file handle when a ``csv.Error`` or ``UnicodeDecodeError`` occurred while initializing (:issue:`39024`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -50,8 +50,14 @@ I/O
 
 Other
 ~~~~~
+
+- The deprecated attributes ``_AXIS_NAMES`` and ``_AXIS_NUMBERS`` of :class:`DataFrame` and :class:`Series` will no longer show up in ``dir`` or ``inspect.getmembers`` calls (:issue:`38740`)
+- Bumped minimum fastparquet version to 0.4.0 to avoid ``AttributeError`` from numba (:issue:`38344`)
+- Bumped minimum pymysql version to 0.8.1 to avoid test failures (:issue:`38344`)
 - Fixed build failure on MacOS 11 in Python 3.9.1 (:issue:`38766`)
 - Added reference to backwards incompatible ``check_freq`` arg of :func:`testing.assert_frame_equal` and :func:`testing.assert_series_equal` in :ref:`pandas 1.1.0 whats new <whatsnew_110.api_breaking.testing.check_freq>` (:issue:`34050`)
+-
+-
 
 .. ---------------------------------------------------------------------------