Merge master

Author: makbigc

.gitignore

@@ -61,6 +61,7 @@ dist
 .coverage
 coverage.xml
 coverage_html_report
+.mypy_cache
 *.pytest_cache
 # hypothesis test database
 .hypothesis/

.travis.yml

@@ -41,7 +41,7 @@ matrix:
 
     - dist: trusty
       env:
-        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true
+        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true
 
     # In allow_failures
     - dist: trusty
@@ -82,15 +82,10 @@ before_install:
 install:
   - echo "install start"
   - ci/prep_cython_cache.sh
-  - ci/install_travis.sh
+  - ci/setup_env.sh
   - ci/submit_cython_cache.sh
   - echo "install done"
 
-before_script:
-  - ci/install_db_travis.sh
-  - export DISPLAY=":99.0"
-  - ci/before_script_travis.sh
-
 script:
   - echo "script start"
   - source activate pandas-dev

README.md

@@ -164,7 +164,7 @@ pip install pandas
 ```
 
 ## Dependencies
-- [NumPy](https://www.numpy.org): 1.12.0 or higher
+- [NumPy](https://www.numpy.org): 1.13.3 or higher
 - [python-dateutil](https://labix.org/python-dateutil): 2.5.0 or higher
 - [pytz](https://pythonhosted.org/pytz): 2015.4 or higher
 

asv_bench/benchmarks/io/csv.py

@@ -4,7 +4,7 @@
 import numpy as np
 import pandas.util.testing as tm
 from pandas import DataFrame, Categorical, date_range, read_csv
-from pandas.compat import cStringIO as StringIO
+from io import StringIO
 
 from ..pandas_vb_common import BaseIO
 

asv_bench/benchmarks/io/excel.py

@@ -1,6 +1,6 @@
+from io import BytesIO
 import numpy as np
 from pandas import DataFrame, date_range, ExcelWriter, read_excel
-from pandas.compat import BytesIO
 import pandas.util.testing as tm
 
 

asv_bench/benchmarks/rolling.py

@@ -113,4 +113,20 @@ def time_quantile(self, constructor, window, dtype, percentile,
         self.roll.quantile(percentile, interpolation=interpolation)
 
 
+class PeakMemFixed(object):
+
+    def setup(self):
+        N = 10
+        arr = 100 * np.random.random(N)
+        self.roll = pd.Series(arr).rolling(10)
+
+    def peakmem_fixed(self):
+        # GH 25926
+        # This is to detect memory leaks in rolling operations.
+        # To save time this is only ran on one method.
+        # 6000 iterations is enough for most types of leaks to be detected
+        for x in range(6000):
+            self.roll.max()
+
+
 from .pandas_vb_common import setup  # noqa: F401

azure-pipelines.yml

@@ -39,9 +39,8 @@ jobs:
   - script: |
       export PATH=$HOME/miniconda3/bin:$PATH
       sudo apt-get install -y libc6-dev-i386
-      ci/incremental/install_miniconda.sh
-      ci/incremental/setup_conda_environment.sh
-    displayName: 'Set up environment'
+      ci/setup_env.sh
+    displayName: 'Setup environment and build pandas'
     condition: true
 
   # Do not require pandas
@@ -59,13 +58,6 @@ jobs:
     displayName: 'Dependencies consistency'
     condition: true
 
-  - script: |
-      export PATH=$HOME/miniconda3/bin:$PATH
-      source activate pandas-dev
-      ci/incremental/build.sh
-    displayName: 'Build'
-    condition: true
-
   # Require pandas
   - script: |
       export PATH=$HOME/miniconda3/bin:$PATH
@@ -88,6 +80,13 @@ jobs:
     displayName: 'Docstring validation'
     condition: true
 
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      source activate pandas-dev
+      ci/code_checks.sh typing
+    displayName: 'Typing validation'
+    condition: true
+
   - script: |
       export PATH=$HOME/miniconda3/bin:$PATH
       source activate pandas-dev

ci/azure/posix.yml

@@ -15,6 +15,18 @@ jobs:
           PATTERN: "not slow and not network"
 
       ${{ if eq(parameters.name, 'Linux') }}:
+        py35_compat:
+          ENV_FILE: ci/deps/azure-35-compat.yaml
+          CONDA_PY: "35"
+          PATTERN: "not slow and not network"
+
+        py36_locale_slow_old_np:
+          ENV_FILE: ci/deps/azure-36-locale.yaml
+          CONDA_PY: "36"
+          PATTERN: "slow"
+          LOCALE_OVERRIDE: "zh_CN.UTF-8"
+          EXTRA_APT: "language-pack-zh-hans"
+
         py36_locale_slow:
           ENV_FILE: ci/deps/azure-36-locale_slow.yaml
           CONDA_PY: "36"
@@ -38,17 +50,9 @@ jobs:
   steps:
     - script: |
         if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386 $EXTRA_APT; fi
-        echo "Installing Miniconda"
-        ci/incremental/install_miniconda.sh
-        export PATH=$HOME/miniconda3/bin:$PATH
-        echo "Setting up Conda environment"
-        ci/incremental/setup_conda_environment.sh
-      displayName: 'Before Install'
-    - script: |
-        export PATH=$HOME/miniconda3/bin:$PATH
-        source activate pandas-dev
-        ci/incremental/build.sh
-      displayName: 'Build'
+        echo "Creating Environment"
+        ci/setup_env.sh
+      displayName: 'Setup environment and build pandas'
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev

ci/azure/windows.yml

@@ -8,10 +8,14 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py36_np14:
+      py36_np15:
         ENV_FILE: ci/deps/azure-windows-36.yaml
         CONDA_PY: "36"
 
+      py37_np141:
+        ENV_FILE: ci/deps/azure-windows-37.yaml
+        CONDA_PY: "37"
+
   steps:
     - task: CondaEnvironment@1
       inputs:

ci/before_script_travis.sh

@@ -16,9 +16,10 @@
 #   $ ./ci/code_checks.sh doctests      # run doctests
 #   $ ./ci/code_checks.sh docstrings    # validate docstring errors
 #   $ ./ci/code_checks.sh dependencies  # check that dependencies are consistent
+#   $ ./ci/code_checks.sh typing	# run static type analysis
 
-[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "dependencies" ]] || \
-    { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|dependencies]"; exit 9999; }
+[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "dependencies" || "$1" == "typing" ]] || \
+    { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|dependencies|typing]"; exit 9999; }
 
 BASE_DIR="$(dirname $0)/.."
 RET=0
@@ -110,15 +111,21 @@ fi
 if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
 
     # Check for imports from pandas.core.common instead of `import pandas.core.common as com`
+    # Check for imports from collections.abc instead of `from collections import abc`
     MSG='Check for non-standard imports' ; echo $MSG
     invgrep -R --include="*.py*" -E "from pandas.core.common import " pandas
+    invgrep -R --include="*.py*" -E "from collections.abc import " pandas
     # invgrep -R --include="*.py*" -E "from numpy import nan " pandas  # GH#24822 not yet implemented since the offending imports have not all been removed
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check for pytest warns' ; echo $MSG
     invgrep -r -E --include '*.py' 'pytest\.warns' pandas/tests/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Check for pytest raises without context' ; echo $MSG
+    invgrep -r -E --include '*.py' "[[:space:]] pytest.raises" pandas/tests/
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     # Check for the following code in testing: `np.testing` and `np.array_equal`
     MSG='Check for invalid testing' ; echo $MSG
     invgrep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/
@@ -175,9 +182,9 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     set -o pipefail
     if [[ "$AZURE" == "true" ]]; then
         # we exclude all c/cpp files as the c/cpp files of pandas code base are tested when Linting .c and .h files
-        ! grep -n '--exclude=*.'{svg,c,cpp,html} -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
+        ! grep -n '--exclude=*.'{svg,c,cpp,html} --exclude-dir=env -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
     else
-        ! grep -n '--exclude=*.'{svg,c,cpp,html}  -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
+        ! grep -n '--exclude=*.'{svg,c,cpp,html} --exclude-dir=env -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
     fi
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 fi
@@ -256,4 +263,16 @@ if [[ -z "$CHECK" || "$CHECK" == "dependencies" ]]; then
 
 fi
 
+### TYPING ###
+if [[ -z "$CHECK" || "$CHECK" == "typing" ]]; then
+
+    echo "mypy --version"
+    mypy --version
+
+    MSG='Performing static analysis using mypy' ; echo $MSG
+    mypy pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+fi
+
+
 exit $RET

ci/code_checks.sh

@@ -0,0 +1,30 @@
+name: pandas-dev
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - beautifulsoup4=4.4.1
+  - bottleneck=1.2.1
+  - jinja2=2.8
+  - numexpr=2.6.2
+  - numpy=1.13.3
+  - openpyxl=2.4.0
+  - pytables=3.4.2
+  - python-dateutil=2.5.0
+  - python=3.5.*
+  - pytz=2015.4
+  - scipy=0.19.0
+  - xlrd=1.0.0
+  - xlsxwriter=0.7.7
+  - xlwt=1.0.0
+  # universal
+  - cython=0.28.2
+  - hypothesis>=3.58.0
+  - pytest-xdist
+  - pytest-mock
+  - isort
+  - pip
+  - pip:
+    # for python 3.5, pytest>=4.0.2 is not available in conda
+    - pytest>=4.0.2
+    - html5lib==1.0b2

ci/deps/azure-35-compat.yaml

@@ -0,0 +1,30 @@
+name: pandas-dev
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - beautifulsoup4==4.5.1
+  - bottleneck=1.2.*
+  - cython=0.28.2
+  - lxml
+  - matplotlib=2.2.2
+  - numpy=1.14.*
+  - openpyxl=2.4.0
+  - python-dateutil
+  - python-blosc
+  - python=3.6.*
+  - pytz=2016.10
+  - scipy
+  - sqlalchemy=1.1.4
+  - xlrd=1.0.0
+  - xlsxwriter=0.9.4
+  - xlwt=1.2.0
+  # universal
+  - pytest>=4.0.2
+  - pytest-xdist
+  - pytest-mock
+  - hypothesis>=3.58.0
+  - isort
+  - pip
+  - pip:
+    - html5lib==1.0b2

ci/deps/azure-36-locale.yaml

@@ -10,14 +10,14 @@ dependencies:
   - ipython
   - jinja2
   - lxml
-  - matplotlib
+  - matplotlib=3.0.*
   - nomkl
   - numexpr
-  - numpy
+  - numpy=1.15.*
   - openpyxl
   - pytables
   - python-dateutil
-  - python=3.6*
+  - python=3.6.*
   - pytz
   - s3fs
   - scipy
@@ -31,5 +31,6 @@ dependencies:
   - pytest-mock
   - moto
   - isort
+  - pip
   - pip:
     - hypothesis>=3.58.0

ci/deps/azure-36-locale_slow.yaml

@@ -16,7 +16,7 @@ dependencies:
   - openpyxl
   - pytables
   - python-dateutil
-  - python=3.7*
+  - python=3.7.*
   - pytz
   - s3fs
   - scipy
@@ -29,6 +29,7 @@ dependencies:
   - pytest-xdist
   - pytest-mock
   - isort
+  - pip
   - pip:
     - hypothesis>=3.58.0
     - moto  # latest moto in conda-forge fails with 3.7, move to conda dependencies when this is fixed

ci/deps/azure-37-locale.yaml

@@ -2,7 +2,7 @@ name: pandas-dev
 channels:
   - defaults
 dependencies:
-  - python=3.7*
+  - python=3.7.*
   - pytz
   - Cython>=0.28.2
   # universal
@@ -11,6 +11,7 @@ dependencies:
   - pytest-mock
   - hypothesis>=3.58.0
   - isort
+  - pip
   - pip:
     - "git+git://github.com/dateutil/dateutil.git"
     - "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"

ci/deps/azure-37-numpydev.yaml

@@ -8,14 +8,14 @@ dependencies:
   - html5lib
   - jinja2
   - lxml
-  - matplotlib=2.2.0
+  - matplotlib=2.2.3
   - nomkl
   - numexpr
-  - numpy=1.12.0
-  - openpyxl=2.5.5
+  - numpy=1.13.3
+  - openpyxl
   - pyarrow
   - pytables
-  - python=3.5*
+  - python=3.5.*
   - pytz
   - xarray
   - xlrd