WIP/CI/TST: Clean up of the tests script

ci/azure/posix.yml

@@ -59,43 +59,33 @@ jobs:
         echo "Creating Environment"
         ci/setup_env.sh
       displayName: 'Setup environment and build pandas'
+
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
         ci/run_tests.sh
       displayName: 'Test'
+
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
+      displayName: 'Build versions'
+
     - task: PublishTestResults@2
       inputs:
-        testResultsFiles: 'test-data-*.xml'
+        testResultsFiles: 'test-data.xml'
         testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }}
-    - powershell: |
-        $junitXml = "test-data-single.xml"
-        $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
-        if ($matches[1] -eq 0)
-        {
-          Write-Host "No test failures in test-data-single"
-        }
-        else
-        {
-          # note that this will produce $LASTEXITCODE=1
-          Write-Error "$($matches[1]) tests failed"
-        }
+      displayName: 'Publish test results'
 
-        $junitXmlMulti = "test-data-multiple.xml"
-        $(Get-Content $junitXmlMulti | Out-String) -match 'failures="(.*?)"'
-        if ($matches[1] -eq 0)
-        {
-          Write-Host "No test failures in test-data-multi"
-        }
-        else
-        {
-          # note that this will produce $LASTEXITCODE=1
-          Write-Error "$($matches[1]) tests failed"
+    - powershell: |
+        $(Get-Content "test-data.xml" | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0) {
+          Write-Host "No test failures in test-data"
+        } else {
+          Write-Error "$($matches[1]) tests failed"  # will produce $LASTEXITCODE=1
         }
       displayName: 'Check for test failures'
+
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev

ci/print_skipped.py

@@ -1,20 +1,13 @@
 #!/usr/bin/env python
-
-import os
-import sys
 import math
 import xml.etree.ElementTree as et
 
 
-def parse_results(filename):
+def main(filename):
     tree = et.parse(filename)
     root = tree.getroot()
-    skipped = []
-
     current_class = ""
-    i = 1
-    assert i - 1 == len(skipped)
-    for el in root.findall("testcase"):
+    for i, el in enumerate(root.findall("testcase")):
         cn = el.attrib["classname"]
         for sk in el.findall("skipped"):
             old_class = current_class
@@ -25,28 +18,15 @@ def parse_results(filename):
             msg = sk.attrib["message"]
             out = ""
             if old_class != current_class:
-                ndigits = int(math.log(i, 10) + 1)
+                ndigits = int(math.log(i + 1, 10) + 1)
 
                 # 4 for : + space + # + space
                 out += "-" * (len(name + msg) + 4 + ndigits) + "\n"
-            out += "#{i} {name}: {msg}".format(i=i, name=name, msg=msg)
-            skipped.append(out)
-            i += 1
-            assert i - 1 == len(skipped)
-    assert i - 1 == len(skipped)
-    # assert len(skipped) == int(root.attrib['skip'])
-    return "\n".join(skipped)
-
-
-def main():
-    test_files = ["test-data-single.xml", "test-data-multiple.xml", "test-data.xml"]
-
-    print("SKIPPED TESTS:")
-    for fn in test_files:
-        if os.path.isfile(fn):
-            print(parse_results(fn))
-    return 0
+            out += "#{i} {name}: {msg}".format(i=i + 1, name=name, msg=msg)
+            yield out
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    print("SKIPPED TESTS:")
+    skipped_tests = main("test-data.xml")
+    print("\n".join(skipped_tests))

ci/run_tests.sh

@@ -2,57 +2,33 @@
 
 set -e
 
-if [ "$DOC" ]; then
-    echo "We are not running pytest as this is a doc-build"
-    exit 0
-fi
-
-# Workaround for pytest-xdist flaky collection order
+# Workaround for pytest-xdist (it collects different tests in the workers if PYTHONHASHSEED is not set)
 # https://github.com/pytest-dev/pytest/issues/920
 # https://github.com/pytest-dev/pytest/issues/1075
 export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
+export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4;
 
 if [ -n "$LOCALE_OVERRIDE" ]; then
     export LC_ALL="$LOCALE_OVERRIDE"
     export LANG="$LOCALE_OVERRIDE"
     PANDAS_LOCALE=`python -c 'import pandas; pandas.get_option("display.encoding")'`
-    if [[ "$LOCALE_OVERIDE" != "$PANDAS_LOCALE" ]]; then
+    if [[ "$LOCALE_OVERRIDE" != "$PANDAS_LOCALE" ]]; then
         echo "pandas could not detect the locale. System locale: $LOCALE_OVERRIDE, pandas detected: $PANDAS_LOCALE"
         # TODO Not really aborting the tests until https://github.com/pandas-dev/pandas/issues/23923 is fixed
         # exit 1
     fi
 fi
-if [[ "not network" == *"$PATTERN"* ]]; then
-    export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4;
-fi
 
-
-if [ -n "$PATTERN" ]; then
-    PATTERN=" and $PATTERN"
+if [ "$COVERAGE" ]; then
+    COVERAGE_FNAME="/tmp/test_coverage.xml"
+    COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME"
 fi
 
-for TYPE in single multiple
-do
-    if [ "$COVERAGE" ]; then
-        COVERAGE_FNAME="/tmp/coc-$TYPE.xml"
-        COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME"
-    fi
-
-    TYPE_PATTERN=$TYPE
-    NUM_JOBS=1
-    if [[ "$TYPE_PATTERN" == "multiple" ]]; then
-        TYPE_PATTERN="not single"
-        NUM_JOBS=2
-    fi
-
-    PYTEST_CMD="pytest -m \"$TYPE_PATTERN$PATTERN\" -n $NUM_JOBS -s --strict --durations=10 --junitxml=test-data-$TYPE.xml $TEST_ARGS $COVERAGE pandas"
-    echo $PYTEST_CMD
-    # if no tests are found (the case of "single and slow"), pytest exits with code 5, and would make the script fail, if not for the below code
-    sh -c "$PYTEST_CMD; ret=\$?; [ \$ret = 5 ] && exit 0 || exit \$ret"
+PYTEST_CMD="pytest -m \"$PATTERN\" -n auto --dist=loadscope -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
+echo $PYTEST_CMD
+sh -c "$PYTEST_CMD"
 
-    if [[ "$COVERAGE" && $? == 0 ]]; then
-        echo "uploading coverage for $TYPE tests"
-        echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME"
-              bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
-    fi
-done
+if [[ "$COVERAGE" && $? == 0 ]]; then
+    echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME"
+          bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
+fi