From 5718fa675bbf0442565c126eebf4f85458500c2c Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Tue, 27 Oct 2020 16:22:21 +1300 Subject: [PATCH 1/7] Use GitHub Action to upload diff images on test failure --- .github/workflows/ci_tests.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 5d941d2bb3b..363b5ea5275 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -117,6 +117,13 @@ jobs: shell: bash -l {0} run: make test PYTEST_EXTRA="-r P" + # Upload diff images on test failure + - name: Upload diff images if any test fails + uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + path: tmp-test-dir-with-unique-name + # Build the documentation - name: Build the documentation shell: bash -l {0} From fde003837dcac0d30ca739e9df42a5515567b9ed Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Tue, 27 Oct 2020 17:29:36 +1300 Subject: [PATCH 2/7] Set a unique artifact name for each OS/Python Version Also trigger a test failure to test if things work. --- .github/workflows/ci_tests.yaml | 1 + pygmt/tests/test_grdimage.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 363b5ea5275..b20d777221b 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -122,6 +122,7 @@ jobs: uses: actions/upload-artifact@v2 if: ${{ failure() }} with: + name: artifact-${{ matrix.os }}-${{ matrix.python-version }} path: tmp-test-dir-with-unique-name # Build the documentation diff --git a/pygmt/tests/test_grdimage.py b/pygmt/tests/test_grdimage.py index 3e2b5b5ec44..51dcb13e5e4 100644 --- a/pygmt/tests/test_grdimage.py +++ b/pygmt/tests/test_grdimage.py @@ -154,7 +154,7 @@ def test_grdimage_central_meridians(grid, proj_type, lon0): # are still slightly different with an RMS error of 25, see issue at # https://github.com/GenericMappingTools/pygmt/issues/390 # TO-DO remove tol=1.5 and pytest.mark.xfail once bug is solved in upstream GMT -@check_figures_equal(tol=1.5) +# @check_figures_equal(tol=1.5) @pytest.mark.parametrize("lat0", [0, 30]) @pytest.mark.parametrize("lon0", [0, 123, 180]) @pytest.mark.parametrize("proj_type", [pytest.param("Q", marks=pytest.mark.xfail), "S"]) From d3c5ba25742681aa56cc4a25a2b37de3716001be Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Tue, 27 Oct 2020 18:12:15 +1300 Subject: [PATCH 3/7] Temporarily trigger test_grdimage_shading_xarray to fail --- pygmt/tests/test_grdimage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pygmt/tests/test_grdimage.py b/pygmt/tests/test_grdimage.py index 51dcb13e5e4..4b422c34a4b 100644 --- a/pygmt/tests/test_grdimage.py +++ b/pygmt/tests/test_grdimage.py @@ -74,14 +74,14 @@ def test_grdimage_file(): return fig +# @pytest.mark.xfail( +# condition=gmt_version <= Version("6.1.1") and sys.platform != "darwin", +# reason="Upstream bug in GMT 6.1.1 that causes this test to fail on Linux/Windows", +# ) @pytest.mark.skipif( gmt_version <= Version("6.1.1") and sys.platform == "darwin", reason="Upstream bug in GMT 6.1.1 that causes segfault on macOS", ) -@pytest.mark.xfail( - condition=gmt_version <= Version("6.1.1") and sys.platform != "darwin", - reason="Upstream bug in GMT 6.1.1 that causes this test to fail on Linux/Windows", -) @check_figures_equal() @pytest.mark.parametrize( "shading", From 4dec78f30fcc48118776af7425efe7ff01d2dc57 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Tue, 27 Oct 2020 20:05:49 +1300 Subject: [PATCH 4/7] Add upload-artifact to GMT Latest tests Also using runner.os instead of matrix.os. --- .github/workflows/ci_tests.yaml | 2 +- .github/workflows/ci_tests_dev.yaml | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index b20d777221b..a4bdc933341 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -122,7 +122,7 @@ jobs: uses: actions/upload-artifact@v2 if: ${{ failure() }} with: - name: artifact-${{ matrix.os }}-${{ matrix.python-version }} + name: artifact-${{ runner.os }}-${{ matrix.python-version }} path: tmp-test-dir-with-unique-name # Build the documentation diff --git a/.github/workflows/ci_tests_dev.yaml b/.github/workflows/ci_tests_dev.yaml index 98790ab96aa..20e2a53c296 100644 --- a/.github/workflows/ci_tests_dev.yaml +++ b/.github/workflows/ci_tests_dev.yaml @@ -86,3 +86,11 @@ jobs: # Run the tests - name: Test with pytest run: make test PYTEST_EXTRA="-r P" + + # Upload diff images on test failure + - name: Upload diff images if any test fails + uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: artifact-GMT ${{ matrix.gmt_git_ref }}-${{ runner.os }} + path: tmp-test-dir-with-unique-name From 0defa075fc2c2758e82de0336006260d408c379f Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Wed, 28 Oct 2020 09:55:19 +1300 Subject: [PATCH 5/7] Revert changes to tests in test_grdimage that were disabled and fix typo Co-Authored-By: Dongdong Tian --- .github/workflows/ci_tests_dev.yaml | 2 +- pygmt/tests/test_grdimage.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_tests_dev.yaml b/.github/workflows/ci_tests_dev.yaml index 20e2a53c296..da78e5eb81c 100644 --- a/.github/workflows/ci_tests_dev.yaml +++ b/.github/workflows/ci_tests_dev.yaml @@ -92,5 +92,5 @@ jobs: uses: actions/upload-artifact@v2 if: ${{ failure() }} with: - name: artifact-GMT ${{ matrix.gmt_git_ref }}-${{ runner.os }} + name: artifact-GMT-${{ matrix.gmt_git_ref }}-${{ runner.os }} path: tmp-test-dir-with-unique-name diff --git a/pygmt/tests/test_grdimage.py b/pygmt/tests/test_grdimage.py index 4b422c34a4b..3e2b5b5ec44 100644 --- a/pygmt/tests/test_grdimage.py +++ b/pygmt/tests/test_grdimage.py @@ -74,14 +74,14 @@ def test_grdimage_file(): return fig -# @pytest.mark.xfail( -# condition=gmt_version <= Version("6.1.1") and sys.platform != "darwin", -# reason="Upstream bug in GMT 6.1.1 that causes this test to fail on Linux/Windows", -# ) @pytest.mark.skipif( gmt_version <= Version("6.1.1") and sys.platform == "darwin", reason="Upstream bug in GMT 6.1.1 that causes segfault on macOS", ) +@pytest.mark.xfail( + condition=gmt_version <= Version("6.1.1") and sys.platform != "darwin", + reason="Upstream bug in GMT 6.1.1 that causes this test to fail on Linux/Windows", +) @check_figures_equal() @pytest.mark.parametrize( "shading", @@ -154,7 +154,7 @@ def test_grdimage_central_meridians(grid, proj_type, lon0): # are still slightly different with an RMS error of 25, see issue at # https://github.com/GenericMappingTools/pygmt/issues/390 # TO-DO remove tol=1.5 and pytest.mark.xfail once bug is solved in upstream GMT -# @check_figures_equal(tol=1.5) +@check_figures_equal(tol=1.5) @pytest.mark.parametrize("lat0", [0, 30]) @pytest.mark.parametrize("lon0", [0, 123, 180]) @pytest.mark.parametrize("proj_type", [pytest.param("Q", marks=pytest.mark.xfail), "S"]) From 7fbb777b5adb226c46f67946a81a7c874f049eaa Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Wed, 28 Oct 2020 09:57:43 +1300 Subject: [PATCH 6/7] Trigger fails on GMT master tests to double check artifacts are uploaded --- pygmt/tests/test_info.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pygmt/tests/test_info.py b/pygmt/tests/test_info.py index e4ce734f7e6..700e7db524f 100644 --- a/pygmt/tests/test_info.py +++ b/pygmt/tests/test_info.py @@ -38,10 +38,10 @@ def test_info_dataframe(): assert output == expected_output -@pytest.mark.xfail( - reason="UNIX timestamps returned instead of ISO datetime, should work on GMT 6.2.0 " - "after https://github.com/GenericMappingTools/gmt/issues/4241 is resolved", -) +# @pytest.mark.xfail( +# reason="UNIX timestamps returned instead of ISO datetime, should work on GMT 6.2.0 " +# "after https://github.com/GenericMappingTools/gmt/issues/4241 is resolved", +# ) def test_info_pandas_dataframe_time_column(): "Make sure info works on pandas.DataFrame inputs with a time column" table = pd.DataFrame( From 9527cf5887a2cd1a21b6290fbf1811bcd10f27a3 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Wed, 28 Oct 2020 09:57:43 +1300 Subject: [PATCH 7/7] Revert "Trigger fails on GMT master tests to double check artifacts are uploaded" This reverts commit 7fbb777b5adb226c46f67946a81a7c874f049eaa. --- pygmt/tests/test_info.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pygmt/tests/test_info.py b/pygmt/tests/test_info.py index 700e7db524f..e4ce734f7e6 100644 --- a/pygmt/tests/test_info.py +++ b/pygmt/tests/test_info.py @@ -38,10 +38,10 @@ def test_info_dataframe(): assert output == expected_output -# @pytest.mark.xfail( -# reason="UNIX timestamps returned instead of ISO datetime, should work on GMT 6.2.0 " -# "after https://github.com/GenericMappingTools/gmt/issues/4241 is resolved", -# ) +@pytest.mark.xfail( + reason="UNIX timestamps returned instead of ISO datetime, should work on GMT 6.2.0 " + "after https://github.com/GenericMappingTools/gmt/issues/4241 is resolved", +) def test_info_pandas_dataframe_time_column(): "Make sure info works on pandas.DataFrame inputs with a time column" table = pd.DataFrame(