From 8b7861441f3a150dbc01fba4465ae577db766328 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 5 Aug 2020 00:26:47 -0400 Subject: [PATCH 1/8] Add testing.check_figures_equal to avoid storing baseline images --- pygmt/exceptions.py | 5 +++++ pygmt/helpers/testing.py | 35 +++++++++++++++++++++++++++++++++++ pygmt/tests/test.py | 7 +++++++ pygmt/tests/test_grdimage.py | 11 +++++++++++ pygmt/tests/test_testing.py | 27 +++++++++++++++++++++++++++ 5 files changed, 85 insertions(+) create mode 100644 pygmt/helpers/testing.py create mode 100644 pygmt/tests/test.py create mode 100644 pygmt/tests/test_testing.py diff --git a/pygmt/exceptions.py b/pygmt/exceptions.py index d5b2c9584ef..fbd429e29f4 100644 --- a/pygmt/exceptions.py +++ b/pygmt/exceptions.py @@ -44,3 +44,8 @@ class GMTVersionError(GMTError): """ Raised when an incompatible version of GMT is being used. """ + +class GMTImageComparisonFailure(AssertionError): + """ + Raised when a comparison between two images fails. + """ diff --git a/pygmt/helpers/testing.py b/pygmt/helpers/testing.py new file mode 100644 index 00000000000..2dc14eb44ea --- /dev/null +++ b/pygmt/helpers/testing.py @@ -0,0 +1,35 @@ +import os +import sys +from pathlib import Path +from matplotlib.testing.compare import compare_images +from ..exceptions import GMTImageComparisonFailure + + +def check_figures_equal(fig_ref, fig_test, fig_prefix=None, tol=0.0): + result_dir = "result_images" + + if not fig_prefix: + try: + fig_prefix = sys._getframe(1).f_code.co_name + except VauleError: + raise GMTInvalidInput("fig_prefix is required.") + + os.makedirs(result_dir, exist_ok=True) + + ref_image_path = os.path.join(result_dir, fig_prefix + '-expected.png') + test_image_path = os.path.join(result_dir, fig_prefix + '.png') + + fig_ref.savefig(ref_image_path) + fig_test.savefig(test_image_path) + + err = compare_images(ref_image_path, test_image_path, tol, in_decorator=True) + + if err is None: # Images are the same + os.remove(ref_image_path) + os.remove(test_image_path) + else: + for key in ["actual", "expected"]: + err[key] = os.path.relpath(err[key]) + raise GMTImageComparisonFailure( + 'images not close (RMS %(rms).3f):\n\t%(actual)s\n\t%(expected)s ' + % err) diff --git a/pygmt/tests/test.py b/pygmt/tests/test.py new file mode 100644 index 00000000000..43d6c94edb8 --- /dev/null +++ b/pygmt/tests/test.py @@ -0,0 +1,7 @@ +import pygmt +from matplotlib.testing.decorators import check_figures_equal + +@check_figures_equal(extensions=['png']) +def test_plot(fig_test, fig_ref): + fig_test.subplots().plot([1, 3, 5]) + fig_ref.subplots().plot([0, 1, 2], [1, 3, 5]) diff --git a/pygmt/tests/test_grdimage.py b/pygmt/tests/test_grdimage.py index 45be76f6d01..9a283c97ee0 100644 --- a/pygmt/tests/test_grdimage.py +++ b/pygmt/tests/test_grdimage.py @@ -8,6 +8,7 @@ from .. import Figure from ..exceptions import GMTInvalidInput from ..datasets import load_earth_relief +from ..helpers.testing import check_figures_equal @pytest.fixture(scope="module", name="grid") @@ -93,3 +94,13 @@ def test_grdimage_over_dateline(xrgrid): xrgrid.gmt.gtype = 1 # geographic coordinate system fig.grdimage(grid=xrgrid, region="g", projection="A0/0/1c", V="i") return fig + + +def test_grdimage_central_longitude(grid): + fig1 = Figure() + fig1.grdimage("@earth_relief_01d_g", projection="W120/15c", cmap='geo') + + fig2 = Figure() + fig2.grdimage(grid, projection="W120/15c", cmap='geo') + + check_figures_equal(fig1, fig2) diff --git a/pygmt/tests/test_testing.py b/pygmt/tests/test_testing.py new file mode 100644 index 00000000000..e26f83adb51 --- /dev/null +++ b/pygmt/tests/test_testing.py @@ -0,0 +1,27 @@ +""" +Test the testing functions for PyGMT +""" +from .. import Figure +from ..helpers.testing import check_figures_equal +from ..exceptions import GMTImageComparisonFailure +import pytest + + +def test_check_figures_equal(): + fig_ref = Figure() + fig_ref.basemap(projection="X10c", region=[0, 10, 0, 10], frame=True) + + fig_test = Figure() + fig_test.basemap(projection="X10c", region=[0, 10, 0, 10], frame=True) + check_figures_equal(fig_ref, fig_test) + + +def test_check_figures_unequal(): + fig_ref = Figure() + fig_ref.basemap(projection="X10c", region=[0, 10, 0, 10], frame=True) + + fig_test = Figure() + fig_test.basemap(projection="X10c", region=[0, 15, 0, 15], frame=True) + + with pytest.raises(GMTImageComparisonFailure): + check_figures_equal(fig_ref, fig_test) From 27e03ed57d4ffcbbc024fa06fea2616e0b60f7cf Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Thu, 3 Sep 2020 14:01:36 +1200 Subject: [PATCH 2/8] Black lint, fix typos and run isort --- pygmt/exceptions.py | 1 + pygmt/helpers/testing.py | 14 ++++++++------ pygmt/tests/test.py | 6 ++++-- pygmt/tests/test_grdimage.py | 4 ++-- pygmt/tests/test_testing.py | 5 +++-- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/pygmt/exceptions.py b/pygmt/exceptions.py index fbd429e29f4..6b72b8cb919 100644 --- a/pygmt/exceptions.py +++ b/pygmt/exceptions.py @@ -45,6 +45,7 @@ class GMTVersionError(GMTError): Raised when an incompatible version of GMT is being used. """ + class GMTImageComparisonFailure(AssertionError): """ Raised when a comparison between two images fails. diff --git a/pygmt/helpers/testing.py b/pygmt/helpers/testing.py index 2dc14eb44ea..572c69c5e32 100644 --- a/pygmt/helpers/testing.py +++ b/pygmt/helpers/testing.py @@ -1,8 +1,10 @@ import os import sys from pathlib import Path + from matplotlib.testing.compare import compare_images -from ..exceptions import GMTImageComparisonFailure + +from ..exceptions import GMTImageComparisonFailure, GMTInvalidInput def check_figures_equal(fig_ref, fig_test, fig_prefix=None, tol=0.0): @@ -11,13 +13,13 @@ def check_figures_equal(fig_ref, fig_test, fig_prefix=None, tol=0.0): if not fig_prefix: try: fig_prefix = sys._getframe(1).f_code.co_name - except VauleError: + except ValueError: raise GMTInvalidInput("fig_prefix is required.") os.makedirs(result_dir, exist_ok=True) - ref_image_path = os.path.join(result_dir, fig_prefix + '-expected.png') - test_image_path = os.path.join(result_dir, fig_prefix + '.png') + ref_image_path = os.path.join(result_dir, fig_prefix + "-expected.png") + test_image_path = os.path.join(result_dir, fig_prefix + ".png") fig_ref.savefig(ref_image_path) fig_test.savefig(test_image_path) @@ -31,5 +33,5 @@ def check_figures_equal(fig_ref, fig_test, fig_prefix=None, tol=0.0): for key in ["actual", "expected"]: err[key] = os.path.relpath(err[key]) raise GMTImageComparisonFailure( - 'images not close (RMS %(rms).3f):\n\t%(actual)s\n\t%(expected)s ' - % err) + "images not close (RMS %(rms).3f):\n\t%(actual)s\n\t%(expected)s " % err + ) diff --git a/pygmt/tests/test.py b/pygmt/tests/test.py index 43d6c94edb8..d740a92f757 100644 --- a/pygmt/tests/test.py +++ b/pygmt/tests/test.py @@ -1,7 +1,9 @@ -import pygmt from matplotlib.testing.decorators import check_figures_equal -@check_figures_equal(extensions=['png']) +import pygmt + + +@check_figures_equal(extensions=["png"]) def test_plot(fig_test, fig_ref): fig_test.subplots().plot([1, 3, 5]) fig_ref.subplots().plot([0, 1, 2], [1, 3, 5]) diff --git a/pygmt/tests/test_grdimage.py b/pygmt/tests/test_grdimage.py index 9a283c97ee0..bd4282b4f02 100644 --- a/pygmt/tests/test_grdimage.py +++ b/pygmt/tests/test_grdimage.py @@ -98,9 +98,9 @@ def test_grdimage_over_dateline(xrgrid): def test_grdimage_central_longitude(grid): fig1 = Figure() - fig1.grdimage("@earth_relief_01d_g", projection="W120/15c", cmap='geo') + fig1.grdimage("@earth_relief_01d_g", projection="W120/15c", cmap="geo") fig2 = Figure() - fig2.grdimage(grid, projection="W120/15c", cmap='geo') + fig2.grdimage(grid, projection="W120/15c", cmap="geo") check_figures_equal(fig1, fig2) diff --git a/pygmt/tests/test_testing.py b/pygmt/tests/test_testing.py index e26f83adb51..4f6cf04c259 100644 --- a/pygmt/tests/test_testing.py +++ b/pygmt/tests/test_testing.py @@ -1,10 +1,11 @@ """ Test the testing functions for PyGMT """ +import pytest + from .. import Figure -from ..helpers.testing import check_figures_equal from ..exceptions import GMTImageComparisonFailure -import pytest +from ..helpers.testing import check_figures_equal def test_check_figures_equal(): From d2ad3f56ba7c2371e4b42c1de998795e4ad7d010 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Thu, 3 Sep 2020 17:39:57 +1200 Subject: [PATCH 3/8] Turn check_figures_equal into a decorator function Also moved test_check_figures_* to a doctest under check_figures_equal. --- pygmt/helpers/__init__.py | 2 +- pygmt/helpers/decorators.py | 88 ++++++++++++++++++++++++++++++++++-- pygmt/helpers/testing.py | 37 --------------- pygmt/tests/test.py | 9 ---- pygmt/tests/test_grdimage.py | 21 ++++----- pygmt/tests/test_testing.py | 28 ------------ 6 files changed, 96 insertions(+), 89 deletions(-) delete mode 100644 pygmt/helpers/testing.py delete mode 100644 pygmt/tests/test.py delete mode 100644 pygmt/tests/test_testing.py diff --git a/pygmt/helpers/__init__.py b/pygmt/helpers/__init__.py index b8a6958816d..48436b86799 100644 --- a/pygmt/helpers/__init__.py +++ b/pygmt/helpers/__init__.py @@ -1,7 +1,7 @@ """ Functions, classes, decorators, and context managers to help wrap GMT modules. """ -from .decorators import fmt_docstring, use_alias, kwargs_to_strings +from .decorators import check_figures_equal, fmt_docstring, kwargs_to_strings, use_alias from .tempfile import GMTTempFile, unique_name from .utils import ( data_kind, diff --git a/pygmt/helpers/decorators.py b/pygmt/helpers/decorators.py index 0a11c37277c..b7fa61a63be 100644 --- a/pygmt/helpers/decorators.py +++ b/pygmt/helpers/decorators.py @@ -5,14 +5,15 @@ arguments, insert common text into docstrings, transform arguments to strings, etc. """ -import textwrap import functools +import os +import textwrap import numpy as np +from matplotlib.testing.compare import compare_images +from ..exceptions import GMTImageComparisonFailure, GMTInvalidInput from .utils import is_nonstr_iter -from ..exceptions import GMTInvalidInput - COMMON_OPTIONS = { "R": """\ @@ -404,3 +405,84 @@ def remove_bools(kwargs): else: new_kwargs[arg] = value return new_kwargs + + +def check_figures_equal(*, result_dir="result_images", tol=0.0): + """ + Decorator for test cases that generate and compare two figures. + + The decorated function must take two arguments, *fig_ref* and *fig_test*, + and draw the reference and test images on them. After the function + returns, the figures are saved and compared. + + Parameters + ---------- + result_dir : str + The directory where the figures will be stored. + tol : float + The RMS threshold above which the test is considered failed. + + Examples + -------- + + >>> import pytest + >>> @check_figures_equal() + ... def test_check_figures_equal(fig_ref, fig_test): + ... fig_ref.basemap(projection="X5c", region=[0, 5, 0, 5], frame=True) + ... fig_test.basemap(projection="X5c", region=[0, 5, 0, 5], frame="af") + >>> test_check_figures_equal() + + >>> import shutil + >>> @check_figures_equal(result_dir="tmp_result_images") + ... def test_check_figures_unequal(fig_ref, fig_test): + ... fig_ref.basemap(projection="X5c", region=[0, 5, 0, 5], frame=True) + ... fig_test.basemap(projection="X5c", region=[0, 3, 0, 3], frame=True) + >>> with pytest.raises(GMTImageComparisonFailure): + ... test_check_figures_unequal() + >>> shutil.rmtree(path="tmp_result_images") + + """ + + def decorator(func): + + os.makedirs(result_dir, exist_ok=True) + + def wrapper(): + try: + from ..figure import Figure # pylint: disable=import-outside-toplevel + + fig_ref = Figure() + fig_test = Figure() + func(fig_ref, fig_test) + ref_image_path = os.path.join( + result_dir, func.__name__ + "-expected.png" + ) + test_image_path = os.path.join(result_dir, func.__name__ + ".png") + fig_ref.savefig(ref_image_path) + fig_test.savefig(test_image_path) + + # Code below is adapted for PyGMT, and is originally based on + # matplotlib.testing.decorators._raise_on_image_difference + err = compare_images( + expected=ref_image_path, + actual=test_image_path, + tol=tol, + in_decorator=True, + ) + if err is None: # Images are the same + os.remove(ref_image_path) + os.remove(test_image_path) + else: # Images are not the same + for key in ["actual", "expected", "diff"]: + err[key] = os.path.relpath(err[key]) + raise GMTImageComparisonFailure( + "images not close (RMS %(rms).3f):\n\t%(actual)s\n\t%(expected)s " + % err + ) + finally: + del fig_ref + del fig_test + + return wrapper + + return decorator diff --git a/pygmt/helpers/testing.py b/pygmt/helpers/testing.py deleted file mode 100644 index 572c69c5e32..00000000000 --- a/pygmt/helpers/testing.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -import sys -from pathlib import Path - -from matplotlib.testing.compare import compare_images - -from ..exceptions import GMTImageComparisonFailure, GMTInvalidInput - - -def check_figures_equal(fig_ref, fig_test, fig_prefix=None, tol=0.0): - result_dir = "result_images" - - if not fig_prefix: - try: - fig_prefix = sys._getframe(1).f_code.co_name - except ValueError: - raise GMTInvalidInput("fig_prefix is required.") - - os.makedirs(result_dir, exist_ok=True) - - ref_image_path = os.path.join(result_dir, fig_prefix + "-expected.png") - test_image_path = os.path.join(result_dir, fig_prefix + ".png") - - fig_ref.savefig(ref_image_path) - fig_test.savefig(test_image_path) - - err = compare_images(ref_image_path, test_image_path, tol, in_decorator=True) - - if err is None: # Images are the same - os.remove(ref_image_path) - os.remove(test_image_path) - else: - for key in ["actual", "expected"]: - err[key] = os.path.relpath(err[key]) - raise GMTImageComparisonFailure( - "images not close (RMS %(rms).3f):\n\t%(actual)s\n\t%(expected)s " % err - ) diff --git a/pygmt/tests/test.py b/pygmt/tests/test.py deleted file mode 100644 index d740a92f757..00000000000 --- a/pygmt/tests/test.py +++ /dev/null @@ -1,9 +0,0 @@ -from matplotlib.testing.decorators import check_figures_equal - -import pygmt - - -@check_figures_equal(extensions=["png"]) -def test_plot(fig_test, fig_ref): - fig_test.subplots().plot([1, 3, 5]) - fig_ref.subplots().plot([0, 1, 2], [1, 3, 5]) diff --git a/pygmt/tests/test_grdimage.py b/pygmt/tests/test_grdimage.py index bd4282b4f02..c8d2f1c6769 100644 --- a/pygmt/tests/test_grdimage.py +++ b/pygmt/tests/test_grdimage.py @@ -2,13 +2,13 @@ Test Figure.grdimage """ import numpy as np -import xarray as xr import pytest +import xarray as xr from .. import Figure -from ..exceptions import GMTInvalidInput from ..datasets import load_earth_relief -from ..helpers.testing import check_figures_equal +from ..exceptions import GMTInvalidInput +from ..helpers import check_figures_equal @pytest.fixture(scope="module", name="grid") @@ -96,11 +96,10 @@ def test_grdimage_over_dateline(xrgrid): return fig -def test_grdimage_central_longitude(grid): - fig1 = Figure() - fig1.grdimage("@earth_relief_01d_g", projection="W120/15c", cmap="geo") - - fig2 = Figure() - fig2.grdimage(grid, projection="W120/15c", cmap="geo") - - check_figures_equal(fig1, fig2) +@check_figures_equal() +def test_grdimage_central_longitude(grid, fig_ref, fig_test): + """ + Test that plotting a grid centred at different longitudes/meridians work. + """ + fig_ref.grdimage("@earth_relief_01d_g", projection="W120/15c", cmap="geo") + fig_test.grdimage(grid, projection="W120/15c", cmap="geo") diff --git a/pygmt/tests/test_testing.py b/pygmt/tests/test_testing.py deleted file mode 100644 index 4f6cf04c259..00000000000 --- a/pygmt/tests/test_testing.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -Test the testing functions for PyGMT -""" -import pytest - -from .. import Figure -from ..exceptions import GMTImageComparisonFailure -from ..helpers.testing import check_figures_equal - - -def test_check_figures_equal(): - fig_ref = Figure() - fig_ref.basemap(projection="X10c", region=[0, 10, 0, 10], frame=True) - - fig_test = Figure() - fig_test.basemap(projection="X10c", region=[0, 10, 0, 10], frame=True) - check_figures_equal(fig_ref, fig_test) - - -def test_check_figures_unequal(): - fig_ref = Figure() - fig_ref.basemap(projection="X10c", region=[0, 10, 0, 10], frame=True) - - fig_test = Figure() - fig_test.basemap(projection="X10c", region=[0, 15, 0, 15], frame=True) - - with pytest.raises(GMTImageComparisonFailure): - check_figures_equal(fig_ref, fig_test) From 3e0d3fb1d7685f8b23775860636900b4e10e5c0f Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Thu, 3 Sep 2020 21:13:26 +1200 Subject: [PATCH 4/8] Ensure pytest fixtures can be used with check_figures_equal decorator Same logic that was implemented in https://github.com/matplotlib/matplotlib/pull/16800 --- pygmt/helpers/decorators.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pygmt/helpers/decorators.py b/pygmt/helpers/decorators.py index b7fa61a63be..180e167fb84 100644 --- a/pygmt/helpers/decorators.py +++ b/pygmt/helpers/decorators.py @@ -6,6 +6,7 @@ etc. """ import functools +import inspect import os import textwrap @@ -446,14 +447,15 @@ def check_figures_equal(*, result_dir="result_images", tol=0.0): def decorator(func): os.makedirs(result_dir, exist_ok=True) + old_sig = inspect.signature(func) - def wrapper(): + def wrapper(*args, **kwargs): try: from ..figure import Figure # pylint: disable=import-outside-toplevel fig_ref = Figure() fig_test = Figure() - func(fig_ref, fig_test) + func(*args, fig_ref=fig_ref, fig_test=fig_test, **kwargs) ref_image_path = os.path.join( result_dir, func.__name__ + "-expected.png" ) @@ -483,6 +485,14 @@ def wrapper(): del fig_ref del fig_test + parameters = [ + param + for param in old_sig.parameters.values() + if param.name not in {"fig_test", "fig_ref"} + ] + new_sig = old_sig.replace(parameters=parameters) + wrapper.__signature__ = new_sig + return wrapper return decorator From 04b3f4154ab35a345c3394cd393e050f306f64ae Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Thu, 3 Sep 2020 22:47:13 +1200 Subject: [PATCH 5/8] Reorder parameters and add docstring note on code origin from matplotlib --- pygmt/helpers/decorators.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pygmt/helpers/decorators.py b/pygmt/helpers/decorators.py index 180e167fb84..4884b883edb 100644 --- a/pygmt/helpers/decorators.py +++ b/pygmt/helpers/decorators.py @@ -408,7 +408,7 @@ def remove_bools(kwargs): return new_kwargs -def check_figures_equal(*, result_dir="result_images", tol=0.0): +def check_figures_equal(*, tol=0.0, result_dir="result_images"): """ Decorator for test cases that generate and compare two figures. @@ -416,12 +416,17 @@ def check_figures_equal(*, result_dir="result_images", tol=0.0): and draw the reference and test images on them. After the function returns, the figures are saved and compared. + This decorator is practically identical to matplotlib's check_figures_equal + function, but adapted for PyGMT figures. See also the original code at + https://matplotlib.org/3.3.1/api/testing_api.html# + matplotlib.testing.decorators.check_figures_equal + Parameters ---------- - result_dir : str - The directory where the figures will be stored. tol : float The RMS threshold above which the test is considered failed. + result_dir : str + The directory where the figures will be stored. Examples -------- From cfe3a24bd3704e416d1e1f86341b716a2e16011f Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Fri, 4 Sep 2020 09:36:28 +1200 Subject: [PATCH 6/8] Move check_figures_equal out of decorators.py and back into testing.py --- pygmt/helpers/__init__.py | 2 +- pygmt/helpers/decorators.py | 103 +--------------------------------- pygmt/helpers/testing.py | 105 +++++++++++++++++++++++++++++++++++ pygmt/tests/test_grdimage.py | 2 +- 4 files changed, 110 insertions(+), 102 deletions(-) create mode 100644 pygmt/helpers/testing.py diff --git a/pygmt/helpers/__init__.py b/pygmt/helpers/__init__.py index 48436b86799..b8a6958816d 100644 --- a/pygmt/helpers/__init__.py +++ b/pygmt/helpers/__init__.py @@ -1,7 +1,7 @@ """ Functions, classes, decorators, and context managers to help wrap GMT modules. """ -from .decorators import check_figures_equal, fmt_docstring, kwargs_to_strings, use_alias +from .decorators import fmt_docstring, use_alias, kwargs_to_strings from .tempfile import GMTTempFile, unique_name from .utils import ( data_kind, diff --git a/pygmt/helpers/decorators.py b/pygmt/helpers/decorators.py index 4884b883edb..0a11c37277c 100644 --- a/pygmt/helpers/decorators.py +++ b/pygmt/helpers/decorators.py @@ -5,16 +5,14 @@ arguments, insert common text into docstrings, transform arguments to strings, etc. """ -import functools -import inspect -import os import textwrap +import functools import numpy as np -from matplotlib.testing.compare import compare_images -from ..exceptions import GMTImageComparisonFailure, GMTInvalidInput from .utils import is_nonstr_iter +from ..exceptions import GMTInvalidInput + COMMON_OPTIONS = { "R": """\ @@ -406,98 +404,3 @@ def remove_bools(kwargs): else: new_kwargs[arg] = value return new_kwargs - - -def check_figures_equal(*, tol=0.0, result_dir="result_images"): - """ - Decorator for test cases that generate and compare two figures. - - The decorated function must take two arguments, *fig_ref* and *fig_test*, - and draw the reference and test images on them. After the function - returns, the figures are saved and compared. - - This decorator is practically identical to matplotlib's check_figures_equal - function, but adapted for PyGMT figures. See also the original code at - https://matplotlib.org/3.3.1/api/testing_api.html# - matplotlib.testing.decorators.check_figures_equal - - Parameters - ---------- - tol : float - The RMS threshold above which the test is considered failed. - result_dir : str - The directory where the figures will be stored. - - Examples - -------- - - >>> import pytest - >>> @check_figures_equal() - ... def test_check_figures_equal(fig_ref, fig_test): - ... fig_ref.basemap(projection="X5c", region=[0, 5, 0, 5], frame=True) - ... fig_test.basemap(projection="X5c", region=[0, 5, 0, 5], frame="af") - >>> test_check_figures_equal() - - >>> import shutil - >>> @check_figures_equal(result_dir="tmp_result_images") - ... def test_check_figures_unequal(fig_ref, fig_test): - ... fig_ref.basemap(projection="X5c", region=[0, 5, 0, 5], frame=True) - ... fig_test.basemap(projection="X5c", region=[0, 3, 0, 3], frame=True) - >>> with pytest.raises(GMTImageComparisonFailure): - ... test_check_figures_unequal() - >>> shutil.rmtree(path="tmp_result_images") - - """ - - def decorator(func): - - os.makedirs(result_dir, exist_ok=True) - old_sig = inspect.signature(func) - - def wrapper(*args, **kwargs): - try: - from ..figure import Figure # pylint: disable=import-outside-toplevel - - fig_ref = Figure() - fig_test = Figure() - func(*args, fig_ref=fig_ref, fig_test=fig_test, **kwargs) - ref_image_path = os.path.join( - result_dir, func.__name__ + "-expected.png" - ) - test_image_path = os.path.join(result_dir, func.__name__ + ".png") - fig_ref.savefig(ref_image_path) - fig_test.savefig(test_image_path) - - # Code below is adapted for PyGMT, and is originally based on - # matplotlib.testing.decorators._raise_on_image_difference - err = compare_images( - expected=ref_image_path, - actual=test_image_path, - tol=tol, - in_decorator=True, - ) - if err is None: # Images are the same - os.remove(ref_image_path) - os.remove(test_image_path) - else: # Images are not the same - for key in ["actual", "expected", "diff"]: - err[key] = os.path.relpath(err[key]) - raise GMTImageComparisonFailure( - "images not close (RMS %(rms).3f):\n\t%(actual)s\n\t%(expected)s " - % err - ) - finally: - del fig_ref - del fig_test - - parameters = [ - param - for param in old_sig.parameters.values() - if param.name not in {"fig_test", "fig_ref"} - ] - new_sig = old_sig.replace(parameters=parameters) - wrapper.__signature__ = new_sig - - return wrapper - - return decorator diff --git a/pygmt/helpers/testing.py b/pygmt/helpers/testing.py new file mode 100644 index 00000000000..5690406bbbf --- /dev/null +++ b/pygmt/helpers/testing.py @@ -0,0 +1,105 @@ +""" +Helper functions for testing. +""" + +import inspect +import os + +from matplotlib.testing.compare import compare_images + +from ..exceptions import GMTImageComparisonFailure +from ..figure import Figure + + +def check_figures_equal(*, tol=0.0, result_dir="result_images"): + """ + Decorator for test cases that generate and compare two figures. + + The decorated function must take two arguments, *fig_ref* and *fig_test*, + and draw the reference and test images on them. After the function + returns, the figures are saved and compared. + + This decorator is practically identical to matplotlib's check_figures_equal + function, but adapted for PyGMT figures. See also the original code at + https://matplotlib.org/3.3.1/api/testing_api.html# + matplotlib.testing.decorators.check_figures_equal + + Parameters + ---------- + tol : float + The RMS threshold above which the test is considered failed. + result_dir : str + The directory where the figures will be stored. + + Examples + -------- + + >>> import pytest + >>> import shutil + + >>> @check_figures_equal(result_dir="tmp_result_images") + ... def test_check_figures_equal(fig_ref, fig_test): + ... fig_ref.basemap(projection="X5c", region=[0, 5, 0, 5], frame=True) + ... fig_test.basemap(projection="X5c", region=[0, 5, 0, 5], frame="af") + >>> test_check_figures_equal() + + >>> @check_figures_equal(result_dir="tmp_result_images") + ... def test_check_figures_unequal(fig_ref, fig_test): + ... fig_ref.basemap(projection="X5c", region=[0, 5, 0, 5], frame=True) + ... fig_test.basemap(projection="X5c", region=[0, 3, 0, 3], frame=True) + >>> with pytest.raises(GMTImageComparisonFailure): + ... test_check_figures_unequal() + + >>> shutil.rmtree(path="tmp_result_images") # cleanup folder if tests pass + """ + + def decorator(func): + + os.makedirs(result_dir, exist_ok=True) + old_sig = inspect.signature(func) + + def wrapper(*args, **kwargs): + try: + fig_ref = Figure() + fig_test = Figure() + func(*args, fig_ref=fig_ref, fig_test=fig_test, **kwargs) + ref_image_path = os.path.join( + result_dir, func.__name__ + "-expected.png" + ) + test_image_path = os.path.join(result_dir, func.__name__ + ".png") + fig_ref.savefig(ref_image_path) + fig_test.savefig(test_image_path) + + # Code below is adapted for PyGMT, and is originally based on + # matplotlib.testing.decorators._raise_on_image_difference + err = compare_images( + expected=ref_image_path, + actual=test_image_path, + tol=tol, + in_decorator=True, + ) + if err is None: # Images are the same + os.remove(ref_image_path) + os.remove(test_image_path) + else: # Images are not the same + for key in ["actual", "expected", "diff"]: + err[key] = os.path.relpath(err[key]) + raise GMTImageComparisonFailure( + "images not close (RMS %(rms).3f):\n\t%(actual)s\n\t%(expected)s " + % err + ) + finally: + del fig_ref + del fig_test + + parameters = [ + param + for param in old_sig.parameters.values() + if param.name not in {"fig_test", "fig_ref"} + ] + new_sig = old_sig.replace(parameters=parameters) + wrapper.__signature__ = new_sig + + return wrapper + + return decorator diff --git a/pygmt/tests/test_grdimage.py b/pygmt/tests/test_grdimage.py index c8d2f1c6769..37b5fca822a 100644 --- a/pygmt/tests/test_grdimage.py +++ b/pygmt/tests/test_grdimage.py @@ -8,7 +8,7 @@ from .. import Figure from ..datasets import load_earth_relief from ..exceptions import GMTInvalidInput -from ..helpers import check_figures_equal +from ..helpers.testing import check_figures_equal @pytest.fixture(scope="module", name="grid") From 1155df0c1c4c1833e5fb209645b5e08ab3deb756 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Fri, 4 Sep 2020 10:41:15 +1200 Subject: [PATCH 7/8] Add notes on using check_figures_equal to MAINTENANCE.md --- CONTRIBUTING.md | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9ccb25bc1c1..f4e54db00cf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -310,8 +310,38 @@ Leave a comment in the PR and we'll help you out. ### Testing plots -We use the [pytest-mpl](https://github.com/matplotlib/pytest-mpl) plug-in to test plot -generating code. +Writing an image-based test is only slightly more difficult than a simple test. +The main consideration is that you must specify the "baseline" or reference +image, and compare it with a "generated" or test image. This is handled using +the *decorator* functions `@check_figures_equal` and +`@pytest.mark.mpl_image_compare` whose usage are further described below. + +#### Using check_figures_equal + +This approach draws the same figure using two different methods (the reference +method and the tested method), and checks that both of them are the same. +It takes two `pygmt.Figure` objects ('fig_ref' and 'fig_test'), generates a png +image, and checks for the Root Mean Square (RMS) error between the two. +Here's an example: + +```python +@check_figures_equal() +def test_my_plotting_case(fig_ref, fig_test): + "Test that my plotting function works" + fig_ref.grdimage("@earth_relief_01d_g", projection="W120/15c", cmap="geo") + fig_test.grdimage(grid, projection="W120/15c", cmap="geo") +``` + +Note: This is the recommended way to test plots whenever possible, such as when +we want to compare a reference GMT plot created from NetCDF files with one +generated by PyGMT that passes through several layers of virtualfile machinery. +Using this method will help save space in the git repository by not having to +store baseline images as with the other method below. + +#### Using mpl_image_compare + +This method uses the [pytest-mpl](https://github.com/matplotlib/pytest-mpl) +plug-in to test plot generating code. Every time the tests are run, `pytest-mpl` compares the generated plots with known correct ones stored in `pygmt/tests/baseline`. If your test created a `pygmt.Figure` object, you can test it by adding a *decorator* and From e6ad74d6a9085eaac9f5a886572540e225e2d4d7 Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Fri, 4 Sep 2020 11:16:17 +1200 Subject: [PATCH 8/8] Extra checks to ensure image files exist or not --- pygmt/helpers/testing.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pygmt/helpers/testing.py b/pygmt/helpers/testing.py index 5690406bbbf..889e7f61efd 100644 --- a/pygmt/helpers/testing.py +++ b/pygmt/helpers/testing.py @@ -42,6 +42,8 @@ def check_figures_equal(*, tol=0.0, result_dir="result_images"): ... fig_ref.basemap(projection="X5c", region=[0, 5, 0, 5], frame=True) ... fig_test.basemap(projection="X5c", region=[0, 5, 0, 5], frame="af") >>> test_check_figures_equal() + >>> assert len(os.listdir("tmp_result_images")) == 0 + >>> shutil.rmtree(path="tmp_result_images") # cleanup folder if tests pass >>> @check_figures_equal(result_dir="tmp_result_images") ... def test_check_figures_unequal(fig_ref, fig_test): @@ -49,7 +51,13 @@ def check_figures_equal(*, tol=0.0, result_dir="result_images"): ... fig_test.basemap(projection="X5c", region=[0, 3, 0, 3], frame=True) >>> with pytest.raises(GMTImageComparisonFailure): ... test_check_figures_unequal() - + >>> for suffix in ["", "-expected", "-failed-diff"]: + ... assert os.path.exists( + ... os.path.join( + ... "tmp_result_images", + ... f"test_check_figures_unequal{suffix}.png", + ... ) + ... ) >>> shutil.rmtree(path="tmp_result_images") # cleanup folder if tests pass """