From a4c21b9ca5eeb7f601c1e3afc66ea4229d24423c Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Thu, 3 Feb 2022 16:47:14 +0100 Subject: [PATCH 01/17] Initial commit --- xwrf/config.yaml | 10 ++++ xwrf/io_plugin.py | 134 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 xwrf/io_plugin.py diff --git a/xwrf/config.yaml b/xwrf/config.yaml index 18646088..b058b950 100644 --- a/xwrf/config.yaml +++ b/xwrf/config.yaml @@ -82,3 +82,13 @@ cf_attribute_map: standard_name: integral_of_surface_upward_heat_flux_in_air_wrt_time ACLHF: standard_name: integral_of_surface_upward_latent_heat_flux_in_air_wrf_time + VAR_SSO: + units: m2 + description: Variance of Subgrid Scale Orography MSL + HGT_M: + units: m + description: GMTED2010 30-arc-second topography height MSL + PRES: + units: Pa + ST: + units: kelvin diff --git a/xwrf/io_plugin.py b/xwrf/io_plugin.py new file mode 100644 index 00000000..7053da1b --- /dev/null +++ b/xwrf/io_plugin.py @@ -0,0 +1,134 @@ +import itertools +import os +import pathlib +import re +import warnings + +import pandas as pd +import xarray as xr + +from .config import config + +_LAT_COORDS = ('XLAT', 'XLAT_M', 'XLAT_U', 'XLAT_V', 'CLAT', 'XLAT_C') + +_LON_COORDS = ('XLONG', 'XLONG_M', 'XLONG_U', 'XLONG_V', 'CLONG', 'XLONG_C') + +_TIME_COORD_VARS = ('XTIME', 'Times', 'Time', 'time') + +_ALL_COORDS = set(itertools.chain(*[_LAT_COORDS, _LON_COORDS, _TIME_COORD_VARS])) + +_BOOLEAN_UNITS_ATTRS = ('-', 'flag', '0/1 Flag') + + +def is_remote_uri(path: str) -> bool: + """Finds URLs of the form protocol:// or protocol:: + This also matches for http[s]://, which were the only remote URLs + supported in <=v0.16.2. + """ + return bool(re.search(r'^[a-z][a-z0-9]*(\://|\:\:)', path)) + + +def _normalize_path(path): + if isinstance(path, pathlib.Path): + path = str(path) + + if isinstance(path, str) and not is_remote_uri(path): + path = os.path.abspath(os.path.expanduser(path)) + + return path + + +def clean(dataset): + """ + Clean up the dataset. + """ + coords = set(dataset.variables).intersection(_ALL_COORDS) + dataset = dataset.set_coords(coords) + for coord in dataset.coords: + attrs = dataset[coord].attrs + encoding = dataset[coord].encoding + if coord in _TIME_COORD_VARS: + try: + dataset[coord].data = pd.to_datetime( + list(map(lambda x: x.decode('utf-8'), dataset[coord].data.tolist())), + format='%Y-%m-%d_%H:%M:%S', + ) + except: + warnings.warn(f'Failed to parse time coordinate: {coord}', stacklevel=2) + + elif coord in (_LON_COORDS + _LAT_COORDS) and dataset[coord].ndim == 3: + + attrs = dataset[coord].attrs + encoding = dataset[coord].encoding + dataset = dataset.assign_coords( + {coord: (dataset[coord].dims[1:], dataset[coord].data[0, :, :])} + ) + dataset[coord].attrs = attrs + dataset[coord].encoding = encoding + + return dataset + + +def make_units_quantify_ready(dataset): + for var in dataset.data_vars: + if dataset[var].attrs.get('units') in _BOOLEAN_UNITS_ATTRS: + dataset[var].attrs.pop('units', None) + + +def modify_attrs_to_cf(dataset): + vars_to_update = set(config.get('cf_attribute_map').keys()).intersection(set(dataset.keys())) + + for var in vars_to_update: + dataset[var].attrs.update(config.get(f'cf_attribute_map.{var}')) + + +class WRFBackendEntrypoint(xr.backends.BackendEntrypoint): + def open_dataset( + self, + filename_or_obj, + mask_and_scale=True, + decode_times=True, + concat_characters=True, + decode_coords=True, + drop_variables=None, + use_cftime=None, + decode_timedelta=None, + group=None, + mode='r', + format='NETCDF4', + clobber=True, + diskless=False, + persist=False, + lock=None, + autoclose=False, + ): + + filename_or_obj = _normalize_path(filename_or_obj) + store = xr.backends.NetCDF4DataStore.open( + filename_or_obj, + mode=mode, + format=format, + clobber=clobber, + diskless=diskless, + persist=persist, + lock=lock, + autoclose=autoclose, + ) + + store_entrypoint = xr.backends.store.StoreBackendEntrypoint() + + with xr.core.utils.close_on_error(store): + dataset = store_entrypoint.open_dataset( + store, + mask_and_scale=mask_and_scale, + decode_times=decode_times, + concat_characters=concat_characters, + decode_coords=decode_coords, + drop_variables=drop_variables, + use_cftime=use_cftime, + decode_timedelta=decode_timedelta, + ) + + make_units_quantify_ready(dataset) + modify_attrs_to_cf(dataset) + return clean(dataset) From 509c96e30e8394778f203ea467be90c6b61d93d6 Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 14 Feb 2022 14:38:14 +0100 Subject: [PATCH 02/17] Amended time parsing and unit handling --- xwrf/accessors.py | 4 +- xwrf/config.yaml | 26 ++++++++- xwrf/io_plugin.py | 134 -------------------------------------------- xwrf/postprocess.py | 15 +++-- 4 files changed, 36 insertions(+), 143 deletions(-) delete mode 100644 xwrf/io_plugin.py diff --git a/xwrf/accessors.py b/xwrf/accessors.py index 8fd110c4..d1711bd3 100644 --- a/xwrf/accessors.py +++ b/xwrf/accessors.py @@ -6,7 +6,7 @@ _collapse_time_dim, _decode_times, _modify_attrs_to_cf, - _remove_units_from_bool_arrays, + _remove_invalid_units, ) @@ -41,7 +41,7 @@ def postprocess(self, decode_times=True) -> xr.Dataset: """ ds = ( self.xarray_obj.pipe(_modify_attrs_to_cf) - .pipe(_remove_units_from_bool_arrays) + .pipe(_remove_invalid_units) .pipe(_collapse_time_dim) ) if decode_times: diff --git a/xwrf/config.yaml b/xwrf/config.yaml index b058b950..92b53abf 100644 --- a/xwrf/config.yaml +++ b/xwrf/config.yaml @@ -22,9 +22,13 @@ time_coords: - Time - time -boolean_units_attrs: +invalid_units_attrs: - '-' - flag + - '0/1 Flag' + - 'whoknows' + - 'category' + - 'none' cf_attribute_map: ZNW: @@ -92,3 +96,23 @@ cf_attribute_map: units: Pa ST: units: kelvin + SM100255: + units: dimensionless + SM028100: + units: dimensionless + SM007028: + units: dimensionless + SM000007: + units: dimensionless + SCB_DOM: + units: dimensionless + SCB_DOM: + units: dimensionless + COSALPHA_V: + units: dimensionless + GREENFRAC: + units: dimensionless + SOILTEMP: + units: kelvin + RH: + units: % diff --git a/xwrf/io_plugin.py b/xwrf/io_plugin.py deleted file mode 100644 index 7053da1b..00000000 --- a/xwrf/io_plugin.py +++ /dev/null @@ -1,134 +0,0 @@ -import itertools -import os -import pathlib -import re -import warnings - -import pandas as pd -import xarray as xr - -from .config import config - -_LAT_COORDS = ('XLAT', 'XLAT_M', 'XLAT_U', 'XLAT_V', 'CLAT', 'XLAT_C') - -_LON_COORDS = ('XLONG', 'XLONG_M', 'XLONG_U', 'XLONG_V', 'CLONG', 'XLONG_C') - -_TIME_COORD_VARS = ('XTIME', 'Times', 'Time', 'time') - -_ALL_COORDS = set(itertools.chain(*[_LAT_COORDS, _LON_COORDS, _TIME_COORD_VARS])) - -_BOOLEAN_UNITS_ATTRS = ('-', 'flag', '0/1 Flag') - - -def is_remote_uri(path: str) -> bool: - """Finds URLs of the form protocol:// or protocol:: - This also matches for http[s]://, which were the only remote URLs - supported in <=v0.16.2. - """ - return bool(re.search(r'^[a-z][a-z0-9]*(\://|\:\:)', path)) - - -def _normalize_path(path): - if isinstance(path, pathlib.Path): - path = str(path) - - if isinstance(path, str) and not is_remote_uri(path): - path = os.path.abspath(os.path.expanduser(path)) - - return path - - -def clean(dataset): - """ - Clean up the dataset. - """ - coords = set(dataset.variables).intersection(_ALL_COORDS) - dataset = dataset.set_coords(coords) - for coord in dataset.coords: - attrs = dataset[coord].attrs - encoding = dataset[coord].encoding - if coord in _TIME_COORD_VARS: - try: - dataset[coord].data = pd.to_datetime( - list(map(lambda x: x.decode('utf-8'), dataset[coord].data.tolist())), - format='%Y-%m-%d_%H:%M:%S', - ) - except: - warnings.warn(f'Failed to parse time coordinate: {coord}', stacklevel=2) - - elif coord in (_LON_COORDS + _LAT_COORDS) and dataset[coord].ndim == 3: - - attrs = dataset[coord].attrs - encoding = dataset[coord].encoding - dataset = dataset.assign_coords( - {coord: (dataset[coord].dims[1:], dataset[coord].data[0, :, :])} - ) - dataset[coord].attrs = attrs - dataset[coord].encoding = encoding - - return dataset - - -def make_units_quantify_ready(dataset): - for var in dataset.data_vars: - if dataset[var].attrs.get('units') in _BOOLEAN_UNITS_ATTRS: - dataset[var].attrs.pop('units', None) - - -def modify_attrs_to_cf(dataset): - vars_to_update = set(config.get('cf_attribute_map').keys()).intersection(set(dataset.keys())) - - for var in vars_to_update: - dataset[var].attrs.update(config.get(f'cf_attribute_map.{var}')) - - -class WRFBackendEntrypoint(xr.backends.BackendEntrypoint): - def open_dataset( - self, - filename_or_obj, - mask_and_scale=True, - decode_times=True, - concat_characters=True, - decode_coords=True, - drop_variables=None, - use_cftime=None, - decode_timedelta=None, - group=None, - mode='r', - format='NETCDF4', - clobber=True, - diskless=False, - persist=False, - lock=None, - autoclose=False, - ): - - filename_or_obj = _normalize_path(filename_or_obj) - store = xr.backends.NetCDF4DataStore.open( - filename_or_obj, - mode=mode, - format=format, - clobber=clobber, - diskless=diskless, - persist=persist, - lock=lock, - autoclose=autoclose, - ) - - store_entrypoint = xr.backends.store.StoreBackendEntrypoint() - - with xr.core.utils.close_on_error(store): - dataset = store_entrypoint.open_dataset( - store, - mask_and_scale=mask_and_scale, - decode_times=decode_times, - concat_characters=concat_characters, - decode_coords=decode_coords, - drop_variables=drop_variables, - use_cftime=use_cftime, - decode_timedelta=decode_timedelta, - ) - - make_units_quantify_ready(dataset) - modify_attrs_to_cf(dataset) - return clean(dataset) diff --git a/xwrf/postprocess.py b/xwrf/postprocess.py index 2efa68c0..26853e26 100644 --- a/xwrf/postprocess.py +++ b/xwrf/postprocess.py @@ -10,21 +10,24 @@ def _decode_times(ds: xr.Dataset) -> xr.Dataset: """ Decode the time variable to datetime64. """ + try: + _time = pd.to_datetime(ds.Times.data.astype('str'), errors='raise', format='%Y-%m-%d_%H:%M:%S') + except ValueError: + _time = pd.to_datetime(ds.Times.data.astype('str'), errors='raise', format='%Y-%m-%dT%H:%M:%S.%f') ds = ds.assign_coords( { - 'Time': pd.to_datetime( - ds.Times.data.astype('str'), errors='raise', format='%Y-%m-%d_%H:%M:%S' - ) + 'Time': _time } ) ds.Time.attrs = {'long_name': 'Time', 'standard_name': 'time'} return ds -def _remove_units_from_bool_arrays(ds: xr.Dataset) -> xr.Dataset: - boolean_units_attrs = config.get('boolean_units_attrs') +def _remove_invalid_units(ds: xr.Dataset) -> xr.Dataset: + invalid_units_attrs = config.get('invalid_units_attrs') + print(invalid_units_attrs) for variable in ds.data_vars: - if ds[variable].attrs.get('units') in boolean_units_attrs: + if ds[variable].attrs.get('units') in invalid_units_attrs: ds[variable].attrs.pop('units', None) return ds From fd34303c96470a282b8f5a8642edeca442018aca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 13:38:59 +0000 Subject: [PATCH 03/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xwrf/postprocess.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/xwrf/postprocess.py b/xwrf/postprocess.py index 26853e26..3f8ee804 100644 --- a/xwrf/postprocess.py +++ b/xwrf/postprocess.py @@ -11,14 +11,14 @@ def _decode_times(ds: xr.Dataset) -> xr.Dataset: Decode the time variable to datetime64. """ try: - _time = pd.to_datetime(ds.Times.data.astype('str'), errors='raise', format='%Y-%m-%d_%H:%M:%S') + _time = pd.to_datetime( + ds.Times.data.astype('str'), errors='raise', format='%Y-%m-%d_%H:%M:%S' + ) except ValueError: - _time = pd.to_datetime(ds.Times.data.astype('str'), errors='raise', format='%Y-%m-%dT%H:%M:%S.%f') - ds = ds.assign_coords( - { - 'Time': _time - } - ) + _time = pd.to_datetime( + ds.Times.data.astype('str'), errors='raise', format='%Y-%m-%dT%H:%M:%S.%f' + ) + ds = ds.assign_coords({'Time': _time}) ds.Time.attrs = {'long_name': 'Time', 'standard_name': 'time'} return ds From 0ff2c699cde57d8060f602523ac0c38ec7ebef6a Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 14 Feb 2022 14:40:09 +0100 Subject: [PATCH 04/17] Amended config --- xwrf/config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/xwrf/config.yaml b/xwrf/config.yaml index 92b53abf..8029f419 100644 --- a/xwrf/config.yaml +++ b/xwrf/config.yaml @@ -106,8 +106,6 @@ cf_attribute_map: units: dimensionless SCB_DOM: units: dimensionless - SCB_DOM: - units: dimensionless COSALPHA_V: units: dimensionless GREENFRAC: From 782a39a41896d22948b8464c88d22162b8d5f327 Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 14 Feb 2022 14:42:19 +0100 Subject: [PATCH 05/17] Further yaml fixes --- xwrf/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xwrf/config.yaml b/xwrf/config.yaml index 8029f419..7a3039f5 100644 --- a/xwrf/config.yaml +++ b/xwrf/config.yaml @@ -113,4 +113,4 @@ cf_attribute_map: SOILTEMP: units: kelvin RH: - units: % + units: '%' From c8240a9615560edd97732eb74be5b413a2afb633 Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 14 Feb 2022 14:50:39 +0100 Subject: [PATCH 06/17] Fixed tests --- tests/test_postprocess.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_postprocess.py b/tests/test_postprocess.py index 7527203c..e5d66fcc 100644 --- a/tests/test_postprocess.py +++ b/tests/test_postprocess.py @@ -22,6 +22,6 @@ def test_cf_attrs_added(dummy_dataset, variable): @pytest.mark.parametrize('variable', ('THIS_IS_AN_IDEAL_RUN', 'SAVE_TOPO_FROM_REAL')) -def test_remove_units_from_bool_arrays(dummy_attrs_only_dataset, variable): - dataset = xwrf.postprocess._remove_units_from_bool_arrays(dummy_attrs_only_dataset) +def test_remove_invalid_units(dummy_attrs_only_dataset, variable): + dataset = xwrf.postprocess._remove_invalid_units(dummy_attrs_only_dataset) assert 'units' not in dataset[variable].attrs From 8da58be19e439028c9d7c4c31795dbfe1992ec9a Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 14 Feb 2022 15:11:08 +0100 Subject: [PATCH 07/17] New units and removed print --- xwrf/config.yaml | 18 ++++++++++++++++-- xwrf/postprocess.py | 1 - 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/xwrf/config.yaml b/xwrf/config.yaml index 7a3039f5..c62e9711 100644 --- a/xwrf/config.yaml +++ b/xwrf/config.yaml @@ -89,9 +89,15 @@ cf_attribute_map: VAR_SSO: units: m2 description: Variance of Subgrid Scale Orography MSL - HGT_M: + HGT_V: units: m - description: GMTED2010 30-arc-second topography height MSL + standard_name: surface_altitude + HGT_V: + units: m + standard_name: surface_altitude + HGT_U: + units: m + standard_name: surface_altitude PRES: units: Pa ST: @@ -104,6 +110,14 @@ cf_attribute_map: units: dimensionless SM000007: units: dimensionless + OL4: + units: dimensionless + OL3: + units: dimensionless + OL2: + units: dimensionless + OL1: + units: dimensionless SCB_DOM: units: dimensionless COSALPHA_V: diff --git a/xwrf/postprocess.py b/xwrf/postprocess.py index 3f8ee804..51ee9088 100644 --- a/xwrf/postprocess.py +++ b/xwrf/postprocess.py @@ -25,7 +25,6 @@ def _decode_times(ds: xr.Dataset) -> xr.Dataset: def _remove_invalid_units(ds: xr.Dataset) -> xr.Dataset: invalid_units_attrs = config.get('invalid_units_attrs') - print(invalid_units_attrs) for variable in ds.data_vars: if ds[variable].attrs.get('units') in invalid_units_attrs: ds[variable].attrs.pop('units', None) From 4b5517fb288e7ae4b9b67e58cd7b5e0d1e0a828b Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 14 Feb 2022 15:12:17 +0100 Subject: [PATCH 08/17] Update --- xwrf/config.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/xwrf/config.yaml b/xwrf/config.yaml index c62e9711..a43f18cb 100644 --- a/xwrf/config.yaml +++ b/xwrf/config.yaml @@ -92,9 +92,6 @@ cf_attribute_map: HGT_V: units: m standard_name: surface_altitude - HGT_V: - units: m - standard_name: surface_altitude HGT_U: units: m standard_name: surface_altitude From 71e3de945bce88805eda74cad2aa5bf7f8301ad1 Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 14 Feb 2022 16:30:09 +0100 Subject: [PATCH 09/17] Revamp of unit handling --- xwrf/accessors.py | 4 ++-- xwrf/config.yaml | 43 ++++++++++++------------------------------- xwrf/postprocess.py | 18 ++++++++++++++---- xwrf/tutorial.py | 1 + 4 files changed, 29 insertions(+), 37 deletions(-) diff --git a/xwrf/accessors.py b/xwrf/accessors.py index d1711bd3..353849c1 100644 --- a/xwrf/accessors.py +++ b/xwrf/accessors.py @@ -6,7 +6,7 @@ _collapse_time_dim, _decode_times, _modify_attrs_to_cf, - _remove_invalid_units, + _make_units_pint_friendly, ) @@ -41,7 +41,7 @@ def postprocess(self, decode_times=True) -> xr.Dataset: """ ds = ( self.xarray_obj.pipe(_modify_attrs_to_cf) - .pipe(_remove_invalid_units) + .pipe(_make_units_pint_friendly) .pipe(_collapse_time_dim) ) if decode_times: diff --git a/xwrf/config.yaml b/xwrf/config.yaml index a43f18cb..c98016b7 100644 --- a/xwrf/config.yaml +++ b/xwrf/config.yaml @@ -22,13 +22,18 @@ time_coords: - Time - time -invalid_units_attrs: - - '-' - - flag - - '0/1 Flag' - - 'whoknows' - - 'category' - - 'none' +unit_harmonization_map: + kelvin: + - Kelvin + dimensionless: + - fraction + invalid: + - '-' + - flag + - '0/1 Flag' + - whoknows + - category + - none cf_attribute_map: ZNW: @@ -99,29 +104,5 @@ cf_attribute_map: units: Pa ST: units: kelvin - SM100255: - units: dimensionless - SM028100: - units: dimensionless - SM007028: - units: dimensionless - SM000007: - units: dimensionless - OL4: - units: dimensionless - OL3: - units: dimensionless - OL2: - units: dimensionless - OL1: - units: dimensionless - SCB_DOM: - units: dimensionless - COSALPHA_V: - units: dimensionless - GREENFRAC: - units: dimensionless - SOILTEMP: - units: kelvin RH: units: '%' diff --git a/xwrf/postprocess.py b/xwrf/postprocess.py index 51ee9088..5e575d4b 100644 --- a/xwrf/postprocess.py +++ b/xwrf/postprocess.py @@ -23,11 +23,21 @@ def _decode_times(ds: xr.Dataset) -> xr.Dataset: return ds -def _remove_invalid_units(ds: xr.Dataset) -> xr.Dataset: - invalid_units_attrs = config.get('invalid_units_attrs') +def _make_units_pint_friendly(ds: xr.Dataset) -> xr.Dataset: + """ + Harmonizes awkward WRF units into pint-friendly ones + """ + # We have to invert the mapping from "new_unit -> wrf_units" to "wrf_unit -> new_unit" + wrf_units_map = { + v: k for (k, val_list) in config.get('unit_harmonization_map').items() for v in val_list + } for variable in ds.data_vars: - if ds[variable].attrs.get('units') in invalid_units_attrs: - ds[variable].attrs.pop('units', None) + if ds[variable].attrs.get('units') in wrf_units_map: + harmonized_unit = wrf_units_map[ds[variable].attrs['units']] + if harmonized_unit == "invalid": + ds[variable].attrs.pop('units', None) + else: + ds[variable].attrs['units'] = harmonized_unit return ds diff --git a/xwrf/tutorial.py b/xwrf/tutorial.py index 45c5ab3c..60a87e56 100644 --- a/xwrf/tutorial.py +++ b/xwrf/tutorial.py @@ -34,6 +34,7 @@ def _construct_cache_dir(path): 'lambert_conformal': 'data/lambert_conformal_sample.nc', 'mercator': 'data/mercator_sample.nc', 'tiny': 'data/tiny.nc', + 'met_em.d01.2005-08-28_12:00:00.nc': 'data/met_em.d01.2005-08-28_12:00:00.nc', } From 1db36b0c34b605471a3abae2781b5637209378d4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 15:30:23 +0000 Subject: [PATCH 10/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xwrf/accessors.py | 2 +- xwrf/postprocess.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xwrf/accessors.py b/xwrf/accessors.py index 353849c1..a9e46c03 100644 --- a/xwrf/accessors.py +++ b/xwrf/accessors.py @@ -5,8 +5,8 @@ from .postprocess import ( _collapse_time_dim, _decode_times, - _modify_attrs_to_cf, _make_units_pint_friendly, + _modify_attrs_to_cf, ) diff --git a/xwrf/postprocess.py b/xwrf/postprocess.py index 5e575d4b..67808c9b 100644 --- a/xwrf/postprocess.py +++ b/xwrf/postprocess.py @@ -34,7 +34,7 @@ def _make_units_pint_friendly(ds: xr.Dataset) -> xr.Dataset: for variable in ds.data_vars: if ds[variable].attrs.get('units') in wrf_units_map: harmonized_unit = wrf_units_map[ds[variable].attrs['units']] - if harmonized_unit == "invalid": + if harmonized_unit == 'invalid': ds[variable].attrs.pop('units', None) else: ds[variable].attrs['units'] = harmonized_unit From 8dd7349919fc7c8969bc8c2df2452c11e0de2936 Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 14 Feb 2022 16:34:10 +0100 Subject: [PATCH 11/17] Forgot to add test --- tests/test_postprocess.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_postprocess.py b/tests/test_postprocess.py index e5d66fcc..d63c9163 100644 --- a/tests/test_postprocess.py +++ b/tests/test_postprocess.py @@ -13,6 +13,11 @@ def dummy_attrs_only_dataset(request): return xwrf.tutorial.open_dataset(request.param) +@pytest.fixture(scope='session', params=['met_em.d01.2005-08-28_12:00:00.nc']) +def met_em_dataset(request): + return xwrf.tutorial.open_dataset(request.param) + + @pytest.mark.parametrize('variable', ('Q2', 'PSFC')) def test_cf_attrs_added(dummy_dataset, variable): dataset = xwrf.postprocess._modify_attrs_to_cf(dummy_dataset) @@ -23,5 +28,11 @@ def test_cf_attrs_added(dummy_dataset, variable): @pytest.mark.parametrize('variable', ('THIS_IS_AN_IDEAL_RUN', 'SAVE_TOPO_FROM_REAL')) def test_remove_invalid_units(dummy_attrs_only_dataset, variable): - dataset = xwrf.postprocess._remove_invalid_units(dummy_attrs_only_dataset) + dataset = xwrf.postprocess._make_units_pint_friendly(dummy_attrs_only_dataset) assert 'units' not in dataset[variable].attrs + + +@pytest.mark.parametrize('variable', ('OL4', 'GREENFRAC')) +def test_met_em_parsing(met_em_dataset, variable): + dataset = xwrf.postprocess._make_units_pint_friendly(met_em_dataset) + assert dataset[variable].attrs['units'] == 'dimensionless' From e9406ce9248abda94edece8fc5921eb69c938a1e Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 14 Feb 2022 16:40:34 +0100 Subject: [PATCH 12/17] Amended mapping for HGT --- xwrf/config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/xwrf/config.yaml b/xwrf/config.yaml index c98016b7..b6843a05 100644 --- a/xwrf/config.yaml +++ b/xwrf/config.yaml @@ -34,6 +34,8 @@ unit_harmonization_map: - whoknows - category - none + meters: + - 'meters MSL' cf_attribute_map: ZNW: @@ -95,10 +97,10 @@ cf_attribute_map: units: m2 description: Variance of Subgrid Scale Orography MSL HGT_V: - units: m standard_name: surface_altitude HGT_U: - units: m + standard_name: surface_altitude + HGT_M: standard_name: surface_altitude PRES: units: Pa From 94efde564e9066f089f8ed74b99df674be091080 Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Mon, 14 Feb 2022 16:59:21 +0100 Subject: [PATCH 13/17] Better key name for met_em dataset --- tests/test_postprocess.py | 2 +- xwrf/tutorial.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_postprocess.py b/tests/test_postprocess.py index d63c9163..b5b58334 100644 --- a/tests/test_postprocess.py +++ b/tests/test_postprocess.py @@ -13,7 +13,7 @@ def dummy_attrs_only_dataset(request): return xwrf.tutorial.open_dataset(request.param) -@pytest.fixture(scope='session', params=['met_em.d01.2005-08-28_12:00:00.nc']) +@pytest.fixture(scope='session', params=['met_em_sample']) def met_em_dataset(request): return xwrf.tutorial.open_dataset(request.param) diff --git a/xwrf/tutorial.py b/xwrf/tutorial.py index 60a87e56..0ac9269f 100644 --- a/xwrf/tutorial.py +++ b/xwrf/tutorial.py @@ -34,7 +34,7 @@ def _construct_cache_dir(path): 'lambert_conformal': 'data/lambert_conformal_sample.nc', 'mercator': 'data/mercator_sample.nc', 'tiny': 'data/tiny.nc', - 'met_em.d01.2005-08-28_12:00:00.nc': 'data/met_em.d01.2005-08-28_12:00:00.nc', + 'met_em_sample': 'data/met_em.d01.2005-08-28_12:00:00.nc', } From 848df46344ca4253bb3c05b3096425865914a0c6 Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Wed, 16 Feb 2022 14:01:52 +0100 Subject: [PATCH 14/17] Apply suggestions by jthielen to retain CF compatibility Co-authored-by: jthielen --- tests/test_postprocess.py | 2 +- xwrf/config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_postprocess.py b/tests/test_postprocess.py index b5b58334..8919cd22 100644 --- a/tests/test_postprocess.py +++ b/tests/test_postprocess.py @@ -35,4 +35,4 @@ def test_remove_invalid_units(dummy_attrs_only_dataset, variable): @pytest.mark.parametrize('variable', ('OL4', 'GREENFRAC')) def test_met_em_parsing(met_em_dataset, variable): dataset = xwrf.postprocess._make_units_pint_friendly(met_em_dataset) - assert dataset[variable].attrs['units'] == 'dimensionless' + assert dataset[variable].attrs['units'] == '1' diff --git a/xwrf/config.yaml b/xwrf/config.yaml index b6843a05..5ecec1d1 100644 --- a/xwrf/config.yaml +++ b/xwrf/config.yaml @@ -25,7 +25,7 @@ time_coords: unit_harmonization_map: kelvin: - Kelvin - dimensionless: + '1': - fraction invalid: - '-' From 9e372551e58f58d11d53e09e70868405a3875b89 Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Wed, 16 Feb 2022 14:31:21 +0100 Subject: [PATCH 15/17] Better config import --- xwrf/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xwrf/__init__.py b/xwrf/__init__.py index 8c1c887b..92850a91 100644 --- a/xwrf/__init__.py +++ b/xwrf/__init__.py @@ -4,7 +4,8 @@ from pkg_resources import DistributionNotFound, get_distribution -from . import config, postprocess, tutorial +from .config import config +from . import postprocess, tutorial from .accessors import WRFDataArrayAccessor, WRFDatasetAccessor try: From 1a33e0a8cde717acd471f6ac13c3832310d36449 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Feb 2022 13:31:51 +0000 Subject: [PATCH 16/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xwrf/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xwrf/__init__.py b/xwrf/__init__.py index 92850a91..b1a05282 100644 --- a/xwrf/__init__.py +++ b/xwrf/__init__.py @@ -4,9 +4,9 @@ from pkg_resources import DistributionNotFound, get_distribution -from .config import config from . import postprocess, tutorial from .accessors import WRFDataArrayAccessor, WRFDatasetAccessor +from .config import config try: __version__ = get_distribution(__name__).version From fa8fbee8af6716d3ab9e70b8e7c3c62c5ddcea1d Mon Sep 17 00:00:00 2001 From: Lukas Pilz Date: Wed, 16 Feb 2022 14:38:13 +0100 Subject: [PATCH 17/17] Fixed config test --- tests/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index 15ef338a..1e5973ef 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,7 +5,7 @@ @pytest.fixture(scope='session') def config(): - return xwrf.config.config + return xwrf.config def test_defaults_exist(config):