diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 0e5438a7fc..fc022cb271 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -487,6 +487,8 @@ of sources and file formats relevant to solar energy modeling. iotools.parse_psm3 iotools.get_pvgis_tmy iotools.read_pvgis_tmy + iotools.get_pvgis_hourly + iotools.read_pvgis_hourly iotools.read_bsrn iotools.get_cams iotools.read_cams diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index 6771e5fb42..0a6a2b30e0 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -103,6 +103,10 @@ Deprecations Enhancements ~~~~~~~~~~~~ +* Added :func:`~pvlib.iotools.read_pvgis_hourly` and + :func:`~pvlib.iotools.get_pvgis_hourly` for reading and retrieving hourly + solar radiation data and PV power output from PVGIS. (:pull:`1186`, + :issue:`849`) * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) * Add :func:`~pvlib.iotools.get_cams`, diff --git a/pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_CM_10kWp_CIS_5_2a_2013_2014.json b/pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_CM_10kWp_CIS_5_2a_2013_2014.json new file mode 100644 index 0000000000..3a27f4f368 --- /dev/null +++ b/pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_CM_10kWp_CIS_5_2a_2013_2014.json @@ -0,0 +1 @@ +{"inputs": {"location": {"latitude": 45.0, "longitude": 8.0, "elevation": 250.0}, "meteo_data": {"radiation_db": "PVGIS-CMSAF", "meteo_db": "ERA-Interim", "year_min": 2013, "year_max": 2014, "use_horizon": true, "horizon_db": null, "horizon_data": "DEM-calculated"}, "mounting_system": {"two_axis": {"slope": {"value": "-", "optimal": "-"}, "azimuth": {"value": "-", "optimal": "-"}}}, "pv_module": {"technology": "CIS", "peak_power": 10.0, "system_loss": 5.0}}, "outputs": {"hourly": [{"time": "20130101:0055", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 3.01, "WS10m": 1.23, "Int": 0.0}, {"time": "20130101:0155", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 2.22, "WS10m": 1.46, "Int": 0.0}, {"time": "20130101:0255", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 1.43, "WS10m": 1.7, "Int": 0.0}, {"time": "20130101:0355", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 0.64, "WS10m": 1.93, "Int": 0.0}, {"time": "20130101:0455", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 0.77, "WS10m": 1.8, "Int": 0.0}, {"time": "20130101:0555", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 0.91, "WS10m": 1.66, "Int": 0.0}, {"time": "20130101:0655", "P": 0.0, "Gb(i)": 0.0, "Gd(i)": 0.0, "Gr(i)": 0.0, "H_sun": 0.0, "T2m": 1.05, "WS10m": 1.53, "Int": 0.0}, {"time": "20130101:0755", "P": 3464.5, "Gb(i)": 270.35, "Gd(i)": 91.27, "Gr(i)": 6.09, "H_sun": 6.12, "T2m": 1.92, "WS10m": 1.44, "Int": 0.0}, {"time": "20130101:0855", "P": 1586.9, "Gb(i)": 80.76, "Gd(i)": 83.95, "Gr(i)": 9.04, "H_sun": 13.28, "T2m": 2.79, "WS10m": 1.36, "Int": 0.0}, {"time": "20130101:0955", "P": 713.3, "Gb(i)": 5.18, "Gd(i)": 70.57, "Gr(i)": 7.31, "H_sun": 18.56, "T2m": 3.66, "WS10m": 1.27, "Int": 0.0}]}, "meta": {"inputs": {"location": {"description": "Selected location", "variables": {"latitude": {"description": "Latitude", "units": "decimal degree"}, "longitude": {"description": "Longitude", "units": "decimal degree"}, "elevation": {"description": "Elevation", "units": "m"}}}, "meteo_data": {"description": "Sources of meteorological data", "variables": {"radiation_db": {"description": "Solar radiation database"}, "meteo_db": {"description": "Database used for meteorological variables other than solar radiation"}, "year_min": {"description": "First year of the calculations"}, "year_max": {"description": "Last year of the calculations"}, "use_horizon": {"description": "Include horizon shadows"}, "horizon_db": {"description": "Source of horizon data"}}}, "mounting_system": {"description": "Mounting system", "choices": "fixed, vertical_axis, inclined_axis, two_axis", "fields": {"slope": {"description": "Inclination angle from the horizontal plane", "units": "degree"}, "azimuth": {"description": "Orientation (azimuth) angle of the (fixed) PV system (0 = S, 90 = W, -90 = E)", "units": "degree"}}}, "pv_module": {"description": "PV module parameters", "variables": {"technology": {"description": "PV technology"}, "peak_power": {"description": "Nominal (peak) power of the PV module", "units": "kW"}, "system_loss": {"description": "Sum of system losses", "units": "%"}}}}, "outputs": {"hourly": {"type": "time series", "timestamp": "hourly averages", "variables": {"P": {"description": "PV system power", "units": "W"}, "Gb(i)": {"description": "Beam (direct) irradiance on the inclined plane (plane of the array)", "units": "W/m2"}, "Gd(i)": {"description": "Diffuse irradiance on the inclined plane (plane of the array)", "units": "W/m2"}, "Gr(i)": {"description": "Reflected irradiance on the inclined plane (plane of the array)", "units": "W/m2"}, "H_sun": {"description": "Sun height", "units": "degree"}, "T2m": {"description": "2-m air temperature", "units": "degree Celsius"}, "WS10m": {"description": "10-m total wind speed", "units": "m/s"}, "Int": {"description": "1 means solar radiation values are reconstructed"}}}}}} \ No newline at end of file diff --git a/pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv b/pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv new file mode 100644 index 0000000000..a71a213a80 --- /dev/null +++ b/pvlib/data/pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv @@ -0,0 +1,35 @@ +Latitude (decimal degrees): 45.000 +Longitude (decimal degrees): 8.000 +Elevation (m): 250 +Radiation database: PVGIS-SARAH + + +Slope: 30 deg. +Azimuth: 0 deg. +time,Gb(i),Gd(i),Gr(i),H_sun,T2m,WS10m,Int +20160101:0010,0.0,0.0,0.0,0.0,3.44,1.43,0.0 +20160101:0110,0.0,0.0,0.0,0.0,2.94,1.47,0.0 +20160101:0210,0.0,0.0,0.0,0.0,2.43,1.51,0.0 +20160101:0310,0.0,0.0,0.0,0.0,1.93,1.54,0.0 +20160101:0410,0.0,0.0,0.0,0.0,2.03,1.62,0.0 +20160101:0510,0.0,0.0,0.0,0.0,2.14,1.69,0.0 +20160101:0610,0.0,0.0,0.0,0.0,2.25,1.77,0.0 +20160101:0710,0.0,0.0,0.0,0.0,3.06,1.49,0.0 +20160101:0810,26.71,8.28,0.21,8.06,3.87,1.22,1.0 +20160101:0910,14.69,5.76,0.16,14.8,4.67,0.95,1.0 +20160101:1010,2.19,0.94,0.03,19.54,5.73,0.77,1.0 +20160101:1110,2.11,0.94,0.03,21.82,6.79,0.58,1.0 +20160101:1210,4.25,1.88,0.05,21.41,7.84,0.4,1.0 +20160101:1310,0.0,0.0,0.0,0.0,7.43,0.72,0.0 + +Gb(i): Beam (direct) irradiance on the inclined plane (plane of the array) (W/m2) +Gd(i): Diffuse irradiance on the inclined plane (plane of the array) (W/m2) +Gr(i): Reflected irradiance on the inclined plane (plane of the array) (W/m2) +H_sun: Sun height (degree) +T2m: 2-m air temperature (degree Celsius) +WS10m: 10-m total wind speed (m/s) +Int: 1 means solar radiation values are reconstructed + + + +PVGIS (c) European Union, 2001-2021 \ No newline at end of file diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index b717c801ca..69786f8630 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -13,6 +13,8 @@ from pvlib.iotools.psm3 import read_psm3 # noqa: F401 from pvlib.iotools.psm3 import parse_psm3 # noqa: F401 from pvlib.iotools.pvgis import get_pvgis_tmy, read_pvgis_tmy # noqa: F401 +from pvlib.iotools.pvgis import read_pvgis_hourly # noqa: F401 +from pvlib.iotools.pvgis import get_pvgis_hourly # noqa: F401 from pvlib.iotools.bsrn import read_bsrn # noqa: F401 from pvlib.iotools.sodapro import get_cams # noqa: F401 from pvlib.iotools.sodapro import read_cams # noqa: F401 diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 90c54ae839..d43d4db87e 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -23,6 +23,345 @@ URL = 'https://re.jrc.ec.europa.eu/api/' +# Dictionary mapping PVGIS names to pvlib names +PVGIS_VARIABLE_MAP = { + 'G(h)': 'ghi', + 'Gb(n)': 'dni', + 'Gd(h)': 'dhi', + 'G(i)': 'poa_global', + 'Gb(i)': 'poa_direct', + 'Gd(i)': 'poa_sky_diffuse', + 'Gr(i)': 'poa_ground_diffuse', + 'H_sun': 'solar_elevation', + 'T2m': 'temp_air', + 'RH': 'relative_humidity', + 'SP': 'pressure', + 'WS10m': 'wind_speed', + 'WD10m': 'wind_direction', +} + + +def get_pvgis_hourly(latitude, longitude, start=None, end=None, + raddatabase=None, components=True, + surface_tilt=0, surface_azimuth=0, + outputformat='json', + usehorizon=True, userhorizon=None, + pvcalculation=False, + peakpower=None, pvtechchoice='crystSi', + mountingplace='free', loss=0, trackingtype=0, + optimal_surface_tilt=False, optimalangles=False, + url=URL, map_variables=True, timeout=30): + """Get hourly solar irradiation and modeled PV power output from PVGIS. + + PVGIS data is freely available at [1]_. + + Parameters + ---------- + latitude: float + In decimal degrees, between -90 and 90, north is positive (ISO 19115) + longitude: float + In decimal degrees, between -180 and 180, east is positive (ISO 19115) + start: int or datetime like, default: None + First year of the radiation time series. Defaults to first year + available. + end: int or datetime like, default: None + Last year of the radiation time series. Defaults to last year + available. + raddatabase: str, default: None + Name of radiation database. Options depend on location, see [3]_. + components: bool, default: True + Output solar radiation components (beam, diffuse, and reflected). + Otherwise only global irradiance is returned. + surface_tilt: float, default: 0 + Tilt angle from horizontal plane. Ignored for two-axis tracking. + surface_azimuth: float, default: 0 + Orientation (azimuth angle) of the (fixed) plane. 0=south, 90=west, + -90: east. Ignored for tracking systems. + usehorizon: bool, default: True + Include effects of horizon + userhorizon: list of float, default: None + Optional user specified elevation of horizon in degrees, at equally + spaced azimuth clockwise from north, only valid if `usehorizon` is + true, if `usehorizon` is true but `userhorizon` is `None` then PVGIS + will calculate the horizon [4]_ + pvcalculation: bool, default: False + Return estimate of hourly PV production. + peakpower: float, default: None + Nominal power of PV system in kW. Required if pvcalculation=True. + pvtechchoice: {'crystSi', 'CIS', 'CdTe', 'Unknown'}, default: 'crystSi' + PV technology. + mountingplace: {'free', 'building'}, default: free + Type of mounting for PV system. Options of 'free' for free-standing + and 'building' for building-integrated. + loss: float, default: 0 + Sum of PV system losses in percent. Required if pvcalculation=True + trackingtype: {0, 1, 2, 3, 4, 5}, default: 0 + Type of suntracking. 0=fixed, 1=single horizontal axis aligned + north-south, 2=two-axis tracking, 3=vertical axis tracking, 4=single + horizontal axis aligned east-west, 5=single inclined axis aligned + north-south. + optimal_surface_tilt: bool, default: False + Calculate the optimum tilt angle. Ignored for two-axis tracking + optimalangles: bool, default: False + Calculate the optimum tilt and azimuth angles. Ignored for two-axis + tracking. + outputformat: str, default: 'json' + Must be in ``['json', 'csv']``. See PVGIS hourly data + documentation [2]_ for more info. + url: str, default: const:`pvlib.iotools.pvgis.URL` + Base url of PVGIS API. ``seriescalc`` is appended to get hourly data + endpoint. + map_variables: bool, default: True + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable PVGIS_VARIABLE_MAP. + timeout: int, default: 30 + Time in seconds to wait for server response before timeout + + Returns + ------- + data : pandas.DataFrame + Time-series of hourly data, see Notes for fields + inputs : dict + Dictionary of the request input parameters + metadata : dict + Dictionary containing metadata + + Raises + ------ + requests.HTTPError + If the request response status is ``HTTP/1.1 400 BAD REQUEST``, then + the error message in the response will be raised as an exception, + otherwise raise whatever ``HTTP/1.1`` error occurred + + Hint + ---- + PVGIS provides access to a number of different solar radiation datasets, + including satellite-based (SARAH, CMSAF, and NSRDB PSM3) and re-analysis + products (ERA5 and COSMO). Each data source has a different geographical + coverage and time stamp convention, e.g., SARAH and CMSAF provide + instantaneous values, whereas values from ERA5 are averages for the hour. + + Notes + ----- + data includes the following fields: + + =========================== ====== ====================================== + raw, mapped Format Description + =========================== ====== ====================================== + *Mapped field names are returned when the map_variables argument is True* + --------------------------------------------------------------------------- + P† float PV system power (W) + G(i), poa_global‡ float Global irradiance on inclined plane (W/m^2) + Gb(i), poa_direct‡ float Beam (direct) irradiance on inclined plane (W/m^2) + Gd(i), poa_sky_diffuse‡ float Diffuse irradiance on inclined plane (W/m^2) + Gr(i), poa_ground_diffuse‡ float Reflected irradiance on inclined plane (W/m^2) + H_sun, solar_elevation float Sun height/elevation (degrees) + T2m, temp_air float Air temperature at 2 m (degrees Celsius) + WS10m, wind_speed float Wind speed at 10 m (m/s) + Int int Solar radiation reconstructed (1/0) + =========================== ====== ====================================== + + †P (PV system power) is only returned when pvcalculation=True. + + ‡Gb(i), Gd(i), and Gr(i) are returned when components=True, otherwise the + sum of the three components, G(i), is returned. + + See Also + -------- + pvlib.iotools.read_pvgis_hourly, pvlib.iotools.get_pvgis_tmy + + References + ---------- + .. [1] `PVGIS `_ + .. [2] `PVGIS Hourly Radiation + `_ + .. [3] `PVGIS Non-interactive service + `_ + .. [4] `PVGIS horizon profile tool + `_ + """ # noqa: E501 + # use requests to format the query string by passing params dictionary + params = {'lat': latitude, 'lon': longitude, 'outputformat': outputformat, + 'angle': surface_tilt, 'aspect': surface_azimuth, + 'pvcalculation': int(pvcalculation), + 'pvtechchoice': pvtechchoice, 'mountingplace': mountingplace, + 'trackingtype': trackingtype, 'components': int(components), + 'usehorizon': int(usehorizon), + 'optimalangles': int(optimalangles), + 'optimalinclination': int(optimalangles), 'loss': loss} + # pvgis only takes 0 for False, and 1 for True, not strings + if userhorizon is not None: + params['userhorizon'] = ','.join(str(x) for x in userhorizon) + if raddatabase is not None: + params['raddatabase'] = raddatabase + if start is not None: + params['startyear'] = start if isinstance(start, int) else start.year + if end is not None: + params['endyear'] = end if isinstance(end, int) else end.year + if peakpower is not None: + params['peakpower'] = peakpower + + # The url endpoint for hourly radiation is 'seriescalc' + res = requests.get(url + 'seriescalc', params=params, timeout=timeout) + # PVGIS returns really well formatted error messages in JSON for HTTP/1.1 + # 400 BAD REQUEST so try to return that if possible, otherwise raise the + # HTTP/1.1 error caught by requests + if not res.ok: + try: + err_msg = res.json() + except Exception: + res.raise_for_status() + else: + raise requests.HTTPError(err_msg['message']) + + return read_pvgis_hourly(io.StringIO(res.text), pvgis_format=outputformat, + map_variables=map_variables) + + +def _parse_pvgis_hourly_json(src, map_variables): + inputs = src['inputs'] + metadata = src['meta'] + data = pd.DataFrame(src['outputs']['hourly']) + data.index = pd.to_datetime(data['time'], format='%Y%m%d:%H%M', utc=True) + data = data.drop('time', axis=1) + data = data.astype(dtype={'Int': 'int'}) # The 'Int' column to be integer + if map_variables: + data = data.rename(columns=PVGIS_VARIABLE_MAP) + return data, inputs, metadata + + +def _parse_pvgis_hourly_csv(src, map_variables): + # The first 4 rows are latitude, longitude, elevation, radiation database + inputs = {} + # 'Latitude (decimal degrees): 45.000\r\n' + inputs['latitude'] = float(src.readline().split(':')[1]) + # 'Longitude (decimal degrees): 8.000\r\n' + inputs['longitude'] = float(src.readline().split(':')[1]) + # Elevation (m): 1389.0\r\n + inputs['elevation'] = float(src.readline().split(':')[1]) + # 'Radiation database: \tPVGIS-SARAH\r\n' + inputs['radiation_database'] = src.readline().split(':')[1].strip() + # Parse through the remaining metadata section (the number of lines for + # this section depends on the requested parameters) + while True: + line = src.readline() + if line.startswith('time,'): # The data header starts with 'time,' + # The last line of the metadata section contains the column names + names = line.strip().split(',') + break + # Only retrieve metadata from non-empty lines + elif line.strip() != '': + inputs[line.split(':')[0]] = line.split(':')[1].strip() + elif line == '': # If end of file is reached + raise ValueError('No data section was detected. File has probably ' + 'been modified since being downloaded from PVGIS') + # Save the entries from the data section to a list, until an empty line is + # reached an empty line. The length of the section depends on the request + data_lines = [] + while True: + line = src.readline() + if line.strip() == '': + break + else: + data_lines.append(line.strip().split(',')) + data = pd.DataFrame(data_lines, columns=names) + data.index = pd.to_datetime(data['time'], format='%Y%m%d:%H%M', utc=True) + data = data.drop('time', axis=1) + if map_variables: + data = data.rename(columns=PVGIS_VARIABLE_MAP) + # All columns should have the dtype=float, except 'Int' which should be + # integer. It is necessary to convert to float, before converting to int + data = data.astype(float).astype(dtype={'Int': 'int'}) + # Generate metadata dictionary containing description of parameters + metadata = {} + for line in src.readlines(): + if ':' in line: + metadata[line.split(':')[0]] = line.split(':')[1].strip() + return data, inputs, metadata + + +def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): + """Read a PVGIS hourly file. + + Parameters + ---------- + filename : str, pathlib.Path, or file-like buffer + Name, path, or buffer of hourly data file downloaded from PVGIS. + pvgis_format : str, default None + Format of PVGIS file or buffer. Equivalent to the ``outputformat`` + parameter in the PVGIS API. If `filename` is a file and + `pvgis_format` is ``None`` then the file extension will be used to + determine the PVGIS format to parse. If `filename` is a buffer, then + `pvgis_format` is required and must be in ``['csv', 'json']``. + map_variables: bool, default True + When true, renames columns of the DataFrame to pvlib variable names + where applicable. See variable PVGIS_VARIABLE_MAP. + + Returns + ------- + data : pandas.DataFrame + the time series data + inputs : dict + the inputs + metadata : dict + metadata + + Raises + ------ + ValueError + if `pvgis_format` is ``None`` and the file extension is neither + ``.csv`` nor ``.json`` or if `pvgis_format` is provided as + input but isn't in ``['csv', 'json']`` + TypeError + if `pvgis_format` is ``None`` and `filename` is a buffer + + See Also + -------- + get_pvgis_hourly, read_pvgis_tmy + """ + # get the PVGIS outputformat + if pvgis_format is None: + # get the file extension from suffix, but remove the dot and make sure + # it's lower case to compare with csv, or json + # NOTE: basic format is not supported for PVGIS Hourly as the data + # format does not include a header + # NOTE: raises TypeError if filename is a buffer + outputformat = Path(filename).suffix[1:].lower() + else: + outputformat = pvgis_format + + # parse the pvgis file based on the output format, either 'json' or 'csv' + # NOTE: json and csv output formats have parsers defined as private + # functions in this module + + # JSON: use Python built-in json module to convert file contents to a + # Python dictionary, and pass the dictionary to the + # _parse_pvgis_hourly_json() function from this module + if outputformat == 'json': + try: + src = json.load(filename) + except AttributeError: # str/path has no .read() attribute + with open(str(filename), 'r') as fbuf: + src = json.load(fbuf) + return _parse_pvgis_hourly_json(src, map_variables=map_variables) + + # CSV: use _parse_pvgis_hourly_csv() + if outputformat == 'csv': + try: + pvgis_data = _parse_pvgis_hourly_csv( + filename, map_variables=map_variables) + except AttributeError: # str/path has no .read() attribute + with open(str(filename), 'r') as fbuf: + pvgis_data = _parse_pvgis_hourly_csv( + fbuf, map_variables=map_variables) + return pvgis_data + + # raise exception if pvgis format isn't in ['csv', 'json'] + err_msg = ( + "pvgis format '{:s}' was unknown, must be either 'json' or 'csv'")\ + .format(outputformat) + raise ValueError(err_msg) + def get_pvgis_tmy(lat, lon, outputformat='json', usehorizon=True, userhorizon=None, startyear=None, endyear=None, url=URL, diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index a6c3c4510b..fc0638ed74 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -4,12 +4,301 @@ import json import numpy as np import pandas as pd +import io import pytest import requests from pvlib.iotools import get_pvgis_tmy, read_pvgis_tmy -from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY +from pvlib.iotools import get_pvgis_hourly, read_pvgis_hourly +from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY, assert_frame_equal + + +# PVGIS Hourly tests +# The test files are actual files from PVGIS where the data section have been +# reduced to only a few lines +testfile_radiation_csv = DATA_DIR / \ + 'pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv' +testfile_pv_json = DATA_DIR / \ + 'pvgis_hourly_Timeseries_45.000_8.000_CM_10kWp_CIS_5_2a_2013_2014.json' + +index_radiation_csv = \ + pd.date_range('20160101 00:10', freq='1h', periods=14, tz='UTC') +index_pv_json = \ + pd.date_range('2013-01-01 00:55', freq='1h', periods=10, tz='UTC') + +columns_radiation_csv = [ + 'Gb(i)', 'Gd(i)', 'Gr(i)', 'H_sun', 'T2m', 'WS10m', 'Int'] +columns_radiation_csv_mapped = [ + 'poa_direct', 'poa_sky_diffuse', 'poa_ground_diffuse', 'solar_elevation', + 'temp_air', 'wind_speed', 'Int'] +columns_pv_json = [ + 'P', 'Gb(i)', 'Gd(i)', 'Gr(i)', 'H_sun', 'T2m', 'WS10m', 'Int'] +columns_pv_json_mapped = [ + 'P', 'poa_direct', 'poa_sky_diffuse', 'poa_ground_diffuse', + 'solar_elevation', 'temp_air', 'wind_speed', 'Int'] + +data_radiation_csv = [ + [0.0, 0.0, 0.0, 0.0, 3.44, 1.43, 0.0], + [0.0, 0.0, 0.0, 0.0, 2.94, 1.47, 0.0], + [0.0, 0.0, 0.0, 0.0, 2.43, 1.51, 0.0], + [0.0, 0.0, 0.0, 0.0, 1.93, 1.54, 0.0], + [0.0, 0.0, 0.0, 0.0, 2.03, 1.62, 0.0], + [0.0, 0.0, 0.0, 0.0, 2.14, 1.69, 0.0], + [0.0, 0.0, 0.0, 0.0, 2.25, 1.77, 0.0], + [0.0, 0.0, 0.0, 0.0, 3.06, 1.49, 0.0], + [26.71, 8.28, 0.21, 8.06, 3.87, 1.22, 1.0], + [14.69, 5.76, 0.16, 14.8, 4.67, 0.95, 1.0], + [2.19, 0.94, 0.03, 19.54, 5.73, 0.77, 1.0], + [2.11, 0.94, 0.03, 21.82, 6.79, 0.58, 1.0], + [4.25, 1.88, 0.05, 21.41, 7.84, 0.4, 1.0], + [0.0, 0.0, 0.0, 0.0, 7.43, 0.72, 0.0]] +data_pv_json = [ + [0.0, 0.0, 0.0, 0.0, 0.0, 3.01, 1.23, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 2.22, 1.46, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 1.43, 1.7, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.64, 1.93, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.77, 1.8, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.91, 1.66, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 1.05, 1.53, 0.0], + [3464.5, 270.35, 91.27, 6.09, 6.12, 1.92, 1.44, 0.0], + [1586.9, 80.76, 83.95, 9.04, 13.28, 2.79, 1.36, 0.0], + [713.3, 5.18, 70.57, 7.31, 18.56, 3.66, 1.27, 0.0]] + +inputs_radiation_csv = {'latitude': 45.0, 'longitude': 8.0, 'elevation': 250.0, + 'radiation_database': 'PVGIS-SARAH', + 'Slope': '30 deg.', 'Azimuth': '0 deg.'} + +metadata_radiation_csv = { + 'Gb(i)': 'Beam (direct) irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: E501 + 'Gd(i)': 'Diffuse irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: E501 + 'Gr(i)': 'Reflected irradiance on the inclined plane (plane of the array) (W/m2)', # noqa: E501 + 'H_sun': 'Sun height (degree)', + 'T2m': '2-m air temperature (degree Celsius)', + 'WS10m': '10-m total wind speed (m/s)', + 'Int': '1 means solar radiation values are reconstructed'} + +inputs_pv_json = { + 'location': {'latitude': 45.0, 'longitude': 8.0, 'elevation': 250.0}, + 'meteo_data': {'radiation_db': 'PVGIS-CMSAF', 'meteo_db': 'ERA-Interim', + 'year_min': 2013, 'year_max': 2014, 'use_horizon': True, + 'horizon_db': None, 'horizon_data': 'DEM-calculated'}, + 'mounting_system': {'two_axis': { + 'slope': {'value': '-', 'optimal': '-'}, + 'azimuth': {'value': '-', 'optimal': '-'}}}, + 'pv_module': {'technology': 'CIS', 'peak_power': 10.0, 'system_loss': 5.0}} + +metadata_pv_json = { + 'inputs': { + 'location': {'description': 'Selected location', 'variables': { + 'latitude': {'description': 'Latitude', 'units': 'decimal degree'}, + 'longitude': {'description': 'Longitude', 'units': 'decimal degree'}, # noqa: E501 + 'elevation': {'description': 'Elevation', 'units': 'm'}}}, + 'meteo_data': { + 'description': 'Sources of meteorological data', + 'variables': { + 'radiation_db': {'description': 'Solar radiation database'}, + 'meteo_db': {'description': 'Database used for meteorological variables other than solar radiation'}, # noqa: E501 + 'year_min': {'description': 'First year of the calculations'}, + 'year_max': {'description': 'Last year of the calculations'}, + 'use_horizon': {'description': 'Include horizon shadows'}, + 'horizon_db': {'description': 'Source of horizon data'}}}, + 'mounting_system': { + 'description': 'Mounting system', + 'choices': 'fixed, vertical_axis, inclined_axis, two_axis', + 'fields': { + 'slope': {'description': 'Inclination angle from the horizontal plane', 'units': 'degree'}, # noqa: E501 + 'azimuth': {'description': 'Orientation (azimuth) angle of the (fixed) PV system (0 = S, 90 = W, -90 = E)', 'units': 'degree'}}}, # noqa: E501 + 'pv_module': { + 'description': 'PV module parameters', + 'variables': { + 'technology': {'description': 'PV technology'}, + 'peak_power': {'description': 'Nominal (peak) power of the PV module', 'units': 'kW'}, # noqa: E501 + 'system_loss': {'description': 'Sum of system losses', 'units': '%'}}}}, # noqa: E501 + 'outputs': { + 'hourly': { + 'type': 'time series', 'timestamp': 'hourly averages', + 'variables': { + 'P': {'description': 'PV system power', 'units': 'W'}, + 'Gb(i)': {'description': 'Beam (direct) irradiance on the inclined plane (plane of the array)', 'units': 'W/m2'}, # noqa: E501 + 'Gd(i)': {'description': 'Diffuse irradiance on the inclined plane (plane of the array)', 'units': 'W/m2'}, # noqa: E501 + 'Gr(i)': {'description': 'Reflected irradiance on the inclined plane (plane of the array)', 'units': 'W/m2'}, # noqa: E501 + 'H_sun': {'description': 'Sun height', 'units': 'degree'}, + 'T2m': {'description': '2-m air temperature', 'units': 'degree Celsius'}, # noqa: E501 + 'WS10m': {'description': '10-m total wind speed', 'units': 'm/s'}, # noqa: E501 + 'Int': {'description': '1 means solar radiation values are reconstructed'}}}}} # noqa: E501 + + +def generate_expected_dataframe(values, columns, index): + """Create dataframe from arrays of values, columns and index, in order to + use this dataframe to compare to. + """ + expected = pd.DataFrame(index=index, data=values, columns=columns) + expected['Int'] = expected['Int'].astype(int) + expected.index.name = 'time' + expected.index.freq = None + return expected + + +@pytest.fixture +def expected_radiation_csv(): + expected = generate_expected_dataframe( + data_radiation_csv, columns_radiation_csv, index_radiation_csv) + return expected + + +@pytest.fixture +def expected_radiation_csv_mapped(): + expected = generate_expected_dataframe( + data_radiation_csv, columns_radiation_csv_mapped, index_radiation_csv) + return expected +@pytest.fixture +def expected_pv_json(): + expected = generate_expected_dataframe( + data_pv_json, columns_pv_json, index_pv_json) + return expected + + +@pytest.fixture +def expected_pv_json_mapped(): + expected = generate_expected_dataframe( + data_pv_json, columns_pv_json_mapped, index_pv_json) + return expected + + +# Test read_pvgis_hourly function using two different files with different +# input arguments (to test variable mapping and pvgis_format) +# pytest request.getfixturevalue is used to simplify the input arguments +@pytest.mark.parametrize('testfile,expected_name,metadata_exp,inputs_exp,map_variables,pvgis_format', [ # noqa: E501 + (testfile_radiation_csv, 'expected_radiation_csv', metadata_radiation_csv, + inputs_radiation_csv, False, None), + (testfile_radiation_csv, 'expected_radiation_csv_mapped', + metadata_radiation_csv, inputs_radiation_csv, True, 'csv'), + (testfile_pv_json, 'expected_pv_json', metadata_pv_json, inputs_pv_json, + False, None), + (testfile_pv_json, 'expected_pv_json_mapped', metadata_pv_json, + inputs_pv_json, True, 'json')]) +def test_read_pvgis_hourly(testfile, expected_name, metadata_exp, + inputs_exp, map_variables, pvgis_format, request): + # Get expected dataframe from fixture + expected = request.getfixturevalue(expected_name) + # Read data from file + out, inputs, metadata = read_pvgis_hourly( + testfile, map_variables=map_variables, pvgis_format=pvgis_format) + # Assert whether dataframe, metadata, and inputs are as expected + assert_frame_equal(out, expected) + assert inputs == inputs_exp + assert metadata == metadata_exp + + +def test_read_pvgis_hourly_bad_extension(): + # Test if ValueError is raised if file extension cannot be recognized and + # pvgis_format is not specified + with pytest.raises(ValueError, match="pvgis format 'txt' was unknown"): + read_pvgis_hourly('filename.txt') + # Test if ValueError is raised if an unkonwn pvgis_format is specified + with pytest.raises(ValueError, match="pvgis format 'txt' was unknown"): + read_pvgis_hourly(testfile_pv_json, pvgis_format='txt') + # Test if TypeError is raised if input is a buffer and pvgis_format=None + with pytest.raises(TypeError, match="expected str, bytes or os.PathLike"): + read_pvgis_hourly(io.StringIO()) + + +args_radiation_csv = { + 'surface_tilt': 30, 'surface_azimuth': 0, 'outputformat': 'csv', + 'usehorizon': False, 'userhorizon': None, 'raddatabase': 'PVGIS-SARAH', + 'start': 2016, 'end': 2016, 'pvcalculation': False, 'components': True} + +url_hourly_radiation_csv = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=45&lon=8&outputformat=csv&angle=30&aspect=0&usehorizon=0&pvtechchoice=crystSi&mountingplace=free&trackingtype=0&components=1&raddatabase=PVGIS-SARAH&startyear=2016&endyear=2016' # noqa: E501 + +args_pv_json = { + 'surface_tilt': 30, 'surface_azimuth': 0, 'outputformat': 'json', + 'usehorizon': True, 'userhorizon': None, 'raddatabase': 'PVGIS-CMSAF', + 'start': pd.Timestamp(2013, 1, 1), 'end': pd.Timestamp(2014, 5, 1), + 'pvcalculation': True, 'peakpower': 10, 'pvtechchoice': 'CIS', 'loss': 5, + 'trackingtype': 2, 'optimalangles': True, 'components': True} + +url_pv_json = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=45&lon=8&outputformat=json&angle=30&aspect=0&pvtechchoice=CIS&mountingplace=free&trackingtype=2&components=1&usehorizon=1&raddatabase=PVGIS-CMSAF&startyear=2013&endyear=2014&pvcalculation=1&peakpower=10&loss=5&optimalangles=1' # noqa: E501 + + +@pytest.mark.parametrize('testfile,expected_name,args,map_variables,url_test', [ # noqa: E501 + (testfile_radiation_csv, 'expected_radiation_csv', + args_radiation_csv, False, url_hourly_radiation_csv), + (testfile_radiation_csv, 'expected_radiation_csv_mapped', + args_radiation_csv, True, url_hourly_radiation_csv), + (testfile_pv_json, 'expected_pv_json', args_pv_json, False, url_pv_json), + (testfile_pv_json, 'expected_pv_json_mapped', args_pv_json, True, + url_pv_json)]) +def test_get_pvgis_hourly(requests_mock, testfile, expected_name, args, + map_variables, url_test, request): + """Test that get_pvgis_hourly generates the correct URI request and that + _parse_pvgis_hourly_json and _parse_pvgis_hourly_csv is called correctly""" + # Open local test file containing McClear monthly data + with open(testfile, 'r') as test_file: + mock_response = test_file.read() + # Specify the full URI of a specific example, this ensures that all of the + # inputs are passing on correctly + requests_mock.get(url_test, text=mock_response) + # Make API call - an error is raised if requested URI does not match + out, inputs, metadata = get_pvgis_hourly( + latitude=45, longitude=8, map_variables=map_variables, **args) + # Get expected dataframe from fixture + expected = request.getfixturevalue(expected_name) + # Compare out and expected dataframes + assert_frame_equal(out, expected) + + +def test_get_pvgis_hourly_bad_status_code(requests_mock): + # Test if a HTTPError is raised if a bad request is returned + requests_mock.get(url_pv_json, status_code=400) + with pytest.raises(requests.HTTPError): + get_pvgis_hourly(latitude=45, longitude=8, **args_pv_json) + # Test if HTTPError is raised and error message is returned if avaiable + requests_mock.get(url_pv_json, status_code=400, + json={'message': 'peakpower Mandatory'}) + with pytest.raises(requests.HTTPError): + get_pvgis_hourly(latitude=45, longitude=8, **args_pv_json) + + +url_bad_outputformat = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=45&lon=8&outputformat=basic&angle=0&aspect=0&pvcalculation=0&pvtechchoice=crystSi&mountingplace=free&trackingtype=0&components=1&usehorizon=1&optimalangles=0&optimalinclination=0&loss=0' # noqa: E501 + + +def test_get_pvgis_hourly_bad_outputformat(requests_mock): + # Test if a ValueError is raised if an unsupported outputformat is used + # E.g. 'basic' is a valid PVGIS format, but is not supported by pvlib + requests_mock.get(url_bad_outputformat) + with pytest.raises(ValueError): + get_pvgis_hourly(latitude=45, longitude=8, outputformat='basic') + + +url_additional_inputs = 'https://re.jrc.ec.europa.eu/api/seriescalc?lat=55.6814&lon=12.5758&outputformat=csv&angle=0&aspect=0&pvcalculation=1&pvtechchoice=crystSi&mountingplace=free&trackingtype=0&components=1&usehorizon=1&optimalangles=1&optimalinclination=1&loss=2&userhorizon=10%2C15%2C20%2C10&peakpower=5' # noqa: E501 + + +def test_get_pvgis_hourly_additional_inputs(requests_mock): + # Test additional inputs, including userhorizons + # Necessary to pass a test file in order for the parser not to fail + with open(testfile_radiation_csv, 'r') as test_file: + mock_response = test_file.read() + requests_mock.get(url_additional_inputs, text=mock_response) + # Make request with userhorizon specified + # Test passes if the request made by get_pvgis_hourly matches exactly the + # url passed to the mock request (url_additional_inputs) + get_pvgis_hourly( + latitude=55.6814, longitude=12.5758, outputformat='csv', + usehorizon=True, userhorizon=[10, 15, 20, 10], pvcalculation=True, + peakpower=5, loss=2, trackingtype=0, components=True, + optimalangles=True) + + +def test_read_pvgis_hourly_empty_file(): + # Check if a IOError is raised if file does not contain a data section + with pytest.raises(ValueError, match='No data section'): + read_pvgis_hourly( + io.StringIO('1:1\n2:2\n3:3\n4:4\n5:5\n'), + pvgis_format='csv') + + +# PVGIS TMY tests @pytest.fixture def expected(): return pd.read_csv(DATA_DIR / 'pvgis_tmy_test.dat', index_col='time(UTC)')