Merge branch 'issue195-processesv2'

Author: soxofaan

.gitmodules

@@ -1,9 +1,6 @@
 [submodule "openeo_driver/specs/openeo-processes/1.0"]
 	path = openeo_driver/specs/openeo-processes/1.x
 	url = https://github.com/Open-EO/openeo-processes.git
-[submodule "openeo_driver/specs/openeo-processes/0.4"]
-	path = openeo_driver/specs/openeo-processes/0.4
-	url = https://github.com/Open-EO/openeo-processes.git
 [submodule "openeo_driver/specs/openeo-api/1.0"]
 	path = openeo_driver/specs/openeo-api/1.x
 	url = https://github.com/Open-EO/openeo-api.git

CHANGELOG.md

@@ -0,0 +1,37 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+This project does not have a real release cycle (yet).
+Upstream projects usually depend on development snapshots of this project.
+Still, to have some kind of indicator of small versus big change,
+we try to bump the version number (in `openeo_driver/_version.py`)
+roughly according to [Semantic Versioning](https://semver.org/).
+
+When adding a feature/bugfix without bumping the version number:
+just describe it under the "In progress" section.
+When bumping the version number in `openeo_driver/_version.py`
+(possibly accompanying a feature/bugfix):
+"close" the "In Progress" section by changing its title to the new version number
+(and describe accompanying changes, if any, under it too)
+and start a new "In Progress" section above it.
+
+
+## In progress
+
+
+
+## 0.70.0
+
+- Initial support for openeo-processes v2.0, when requesting version 1.2 of the openEO API ([#195](https://github.com/Open-EO/openeo-python-driver/issues/195))
+- Drop support for 0.4 version of openeo-processes ([#47](https://github.com/Open-EO/openeo-python-driver/issues/47))
+
+
+## 0.69.1
+
+- Add backoff to ensure EJR deletion ([#163](https://github.com/Open-EO/openeo-python-driver/issues/163))
+
+
+## 0.69.0
+
+- Support job deletion in EJR ([#163](https://github.com/Open-EO/openeo-python-driver/issues/163))

openeo_driver/ProcessGraphDeserializer.py

@@ -1 +1 @@
-__version__ = "0.69.2a1"
+__version__ = "0.70.0a1"

openeo_driver/_version.py

@@ -57,6 +57,7 @@ def returns(self, description: str, schema: dict) -> 'ProcessSpec':
 
     def to_dict_040(self) -> dict:
         """Generate process spec as (JSON-able) dictionary (API 0.4.0 style)."""
+        # TODO #47 drop this
         if len(self._parameters) == 0:
             warnings.warn("Process with no parameters")
         assert self._returns is not None
@@ -142,7 +143,8 @@ def add_process(self, name: str, function: Callable = None, spec: dict = None, n
         if self.contains(name, namespace):
             raise ProcessRegistryException(f"Process {name!r} already defined in namespace {namespace!r}")
         if spec:
-            assert name == spec['id']
+            if name != spec["id"]:
+                raise ProcessRegistryException(f"Process {name!r} has unexpected id {spec['id']!r}")
         if function and self._argument_names:
             sig = inspect.signature(function)
             arg_names = [n for n, p in sig.parameters.items() if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD]
@@ -188,7 +190,7 @@ def add_simple_function(
         self, f: Optional[Callable] = None, name: Optional[str] = None, spec: Optional[dict] = None
     ):
         """
-        Register a simple function that uses normal arguments instead of `args: dict, env: EvalEnv`:
+        Register a simple function that uses normal arguments instead of `args: ProcessArgs, env: EvalEnv`:
         wrap it in a wrapper that automatically extracts these arguments
         :param f:
         :param name: process_id (when guessing from `f.__name__` doesn't work)
@@ -265,8 +267,9 @@ def get_function(self, name: str, namespace: str = DEFAULT_NAMESPACE) -> Callabl
         return self._get(name, namespace).function
 
 
-# Type annotation for an argument value
+# Type annotation aliases
 ArgumentValue = Any
+Validator = Callable[[Any], bool]
 
 
 class ProcessArgs(dict):
@@ -294,7 +297,7 @@ def get_required(
         name: str,
         *,
         expected_type: Optional[Union[type, Tuple[type, ...]]] = None,
-        validator: Optional[Callable[[Any], bool]] = None,
+        validator: Optional[Validator] = None,
     ) -> ArgumentValue:
         """
         Get a required argument by name.
@@ -315,33 +318,35 @@ def _check_value(
         name: str,
         value: Any,
         expected_type: Optional[Union[type, Tuple[type, ...]]] = None,
-        validator: Optional[Callable[[Any], bool]] = None,
+        validator: Optional[Validator] = None,
     ):
         if expected_type:
             if not isinstance(value, expected_type):
                 raise ProcessParameterInvalidException(
                     parameter=name, process=self.process_id, reason=f"Expected {expected_type} but got {type(value)}."
                 )
         if validator:
+            reason = None
             try:
                 valid = validator(value)
-                reason = "Failed validation."
             except OpenEOApiException:
                 # Preserve original OpenEOApiException
                 raise
             except Exception as e:
                 valid = False
                 reason = str(e)
             if not valid:
-                raise ProcessParameterInvalidException(parameter=name, process=self.process_id, reason=reason)
+                raise ProcessParameterInvalidException(
+                    parameter=name, process=self.process_id, reason=reason or "Failed validation."
+                )
 
     def get_optional(
         self,
         name: str,
         default: Union[Any, Callable[[], Any]] = None,
         *,
         expected_type: Optional[Union[type, Tuple[type, ...]]] = None,
-        validator: Optional[Callable[[Any], bool]] = None,
+        validator: Optional[Validator] = None,
     ) -> ArgumentValue:
         """
         Get an optional argument with default
@@ -364,7 +369,7 @@ def get_deep(
         self,
         *steps: str,
         expected_type: Optional[Union[type, Tuple[type, ...]]] = None,
-        validator: Optional[Callable[[Any], bool]] = None,
+        validator: Optional[Validator] = None,
     ) -> ArgumentValue:
         """
         Walk recursively through a dictionary to get to a value.
@@ -431,10 +436,26 @@ def get_enum(self, name: str, options: Collection[ArgumentValue]) -> ArgumentVal
         return value
 
     @staticmethod
-    def validator_one_of(options: list, show_value: bool = True):
-        """Build a validator function that check that the value is in given list"""
+    def validator_generic(condition: Callable[[Any], bool], error_message: str) -> Validator:
+        """
+        Build validator function based on a condition (another validator)
+        and a custom error message when validation returns False.
+        (supports interpolation of actual value with "{actual}").
+        """
 
         def validator(value):
+            valid = condition(value)
+            if not valid:
+                raise ValueError(error_message.format(actual=value))
+            return valid
+
+        return validator
+
+    @staticmethod
+    def validator_one_of(options: list, show_value: bool = True) -> Validator:
+        """Build a validator function that check that the value is in given list"""
+
+        def validator(value) -> bool:
             if value not in options:
                 if show_value:
                     message = f"Must be one of {options!r} but got {value!r}."
@@ -446,7 +467,7 @@ def validator(value):
         return validator
 
     @staticmethod
-    def validator_file_format(formats: Union[List[str], Dict[str, dict]]):
+    def validator_file_format(formats: Union[List[str], Dict[str, dict]]) -> Validator:
         """
         Build validator for input/output format (case-insensitive check)
 
@@ -455,7 +476,7 @@ def validator_file_format(formats: Union[List[str], Dict[str, dict]]):
         formats = list(formats)
         options = set(f.lower() for f in formats)
 
-        def validator(value: str):
+        def validator(value: str) -> bool:
             if value.lower() not in options:
                 raise OpenEOApiException(
                     message=f"Invalid file format {value!r}. Allowed formats: {', '.join(formats)}",
@@ -469,10 +490,10 @@ def validator(value: str):
     @staticmethod
     def validator_geojson_dict(
         allowed_types: Optional[Collection[str]] = None,
-    ):
+    ) -> Validator:
         """Build validator to verify that provided structure looks like a GeoJSON-style object"""
 
-        def validator(value):
+        def validator(value) -> bool:
             issues = validate_geojson_basic(value=value, allowed_types=allowed_types, raise_exception=False)
             if issues:
                 raise ValueError(f"Invalid GeoJSON: {', '.join(issues)}.")

openeo_driver/processes.py

@@ -556,7 +556,6 @@ def stats(band):
 
     def to_csv(self, destination=None):
         csv_paths = glob.glob(self._csv_dir + "/*.csv")
-        print(csv_paths)
         # TODO: assumption there is only one CSV?
         if(destination == None):
             return csv_paths[0]

openeo_driver/save_result.py

@@ -1 +1,2 @@
+0.4
 1.0

openeo_driver/specs/openeo-processes/.gitignore

@@ -46,7 +46,7 @@
         "flask",
         "werkzeug>=1.0.1,<2.3.0",  # https://github.com/Open-EO/openeo-python-driver/issues/187
         "requests>=2.28.0",
-        "openeo>=0.15.0",
+        "openeo>=0.24.0.a2.dev",
         "openeo_processes==0.0.4",  # 0.0.4 is special build/release, also see https://github.com/Open-EO/openeo-python-driver/issues/152
         "gunicorn>=20.0.1",
         "numpy>=1.22.0",

openeo_driver/specs/openeo-processes/0.4

@@ -0,0 +1,51 @@
+{
+  "collection": {
+    "process_id": "load_collection",
+    "arguments": {"id": "S2_FOOBAR"}
+  },
+  "apply": {
+    "process_id": "apply_polygon",
+    "arguments": {
+      "data": {"from_node": "collection"},
+      "chunks": {
+        "type": "FeatureCollection",
+        "features": [
+          {
+            "type": "Feature",
+            "properties": {},
+            "geometry": {
+              "type": "Polygon",
+              "coordinates": [[[1, 5], [2, 5], [2, 6], [1, 6], [1, 5]]]
+            }
+          },
+          {
+            "type": "Feature",
+            "properties": {},
+            "geometry": {
+              "type": "Polygon",
+              "coordinates": [
+                [[10, 15], [12, 15], [12, 16], [10, 16], [10, 15]]
+              ]
+            }
+          }
+        ]
+      },
+      "mask_value": -5,
+      "process": {
+        "process_graph": {
+          "runudf1": {
+            "process_id": "run_udf",
+            "arguments": {
+              "data": {"from_parameter": "data"},
+              "udf": "print('hello world')",
+              "runtime": "Python",
+              "context": {"param": {"from_parameter": "udfparam"}}
+            },
+            "result": true
+          }
+        }
+      }
+    },
+    "result": true
+  }
+}

setup.py

@@ -13,7 +13,6 @@
 
 def test_period_to_intervals():
     weekly_intervals = _period_to_intervals("2021-06-08", "2021-06-24", "week")
-    print(list(weekly_intervals))
     weekly_intervals = [(i[0].isoformat(), i[1].isoformat()) for i in weekly_intervals]
     assert 3 == len(weekly_intervals)
     assert weekly_intervals[0] == ('2021-06-06T00:00:00', '2021-06-13T00:00:00')
@@ -59,7 +58,6 @@ def test_period_to_intervals_monthly_tz():
 
 def test_period_to_intervals_yearly():
     intervals = _period_to_intervals("2018-06-08", "2021-08-24", "year")
-    print(list(intervals))
     intervals = [(i[0].isoformat(), i[1].isoformat()) for i in intervals]
     assert 4 == len(intervals)
     assert intervals[0] == ('2018-01-01T00:00:00', '2019-01-01T00:00:00')
@@ -70,7 +68,6 @@ def test_period_to_intervals_yearly():
 
 def test_period_to_intervals_monthly_full_year():
     intervals = _period_to_intervals("2020-01-01", "2021-01-01", "month")
-    print(list(intervals))
     intervals = [(i[0].isoformat(), i[1].isoformat()) for i in intervals]
     assert 12 == len(intervals)
     assert intervals[0] == ('2020-01-01T00:00:00', '2020-02-01T00:00:00')
@@ -81,7 +78,6 @@ def test_period_to_intervals_monthly_full_year():
 
 def test_period_to_intervals_daily():
     intervals = _period_to_intervals("2021-06-08", "2021-06-11", "day")
-    print(list(intervals))
     intervals = [(i[0].isoformat(), i[1].isoformat()) for i in intervals]
     assert 4 == len(intervals)
     assert intervals[0] == ('2021-06-07T00:00:00', '2021-06-08T00:00:00')
@@ -92,7 +88,6 @@ def test_period_to_intervals_daily():
 
 def test_period_to_intervals_dekad():
     intervals = _period_to_intervals("2021-06-08", "2021-07-20", "dekad")
-    print(list(intervals))
     intervals = [(i[0].isoformat(), i[1].isoformat()) for i in intervals]
     assert 5 == len(intervals)
     assert intervals[0] == ('2021-06-01T00:00:00', '2021-06-11T00:00:00')
@@ -104,7 +99,6 @@ def test_period_to_intervals_dekad():
 
 def test_period_to_intervals_dekad_first_of_month():
     intervals = _period_to_intervals("2021-06-01", "2021-07-20", "dekad")
-    print(list(intervals))
     intervals = [(i[0].isoformat(), i[1].isoformat()) for i in intervals]
     assert 5 == len(intervals)
     assert intervals[0] == ('2021-06-01T00:00:00', '2021-06-11T00:00:00')

tests/data/pg/1.0/apply_polygon.json

@@ -615,6 +615,43 @@ def test_get_enum(self):
         ):
             _ = args.get_enum("color", options=["R", "G", "B"])
 
+    def test_validator_generic(self):
+        args = ProcessArgs({"size": 11}, process_id="wibble")
+
+        validator = ProcessArgs.validator_generic(lambda v: v > 1, error_message="Should be stricly positive.")
+        value = args.get_required("size", expected_type=int, validator=validator)
+        assert value == 11
+
+        validator = ProcessArgs.validator_generic(lambda v: v % 2 == 0, error_message="Should be even.")
+        with pytest.raises(
+            ProcessParameterInvalidException,
+            match=re.escape("The value passed for parameter 'size' in process 'wibble' is invalid: Should be even."),
+        ):
+            _ = args.get_required("size", expected_type=int, validator=validator)
+
+        validator = ProcessArgs.validator_generic(
+            lambda v: v % 2 == 0, error_message="Should be even but got {actual}."
+        )
+        with pytest.raises(
+            ProcessParameterInvalidException,
+            match=re.escape(
+                "The value passed for parameter 'size' in process 'wibble' is invalid: Should be even but got 11."
+            ),
+        ):
+            _ = args.get_required("size", expected_type=int, validator=validator)
+
+    def test_validator_one_of(self):
+        args = ProcessArgs({"color": "red", "size": 5}, process_id="wibble")
+        with pytest.raises(
+            ProcessParameterInvalidException,
+            match=re.escape(
+                "The value passed for parameter 'color' in process 'wibble' is invalid: Must be one of ['yellow', 'violet'] but got 'red'."
+            ),
+        ):
+            _ = args.get_required(
+                "color", expected_type=str, validator=ProcessArgs.validator_one_of(["yellow", "violet"])
+            )
+
     def test_validator_geojson_dict(self):
         polygon = {"type": "Polygon", "coordinates": [[1, 2]]}
         args = ProcessArgs({"geometry": polygon, "color": "red"}, process_id="wibble")

tests/test_ProcessGraphDeserializer.py

@@ -38,11 +38,9 @@ def test_aggregate_polygon_result_basic(tmp_path):
     assert ["red", "green", "blue"] == [b['name'] for b in theAsset['bands']]
 
     timeseries_ds = xr.open_dataset(filename)
-    print(timeseries_ds)
     assert_array_equal(timeseries_ds.red.coords['t'].data, np.asarray([ np.datetime64('2019-10-15T08:15:45'),np.datetime64('2019-11-11T01:11:11')]))
     timeseries_ds.red.sel(feature=1)
     timeseries_ds.red.sel( t='2019-10-16')
-    print(timeseries_ds)
     assert_array_equal(
         4, timeseries_ds.red.sel(feature=1).sel(t="2019-10-15T08:15:45").data
     )
@@ -146,7 +144,6 @@ def test_aggregate_polygon_result_CSV(tmp_path):
     assert 100.0 == theAsset['raster:bands'][0]["statistics"]['valid_percent']
 
     timeseries_ds = xr.open_dataset(filename)
-    print(timeseries_ds)
 
     assert_array_equal(timeseries_ds.red.coords['t'].data, np.asarray([ np.datetime64('2017-09-05T00:00:00'),np.datetime64('2017-09-06T00:00:00')]))