From 08ec48422a6ab7a4cc5b81ef6d519174f728613a Mon Sep 17 00:00:00 2001 From: Manu Goudar Date: Tue, 3 Jun 2025 11:45:02 +0200 Subject: [PATCH 01/12] Add parcel delineation UDP --- .../parcel_delineation.json | 39 +++ .../parcel_delineation/openeo_udp/README.md | 4 + .../parcel_delineation/openeo_udp/generate.py | 93 ++++++ .../openeo_udp/parcel_delineation.json | 302 ++++++++++++++++++ .../records/parcel_delineation.json | 114 +++++++ 5 files changed, 552 insertions(+) create mode 100644 algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json create mode 100644 algorithm_catalog/vito/parcel_delineation/openeo_udp/README.md create mode 100644 algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py create mode 100644 algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json create mode 100644 algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json diff --git a/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json new file mode 100644 index 0000000..c4dc40c --- /dev/null +++ b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json @@ -0,0 +1,39 @@ +[ + { + "id": "parcel_delineation", + "type": "openeo", + "description": "Parcel Delineation based on ML using Sentinal-2", + "backend": "openeo.dataspace.copernicus.eu", + "process_graph": { + "parcel_delineation1": { + "process_id": "parcel_delineation", + "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json" + "arguments": { + "spatial_extent": { + "west": 5.0, + "south": 51.2, + "east": 5.1, + "north": 51.3 + }, + "temporal_extent": [ + "2021-01-01", + "2021-12-31" + ] + }, + "result": true + } + }, + "job_options": { + "udf-dependency-archives": [ + "https://artifactory.vgt.vito.be:443/auxdata-public/openeo/onnx_dependencies.zip#onnx_deps", + "https://artifactory.vgt.vito.be:443/artifactory/auxdata-public/openeo/parcelDelination/BelgiumCropMap_unet_3BandsGenerator_Models.zip#onnx_models" + ], + "driver-memory": "500m", + "driver-memoryOverhead": "1000m", + "executor-memory": "1000m", + "executor-memoryOverhead": "500m", + "python-memory": "4000m" + }, + "reference_data": {} + } +] diff --git a/algorithm_catalog/vito/parcel_delineation/openeo_udp/README.md b/algorithm_catalog/vito/parcel_delineation/openeo_udp/README.md new file mode 100644 index 0000000..579b0a7 --- /dev/null +++ b/algorithm_catalog/vito/parcel_delineation/openeo_udp/README.md @@ -0,0 +1,4 @@ +# Parcel delineation +This is an [openEO](https://openeo.org/) example for delineating agricultural parcels based on a neural network, using Sentinel-2 input data. + +[VITO Remote Sensing](https://remotesensing.vito.be) diff --git a/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py b/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py new file mode 100644 index 0000000..8438819 --- /dev/null +++ b/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py @@ -0,0 +1,93 @@ +import json +from pathlib import Path +import openeo +from openeo.api.process import Parameter +from openeo.rest.udp import build_process_dict + + +def generate() -> dict: + # DEFINE PARAMETERS + # define spatial_extent + spatial_extent = Parameter.bounding_box( + name="spatial_extent", default={"west": 5.0, "south": 51.2, "east": 5.1, "north": 51.3} + ) + # define temporal_extent + temporal_extent = Parameter.temporal_interval(name="temporal_extent", default=["2021-01-01", "2021-12-31"]) + + # backend to connect and load + backend_url = "openeo.dataspace.copernicus.eu/" + conn = openeo.connect(backend_url).authenticate_oidc() + + # Compute cloud mask, and filter input data based on cloud mask. + # compute cloud mask using the SCL band + scl = conn.load_collection( + "SENTINEL2_L2A", + temporal_extent=temporal_extent, + spatial_extent=spatial_extent, + bands=["SCL"], + max_cloud_cover=10, + ) + cloud_mask = scl.process( + "to_scl_dilation_mask", + data=scl, + kernel1_size=17, + kernel2_size=77, + mask1_values=[2, 4, 5, 6, 7], + mask2_values=[3, 8, 9, 10, 11], + erosion_kernel_size=3, + ) + + # Load s2 bands and set max cloud cover to be less than 10% + s2_bands = conn.load_collection( + collection_id="SENTINEL2_L2A", + spatial_extent=spatial_extent, + temporal_extent=temporal_extent, + bands=["B04", "B08"], + max_cloud_cover=10, + ) + # mask data with cloud mask + s2_bands_masked = s2_bands.mask(cloud_mask) + + # The delineation will be estimated based on the NDVI. The `ndvi` process can be used for these calculations. + ndviband = s2_bands_masked.ndvi(red="B04", nir="B08") + + # Apply ML algorithm + # apply a neural network, requires 128x128 pixel 'chunks' as input. + segment_udf = openeo.UDF.from_url( + "https://raw.githubusercontent.com/Open-EO/openeo-community-examples/refs/heads/main/python/ParcelDelineation/udf_segmentation.py" + ) + segmentationband = ndviband.apply_neighborhood( + process=segment_udf, + size=[{"dimension": "x", "value": 64, "unit": "px"}, {"dimension": "y", "value": 64, "unit": "px"}], + overlap=[{"dimension": "x", "value": 32, "unit": "px"}, {"dimension": "y", "value": 32, "unit": "px"}], + ) + + # Postprocess the output from the neural network using a sobel filter and + # Felzenszwalb's algorithm, which are then merged. + segment_postprocess_udf = openeo.UDF.from_url( + "https://raw.githubusercontent.com/Open-EO/openeo-community-examples/refs/heads/main/python/ParcelDelineation/udf_sobel_felzenszwalb.py" + ) + sobel_felzenszwalb = segmentationband.apply_neighborhood( + process=segment_postprocess_udf, + size=[{"dimension": "x", "value": 2048, "unit": "px"}, {"dimension": "y", "value": 2048, "unit": "px"}], + overlap=[{"dimension": "x", "value": 0, "unit": "px"}, {"dimension": "y", "value": 0, "unit": "px"}], + ) + # Build the process dictionary + return build_process_dict( + process_graph=sobel_felzenszwalb, + process_id="parcel_delineation", + summary="Parcel delineation using Sentinel-2 data retrieved from the CDSE and processed on openEO.", + description="Parcel delineation using Sentinel-2", + parameters=[spatial_extent, temporal_extent], + ) + + +if __name__ == "__main__": + # save the generated process to a file + output_path = Path(__file__).parent + print(output_path) + output_path.mkdir(parents=True, exist_ok=True) + + # Save the generated process to a file + with open(output_path / "parcel_delineation.json", "w") as f: + json.dump(generate(), f, indent=2) diff --git a/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json new file mode 100644 index 0000000..24f381d --- /dev/null +++ b/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json @@ -0,0 +1,302 @@ +{ + "process_graph": { + "loadcollection1": { + "process_id": "load_collection", + "arguments": { + "bands": [ + "B04", + "B08" + ], + "id": "SENTINEL2_L2A", + "properties": { + "eo:cloud_cover": { + "process_graph": { + "lte1": { + "process_id": "lte", + "arguments": { + "x": { + "from_parameter": "value" + }, + "y": 10 + }, + "result": true + } + } + } + }, + "spatial_extent": { + "from_parameter": "spatial_extent" + }, + "temporal_extent": { + "from_parameter": "temporal_extent" + } + } + }, + "loadcollection2": { + "process_id": "load_collection", + "arguments": { + "bands": [ + "SCL" + ], + "id": "SENTINEL2_L2A", + "properties": { + "eo:cloud_cover": { + "process_graph": { + "lte2": { + "process_id": "lte", + "arguments": { + "x": { + "from_parameter": "value" + }, + "y": 10 + }, + "result": true + } + } + } + }, + "spatial_extent": { + "from_parameter": "spatial_extent" + }, + "temporal_extent": { + "from_parameter": "temporal_extent" + } + } + }, + "toscldilationmask1": { + "process_id": "to_scl_dilation_mask", + "arguments": { + "data": { + "from_node": "loadcollection2" + }, + "erosion_kernel_size": 3, + "kernel1_size": 17, + "kernel2_size": 77, + "mask1_values": [ + 2, + 4, + 5, + 6, + 7 + ], + "mask2_values": [ + 3, + 8, + 9, + 10, + 11 + ] + } + }, + "mask1": { + "process_id": "mask", + "arguments": { + "data": { + "from_node": "loadcollection1" + }, + "mask": { + "from_node": "toscldilationmask1" + } + } + }, + "ndvi1": { + "process_id": "ndvi", + "arguments": { + "data": { + "from_node": "mask1" + }, + "nir": "B08", + "red": "B04" + } + }, + "applyneighborhood1": { + "process_id": "apply_neighborhood", + "arguments": { + "data": { + "from_node": "ndvi1" + }, + "overlap": [ + { + "dimension": "x", + "value": 32, + "unit": "px" + }, + { + "dimension": "y", + "value": 32, + "unit": "px" + } + ], + "process": { + "process_graph": { + "runudf1": { + "process_id": "run_udf", + "arguments": { + "data": { + "from_parameter": "data" + }, + "runtime": "Python", + "udf": "from functools import lru_cache\nimport gc\nimport sys\nfrom typing import Dict, Tuple\nfrom random import seed, sample\nfrom xarray import DataArray, zeros_like\nfrom openeo.udf import inspect\n\n# Add the onnx dependencies to the path\nsys.path.insert(1, \"onnx_deps\")\nimport onnxruntime as ort\n\n\nmodel_names = frozenset(\n [\n \"BelgiumCropMap_unet_3BandsGenerator_Network1.onnx\",\n \"BelgiumCropMap_unet_3BandsGenerator_Network2.onnx\",\n \"BelgiumCropMap_unet_3BandsGenerator_Network3.onnx\",\n ]\n)\n\n\n@lru_cache(maxsize=1)\ndef load_ort_sessions(names):\n \"\"\"\n Load the models and make the prediction functions.\n The lru_cache avoids loading the model multiple times on the same worker.\n\n @param modeldir: Model directory\n @return: Loaded model sessions\n \"\"\"\n # inspect(message=\"Loading convolutional neural networks as ONNX runtime sessions ...\")\n return [ort.InferenceSession(f\"onnx_models/{model_name}\") for model_name in names]\n\n\ndef process_window_onnx(ndvi_stack: DataArray, patch_size=128) -> DataArray:\n \"\"\"Compute prediction.\n\n Compute predictions using ML models. ML models takes three inputs images and predicts\n one image. Four predictions are made per model using three random images. Three images\n are considered to save computational time. Final result is median of these predictions.\n\n Parameters\n ----------\n ndvi_stack : DataArray\n ndvi data\n patch_size : Int\n Size of the sample\n\n Returns\n -------\n xr.DataArray\n Machine learning prediction.\n \"\"\"\n # Do 12 predictions: use 3 networks, and for each take 3 random NDVI images and repeat 4 times\n ort_sessions = load_ort_sessions(model_names) # get models\n\n predictions_per_model = 4\n no_rand_images = 3 # Number of random images that are needed for input\n no_images = ndvi_stack.t.shape[0]\n\n # Range of index of images for random index selection\n images_range = range(no_images)\n # List of all predictions\n prediction = []\n for ort_session in ort_sessions:\n # make 4 predictions per model\n for i in range(predictions_per_model):\n # initialize a predicter array\n # Seed to lead to a reproducible results.\n seed(i)\n # Random selection of 3 images for input\n idx = sample(images_range, k=no_rand_images)\n # log a message that the selected indices are not at least a week away\n if len(set((ndvi_stack.isel(t=idx)).t.dt.isocalendar().week.data)) != no_rand_images:\n inspect(message=\"Time difference is not larger than a week for good parcel delineation\")\n\n # re-shape the input data for ML input\n input_data = ndvi_stack.isel(t=idx).data.reshape(1, patch_size * patch_size, no_rand_images)\n ort_inputs = {ort_session.get_inputs()[0].name: input_data}\n\n # Run ML to predict\n ort_outputs = ort_session.run(None, ort_inputs)\n # reshape ort_outputs and append it to prediction list\n prediction.append(ort_outputs[0].reshape((patch_size, patch_size)))\n\n # free up some memory to avoid memory errors\n gc.collect()\n\n # Create a DataArray of all predictions\n all_predictions = DataArray(\n prediction,\n dims=[\"predict\", \"x\", \"y\"],\n coords={\n \"predict\": range(len(prediction)),\n \"x\": ndvi_stack.coords[\"x\"],\n \"y\": ndvi_stack.coords[\"y\"],\n },\n )\n # final prediction is the median of all predictions per pixel\n return all_predictions.median(dim=\"predict\")\n\n\ndef get_valid_ml_inputs(nvdi_stack_data: DataArray, sum_invalid, min_images: int) -> DataArray:\n \"\"\"Machine learning inputs\n\n Extract ML inputs based on how good the data is\n\n \"\"\"\n if (sum_invalid.data == 0).sum() >= min_images:\n good_data = nvdi_stack_data.sel(t=sum_invalid[sum_invalid.data == 0].t)\n else: # select the 4 best time samples with least amount of invalid pixels.\n good_data = nvdi_stack_data.sel(t=sum_invalid.sortby(sum_invalid).t[:min_images])\n return good_data\n\n\ndef preprocess_datacube(cubearray: DataArray, min_images: int) -> Tuple[bool, DataArray]:\n \"\"\"Preprocess data for machine learning.\n\n Preprocess data by clamping NVDI values and first check if the\n data is valid for machine learning and then check if there is good\n data to perform machine learning.\n\n Parameters\n ----------\n cubearray : xr.DataArray\n Input datacube\n min_images : int\n Minimum number of samples to consider for machine learning.\n\n Returns\n -------\n bool\n True refers to data is invalid for machine learning.\n xr.DataArray\n If above bool is False, return data for machine learning else returns a\n sample containing nan (similar to machine learning output).\n \"\"\"\n # Preprocessing data\n # check if bands is in the dims and select the first index\n if \"bands\" in cubearray.dims:\n nvdi_stack = cubearray.isel(bands=0)\n else:\n nvdi_stack = cubearray\n # Clamp out of range NDVI values\n nvdi_stack = nvdi_stack.where(lambda nvdi_stack: nvdi_stack < 0.92, 0.92)\n nvdi_stack = nvdi_stack.where(lambda nvdi_stack: nvdi_stack > -0.08)\n nvdi_stack += 0.08\n # Count the amount of invalid pixels in each time sample.\n sum_invalid = nvdi_stack.isnull().sum(dim=[\"x\", \"y\"])\n # Check % of invalid pixels in each time sample by using mean\n sum_invalid_mean = nvdi_stack.isnull().mean(dim=[\"x\", \"y\"])\n # Fill the invalid pixels with value 0\n nvdi_stack_data = nvdi_stack.fillna(0)\n\n # Check if data is valid for machine learning. If invalid, return True and\n # an DataArray of nan values (similar to the machine learning output)\n # The number of invalid time sample less then min images\n if (sum_invalid_mean.data < 1).sum() <= min_images:\n inspect(message=\"Input data is invalid for this window -> skipping!\")\n # create a nan dataset and return\n nan_data = zeros_like(nvdi_stack.sel(t=sum_invalid_mean.t[0], drop=True))\n nan_data = nan_data.where(lambda nan_data: nan_data > 1)\n return True, nan_data\n # Data selection: valid data for machine learning\n # select time samples where there are no invalid pixels\n good_data = get_valid_ml_inputs(nvdi_stack_data, sum_invalid, min_images)\n return False, good_data.transpose(\"x\", \"y\", \"t\")\n\n\ndef apply_datacube(cube: DataArray, context: Dict) -> DataArray:\n # select atleast best 4 temporal images of ndvi for ML\n min_images = 4\n # preprocess the datacube\n invalid_data, ndvi_stack = preprocess_datacube(cube, min_images)\n # If data is invalid, there is no need to run prediction algorithm so\n # return prediction as nan DataArray and reintroduce time and bands dimensions\n if invalid_data:\n return ndvi_stack.expand_dims(dim={\"t\": [(cube.t.dt.year.values[0])], \"bands\": [\"prediction\"]})\n # Machine learning prediction: process the window\n result = process_window_onnx(ndvi_stack)\n # Reintroduce time and bands dimensions\n result_xarray = result.expand_dims(dim={\"t\": [(cube.t.dt.year.values[0])], \"bands\": [\"prediction\"]})\n # Return the resulting xarray\n return result_xarray\n" + }, + "result": true + } + } + }, + "size": [ + { + "dimension": "x", + "value": 64, + "unit": "px" + }, + { + "dimension": "y", + "value": 64, + "unit": "px" + } + ] + } + }, + "applyneighborhood2": { + "process_id": "apply_neighborhood", + "arguments": { + "data": { + "from_node": "applyneighborhood1" + }, + "overlap": [ + { + "dimension": "x", + "value": 0, + "unit": "px" + }, + { + "dimension": "y", + "value": 0, + "unit": "px" + } + ], + "process": { + "process_graph": { + "runudf2": { + "process_id": "run_udf", + "arguments": { + "data": { + "from_parameter": "data" + }, + "runtime": "Python", + "udf": "from xarray import DataArray\nfrom skimage import segmentation, graph\nfrom skimage.filters import sobel\nfrom typing import Dict\nfrom openeo.udf import inspect\n\n\ndef apply_datacube(cube: DataArray, context: Dict) -> DataArray:\n inspect(message=f\"Dimensions of the final datacube {cube.dims}\")\n # get the underlying array without the bands and t dimension\n image_data = cube.squeeze(\"t\", drop=True).squeeze(\"bands\", drop=True).values\n # compute edges\n edges = sobel(image_data)\n # Perform felzenszwalb segmentation\n segment = segmentation.felzenszwalb(image_data, scale=120, sigma=0.0, min_size=30, channel_axis=None)\n # Perform the rag boundary analysis and merge the segments\n bgraph = graph.rag_boundary(segment, edges)\n # merging segments\n mergedsegment = graph.cut_threshold(segment, bgraph, 0.15, in_place=False)\n # create a data cube and perform masking operations\n output_arr = DataArray(mergedsegment.reshape(cube.shape), dims=cube.dims, coords=cube.coords)\n output_arr = output_arr.where(cube >= 0.3) # Mask the output pixels based on the cube values <0.3\n output_arr = output_arr.where(output_arr >= 0) # Mask all values less than or equal to zero\n return output_arr\n" + }, + "result": true + } + } + }, + "size": [ + { + "dimension": "x", + "value": 2048, + "unit": "px" + }, + { + "dimension": "y", + "value": 2048, + "unit": "px" + } + ] + }, + "result": true + } + }, + "id": "parcel_delineation", + "summary": "Parcel delineation using Sentinel-2 data retrieved from the CDSE and processed on openEO.", + "description": "Parcel delineation using Sentinel-2", + "parameters": [ + { + "name": "spatial_extent", + "description": "Spatial extent specified as a bounding box with 'west', 'south', 'east' and 'north' fields.", + "schema": { + "type": "object", + "subtype": "bounding-box", + "required": [ + "west", + "south", + "east", + "north" + ], + "properties": { + "west": { + "type": "number", + "description": "West (lower left corner, coordinate axis 1)." + }, + "south": { + "type": "number", + "description": "South (lower left corner, coordinate axis 2)." + }, + "east": { + "type": "number", + "description": "East (upper right corner, coordinate axis 1)." + }, + "north": { + "type": "number", + "description": "North (upper right corner, coordinate axis 2)." + }, + "crs": { + "description": "Coordinate reference system of the extent, specified as as [EPSG code](http://www.epsg-registry.org/) or [WKT2 CRS string](http://docs.opengeospatial.org/is/18-010r7/18-010r7.html). Defaults to `4326` (EPSG code 4326) unless the client explicitly requests a different coordinate reference system.", + "anyOf": [ + { + "type": "integer", + "subtype": "epsg-code", + "title": "EPSG Code", + "minimum": 1000 + }, + { + "type": "string", + "subtype": "wkt2-definition", + "title": "WKT2 definition" + } + ], + "default": 4326 + } + } + }, + "default": { + "west": 5.0, + "south": 51.2, + "east": 5.1, + "north": 51.3 + }, + "optional": true + }, + { + "name": "temporal_extent", + "description": "Temporal extent specified as two-element array with start and end date/date-time.", + "schema": { + "type": "array", + "subtype": "temporal-interval", + "uniqueItems": true, + "minItems": 2, + "maxItems": 2, + "items": { + "anyOf": [ + { + "type": "string", + "subtype": "date-time", + "format": "date-time" + }, + { + "type": "string", + "subtype": "date", + "format": "date" + }, + { + "type": "null" + } + ] + } + }, + "default": [ + "2021-01-01", + "2021-12-31" + ], + "optional": true + } + ] +} \ No newline at end of file diff --git a/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json new file mode 100644 index 0000000..231aaad --- /dev/null +++ b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json @@ -0,0 +1,114 @@ +{ + "id": "parcel_delineation", + "type": "Feature", + "conformsTo": [ + "http://www.opengis.net/spec/ogcapi-records-1/1.0/req/record-core" + ], + "geometry": null, + "properties": { + "created": "2025-06-02T00:00:00Z", + "updated": "2025-06-02T00:00:00Z", + "type": "apex_algorithm", + "title": "Parcel Delination based on ML using Sentinal-2", + "description": "An openEO process example for delineating agricultural parcels based on ML using Sentinel-2 data.", + "cost_estimate": 0.11, + "cost_unit": "platform credits per km²", + "keywords": [ + "agricultural parcels", + "delineation" + ], + "language": { + "code": "en-US", + "name": "English (United States)" + }, + "languages": [ + { + "code": "en-US", + "name": "English (United States)" + } + ], + "contacts": [ + { + "name": "Kristoff Van Tricht", + "position": "Researcher", + "organization": "VITO", + "links": [ + { + "href": "https://www.vito.be/", + "title": "VITO Website", + "rel": "about", + "type": "text/html" + }, + { + "href": "https://github.com/kvantricht", + "title": "GitHub", + "rel": "about", + "type": "text/html" + } + ], + "contactInstructions": "Contact via VITO", + "roles": [ + "principal investigator" + ] + }, + { + "name": "VITO", + "links": [ + { + "href": "https://www.vito.be/", + "title": "VITO Website", + "rel": "about", + "type": "text/html" + } + ], + "contactInstructions": "SEE WEBSITE", + "roles": [ + "processor" + ] + } + ], + "themes": [ + { + "concepts": [ + { + "id": "Sentinel-2" + } + ], + "scheme": "https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/sciencekeywords" + } + ], + "formats": [ + { + "name": "GeoTIFF" + } + ], + "license": "CC-BY-4.0" + }, + "linkTemplates": [], + "links": [ + { + "rel": "application", + "type": "application/vnd.openeo+json;type=process", + "title": "openEO Process Definition", + "href": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json" + }, + { + "rel": "code", + "type": "text/html", + "title": "Git source repository", + "href": "https://github.com/Open-EO/openeo-community-examples/tree/main/python/ParcelDelineation/" + }, + { + "rel": "service", + "type": "application/json", + "title": "CDSE openEO federation", + "href": "https://openeofed.dataspace.copernicus.eu" + }, + { + "rel": "webapp", + "type": "text/html", + "title": "OpenEO Web Editor", + "href": "https://editor.openeo.org/?wizard=UDP&wizard~process=parceldelination&wizard~processUrl=https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json&server=https://openeo.dataspace.copernicus.eu" + } + ] +} From 70826f33e045f8296a2b9b5e4a7489c8b594168e Mon Sep 17 00:00:00 2001 From: Manu Goudar Date: Tue, 3 Jun 2025 11:49:11 +0200 Subject: [PATCH 02/12] Small comma fix --- .../benchmark_scenarios/parcel_delineation.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json index c4dc40c..8a63d8f 100644 --- a/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json @@ -7,7 +7,7 @@ "process_graph": { "parcel_delineation1": { "process_id": "parcel_delineation", - "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json" + "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json", "arguments": { "spatial_extent": { "west": 5.0, From 7d4eef10a6bab31d334fa19882e49a84eb233ae2 Mon Sep 17 00:00:00 2001 From: Manu Goudar Date: Tue, 3 Jun 2025 12:56:17 +0200 Subject: [PATCH 03/12] Add job_options to UDP. --- .../benchmark_scenarios/parcel_delineation.json | 13 +------------ .../vito/parcel_delineation/openeo_udp/generate.py | 14 ++++++++++++++ .../openeo_udp/parcel_delineation.json | 11 +++++++++++ .../records/parcel_delineation.json | 4 ++-- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json index 8a63d8f..5279024 100644 --- a/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json @@ -7,7 +7,7 @@ "process_graph": { "parcel_delineation1": { "process_id": "parcel_delineation", - "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json", + "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/parceldelineation/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json" "arguments": { "spatial_extent": { "west": 5.0, @@ -23,17 +23,6 @@ "result": true } }, - "job_options": { - "udf-dependency-archives": [ - "https://artifactory.vgt.vito.be:443/auxdata-public/openeo/onnx_dependencies.zip#onnx_deps", - "https://artifactory.vgt.vito.be:443/artifactory/auxdata-public/openeo/parcelDelination/BelgiumCropMap_unet_3BandsGenerator_Models.zip#onnx_models" - ], - "driver-memory": "500m", - "driver-memoryOverhead": "1000m", - "executor-memory": "1000m", - "executor-memoryOverhead": "500m", - "python-memory": "4000m" - }, "reference_data": {} } ] diff --git a/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py b/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py index 8438819..a7d0c0f 100644 --- a/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py +++ b/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py @@ -72,6 +72,18 @@ def generate() -> dict: size=[{"dimension": "x", "value": 2048, "unit": "px"}, {"dimension": "y", "value": 2048, "unit": "px"}], overlap=[{"dimension": "x", "value": 0, "unit": "px"}, {"dimension": "y", "value": 0, "unit": "px"}], ) + job_options = { + "udf-dependency-archives": [ + "https://artifactory.vgt.vito.be:443/auxdata-public/openeo/onnx_dependencies.zip#onnx_deps", + "https://artifactory.vgt.vito.be:443/artifactory/auxdata-public/openeo/parcelDelination/BelgiumCropMap_unet_3BandsGenerator_Models.zip#onnx_models" + ], + "driver-memory": "500m", + "driver-memoryOverhead": "1000m", + "executor-memory": "1000m", + "executor-memoryOverhead": "500m", + "python-memory": "4000m" + } + # Build the process dictionary return build_process_dict( process_graph=sobel_felzenszwalb, @@ -79,9 +91,11 @@ def generate() -> dict: summary="Parcel delineation using Sentinel-2 data retrieved from the CDSE and processed on openEO.", description="Parcel delineation using Sentinel-2", parameters=[spatial_extent, temporal_extent], + default_job_options=job_options, ) + if __name__ == "__main__": # save the generated process to a file output_path = Path(__file__).parent diff --git a/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json index 24f381d..e6a43e2 100644 --- a/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json @@ -208,6 +208,17 @@ "id": "parcel_delineation", "summary": "Parcel delineation using Sentinel-2 data retrieved from the CDSE and processed on openEO.", "description": "Parcel delineation using Sentinel-2", + "default_job_options": { + "udf-dependency-archives": [ + "https://artifactory.vgt.vito.be:443/auxdata-public/openeo/onnx_dependencies.zip#onnx_deps", + "https://artifactory.vgt.vito.be:443/artifactory/auxdata-public/openeo/parcelDelination/BelgiumCropMap_unet_3BandsGenerator_Models.zip#onnx_models" + ], + "driver-memory": "500m", + "driver-memoryOverhead": "1000m", + "executor-memory": "1000m", + "executor-memoryOverhead": "500m", + "python-memory": "4000m" + }, "parameters": [ { "name": "spatial_extent", diff --git a/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json index 231aaad..4cdce50 100644 --- a/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json @@ -90,7 +90,7 @@ "rel": "application", "type": "application/vnd.openeo+json;type=process", "title": "openEO Process Definition", - "href": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json" + "href": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/parceldelineation/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json" }, { "rel": "code", @@ -108,7 +108,7 @@ "rel": "webapp", "type": "text/html", "title": "OpenEO Web Editor", - "href": "https://editor.openeo.org/?wizard=UDP&wizard~process=parceldelination&wizard~processUrl=https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json&server=https://openeo.dataspace.copernicus.eu" + "href": "https://editor.openeo.org/?wizard=UDP&wizard~process=parceldelination&wizard~processUrl=https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/parceldelineation/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json&server=https://openeo.dataspace.copernicus.eu" } ] } From 8b166766dc38c675d6b43fb68f64d5c6a8f640e3 Mon Sep 17 00:00:00 2001 From: Manu Goudar Date: Tue, 3 Jun 2025 12:59:58 +0200 Subject: [PATCH 04/12] fix comma. --- .../benchmark_scenarios/parcel_delineation.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json index 5279024..e0577b8 100644 --- a/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json @@ -7,7 +7,7 @@ "process_graph": { "parcel_delineation1": { "process_id": "parcel_delineation", - "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/parceldelineation/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json" + "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/parceldelineation/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json", "arguments": { "spatial_extent": { "west": 5.0, From 69276f373c4590acbf35fb4867632582f033a13d Mon Sep 17 00:00:00 2001 From: Manu Goudar Date: Tue, 3 Jun 2025 13:29:33 +0200 Subject: [PATCH 05/12] Fix file paths. --- .../benchmark_scenarios/parcel_delineation.json | 2 +- .../vito/parcel_delineation/records/parcel_delineation.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json index e0577b8..5b1a68d 100644 --- a/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json @@ -7,7 +7,7 @@ "process_graph": { "parcel_delineation1": { "process_id": "parcel_delineation", - "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/parceldelineation/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json", + "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json", "arguments": { "spatial_extent": { "west": 5.0, diff --git a/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json index 4cdce50..231aaad 100644 --- a/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json @@ -90,7 +90,7 @@ "rel": "application", "type": "application/vnd.openeo+json;type=process", "title": "openEO Process Definition", - "href": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/parceldelineation/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json" + "href": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json" }, { "rel": "code", @@ -108,7 +108,7 @@ "rel": "webapp", "type": "text/html", "title": "OpenEO Web Editor", - "href": "https://editor.openeo.org/?wizard=UDP&wizard~process=parceldelination&wizard~processUrl=https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/parceldelineation/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json&server=https://openeo.dataspace.copernicus.eu" + "href": "https://editor.openeo.org/?wizard=UDP&wizard~process=parceldelination&wizard~processUrl=https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json&server=https://openeo.dataspace.copernicus.eu" } ] } From 5e05e994977804afcf7f05782c5de2f12bbb94d7 Mon Sep 17 00:00:00 2001 From: Manu Goudar Date: Tue, 3 Jun 2025 14:14:30 +0200 Subject: [PATCH 06/12] Correct name. --- .../vito/parcel_delineation/records/parcel_delineation.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json index 231aaad..b42eb39 100644 --- a/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json @@ -29,7 +29,7 @@ ], "contacts": [ { - "name": "Kristoff Van Tricht", + "name": "Kristof Van Tricht", "position": "Researcher", "organization": "VITO", "links": [ From e5d5e1128b091f3086adb7ed31e6d9e755087097 Mon Sep 17 00:00:00 2001 From: Manu Goudar Date: Tue, 3 Jun 2025 15:43:58 +0200 Subject: [PATCH 07/12] Add UDFs locally. --- .../parcel_delineation/openeo_udp/generate.py | 25 +-- .../openeo_udp/udf_segmentation.py | 185 ++++++++++++++++++ .../openeo_udp/udf_sobel_felzenszwalb.py | 24 +++ 3 files changed, 219 insertions(+), 15 deletions(-) create mode 100644 algorithm_catalog/vito/parcel_delineation/openeo_udp/udf_segmentation.py create mode 100644 algorithm_catalog/vito/parcel_delineation/openeo_udp/udf_sobel_felzenszwalb.py diff --git a/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py b/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py index a7d0c0f..2efc864 100644 --- a/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py +++ b/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py @@ -53,9 +53,7 @@ def generate() -> dict: # Apply ML algorithm # apply a neural network, requires 128x128 pixel 'chunks' as input. - segment_udf = openeo.UDF.from_url( - "https://raw.githubusercontent.com/Open-EO/openeo-community-examples/refs/heads/main/python/ParcelDelineation/udf_segmentation.py" - ) + segment_udf = openeo.UDF.from_file("udf_segmentation.py") segmentationband = ndviband.apply_neighborhood( process=segment_udf, size=[{"dimension": "x", "value": 64, "unit": "px"}, {"dimension": "y", "value": 64, "unit": "px"}], @@ -64,9 +62,7 @@ def generate() -> dict: # Postprocess the output from the neural network using a sobel filter and # Felzenszwalb's algorithm, which are then merged. - segment_postprocess_udf = openeo.UDF.from_url( - "https://raw.githubusercontent.com/Open-EO/openeo-community-examples/refs/heads/main/python/ParcelDelineation/udf_sobel_felzenszwalb.py" - ) + segment_postprocess_udf = openeo.UDF.from_file("udf_sobel_felzenszwalb.py") sobel_felzenszwalb = segmentationband.apply_neighborhood( process=segment_postprocess_udf, size=[{"dimension": "x", "value": 2048, "unit": "px"}, {"dimension": "y", "value": 2048, "unit": "px"}], @@ -74,14 +70,14 @@ def generate() -> dict: ) job_options = { "udf-dependency-archives": [ - "https://artifactory.vgt.vito.be:443/auxdata-public/openeo/onnx_dependencies.zip#onnx_deps", - "https://artifactory.vgt.vito.be:443/artifactory/auxdata-public/openeo/parcelDelination/BelgiumCropMap_unet_3BandsGenerator_Models.zip#onnx_models" - ], - "driver-memory": "500m", - "driver-memoryOverhead": "1000m", - "executor-memory": "1000m", - "executor-memoryOverhead": "500m", - "python-memory": "4000m" + "https://artifactory.vgt.vito.be/auxdata-public/openeo/onnx_dependencies.zip#onnx_deps", + "https://artifactory.vgt.vito.be/artifactory/auxdata-public/openeo/parcelDelination/BelgiumCropMap_unet_3BandsGenerator_Models.zip#onnx_models", + ], + "driver-memory": "500m", + "driver-memoryOverhead": "1000m", + "executor-memory": "1000m", + "executor-memoryOverhead": "500m", + "python-memory": "4000m", } # Build the process dictionary @@ -95,7 +91,6 @@ def generate() -> dict: ) - if __name__ == "__main__": # save the generated process to a file output_path = Path(__file__).parent diff --git a/algorithm_catalog/vito/parcel_delineation/openeo_udp/udf_segmentation.py b/algorithm_catalog/vito/parcel_delineation/openeo_udp/udf_segmentation.py new file mode 100644 index 0000000..f219797 --- /dev/null +++ b/algorithm_catalog/vito/parcel_delineation/openeo_udp/udf_segmentation.py @@ -0,0 +1,185 @@ +from functools import lru_cache +import gc +import sys +from typing import Dict, Tuple +from random import seed, sample +from xarray import DataArray, zeros_like +from openeo.udf import inspect + +# Add the onnx dependencies to the path +sys.path.insert(1, "onnx_deps") +import onnxruntime as ort + + +model_names = frozenset( + [ + "BelgiumCropMap_unet_3BandsGenerator_Network1.onnx", + "BelgiumCropMap_unet_3BandsGenerator_Network2.onnx", + "BelgiumCropMap_unet_3BandsGenerator_Network3.onnx", + ] +) + + +@lru_cache(maxsize=1) +def load_ort_sessions(names): + """ + Load the models and make the prediction functions. + The lru_cache avoids loading the model multiple times on the same worker. + + @param modeldir: Model directory + @return: Loaded model sessions + """ + # inspect(message="Loading convolutional neural networks as ONNX runtime sessions ...") + return [ort.InferenceSession(f"onnx_models/{model_name}") for model_name in names] + + +def process_window_onnx(ndvi_stack: DataArray, patch_size=128) -> DataArray: + """Compute prediction. + + Compute predictions using ML models. ML models takes three inputs images and predicts + one image. Four predictions are made per model using three random images. Three images + are considered to save computational time. Final result is median of these predictions. + + Parameters + ---------- + ndvi_stack : DataArray + ndvi data + patch_size : Int + Size of the sample + + Returns + ------- + xr.DataArray + Machine learning prediction. + """ + # Do 12 predictions: use 3 networks, and for each take 3 random NDVI images and repeat 4 times + ort_sessions = load_ort_sessions(model_names) # get models + + predictions_per_model = 4 + no_rand_images = 3 # Number of random images that are needed for input + no_images = ndvi_stack.t.shape[0] + + # Range of index of images for random index selection + images_range = range(no_images) + # List of all predictions + prediction = [] + for ort_session in ort_sessions: + # make 4 predictions per model + for i in range(predictions_per_model): + # initialize a predicter array + # Seed to lead to a reproducible results. + seed(i) + # Random selection of 3 images for input + idx = sample(images_range, k=no_rand_images) + # log a message that the selected indices are not at least a week away + if len(set((ndvi_stack.isel(t=idx)).t.dt.isocalendar().week.data)) != no_rand_images: + inspect(message="Time difference is not larger than a week for good parcel delineation") + + # re-shape the input data for ML input + input_data = ndvi_stack.isel(t=idx).data.reshape(1, patch_size * patch_size, no_rand_images) + ort_inputs = {ort_session.get_inputs()[0].name: input_data} + + # Run ML to predict + ort_outputs = ort_session.run(None, ort_inputs) + # reshape ort_outputs and append it to prediction list + prediction.append(ort_outputs[0].reshape((patch_size, patch_size))) + + # free up some memory to avoid memory errors + gc.collect() + + # Create a DataArray of all predictions + all_predictions = DataArray( + prediction, + dims=["predict", "x", "y"], + coords={ + "predict": range(len(prediction)), + "x": ndvi_stack.coords["x"], + "y": ndvi_stack.coords["y"], + }, + ) + # final prediction is the median of all predictions per pixel + return all_predictions.median(dim="predict") + + +def get_valid_ml_inputs(nvdi_stack_data: DataArray, sum_invalid, min_images: int) -> DataArray: + """Machine learning inputs + + Extract ML inputs based on how good the data is + + """ + if (sum_invalid.data == 0).sum() >= min_images: + good_data = nvdi_stack_data.sel(t=sum_invalid[sum_invalid.data == 0].t) + else: # select the 4 best time samples with least amount of invalid pixels. + good_data = nvdi_stack_data.sel(t=sum_invalid.sortby(sum_invalid).t[:min_images]) + return good_data + + +def preprocess_datacube(cubearray: DataArray, min_images: int) -> Tuple[bool, DataArray]: + """Preprocess data for machine learning. + + Preprocess data by clamping NVDI values and first check if the + data is valid for machine learning and then check if there is good + data to perform machine learning. + + Parameters + ---------- + cubearray : xr.DataArray + Input datacube + min_images : int + Minimum number of samples to consider for machine learning. + + Returns + ------- + bool + True refers to data is invalid for machine learning. + xr.DataArray + If above bool is False, return data for machine learning else returns a + sample containing nan (similar to machine learning output). + """ + # Preprocessing data + # check if bands is in the dims and select the first index + if "bands" in cubearray.dims: + nvdi_stack = cubearray.isel(bands=0) + else: + nvdi_stack = cubearray + # Clamp out of range NDVI values + nvdi_stack = nvdi_stack.where(lambda nvdi_stack: nvdi_stack < 0.92, 0.92) + nvdi_stack = nvdi_stack.where(lambda nvdi_stack: nvdi_stack > -0.08) + nvdi_stack += 0.08 + # Count the amount of invalid pixels in each time sample. + sum_invalid = nvdi_stack.isnull().sum(dim=["x", "y"]) + # Check % of invalid pixels in each time sample by using mean + sum_invalid_mean = nvdi_stack.isnull().mean(dim=["x", "y"]) + # Fill the invalid pixels with value 0 + nvdi_stack_data = nvdi_stack.fillna(0) + + # Check if data is valid for machine learning. If invalid, return True and + # an DataArray of nan values (similar to the machine learning output) + # The number of invalid time sample less then min images + if (sum_invalid_mean.data < 1).sum() <= min_images: + inspect(message="Input data is invalid for this window -> skipping!") + # create a nan dataset and return + nan_data = zeros_like(nvdi_stack.sel(t=sum_invalid_mean.t[0], drop=True)) + nan_data = nan_data.where(lambda nan_data: nan_data > 1) + return True, nan_data + # Data selection: valid data for machine learning + # select time samples where there are no invalid pixels + good_data = get_valid_ml_inputs(nvdi_stack_data, sum_invalid, min_images) + return False, good_data.transpose("x", "y", "t") + + +def apply_datacube(cube: DataArray, context: Dict) -> DataArray: + # select atleast best 4 temporal images of ndvi for ML + min_images = 4 + # preprocess the datacube + invalid_data, ndvi_stack = preprocess_datacube(cube, min_images) + # If data is invalid, there is no need to run prediction algorithm so + # return prediction as nan DataArray and reintroduce time and bands dimensions + if invalid_data: + return ndvi_stack.expand_dims(dim={"t": [(cube.t.dt.year.values[0])], "bands": ["prediction"]}) + # Machine learning prediction: process the window + result = process_window_onnx(ndvi_stack) + # Reintroduce time and bands dimensions + result_xarray = result.expand_dims(dim={"t": [(cube.t.dt.year.values[0])], "bands": ["prediction"]}) + # Return the resulting xarray + return result_xarray diff --git a/algorithm_catalog/vito/parcel_delineation/openeo_udp/udf_sobel_felzenszwalb.py b/algorithm_catalog/vito/parcel_delineation/openeo_udp/udf_sobel_felzenszwalb.py new file mode 100644 index 0000000..84202da --- /dev/null +++ b/algorithm_catalog/vito/parcel_delineation/openeo_udp/udf_sobel_felzenszwalb.py @@ -0,0 +1,24 @@ +from xarray import DataArray +from skimage import segmentation, graph +from skimage.filters import sobel +from typing import Dict +from openeo.udf import inspect + + +def apply_datacube(cube: DataArray, context: Dict) -> DataArray: + inspect(message=f"Dimensions of the final datacube {cube.dims}") + # get the underlying array without the bands and t dimension + image_data = cube.squeeze("t", drop=True).squeeze("bands", drop=True).values + # compute edges + edges = sobel(image_data) + # Perform felzenszwalb segmentation + segment = segmentation.felzenszwalb(image_data, scale=120, sigma=0.0, min_size=30, channel_axis=None) + # Perform the rag boundary analysis and merge the segments + bgraph = graph.rag_boundary(segment, edges) + # merging segments + mergedsegment = graph.cut_threshold(segment, bgraph, 0.15, in_place=False) + # create a data cube and perform masking operations + output_arr = DataArray(mergedsegment.reshape(cube.shape), dims=cube.dims, coords=cube.coords) + output_arr = output_arr.where(cube >= 0.3) # Mask the output pixels based on the cube values <0.3 + output_arr = output_arr.where(output_arr >= 0) # Mask all values less than or equal to zero + return output_arr From 4edc975de279d2800e5d5367032340090d554761 Mon Sep 17 00:00:00 2001 From: Manu Goudar Date: Thu, 5 Jun 2025 12:05:56 +0200 Subject: [PATCH 08/12] Add conformsto schema --- .../vito/parcel_delineation/records/parcel_delineation.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json index b42eb39..360fff2 100644 --- a/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json @@ -2,7 +2,8 @@ "id": "parcel_delineation", "type": "Feature", "conformsTo": [ - "http://www.opengis.net/spec/ogcapi-records-1/1.0/req/record-core" + "http://www.opengis.net/spec/ogcapi-records-1/1.0/req/record-core", + "https://apex.esa.int/core/openeo-udp" ], "geometry": null, "properties": { From d5f956cd12814036679fb05de0c640b9fb43a3cb Mon Sep 17 00:00:00 2001 From: Manu Goudar Date: Thu, 5 Jun 2025 13:59:46 +0200 Subject: [PATCH 09/12] Fix file links. --- .../benchmark_scenarios/parcel_delineation.json | 2 +- .../vito/parcel_delineation/records/parcel_delineation.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json index 5b1a68d..22dc152 100644 --- a/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json @@ -7,7 +7,7 @@ "process_graph": { "parcel_delineation1": { "process_id": "parcel_delineation", - "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json", + "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/4edc975de279d2800e5d5367032340090d554761/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json", "arguments": { "spatial_extent": { "west": 5.0, diff --git a/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json index 360fff2..502ef55 100644 --- a/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json @@ -91,7 +91,7 @@ "rel": "application", "type": "application/vnd.openeo+json;type=process", "title": "openEO Process Definition", - "href": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json" + "href": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/4edc975de279d2800e5d5367032340090d554761/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json" }, { "rel": "code", @@ -109,7 +109,7 @@ "rel": "webapp", "type": "text/html", "title": "OpenEO Web Editor", - "href": "https://editor.openeo.org/?wizard=UDP&wizard~process=parceldelination&wizard~processUrl=https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json&server=https://openeo.dataspace.copernicus.eu" + "href": "https://editor.openeo.org/?wizard=UDP&wizard~process=parceldelination&wizard~processUrl=https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/4edc975de279d2800e5d5367032340090d554761/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json&server=https://openeo.dataspace.copernicus.eu" } ] } From 6506c6621ed2ce3c7bba635b03d7d7a2fa20cc92 Mon Sep 17 00:00:00 2001 From: Manu Goudar Date: Thu, 5 Jun 2025 14:15:50 +0200 Subject: [PATCH 10/12] fix service type --- .../vito/parcel_delineation/records/parcel_delineation.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json index 502ef55..97c3371 100644 --- a/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/records/parcel_delineation.json @@ -9,7 +9,7 @@ "properties": { "created": "2025-06-02T00:00:00Z", "updated": "2025-06-02T00:00:00Z", - "type": "apex_algorithm", + "type": "service", "title": "Parcel Delination based on ML using Sentinal-2", "description": "An openEO process example for delineating agricultural parcels based on ML using Sentinel-2 data.", "cost_estimate": 0.11, From b828eecc13f1bd4d012576d746329041cf840c61 Mon Sep 17 00:00:00 2001 From: Manu Goudar Date: Fri, 6 Jun 2025 15:11:44 +0200 Subject: [PATCH 11/12] Increase memory. --- .../vito/parcel_delineation/openeo_udp/generate.py | 2 +- .../parcel_delineation/openeo_udp/parcel_delineation.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py b/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py index 2efc864..e6d3cf3 100644 --- a/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py +++ b/algorithm_catalog/vito/parcel_delineation/openeo_udp/generate.py @@ -77,7 +77,7 @@ def generate() -> dict: "driver-memoryOverhead": "1000m", "executor-memory": "1000m", "executor-memoryOverhead": "500m", - "python-memory": "4000m", + "python-memory": "4200m", } # Build the process dictionary diff --git a/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json index e6a43e2..3338e28 100644 --- a/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json @@ -210,14 +210,14 @@ "description": "Parcel delineation using Sentinel-2", "default_job_options": { "udf-dependency-archives": [ - "https://artifactory.vgt.vito.be:443/auxdata-public/openeo/onnx_dependencies.zip#onnx_deps", - "https://artifactory.vgt.vito.be:443/artifactory/auxdata-public/openeo/parcelDelination/BelgiumCropMap_unet_3BandsGenerator_Models.zip#onnx_models" + "https://artifactory.vgt.vito.be/auxdata-public/openeo/onnx_dependencies.zip#onnx_deps", + "https://artifactory.vgt.vito.be/artifactory/auxdata-public/openeo/parcelDelination/BelgiumCropMap_unet_3BandsGenerator_Models.zip#onnx_models" ], "driver-memory": "500m", "driver-memoryOverhead": "1000m", "executor-memory": "1000m", "executor-memoryOverhead": "500m", - "python-memory": "4000m" + "python-memory": "4200m" }, "parameters": [ { From a07a875e3e5a41f0532f5ceae5507787b56d4e7c Mon Sep 17 00:00:00 2001 From: Manu Goudar Date: Fri, 6 Jun 2025 15:24:12 +0200 Subject: [PATCH 12/12] Update file link. --- .../benchmark_scenarios/parcel_delineation.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json index 22dc152..5ab57cd 100644 --- a/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json +++ b/algorithm_catalog/vito/parcel_delineation/benchmark_scenarios/parcel_delineation.json @@ -7,7 +7,7 @@ "process_graph": { "parcel_delineation1": { "process_id": "parcel_delineation", - "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/4edc975de279d2800e5d5367032340090d554761/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json", + "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/b828eecc13f1bd4d012576d746329041cf840c61/algorithm_catalog/vito/parcel_delineation/openeo_udp/parcel_delineation.json", "arguments": { "spatial_extent": { "west": 5.0,