Issue #668 support federation extension on job and service listings

Author: soxofaan

docs/api.rst

@@ -79,6 +79,9 @@ openeo.rest.models
 .. automodule:: openeo.rest.models.federation_extension
     :members: FederationExtension
 
+.. automodule:: openeo.rest.models.logs
+    :members: LogEntry, normalize_log_level
+
 
 openeo.api.process
 --------------------
@@ -87,12 +90,6 @@ openeo.api.process
     :members: Parameter
 
 
-openeo.api.logs
------------------
-
-.. automodule:: openeo.api.logs
-    :members: LogEntry, normalize_log_level
-
 
 openeo.rest.connection
 ----------------------

openeo/api/logs.py

@@ -1,99 +1,13 @@
-import logging
-from typing import Optional, Union
+import warnings
 
+from openeo.internal.warnings import UserDeprecationWarning
+from openeo.rest.models.logs import LogEntry, log_level_name, normalize_log_level
 
-class LogEntry(dict):
-    """
-    Log message and info for jobs and services
+warnings.warn(
+    message="Submodule `openeo.api.logs` is deprecated in favor of `openeo.rest.models.logs`.",
+    category=UserDeprecationWarning,
+    stacklevel=2,
+)
 
-    Fields:
-        - ``id``: Unique ID for the log, string, REQUIRED
-        - ``code``: Error code, string, optional
-        - ``level``: Severity level, string (error, warning, info or debug), REQUIRED
-        - ``message``: Error message, string, REQUIRED
-        - ``time``: Date and time of the error event as RFC3339 date-time, string, available since API 1.1.0
-        - ``path``: A "stack trace" for the process, array of dicts
-        - ``links``: Related links, array of dicts
-        - ``usage``: Usage metrics available as property 'usage', dict, available since API 1.1.0
-          May contain the following metrics: cpu, memory, duration, network, disk, storage and other custom ones
-          Each of the metrics is also a dict with the following parts: value (numeric) and unit (string)
-        - ``data``: Arbitrary data the user wants to "log" for debugging purposes.
-          Please note that this property may not exist as there's a difference
-          between None and non-existing. None for example refers to no-data in
-          many cases while the absence of the property means that the user did
-          not provide any data for debugging.
-    """
 
-    _required = {"id", "level", "message"}
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        # Check required fields
-        missing = self._required.difference(self.keys())
-        if missing:
-            raise ValueError("Missing required fields: {m}".format(m=sorted(missing)))
-
-    @property
-    def id(self):
-        return self["id"]
-
-    # Legacy alias
-    log_id = id
-
-    @property
-    def message(self):
-        return self["message"]
-
-    @property
-    def level(self):
-        return self["level"]
-
-    # TODO: add properties for "code", "time", "path", "links" and "data" with sensible defaults?
-
-
-def normalize_log_level(
-    log_level: Union[int, str, None], default: int = logging.DEBUG
-) -> int:
-    """
-    Helper function to convert a openEO API log level (e.g. string "error")
-    to the integer constants defined in Python's standard library ``logging`` module (e.g. ``logging.ERROR``).
-
-    :param log_level: log level to normalize: a log level string in the style of
-        the openEO API ("error", "warning", "info", or "debug"),
-        an integer value (e.g. a ``logging`` constant), or ``None``.
-
-    :param default: fallback log level to return on unknown log level strings or ``None`` input.
-
-    :raises TypeError: when log_level is any other type than str, an int or None.
-    :return: One of the following log level constants from the standard module ``logging``:
-        ``logging.ERROR``, ``logging.WARNING``, ``logging.INFO``, or ``logging.DEBUG`` .
-    """
-    if isinstance(log_level, str):
-        log_level = log_level.upper()
-        if log_level in ["CRITICAL", "ERROR", "FATAL"]:
-            return logging.ERROR
-        elif log_level in ["WARNING", "WARN"]:
-            return logging.WARNING
-        elif log_level == "INFO":
-            return logging.INFO
-        elif log_level == "DEBUG":
-            return logging.DEBUG
-        else:
-            return default
-    elif isinstance(log_level, int):
-        return log_level
-    elif log_level is None:
-        return default
-    else:
-        raise TypeError(
-            f"Value for log_level is not an int or str: type={type(log_level)}, value={log_level!r}"
-        )
-
-
-def log_level_name(log_level: Union[int, str, None]) -> str:
-    """
-    Get the name of a normalized log level.
-    This value conforms to log level names used in the openEO API.
-    """
-    return logging.getLevelName(normalize_log_level(log_level)).lower()
+__all__ = ["LogEntry", "normalize_log_level", "log_level_name"]

openeo/rest/job.py

@@ -10,7 +10,6 @@
 
 import requests
 
-from openeo.api.logs import LogEntry, log_level_name, normalize_log_level
 from openeo.internal.documentation import openeo_endpoint
 from openeo.internal.jupyter import (
     VisualDict,
@@ -26,6 +25,8 @@
     OpenEoApiPlainError,
     OpenEoClientException,
 )
+from openeo.rest.models.general import LogsResponse
+from openeo.rest.models.logs import LogEntry, log_level_name, normalize_log_level
 from openeo.util import ensure_dir
 
 if typing.TYPE_CHECKING:
@@ -184,9 +185,7 @@ def get_results(self) -> JobResults:
         """
         return JobResults(job=self)
 
-    def logs(
-        self, offset: Optional[str] = None, level: Optional[Union[str, int]] = None
-    ) -> List[LogEntry]:
+    def logs(self, offset: Optional[str] = None, level: Union[str, int, None] = None) -> LogsResponse:
         """Retrieve job logs.
 
         :param offset: The last identifier (property ``id`` of a LogEntry) the client has received.
@@ -217,22 +216,8 @@ def logs(
             params["offset"] = offset
         if level is not None:
             params["level"] = log_level_name(level)
-        response = self.connection.get(url, params=params, expected_status=200)
-        logs = response.json()["logs"]
-
-        # Only filter logs when specified.
-        # We should still support client-side log_level filtering because not all backends
-        # support the minimum log level parameter.
-        if level is not None:
-            log_level = normalize_log_level(level)
-            logs = (
-                log
-                for log in logs
-                if normalize_log_level(log.get("level")) >= log_level
-            )
-
-        entries = [LogEntry(log) for log in logs]
-        return VisualList("logs", data=entries)
+        response_data = self.connection.get(url, params=params, expected_status=200).json()
+        return LogsResponse(response_data=response_data, log_level=level)
 
     def run_synchronous(
         self,

openeo/rest/models/federation_extension.py

@@ -20,14 +20,23 @@ def missing(self) -> Union[List[str], None]:
         Get the ``federation:missing`` property (if any) of the resource,
         which lists back-ends that were not available during the request.
 
+        Example usage with collection listing request
+        (using :py:meth:`~openeo.rest.connection.Connection.list_collections()`):
+
+        .. code-block:: pycon
+
+            >>> collections = connection.list_collections()
+            >>> collections.ext_federation.missing
+            ["backend1"]
+
         :return: list of back-end IDs that were not available.
             Or None, when ``federation:missing`` is not present in response.
         """
         return self._data.get("federation:missing", None)
 
     def warn_on_missing(self, resource_name: str) -> None:
         """
-        Warn about presence of ``federation:missing`` in the resource.
+        Warn about presence of non-empty ``federation:missing`` in the resource.
         """
         missing = self.missing
         if missing:

openeo/rest/models/general.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 
+import functools
 from dataclasses import dataclass
 from typing import List, Optional, Union
 
 from openeo.internal.jupyter import render_component
 from openeo.rest.models.federation_extension import FederationExtension
+from openeo.rest.models.logs import LogEntry, normalize_log_level
 
 
 @dataclass(frozen=True)
@@ -34,12 +36,15 @@ class CollectionListingResponse(list):
     from a ``GET /collections`` request.
 
     .. note::
-        This object mimics a simple list of collection metadata dictionaries,
+        This object mimics, for backward compatibility reasons,
+        the interface of simple list of collection metadata dictionaries (``List[dict]``),
         which was the original return API of
         :py:meth:`~openeo.rest.connection.Connection.list_collections()`,
         but now also provides methods/properties to access additional response data.
 
     :param response_data: response data from a ``GET /collections`` request
+    :param warn_on_federation_missing: whether to automatically warn
+        about missing federation components.
 
     .. seealso:: :py:meth:`openeo.rest.connection.Connection.list_collections()`
 
@@ -75,12 +80,14 @@ class ProcessListingResponse(list):
     from a ``GET /processes`` request.
 
     .. note::
-        This object mimics a simple list of collection metadata dictionaries,
-        which was the original return API of
+        This object mimics, for backward compatibility reasons,
+        the interface of simple list of process metadata dictionaries (``List[dict]``),
         :py:meth:`~openeo.rest.connection.Connection.list_processes()`,
         but now also provides methods/properties to access additional response data.
 
     :param response_data: response data from a ``GET /processes`` request
+    :param warn_on_federation_missing: whether to automatically warn
+        about missing federation components.
 
     .. seealso:: :py:meth:`openeo.rest.connection.Connection.list_processes()`
 
@@ -118,12 +125,15 @@ class JobListingResponse(list):
     from a ``GET /jobs`` request.
 
     .. note::
-        This object mimics a simple ``List[dict]`` with job metadata,
+        This object mimics, for backward compatibility reasons,
+        the interface of simple list of job metadata dictionaries (``List[dict]``),
         which was the original return API of
         :py:meth:`~openeo.rest.connection.Connection.list_jobs()`,
         but now also provides methods/properties to access additional response data.
 
     :param response_data: response data from a ``GET /jobs`` request
+    :param warn_on_federation_missing: whether to automatically warn
+        about missing federation components.
 
     .. seealso:: :py:meth:`openeo.rest.connection.Connection.list_jobs()`
 
@@ -151,3 +161,76 @@ def links(self) -> List[Link]:
     def ext_federation(self) -> FederationExtension:
         """Accessor for federation extension data related to this resource."""
         return FederationExtension(self._data)
+
+
+class LogsResponse(list):
+    """
+    Container for job/service logs as received
+    from a ``GET /jobs/{job_id}/logs`` or ``GET /services/{service_id}/logs`` request.
+
+    .. note::
+        This object mimics, for backward compatibility reasons,
+        the interface of a simple list (``List[LogEntry]``)
+        which was the original return API of
+        :py:meth:`~openeo.rest.job.BatchJob.logs()`
+        and :py:meth:`~openeo.rest.service.Service.logs()`,
+        but now also provides methods/properties to access additional response data.
+
+    :param response_data: response data from a ``GET /jobs/{job_id}/logs``
+        or ``GET /services/{service_id}/logs`` request.
+    :param warn_on_federation_missing: whether to automatically warn
+        about missing federation components.
+
+    .. seealso:: :py:meth:`~openeo.rest.job.BatchJob.logs()`
+        and :py:meth:`~openeo.rest.service.Service.logs()`
+
+    .. versionadded:: 0.38.0
+    """
+
+    __slots__ = ["_data"]
+
+    def __init__(
+        self, response_data: dict, *, log_level: Optional[str] = None, warn_on_federation_missing: bool = True
+    ):
+        self._data = response_data
+
+        logs = response_data.get("logs", [])
+        # Extra client-side level filtering (in case the back-end does not support that)
+        if log_level:
+
+            @functools.lru_cache
+            def accept_level(level: str) -> bool:
+                return normalize_log_level(level) >= normalize_log_level(log_level)
+
+            if (
+                # Backend does not list effective lowest level
+                "level" not in response_data
+                # Or effective lowest level is still too low
+                or not accept_level(response_data["level"])
+            ):
+                logs = (log for log in logs if accept_level(log.get("level")))
+        logs = [LogEntry(log) for log in logs]
+
+        # Mimic original list of process metadata dictionaries
+        super().__init__(logs)
+
+        if warn_on_federation_missing:
+            self.ext_federation.warn_on_missing(resource_name="log listing")
+
+    def _repr_html_(self):
+        return render_component(component="logs", data=self)
+
+    @property
+    def logs(self) -> List[LogEntry]:
+        """Get the log entries."""
+        return self
+
+    @property
+    def links(self) -> List[Link]:
+        """Get links related to this resource."""
+        return [Link.from_dict(d) for d in self._data.get("links", [])]
+
+    @property
+    def ext_federation(self) -> FederationExtension:
+        """Accessor for federation extension data related to this resource."""
+        return FederationExtension(self._data)