feat(logger): add DatadogLogFormatter and observability provider (#2183)

Author: heitorlessa

aws_lambda_powertools/logging/formatter.py

@@ -139,11 +139,11 @@ def __init__(
         if self.utc:
             self.converter = time.gmtime
 
-        super(LambdaPowertoolsFormatter, self).__init__(datefmt=self.datefmt)
-
         self.keys_combined = {**self._build_default_keys(), **kwargs}
         self.log_format.update(**self.keys_combined)
 
+        super().__init__(datefmt=self.datefmt)
+
     def serialize(self, log: Dict) -> str:
         """Serialize structured log dict to JSON str"""
         return self.json_serializer(log)

aws_lambda_powertools/logging/formatters/__init__.py

@@ -0,0 +1,5 @@
+"""Built-in Logger formatters for Observability Providers that require custom config."""
+
+# NOTE: we don't expose formatters directly (barrel import)
+# as we cannot know if they'll need additional dependencies in the future
+# so we isolate to avoid a performance hit and workarounds like lazy imports

aws_lambda_powertools/logging/formatters/datadog.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from aws_lambda_powertools.logging.formatter import LambdaPowertoolsFormatter
+
+
+class DatadogLogFormatter(LambdaPowertoolsFormatter):
+    def __init__(
+        self,
+        json_serializer: Callable[[dict], str] | None = None,
+        json_deserializer: Callable[[dict | str | bool | int | float], str] | None = None,
+        json_default: Callable[[Any], Any] | None = None,
+        datefmt: str | None = None,
+        use_datetime_directive: bool = False,
+        log_record_order: list[str] | None = None,
+        utc: bool = False,
+        use_rfc3339: bool = True,  # NOTE: The only change from our base formatter
+        **kwargs,
+    ):
+        """Datadog formatter to comply with Datadog log parsing
+
+        Changes compared to the default Logger Formatter:
+
+        - timestamp format to use RFC3339 e.g., "2023-05-01T15:34:26.841+0200"
+
+
+        Parameters
+        ----------
+        log_record_order : list[str] | None, optional
+            _description_, by default None
+
+        Parameters
+        ----------
+        json_serializer : Callable, optional
+            function to serialize `obj` to a JSON formatted `str`, by default json.dumps
+        json_deserializer : Callable, optional
+            function to deserialize `str`, `bytes`, bytearray` containing a JSON document to a Python `obj`,
+            by default json.loads
+        json_default : Callable, optional
+            function to coerce unserializable values, by default str
+
+            Only used when no custom JSON encoder is set
+
+        datefmt : str, optional
+            String directives (strftime) to format log timestamp.
+
+            See https://docs.python.org/3/library/time.html#time.strftime or
+        use_datetime_directive: str, optional
+            Interpret `datefmt` as a format string for `datetime.datetime.strftime`, rather than
+            `time.strftime` - Only useful when used alongside `datefmt`.
+
+            See https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior . This
+            also supports a custom %F directive for milliseconds.
+
+        log_record_order : list, optional
+            set order of log keys when logging, by default ["level", "location", "message", "timestamp"]
+
+        utc : bool, optional
+            set logging timestamp to UTC, by default False to continue to use local time as per stdlib
+        use_rfc3339: bool, optional
+            Whether to use a popular dateformat that complies with both RFC3339 and ISO8601.
+            e.g., 2022-10-27T16:27:43.738+02:00.
+        kwargs
+            Key-value to persist in all log messages
+        """
+        super().__init__(
+            json_serializer=json_serializer,
+            json_deserializer=json_deserializer,
+            json_default=json_default,
+            datefmt=datefmt,
+            use_datetime_directive=use_datetime_directive,
+            log_record_order=log_record_order,
+            utc=utc,
+            use_rfc3339=use_rfc3339,
+            **kwargs,
+        )

docs/core/logger.md

@@ -445,6 +445,26 @@ If you prefer configuring it separately, or you'd want to bring this JSON Format
 --8<-- "examples/logger/src/powertools_formatter_setup.py"
 ```
 
+### Observability providers
+
+!!! note "In this context, an observability provider is an [AWS Lambda Partner](https://go.aws/3HtU6CZ){target="_blank"} offering a platform for logging, metrics, traces, etc."
+
+You can send logs to the observability provider of your choice via [Lambda Extensions](https://aws.amazon.com/blogs/compute/using-aws-lambda-extensions-to-send-logs-to-custom-destinations/){target="_blank"}. In most cases, you shouldn't need any custom Logger configuration, and logs will be shipped async without any performance impact.
+
+#### Built-in formatters
+
+In rare circumstances where JSON logs are not parsed correctly by your provider, we offer built-in formatters to make this transition easier.
+
+| Provider | Formatter             | Notes                                                |
+| -------- | --------------------- | ---------------------------------------------------- |
+| Datadog  | `DatadogLogFormatter` | Modifies default timestamp to use RFC3339 by default |
+
+You can use import and use them as any other Logger formatter via `logger_formatter` parameter:
+
+```python hl_lines="2 4" title="Using built-in Logger Formatters"
+--8<-- "examples/logger/src/observability_provider_builtin_formatters.py"
+```
+
 ### Migrating from other Loggers
 
 If you're migrating from other Loggers, there are few key points to be aware of: [Service parameter](#the-service-parameter), [Inheriting Loggers](#inheriting-loggers), [Overriding Log records](#overriding-log-records), and [Logging exceptions](#logging-exceptions).

examples/logger/src/observability_provider_builtin_formatters.py

@@ -0,0 +1,5 @@
+from aws_lambda_powertools import Logger
+from aws_lambda_powertools.logging.formatters.datadog import DatadogLogFormatter
+
+logger = Logger(service="payment", logger_formatter=DatadogLogFormatter())
+logger.info("hello")

tests/functional/test_logger_powertools_formatter.py

@@ -3,12 +3,14 @@
 import json
 import os
 import random
+import re
 import string
 import time
 
 import pytest
 
 from aws_lambda_powertools import Logger
+from aws_lambda_powertools.logging.formatters.datadog import DatadogLogFormatter
 
 
 @pytest.fixture
@@ -22,6 +24,10 @@ def service_name():
     return "".join(random.SystemRandom().choice(chars) for _ in range(15))
 
 
+def capture_logging_output(stdout):
+    return json.loads(stdout.getvalue().strip())
+
+
 @pytest.mark.parametrize("level", ["DEBUG", "WARNING", "ERROR", "INFO", "CRITICAL"])
 def test_setup_with_valid_log_levels(stdout, level, service_name):
     logger = Logger(service=service_name, level=level, stream=stdout, request_id="request id!", another="value")
@@ -309,3 +315,17 @@ def test_log_json_pretty_indent(stdout, service_name, monkeypatch):
     # THEN the json should contain more than line
     new_lines = stdout.getvalue().count(os.linesep)
     assert new_lines > 1
+
+
+def test_datadog_formatter_use_rfc3339_date(stdout, service_name):
+    # GIVEN Datadog Log Formatter is used
+    logger = Logger(service=service_name, stream=stdout, logger_formatter=DatadogLogFormatter())
+    RFC3339_REGEX = r"^((?:(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2}(?:\.\d+)?))(Z|[\+-]\d{2}:\d{2})?)$"
+
+    # WHEN a log statement happens
+    logger.info({})
+
+    # THEN the timestamp uses RFC3339 by default
+    log = capture_logging_output(stdout)
+
+    assert re.fullmatch(RFC3339_REGEX, log["timestamp"])  # "2022-10-27T17:42:26.841+0200"