refactor(metrics): optimize validation and serialization (#307)

* feat: allow metric validation to be disabled

* refactor: use native python validation over JSON Schema

* test: adjust perf bar to flaky/CI machines

* chore: flaky CI

* chore: update lock to test flaky CI build

* chore: ignore 3.6 to test flaky CI

* chore: re-add 3.6 to test flaky CI

Author: heitorlessa

aws_lambda_powertools/metrics/base.py

@@ -9,14 +9,12 @@
 
 from ..shared import constants
 from ..shared.functions import resolve_env_var_choice
-from ..shared.lazy_import import LazyLoader
 from .exceptions import MetricUnitError, MetricValueError, SchemaValidationError
-from .schema import CLOUDWATCH_EMF_SCHEMA
 
-fastjsonschema = LazyLoader("fastjsonschema", globals(), "fastjsonschema")
 logger = logging.getLogger(__name__)
 
 MAX_METRICS = 100
+MAX_DIMENSIONS = 9
 
 
 class MetricUnit(Enum):
@@ -180,6 +178,12 @@ def serialize_metric_set(self, metrics: Dict = None, dimensions: Dict = None, me
         if self.service and not self.dimension_set.get("service"):
             self.dimension_set["service"] = self.service
 
+        if len(metrics) == 0:
+            raise SchemaValidationError("Must contain at least one metric.")
+
+        if self.namespace is None:
+            raise SchemaValidationError("Must contain a metric namespace.")
+
         logger.debug({"details": "Serializing metrics", "metrics": metrics, "dimensions": dimensions})
 
         metric_names_and_units: List[Dict[str, str]] = []  # [ { "Name": "metric_name", "Unit": "Count" } ]
@@ -209,12 +213,6 @@ def serialize_metric_set(self, metrics: Dict = None, dimensions: Dict = None, me
             **metric_names_and_values,  # "single_metric": 1.0
         }
 
-        try:
-            logger.debug("Validating serialized metrics against CloudWatch EMF schema")
-            fastjsonschema.validate(definition=CLOUDWATCH_EMF_SCHEMA, data=embedded_metrics_object)
-        except fastjsonschema.JsonSchemaException as e:
-            message = f"Invalid format. Error: {e.message}, Invalid item: {e.name}"  # noqa: B306, E501
-            raise SchemaValidationError(message)
         return embedded_metrics_object
 
     def add_dimension(self, name: str, value: str):
@@ -234,7 +232,10 @@ def add_dimension(self, name: str, value: str):
             Dimension value
         """
         logger.debug(f"Adding dimension: {name}:{value}")
-
+        if len(self.dimension_set) == 9:
+            raise SchemaValidationError(
+                f"Maximum number of dimensions exceeded ({MAX_DIMENSIONS}): Unable to add dimension {name}."
+            )
         # Cast value to str according to EMF spec
         # Majority of values are expected to be string already, so
         # checking before casting improves performance in most cases
@@ -295,7 +296,7 @@ def __extract_metric_unit_value(self, unit: Union[str, MetricUnit]) -> str:
             if unit in self._metric_unit_options:
                 unit = MetricUnit[unit].value
 
-            if unit not in self._metric_units:  # str correta
+            if unit not in self._metric_units:
                 raise MetricUnitError(
                     f"Invalid metric unit '{unit}', expected either option: {self._metric_unit_options}"
                 )

aws_lambda_powertools/metrics/metric.py

@@ -102,8 +102,12 @@ def single_metric(name: str, unit: MetricUnit, value: float, namespace: str = No
 
     Raises
     ------
-    e
-        Propagate error received
+    MetricUnitError
+        When metric metric isn't supported by CloudWatch
+    MetricValueError
+        When metric value isn't a number
+    SchemaValidationError
+        When metric object fails EMF schema validation
     """
     metric_set = None
     try:
@@ -112,4 +116,4 @@ def single_metric(name: str, unit: MetricUnit, value: float, namespace: str = No
         yield metric
         metric_set: Dict = metric.serialize_metric_set()
     finally:
-        print(json.dumps(metric_set))
+        print(json.dumps(metric_set, separators=(",", ":")))

aws_lambda_powertools/metrics/metrics.py

@@ -57,13 +57,19 @@ def do_something():
 
     Parameters
     ----------
-    MetricManager : MetricManager
-        Inherits from `aws_lambda_powertools.metrics.base.MetricManager`
+    service : str, optional
+        service name to be used as metric dimension, by default "service_undefined"
+    namespace : str
+        Namespace for metrics
 
     Raises
     ------
-    e
-        Propagate error received
+    MetricUnitError
+        When metric metric isn't supported by CloudWatch
+    MetricValueError
+        When metric value isn't a number
+    SchemaValidationError
+        When metric object fails EMF schema validation
     """
 
     _metrics = {}
@@ -150,7 +156,7 @@ def decorate(event, context):
                 else:
                     metrics = self.serialize_metric_set()
                     self.clear_metrics()
-                    print(json.dumps(metrics))
+                    print(json.dumps(metrics, separators=(",", ":")))
 
             return response
 

aws_lambda_powertools/metrics/schema.py

@@ -77,7 +77,7 @@ def metadata() -> Dict[str, str]:
 
 
 @pytest.fixture
-def a_hundred_metrics(namespace=namespace) -> List[Dict[str, str]]:
+def a_hundred_metrics() -> List[Dict[str, str]]:
     return [{"name": f"metric_{i}", "unit": "Count", "value": 1} for i in range(100)]
 
 
@@ -257,7 +257,7 @@ def test_schema_validation_no_namespace(metric, dimension):
     # GIVEN we don't add any namespace
     # WHEN we attempt to serialize a valid EMF object
     # THEN it should fail namespace validation
-    with pytest.raises(SchemaValidationError, match=".*Namespace must be string"):
+    with pytest.raises(SchemaValidationError, match="Must contain a metric namespace."):
         with single_metric(**metric) as my_metric:
             my_metric.add_dimension(**dimension)
 
@@ -278,7 +278,7 @@ def test_schema_no_metrics(service, namespace):
     my_metrics = Metrics(service=service, namespace=namespace)
 
     # THEN it should fail validation and raise SchemaValidationError
-    with pytest.raises(SchemaValidationError, match=".*Metrics must contain at least 1 items"):
+    with pytest.raises(SchemaValidationError, match="Must contain at least one metric."):
         my_metrics.serialize_metric_set()
 
 
@@ -288,7 +288,7 @@ def test_exceed_number_of_dimensions(metric, namespace):
 
     # WHEN we attempt to serialize them into a valid EMF object
     # THEN it should fail validation and raise SchemaValidationError
-    with pytest.raises(SchemaValidationError, match="must contain less than or equal to 9 items"):
+    with pytest.raises(SchemaValidationError, match="Maximum number of dimensions exceeded.*"):
         with single_metric(**metric, namespace=namespace) as my_metric:
             for dimension in dimensions:
                 my_metric.add_dimension(**dimension)
@@ -328,7 +328,7 @@ def lambda_handler(evt, context):
 
     # THEN the raised exception should be SchemaValidationError
     # and specifically about the lack of Metrics
-    with pytest.raises(SchemaValidationError, match=".*Metrics must contain at least 1 items"):
+    with pytest.raises(SchemaValidationError, match="Must contain at least one metric."):
         lambda_handler({}, {})
 
 

poetry.lock

@@ -0,0 +1,18 @@
+import time
+from contextlib import contextmanager
+from typing import Generator
+
+
+@contextmanager
+def timing() -> Generator:
+    """ "Generator to quickly time operations. It can add 5ms so take that into account in elapsed time
+
+    Examples
+    --------
+
+        with timing() as t:
+            print("something")
+        elapsed = t()
+    """
+    start = time.perf_counter()
+    yield lambda: time.perf_counter() - start  # gen as lambda to calculate elapsed time

tests/functional/test_metrics.py

@@ -0,0 +1,87 @@
+import json
+import time
+from contextlib import contextmanager
+from typing import Dict, Generator
+
+import pytest
+
+from aws_lambda_powertools import Metrics
+from aws_lambda_powertools.metrics import MetricUnit
+from aws_lambda_powertools.metrics import metrics as metrics_global
+
+# adjusted for slower machines in CI too
+METRICS_VALIDATION_SLA: float = 0.0013
+METRICS_SERIALIZATION_SLA: float = 0.0013
+
+
+@contextmanager
+def timing() -> Generator:
+    """ "Generator to quickly time operations. It can add 5ms so take that into account in elapsed time
+
+    Examples
+    --------
+
+        with timing() as t:
+            print("something")
+        elapsed = t()
+    """
+    start = time.perf_counter()
+    yield lambda: time.perf_counter() - start  # gen as lambda to calculate elapsed time
+
+
+@pytest.fixture(scope="function", autouse=True)
+def reset_metric_set():
+    metrics = Metrics()
+    metrics.clear_metrics()
+    metrics_global.is_cold_start = True  # ensure each test has cold start
+    yield
+
+
+@pytest.fixture
+def namespace() -> str:
+    return "test_namespace"
+
+
+@pytest.fixture
+def metric() -> Dict[str, str]:
+    return {"name": "single_metric", "unit": MetricUnit.Count, "value": 1}
+
+
+def add_max_metrics_before_serialization(metrics_instance: Metrics):
+    metrics_instance.add_dimension(name="test_dimension", value="test")
+
+    for i in range(99):
+        metrics_instance.add_metric(name=f"metric_{i}", unit="Count", value=1)
+
+
+@pytest.mark.perf
+def test_metrics_large_operation_without_json_serialization_sla(namespace):
+    # GIVEN Metrics is initialized
+    my_metrics = Metrics(namespace=namespace)
+
+    # WHEN we add and serialize 99 metrics
+    with timing() as t:
+        add_max_metrics_before_serialization(metrics_instance=my_metrics)
+        my_metrics.serialize_metric_set()
+
+    # THEN completion time should be below our validation SLA
+    elapsed = t()
+    if elapsed > METRICS_VALIDATION_SLA:
+        pytest.fail(f"Metric validation should be below {METRICS_VALIDATION_SLA}s: {elapsed}")
+
+
+@pytest.mark.perf
+def test_metrics_large_operation_and_json_serialization_sla(namespace):
+    # GIVEN Metrics is initialized with validation disabled
+    my_metrics = Metrics(namespace=namespace)
+
+    # WHEN we add and serialize 99 metrics
+    with timing() as t:
+        add_max_metrics_before_serialization(metrics_instance=my_metrics)
+        metrics = my_metrics.serialize_metric_set()
+        print(json.dumps(metrics, separators=(",", ":")))
+
+    # THEN completion time should be below our serialization SLA
+    elapsed = t()
+    if elapsed > METRICS_SERIALIZATION_SLA:
+        pytest.fail(f"Metric serialization should be below {METRICS_SERIALIZATION_SLA}s: {elapsed}")