improv: Better namespace/dimension handling for Metrics

aws_lambda_powertools/metrics/base.py

@@ -35,7 +35,7 @@ class MetricManager:
 
     Environment variables
     ---------------------
-    POWERTOOLS_SERVICE_NAME : str
+    POWERTOOLS_METRICS_NAMESPACE : str
         metric namespace to be set for all metrics
 
     Raises
@@ -53,7 +53,7 @@ class MetricManager:
     def __init__(self, metric_set: Dict[str, str] = None, dimension_set: Dict = None, namespace: str = None):
         self.metric_set = metric_set if metric_set is not None else {}
         self.dimension_set = dimension_set if dimension_set is not None else {}
-        self.namespace = namespace or os.getenv("POWERTOOLS_SERVICE_NAME")
+        self.namespace = namespace or os.getenv("POWERTOOLS_METRICS_NAMESPACE")
         self._metric_units = [unit.value for unit in MetricUnit]
         self._metric_unit_options = list(MetricUnit.__members__)
 

aws_lambda_powertools/metrics/metric.py

@@ -21,7 +21,7 @@ class SingleMetric(MetricManager):
 
     Environment variables
     ---------------------
-    POWERTOOLS_SERVICE_NAME : str
+    POWERTOOLS_METRICS_NAMESPACE : str
         metric namespace
 
     Example
@@ -30,7 +30,7 @@ class SingleMetric(MetricManager):
 
         from aws_lambda_powertools.metrics import SingleMetric, MetricUnit
         import json
-        metric = Single_Metric(service="ServerlessAirline")
+        metric = Single_Metric(namespace="ServerlessAirline")
 
         metric.add_metric(name="ColdStart", unit=MetricUnit.Count, value=1)
         metric.add_dimension(name="function_version", value=47)
@@ -62,7 +62,7 @@ def add_metric(self, name: str, unit: MetricUnit, value: float):
 
 
 @contextmanager
-def single_metric(name: str, unit: MetricUnit, value: float, service: str = None):
+def single_metric(name: str, unit: MetricUnit, value: float, namespace: str = None):
     """Context manager to simplify creation of a single metric
 
     Example
@@ -71,12 +71,12 @@ def single_metric(name: str, unit: MetricUnit, value: float, service: str = None
 
         from aws_lambda_powertools.metrics import single_metric, MetricUnit
 
-        with single_metric(name="ColdStart", unit=MetricUnit.Count, value=1, service="ServerlessAirline") as metric:
+        with single_metric(name="ColdStart", unit=MetricUnit.Count, value=1, namespace="ServerlessAirline") as metric:
                 metric.add_dimension(name="function_version", value=47)
 
     **Same as above but set namespace using environment variable**
 
-        $ export POWERTOOLS_SERVICE_NAME="ServerlessAirline"
+        $ export POWERTOOLS_METRICS_NAMESPACE="ServerlessAirline"
 
         from aws_lambda_powertools.metrics import single_metric, MetricUnit
 
@@ -91,8 +91,8 @@ def single_metric(name: str, unit: MetricUnit, value: float, service: str = None
         `aws_lambda_powertools.helper.models.MetricUnit`
     value : float
         Metric value
-    service: str
-        Service name used as namespace
+    namespace: str
+        Namespace for metrics
 
     Yields
     -------
@@ -106,7 +106,7 @@ def single_metric(name: str, unit: MetricUnit, value: float, service: str = None
     """
     metric_set = None
     try:
-        metric: SingleMetric = SingleMetric(namespace=service)
+        metric: SingleMetric = SingleMetric(namespace=namespace)
         metric.add_metric(name=name, unit=unit, value=value)
         yield metric
         logger.debug("Serializing single metric")

aws_lambda_powertools/metrics/metrics.py

@@ -1,6 +1,7 @@
 import functools
 import json
 import logging
+import os
 from typing import Any, Callable
 
 from aws_lambda_powertools.metrics.base import MetricManager
@@ -29,10 +30,9 @@ class Metrics(MetricManager):
 
         from aws_lambda_powertools.metrics import Metrics
 
-        metrics = Metrics(service="ServerlessAirline")
+        metrics = Metrics(namespace="ServerlessAirline", service="payment")
         metrics.add_metric(name="ColdStart", unit=MetricUnit.Count, value=1)
         metrics.add_metric(name="BookingConfirmation", unit="Count", value=1)
-        metrics.add_dimension(name="service", value="booking")
         metrics.add_dimension(name="function_version", value="$LATEST")
         ...
 
@@ -47,8 +47,10 @@ def do_something():
 
     Environment variables
     ---------------------
-    POWERTOOLS_SERVICE_NAME : str
+    POWERTOOLS_METRICS_NAMESPACE : str
         metric namespace
+    POWERTOOLS_SERVICE_NAME : str
+        service name used for default dimension
 
     Parameters
     ----------
@@ -64,13 +66,14 @@ def do_something():
     _metrics = {}
     _dimensions = {}
 
-    def __init__(
-        self, service: str = None,
-    ):
+    def __init__(self, service: str = None, namespace: str = None):
         self.metric_set = self._metrics
         self.dimension_set = self._dimensions
-        self.service = service
-        super().__init__(metric_set=self.metric_set, dimension_set=self.dimension_set, namespace=self.service)
+        self.service = service or os.environ.get("POWERTOOLS_SERVICE_NAME")
+        self.namespace = namespace
+        if self.service:
+            self.dimension_set["service"] = self.service
+        super().__init__(metric_set=self.metric_set, dimension_set=self.dimension_set, namespace=self.namespace)
 
     def clear_metrics(self):
         logger.debug("Clearing out existing metric set from memory")

docs/content/core/metrics.mdx

@@ -16,7 +16,7 @@ Metrics creates custom metrics asynchronously via logging metrics to standard ou
 
 ## Initialization
 
-Set `POWERTOOLS_SERVICE_NAME` env var as a start - Here is an example using AWS Serverless Application Model (SAM)
+Set `POWERTOOLS_SERVICE_NAME` and `POWERTOOLS_METRICS_NAMESPACE` env vars as a start - Here is an example using AWS Serverless Application Model (SAM)
 
 ```yaml:title=template.yaml
 Resources:
@@ -27,37 +27,39 @@ Resources:
         Runtime: python3.8
         Environment:
             Variables:
-                POWERTOOLS_SERVICE_NAME: ServerlessAirline # highlight-line
+                POWERTOOLS_SERVICE_NAME: payment # highlight-line
+                POWERTOOLS_METRICS_NAMESPACE: ServerlessAirline # highlight-line
 ```
 
 We recommend you use your application or main service as a metric namespace.
-You can explicitly set a namespace name via `service` param or via `POWERTOOLS_SERVICE_NAME` env var. This sets **namespace** key that will be used for all metrics.
+You can explicitly set a namespace name via `namespace` param or via `POWERTOOLS_METRICS_NAMESPACE` env var. This sets **namespace** key that will be used for all metrics.
+You can also pass a service name via `service` param or `POWERTOOLS_SERVICE_NAME` env var. This will create a dimension with the service name.
 
 ```python:title=app.py
 from aws_lambda_powertools.metrics import Metrics, MetricUnit
 
-# POWERTOOLS_SERVICE_NAME defined
+# POWERTOOLS_METRICS_NAMESPACE and POWERTOOLS_SERVICE_NAME defined
 metrics = Metrics() # highlight-line 
 
 # Explicit definition
-Metrics(service="ServerlessAirline")  # sets namespace to "ServerlessAirline"
+Metrics(namespace="ServerlessAirline", service="orders")  # creates a default dimension {"service": "orders"} under the namespace "ServerlessAirline"
 
 
 ```
 
-You can initialize Metrics anywhere in your code as many time as you need - It'll keep track of your aggregate metrics in memory.
+You can initialize Metrics anywhere in your code as many times as you need - It'll keep track of your aggregate metrics in memory.
 
 ## Creating metrics
 
-You can create metrics using `add_metric`, and set dimensions for all your aggregate metrics using `add_dimension`.
+You can create metrics using `add_metric`, and manually create dimensions for all your aggregate metrics using `add_dimension`.
 
 ```python:title=app.py
 from aws_lambda_powertools.metrics import Metrics, MetricUnit
 
-metrics = Metrics(service="ExampleService")
+metrics = Metrics(namespace="ExampleApplication", service="booking")
 # highlight-start
-metrics.add_metric(name="ColdStart", unit=MetricUnit.Count, value=1)
-metrics.add_dimension(name="service", value="booking")
+metrics.add_dimension(name="environment", value="prod")
+metrics.add_metric(name="SuccessfulBooking", unit=MetricUnit.Count, value=1)
 # highlight-end
 ```
 
@@ -79,7 +81,7 @@ CloudWatch EMF uses the same dimensions across all your metrics. Use `single_met
 ```python:title=single_metric.py
 from aws_lambda_powertools.metrics import MetricUnit, single_metric
 
-with single_metric(name="ColdStart", unit=MetricUnit.Count, value=1, service="ExampleService") as metric: # highlight-line
+with single_metric(name="ColdStart", unit=MetricUnit.Count, value=1, namespace="ExampleApplication") as metric: # highlight-line
     metric.add_dimension(name="function_context", value="$LATEST")
     ...
 ```
@@ -115,15 +117,14 @@ def lambda_handler(evt, ctx):
 ```python:title=lambda_handler_nested_middlewares.py
 from aws_lambda_powertools.metrics import Metrics, MetricUnit
 
-metrics = Metrics(service="ExampleService")
+metrics = Metrics(namespace="ExampleApplication", service="booking")
 metrics.add_metric(name="ColdStart", unit="Count", value=1)
 
 # highlight-start
 @metrics.log_metrics
 @tracer.capture_lambda_handler
 # highlight-end
 def lambda_handler(evt, ctx):
-    metrics.add_dimension(name="service", value="booking")
     metrics.add_metric(name="BookingConfirmation", unit="Count", value=1)
     ...
 ```
@@ -136,9 +137,8 @@ If you prefer not to use `log_metrics` because you might want to encapsulate add
 import json
 from aws_lambda_powertools.metrics import Metrics, MetricUnit
 
-metrics = Metrics(service="ExampleService")
+metrics = Metrics(namespace="ExampleApplication", service="booking")
 metrics.add_metric(name="ColdStart", unit="Count", value=1)
-metrics.add_dimension(name="service", value="booking")
 
 # highlight-start
 your_metrics_object = metrics.serialize_metric_set() 
@@ -149,10 +149,11 @@ print(json.dumps(your_metrics_object))
 
 ## Testing your code
 
-Use `POWERTOOLS_SERVICE_NAME` env var when unit testing your code to ensure a metric namespace object is created, and your code doesn't fail validation.
+Use `POWERTOOLS_METRICS_NAMESPACE` and `POWERTOOLS_SERVICE_NAME` env vars when unit testing your code to ensure metric namespace and dimension objects are created, and your code doesn't fail validation.
 
 ```bash:title=pytest_metric_namespace.sh
-POWERTOOLS_SERVICE_NAME="Example" python -m pytest
+
+POWERTOOLS_SERVICE_NAME="Example" POWERTOOLS_METRICS_NAMESPACE="Application" python -m pytest
 ```
 
-You can ignore this if you are explicitly setting namespace by passing a service name when initializing Metrics: `metrics = Metrics(service=ServiceName)`.
+You can ignore this if you are explicitly setting namespace/default dimension by passing the `namespace` and `service` parameters when initializing Metrics: `metrics = Metrics(namespace=ApplicationName, service=ServiceName)`.

docs/content/index.mdx

@@ -36,7 +36,8 @@ _`*` Core utilities are Tracer, Logger and Metrics. Optional utilities may vary
 
 Environment variable | Description | Utility
 ------------------------------------------------- | --------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | -------------------------------------------------
-**POWERTOOLS_SERVICE_NAME** | Sets service name used for tracing namespace, metrics namespace and structured logging | all
+**POWERTOOLS_SERVICE_NAME** | Sets service name used for tracing namespace, metrics dimension and structured logging | all
+**POWERTOOLS_METRICS_NAMESPACE** | Sets namespace used for metrics | [Metrics](./core/metrics)
 **POWERTOOLS_TRACE_DISABLED** | Disables tracing | [Tracing](./core/tracer)
 **POWERTOOLS_TRACE_MIDDLEWARES** | Creates sub-segment for each custom middleware | [middleware_factory](./utilities/middleware_factory)
 **POWERTOOLS_LOGGER_LOG_EVENT** | Logs incoming event | [Logging](./core/logger)

example/README.md

@@ -9,9 +9,9 @@ This example uses both [tracing](https://github.com/awslabs/aws-lambda-powertool
 * **Deploy**: `sam deploy --guided`
 * **Unit Tests**: We recommend proceeding with the following commands in a virtual environment
   - **Install deps**: `pip install -r hello_world/requirements.txt && pip install -r requirements-dev.txt`
-  - **Run tests with tracing disabled and namespace set**
-    - `POWERTOOLS_SERVICE_NAME="Example" POWERTOOLS_TRACE_DISABLED=1 python -m pytest`
-    - Both are necessary because `app.py` initializes them in the global scope, since both Tracer and Metrics will be initialized and configured during import time. For unit tests, we could always patch and explicitly config but env vars do just fine for this example.
+  - **Run tests with namespace and service set, and tracing disabled**
+    - `POWERTOOLS_METRICS_NAMESPACE="Example" POWERTOOLS_SERVICE_NAME="Example" POWERTOOLS_TRACE_DISABLED=1 python -m pytest`
+    - These are necessary because `app.py` initializes them in the global scope, since both Tracer and Metrics will be initialized and configured during import time. For unit tests, we could always patch and explicitly config but env vars do just fine for this example.
 
 # Example code
 

example/template.yaml

@@ -24,6 +24,7 @@ Resources:
           POWERTOOLS_TRACE_DISABLED: "false"          # Explicitly disables tracing, default
           POWERTOOLS_LOGGER_LOG_EVENT: "false"        # Logs incoming event, default
           POWERTOOLS_LOGGER_SAMPLE_RATE: "0"          # Debug log sampling percentage, default
+          POWERTOOLS_METRICS_NAMESPACE: "Example"     # Metric Namespace
           LOG_LEVEL: INFO                             # Log level for Logger (INFO, DEBUG, etc.), default
       Events:
         HelloWorld:

tests/functional/test_metrics.py

@@ -162,25 +162,51 @@ def lambda_handler(evt, ctx):
 
 
 def test_namespace_env_var(monkeypatch, capsys, metric, dimension, namespace):
-    # GIVEN we use POWERTOOLS_SERVICE_NAME
-    monkeypatch.setenv("POWERTOOLS_SERVICE_NAME", namespace["name"])
+    # GIVEN we use POWERTOOLS_METRICS_NAMESPACE
+    monkeypatch.setenv("POWERTOOLS_METRICS_NAMESPACE", namespace["name"])
 
     # WHEN creating a metric but don't explicitly
     # add a namespace
     with single_metric(**metric) as my_metrics:
         my_metrics.add_dimension(**dimension)
-        monkeypatch.delenv("POWERTOOLS_SERVICE_NAME")
+        monkeypatch.delenv("POWERTOOLS_METRICS_NAMESPACE")
 
     output = json.loads(capsys.readouterr().out.strip())
     expected = serialize_single_metric(metric=metric, dimension=dimension, namespace=namespace)
 
     remove_timestamp(metrics=[output, expected])  # Timestamp will always be different
 
     # THEN we should add a namespace implicitly
-    # with the value of POWERTOOLS_SERVICE_NAME env var
+    # with the value of POWERTOOLS_METRICS_NAMESPACE env var
     assert expected["_aws"] == output["_aws"]
 
 
+def test_service_env_var(monkeypatch, capsys, metric, namespace):
+    # GIVEN we use POWERTOOLS_SERVICE_NAME
+    monkeypatch.setenv("POWERTOOLS_SERVICE_NAME", "test_service")
+    my_metrics = Metrics(namespace=namespace["name"])
+
+    # WHEN creating a metric but don't explicitly
+    # add a dimension
+    @my_metrics.log_metrics
+    def lambda_handler(evt, context):
+        my_metrics.add_metric(**metric)
+        return True
+
+    lambda_handler({}, {})
+
+    monkeypatch.delenv("POWERTOOLS_SERVICE_NAME")
+
+    output = json.loads(capsys.readouterr().out.strip())
+    expected_dimension = {"name": "service", "value": "test_service"}
+    expected = serialize_single_metric(metric=metric, dimension=expected_dimension, namespace=namespace)
+
+    remove_timestamp(metrics=[output, expected])  # Timestamp will always be different
+
+    # THEN metrics should be logged using the implicitly created "service" dimension
+    assert expected == output
+
+
 def test_metrics_spillover(monkeypatch, capsys, metric, dimension, namespace, a_hundred_metrics):
     # GIVEN Metrics is initialized and we have over a hundred metrics to add
     my_metrics = Metrics()
@@ -243,8 +269,8 @@ def test_incorrect_metric_unit(metric, dimension, namespace):
 
 
 def test_schema_no_namespace(metric, dimension):
-    # GIVEN we don't add any metric or dimension
-    # but a namespace
+    # GIVEN we add any metric or dimension
+    # but no namespace
 
     # WHEN we attempt to serialize a valid EMF object
     # THEN it should fail validation and raise SchemaValidationError
@@ -421,9 +447,9 @@ def test_add_namespace_warns_for_deprecation(capsys, metrics, dimensions, namesp
         my_metrics.add_namespace(**namespace)
 
 
-def test_log_metrics_with_explicit_service(capsys, metrics, dimensions):
+def test_log_metrics_with_explicit_namespace(capsys, metrics, dimensions, namespace):
     # GIVEN Metrics is initialized with service specified
-    my_metrics = Metrics(service="test_service")
+    my_metrics = Metrics(service="test_service", namespace=namespace["name"])
     for metric in metrics:
         my_metrics.add_metric(**metric)
     for dimension in dimensions:
@@ -438,18 +464,76 @@ def lambda_handler(evt, ctx):
     lambda_handler({}, {})
 
     output = json.loads(capsys.readouterr().out.strip())
-    expected = serialize_metrics(metrics=metrics, dimensions=dimensions, namespace={"name": "test_service"})
+
+    dimensions.insert(0, {"name": "service", "value": "test_service"})
+    expected = serialize_metrics(metrics=metrics, dimensions=dimensions, namespace=namespace)
 
     remove_timestamp(metrics=[output, expected])  # Timestamp will always be different
 
     # THEN we should have no exceptions and the namespace should be set to the name provided in the
     # service passed to Metrics constructor
-    assert expected["_aws"] == output["_aws"]
+    assert expected == output
 
 
-def test_log_metrics_with_namespace_overridden(capsys, metrics, dimensions):
+def test_log_metrics_with_implicit_dimensions(capsys, metrics):
+    # GIVEN Metrics is initialized with service specified
+    my_metrics = Metrics(service="test_service", namespace="test_application")
+    for metric in metrics:
+        my_metrics.add_metric(**metric)
+
+    # WHEN we utilize log_metrics to serialize and don't explicitly add any dimensions
+    @my_metrics.log_metrics
+    def lambda_handler(evt, ctx):
+        return True
+
+    lambda_handler({}, {})
+
+    output = json.loads(capsys.readouterr().out.strip())
+
+    expected_dimensions = [{"name": "service", "value": "test_service"}]
+    expected = serialize_metrics(
+        metrics=metrics, dimensions=expected_dimensions, namespace={"name": "test_application"}
+    )
+
+    remove_timestamp(metrics=[output, expected])  # Timestamp will always be different
+
+    # THEN we should have no exceptions and the dimensions should be set to the name provided in the
+    # service passed to Metrics constructor
+    assert expected == output
+
+
+def test_log_metrics_with_renamed_service(capsys, metrics):
     # GIVEN Metrics is initialized with service specified
-    my_metrics = Metrics(service="test_service")
+    my_metrics = Metrics(service="test_service", namespace="test_application")
+    for metric in metrics:
+        my_metrics.add_metric(**metric)
+
+    # WHEN we manually call add_dimension to change the value of the service dimension
+    my_metrics.add_dimension(name="service", value="another_test_service")
+
+    @my_metrics.log_metrics
+    def lambda_handler(evt, ctx):
+        return True
+
+    lambda_handler({}, {})
+
+    output = json.loads(capsys.readouterr().out.strip())
+
+    expected_dimensions = [{"name": "service", "value": "test_service"}]
+    expected = serialize_metrics(
+        metrics=metrics, dimensions=expected_dimensions, namespace={"name": "test_application"}
+    )
+
+    remove_timestamp(metrics=[output, expected])  # Timestamp will always be different
+
+    # THEN we should have no exceptions and the dimensions should be set to the name provided in the
+    # add_dimension call
+    assert output["service"] == "another_test_service"
+
+
+def test_log_metrics_with_namespace_overridden(capsys, metrics, dimensions):
+    # GIVEN Metrics is initialized with namespace specified
+    my_metrics = Metrics(namespace="test_service")
     for metric in metrics:
         my_metrics.add_metric(**metric)
     for dimension in dimensions:
@@ -470,10 +554,10 @@ def lambda_handler(evt, ctx):
 
 
 def test_single_metric_with_service(capsys, metric, dimension):
-    # GIVEN we pass service parameter to single_metric
+    # GIVEN we pass namespace parameter to single_metric
 
     # WHEN creating a metric
-    with single_metric(**metric, service="test_service") as my_metrics:
+    with single_metric(**metric, namespace="test_service") as my_metrics:
         my_metrics.add_dimension(**dimension)
 
     output = json.loads(capsys.readouterr().out.strip())
@@ -485,17 +569,17 @@ def test_single_metric_with_service(capsys, metric, dimension):
     assert expected["_aws"] == output["_aws"]
 
 
-def test_namespace_var_precedence(monkeypatch, capsys, metric, dimension):
-    # GIVEN we use POWERTOOLS_SERVICE_NAME
-    monkeypatch.setenv("POWERTOOLS_SERVICE_NAME", "test_service_env_var")
+def test_namespace_var_precedence(monkeypatch, capsys, metric, dimension, namespace):
+    # GIVEN we use POWERTOOLS_METRICS_NAMESPACE
+    monkeypatch.setenv("POWERTOOLS_METRICS_NAMESPACE", namespace["name"])
 
-    # WHEN creating a metric and explicitly set a service name
-    with single_metric(**metric, service="test_service_explicit") as my_metrics:
+    # WHEN creating a metric and explicitly set a namespace
+    with single_metric(**metric, namespace=namespace["name"]) as my_metrics:
         my_metrics.add_dimension(**dimension)
-        monkeypatch.delenv("POWERTOOLS_SERVICE_NAME")
+        monkeypatch.delenv("POWERTOOLS_METRICS_NAMESPACE")
 
     output = json.loads(capsys.readouterr().out.strip())
-    expected = serialize_single_metric(metric=metric, dimension=dimension, namespace={"name": "test_service_explicit"})
+    expected = serialize_single_metric(metric=metric, dimension=dimension, namespace=namespace)
 
     remove_timestamp(metrics=[output, expected])  # Timestamp will always be different