From a1d50597fa105f82d508283b2fee2003ec280b76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vilius=20=C5=A0umskas?= Date: Wed, 2 Apr 2025 13:29:48 +0300 Subject: [PATCH 1/9] Adjust GcpLayout JSON to latest format First, it formats the log timestamp field to the correct format recognized by Fluent-Bit (component of Google Cloud Logging) and Google Ops Agent. Secondly, severity field now must be prefixed with logging.googleapis.com. Third, counter cannot be used for insertId as it is duplicated on different threads. And the last but not the least, exception, thread and logger fields are pretty standard when logging via Logback's JSON layout and Google's Spring GCP libraries. Field name changes now match these other loggers. --- .../layout/template/json/GcpLayoutTest.java | 22 +++------- .../src/main/resources/GcpLayout.json | 42 +++++++++---------- 2 files changed, 24 insertions(+), 40 deletions(-) diff --git a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java index c257b1c8af8..2f2dda1302a 100644 --- a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java +++ b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java @@ -149,40 +149,28 @@ private static void verifySerialization(final LogEvent logEvent) { } // Verify insert id. - assertThat(accessor.getString("logging.googleapis.com/insertId")).matches("[-]?[0-9]+"); + assertThat(accessor.getString("logging.googleapis.com/insertId")).matches("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"); // Verify exception. if (exception != null) { - // Verify exception class. - assertThat(accessor.getString(new String[] {"_exception", "class"})) - .isEqualTo(exception.getClass().getCanonicalName()); - - // Verify exception message. - assertThat(accessor.getString(new String[] {"_exception", "message"})) - .isEqualTo(exception.getMessage()); - // Verify exception stack trace. - assertThat(accessor.getString(new String[] {"_exception", "stackTrace"})) + assertThat(accessor.getString("exception")) .contains(exception.getLocalizedMessage()) .contains("at org.apache.logging.log4j.layout.template.json") .contains("at " + JAVA_BASE_PREFIX + "java.lang.reflect.Method") .contains("at org.junit.platform.engine"); } else { - assertThat(accessor.getObject(new String[] {"_exception", "class"})) - .isNull(); - assertThat(accessor.getObject(new String[] {"_exception", "message"})) - .isNull(); - assertThat(accessor.getString(new String[] {"_exception", "stackTrace"})) + assertThat(accessor.getString("exception")) .isEmpty(); } // Verify thread name. - assertThat(accessor.getString("_thread")).isEqualTo(logEvent.getThreadName()); + assertThat(accessor.getString("thread")).isEqualTo(logEvent.getThreadName()); // Verify logger name. - assertThat(accessor.getString("_logger")).isEqualTo(logEvent.getLoggerName()); + assertThat(accessor.getString("logger")).isEqualTo(logEvent.getLoggerName()); }); } diff --git a/log4j-layout-template-json/src/main/resources/GcpLayout.json b/log4j-layout-template-json/src/main/resources/GcpLayout.json index f00c84d981e..5f61539d8a4 100644 --- a/log4j-layout-template-json/src/main/resources/GcpLayout.json +++ b/log4j-layout-template-json/src/main/resources/GcpLayout.json @@ -1,13 +1,18 @@ { - "timestamp": { + "timestampSeconds": { "$resolver": "timestamp", - "pattern": { - "format": "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", - "timeZone": "UTC", - "locale": "en_US" + "epoch": { + "unit": "secs", + "rounded": true } }, - "severity": { + "timestampNanos": { + "$resolver": "timestamp", + "epoch": { + "unit": "secs.nanos" + } + }, + "logging.googleapis.com/severity": { "$resolver": "pattern", "pattern": "%level{WARN=WARNING, TRACE=DEBUG, FATAL=EMERGENCY}", "stackTraceEnabled": false @@ -37,8 +42,9 @@ } }, "logging.googleapis.com/insertId": { - "$resolver": "counter", - "stringified": true + "$resolver": "pattern", + "pattern": "%uuid{TIME}", + "stackTraceEnabled": false }, "logging.googleapis.com/trace": { "$resolver": "mdc", @@ -49,25 +55,15 @@ "key": "span_id" }, "logging.googleapis.com/trace_sampled": true, - "_exception": { - "class": { - "$resolver": "exception", - "field": "className" - }, - "message": { - "$resolver": "exception", - "field": "message" - }, - "stackTrace": { - "$resolver": "pattern", - "pattern": "%xEx" - } + "exception": { + "$resolver": "pattern", + "pattern": "%xEx" }, - "_thread": { + "thread": { "$resolver": "thread", "field": "name" }, - "_logger": { + "logger": { "$resolver": "logger", "field": "name" } From e607ad421ceb931bb0ac344504c664c142008edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vilius=20=C5=A0umskas?= Date: Wed, 2 Apr 2025 17:45:26 +0300 Subject: [PATCH 2/9] revert severity changes, remove insertId --- .../src/main/resources/GcpLayout.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/log4j-layout-template-json/src/main/resources/GcpLayout.json b/log4j-layout-template-json/src/main/resources/GcpLayout.json index 5f61539d8a4..244f8b8032b 100644 --- a/log4j-layout-template-json/src/main/resources/GcpLayout.json +++ b/log4j-layout-template-json/src/main/resources/GcpLayout.json @@ -12,7 +12,7 @@ "unit": "secs.nanos" } }, - "logging.googleapis.com/severity": { + "severity": { "$resolver": "pattern", "pattern": "%level{WARN=WARNING, TRACE=DEBUG, FATAL=EMERGENCY}", "stackTraceEnabled": false @@ -41,11 +41,6 @@ "stackTraceEnabled": false } }, - "logging.googleapis.com/insertId": { - "$resolver": "pattern", - "pattern": "%uuid{TIME}", - "stackTraceEnabled": false - }, "logging.googleapis.com/trace": { "$resolver": "mdc", "key": "trace_id" From 8820bfe8468748cf46c05dd3e82fd36bebe188ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vilius=20=C5=A0umskas?= Date: Wed, 2 Apr 2025 17:47:11 +0300 Subject: [PATCH 3/9] Remove insertid from tests --- .../logging/log4j/layout/template/json/GcpLayoutTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java index 2f2dda1302a..c111a3a4cd6 100644 --- a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java +++ b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java @@ -148,9 +148,6 @@ private static void verifySerialization(final LogEvent logEvent) { .isEmpty(); } - // Verify insert id. - assertThat(accessor.getString("logging.googleapis.com/insertId")).matches("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"); - // Verify exception. if (exception != null) { From 923642944667bee0e26c3931582ccfa7fd5d22e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vilius=20=C5=A0umskas?= Date: Wed, 2 Apr 2025 18:38:23 +0300 Subject: [PATCH 4/9] fix spotless error --- .../logging/log4j/layout/template/json/GcpLayoutTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java index c111a3a4cd6..4167532478d 100644 --- a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java +++ b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java @@ -159,8 +159,7 @@ private static void verifySerialization(final LogEvent logEvent) { .contains("at org.junit.platform.engine"); } else { - assertThat(accessor.getString("exception")) - .isEmpty(); + assertThat(accessor.getString("exception")).isEmpty(); } // Verify thread name. From 23ef884930880482ccc50e40f1e3a05a2ca97687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vilius=20=C5=A0umskas?= Date: Wed, 16 Apr 2025 13:40:54 +0300 Subject: [PATCH 5/9] Switch exception field to use exception resolver --- .../src/main/resources/GcpLayout.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/log4j-layout-template-json/src/main/resources/GcpLayout.json b/log4j-layout-template-json/src/main/resources/GcpLayout.json index 244f8b8032b..75bf58a7896 100644 --- a/log4j-layout-template-json/src/main/resources/GcpLayout.json +++ b/log4j-layout-template-json/src/main/resources/GcpLayout.json @@ -51,8 +51,11 @@ }, "logging.googleapis.com/trace_sampled": true, "exception": { - "$resolver": "pattern", - "pattern": "%xEx" + "$resolver": "exception", + "field": "stackTrace", + "stackTrace": { + "stringified": true + } }, "thread": { "$resolver": "thread", From 9c357908e4512bc4410e14a684b9599b923add6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vilius=20=C5=A0umskas?= Date: Thu, 17 Apr 2025 13:04:26 +0300 Subject: [PATCH 6/9] try to fix timestamp tests --- .../layout/template/json/GcpLayoutTest.java | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java index 4167532478d..b1bb86c0415 100644 --- a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java +++ b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java @@ -21,11 +21,6 @@ import static org.apache.logging.log4j.layout.template.json.TestHelpers.usingSerializedLogEventAccessor; import static org.assertj.core.api.Assertions.assertThat; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Locale; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.core.impl.ContextDataFactory; @@ -44,9 +39,6 @@ class GcpLayoutTest { private static final int LOG_EVENT_COUNT = 1_000; - private static final DateTimeFormatter DATE_TIME_FORMATTER = - DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US); - @Test void test_lite_log_events() { LogEventFixture.createLiteLogEvents(LOG_EVENT_COUNT).forEach(GcpLayoutTest::verifySerialization); @@ -84,8 +76,9 @@ private static void verifySerialization(final LogEvent logEvent) { usingSerializedLogEventAccessor(LAYOUT, logEvent, accessor -> { // Verify timestamp. - final String expectedTimestamp = formatLogEventInstant(logEvent); - assertThat(accessor.getString("timestamp")).isEqualTo(expectedTimestamp); + final org.apache.logging.log4j.core.time.Instant instant = logEvent.getInstant(); + assertThat(accessor.getInteger("timestampSeconds")).isEqualTo(instant.getEpochSecond()); + assertThat(accessor.getInteger("timestampNanos")).isEqualTo(instant.getNanoOfSecond()); // Verify severity. final Level level = logEvent.getLevel(); @@ -169,11 +162,4 @@ private static void verifySerialization(final LogEvent logEvent) { assertThat(accessor.getString("logger")).isEqualTo(logEvent.getLoggerName()); }); } - - private static String formatLogEventInstant(final LogEvent logEvent) { - final org.apache.logging.log4j.core.time.Instant instant = logEvent.getInstant(); - final ZonedDateTime dateTime = Instant.ofEpochSecond(instant.getEpochSecond(), instant.getNanoOfSecond()) - .atZone(ZoneId.of("UTC")); - return DATE_TIME_FORMATTER.format(dateTime); - } } From 5e0656e9f7f57a2bb942a82609d988b13aaa3d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vilius=20=C5=A0umskas?= Date: Mon, 21 Apr 2025 02:28:01 +0300 Subject: [PATCH 7/9] Fix tests with empty exceptions --- .../logging/log4j/layout/template/json/GcpLayoutTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java index b1bb86c0415..6501259cfb2 100644 --- a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java +++ b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/GcpLayoutTest.java @@ -152,7 +152,7 @@ private static void verifySerialization(final LogEvent logEvent) { .contains("at org.junit.platform.engine"); } else { - assertThat(accessor.getString("exception")).isEmpty(); + assertThat(accessor.getString("exception")).isNull(); } // Verify thread name. From 9e8a03bccc5ff02db7feb2cb96529cc2208e773d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vilius=20=C5=A0umskas?= Date: Wed, 23 Apr 2025 17:17:53 +0300 Subject: [PATCH 8/9] Add changelog --- src/changelog/.2.x.x/3586_improve_GcpLayout.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 src/changelog/.2.x.x/3586_improve_GcpLayout.xml diff --git a/src/changelog/.2.x.x/3586_improve_GcpLayout.xml b/src/changelog/.2.x.x/3586_improve_GcpLayout.xml new file mode 100644 index 00000000000..d92297717ff --- /dev/null +++ b/src/changelog/.2.x.x/3586_improve_GcpLayout.xml @@ -0,0 +1,10 @@ + + + + + Adjust GcpLayout JSON template to support automatic timestamp recognition by the Google Cloud Logging. This also changes `exception`, `thread`, `logger` fields a bit, and removes `insertId` field. + + From e31e122bb902217672ef1d61276f6558972dcae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Volkan=20Yaz=C4=B1c=C4=B1?= Date: Sat, 3 May 2025 19:49:12 +0200 Subject: [PATCH 9/9] Improve changelog. --- src/changelog/.2.x.x/3586_improve_GcpLayout.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/changelog/.2.x.x/3586_improve_GcpLayout.xml b/src/changelog/.2.x.x/3586_improve_GcpLayout.xml index d92297717ff..00e0fc7a69b 100644 --- a/src/changelog/.2.x.x/3586_improve_GcpLayout.xml +++ b/src/changelog/.2.x.x/3586_improve_GcpLayout.xml @@ -5,6 +5,6 @@ type="changed"> - Adjust GcpLayout JSON template to support automatic timestamp recognition by the Google Cloud Logging. This also changes `exception`, `thread`, `logger` fields a bit, and removes `insertId` field. + Update `GcpLayout.json` JSON Template Layout event template to support automatic timestamp recognition by the Google Cloud Logging. This also changes `exception`, `thread`, `logger` fields, and removes `insertId` field.