From edbbd52c346fe60da43c9e5f50342ffdbdb5ed27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Laguna?= Date: Mon, 29 Sep 2025 13:14:10 +0200 Subject: [PATCH] Add StructuredLogginJsonEncoder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rubén Laguna --- .../logback/classic/encoder/JsonEncoder.java | 99 ++++++----- .../encoder/StructuredLoggingJsonEncoder.java | 106 ++++++++++++ .../StructuredLoggingJsonEncoderTest.java | 157 ++++++++++++++++++ 3 files changed, 319 insertions(+), 43 deletions(-) create mode 100644 logback-classic/src/main/java/ch/qos/logback/classic/encoder/StructuredLoggingJsonEncoder.java create mode 100644 logback-classic/src/test/java/ch/qos/logback/classic/encoder/StructuredLoggingJsonEncoderTest.java diff --git a/logback-classic/src/main/java/ch/qos/logback/classic/encoder/JsonEncoder.java b/logback-classic/src/main/java/ch/qos/logback/classic/encoder/JsonEncoder.java index 18c61461c5..8c04dd3774 100644 --- a/logback-classic/src/main/java/ch/qos/logback/classic/encoder/JsonEncoder.java +++ b/logback-classic/src/main/java/ch/qos/logback/classic/encoder/JsonEncoder.java @@ -89,37 +89,37 @@ public class JsonEncoder extends EncoderBase { public static final String STEP_ARRAY_NAME_ATTRIBUTE = "stepArray"; - private static final char OPEN_OBJ = '{'; - private static final char CLOSE_OBJ = '}'; - private static final char OPEN_ARRAY = '['; - private static final char CLOSE_ARRAY = ']'; + protected static final char OPEN_OBJ = '{'; + protected static final char CLOSE_OBJ = '}'; + protected static final char OPEN_ARRAY = '['; + protected static final char CLOSE_ARRAY = ']'; - private static final char QUOTE = DOUBLE_QUOTE_CHAR; - private static final char SP = ' '; - private static final char ENTRY_SEPARATOR = COLON_CHAR; + protected static final char QUOTE = DOUBLE_QUOTE_CHAR; + protected static final char SP = ' '; + protected static final char ENTRY_SEPARATOR = COLON_CHAR; - private static final String COL_SP = ": "; + protected static final String COL_SP = ": "; - private static final String QUOTE_COL = "\":"; + protected static final String QUOTE_COL = "\":"; - private static final char VALUE_SEPARATOR = COMMA_CHAR; + protected static final char VALUE_SEPARATOR = COMMA_CHAR; - private boolean withSequenceNumber = true; + protected boolean withSequenceNumber = true; - private boolean withTimestamp = true; - private boolean withNanoseconds = true; + protected boolean withTimestamp = true; + protected boolean withNanoseconds = true; - private boolean withLevel = true; - private boolean withThreadName = true; - private boolean withLoggerName = true; - private boolean withContext = true; - private boolean withMarkers = true; - private boolean withMDC = true; - private boolean withKVPList = true; - private boolean withMessage = true; - private boolean withArguments = true; - private boolean withThrowable = true; - private boolean withFormattedMessage = false; + protected boolean withLevel = true; + protected boolean withThreadName = true; + protected boolean withLoggerName = true; + protected boolean withContext = true; + protected boolean withMarkers = true; + protected boolean withMDC = true; + protected boolean withKVPList = true; + protected boolean withMessage = true; + protected boolean withArguments = true; + protected boolean withThrowable = true; + protected boolean withFormattedMessage = false; @Override public byte[] headerBytes() { @@ -138,7 +138,7 @@ public byte[] encode(ILoggingEvent event) { if (withTimestamp) { appendValueSeparator(sb, withSequenceNumber); - appenderMemberWithLongValue(sb, TIMESTAMP_ATTR_NAME, event.getTimeStamp()); + appenderTimestamp(sb, event); } if (withNanoseconds) { @@ -179,7 +179,7 @@ public byte[] encode(ILoggingEvent event) { if (withMessage) { sb.append(VALUE_SEPARATOR); - appenderMember(sb, MESSAGE_ATTR_NAME, jsonEscape(event.getMessage())); + appenderMessage(sb, event); } if (withFormattedMessage) { @@ -194,12 +194,25 @@ public byte[] encode(ILoggingEvent event) { if (withThrowable) appendThrowableProxy(sb, THROWABLE_ATTR_NAME, event.getThrowableProxy()); + appenderExtra(sb, event); + sb.append(CLOSE_OBJ); sb.append(CoreConstants.JSON_LINE_SEPARATOR); return sb.toString().getBytes(UTF_8_CHARSET); } - void appendValueSeparator(StringBuilder sb, boolean... subsequentConditionals) { + protected void appenderMessage(StringBuilder sb, ILoggingEvent event) { + appenderMember(sb, MESSAGE_ATTR_NAME, jsonEscape(event.getMessage())); + } + + protected void appenderExtra(StringBuilder sb, ILoggingEvent event) { + } + + protected void appenderTimestamp(StringBuilder sb, ILoggingEvent event) { + appenderMemberWithLongValue(sb, TIMESTAMP_ATTR_NAME, event.getTimeStamp()); + } + + protected void appendValueSeparator(StringBuilder sb, boolean... subsequentConditionals) { boolean enabled = false; for (boolean subsequent : subsequentConditionals) { if (subsequent) { @@ -212,7 +225,7 @@ void appendValueSeparator(StringBuilder sb, boolean... subsequentConditionals) { sb.append(VALUE_SEPARATOR); } - private void appendLoggerContext(StringBuilder sb, LoggerContextVO loggerContextVO) { + protected void appendLoggerContext(StringBuilder sb, LoggerContextVO loggerContextVO) { sb.append(QUOTE).append(CONTEXT_ATTR_NAME).append(QUOTE_COL); if (loggerContextVO == null) { @@ -231,7 +244,7 @@ private void appendLoggerContext(StringBuilder sb, LoggerContextVO loggerContext } - private void appendMap(StringBuilder sb, String attrName, Map map) { + protected void appendMap(StringBuilder sb, String attrName, Map map) { sb.append(QUOTE).append(attrName).append(QUOTE_COL); if (map == null) { sb.append(NULL_STR); @@ -253,11 +266,11 @@ private void appendMap(StringBuilder sb, String attrName, Map ma sb.append(CLOSE_OBJ); } - private void appendThrowableProxy(StringBuilder sb, String attributeName, IThrowableProxy itp) { + protected void appendThrowableProxy(StringBuilder sb, String attributeName, IThrowableProxy itp) { appendThrowableProxy(sb, attributeName, itp, true); } - private void appendThrowableProxy(StringBuilder sb, String attributeName, IThrowableProxy itp, boolean appendValueSeparator) { + protected void appendThrowableProxy(StringBuilder sb, String attributeName, IThrowableProxy itp, boolean appendValueSeparator) { if (appendValueSeparator) sb.append(VALUE_SEPARATOR); @@ -316,7 +329,7 @@ private void appendThrowableProxy(StringBuilder sb, String attributeName, IThrow } - private void appendSTEPArray(StringBuilder sb, StackTraceElementProxy[] stepArray, int commonFrames) { + protected void appendSTEPArray(StringBuilder sb, StackTraceElementProxy[] stepArray, int commonFrames) { sb.append(QUOTE).append(STEP_ARRAY_NAME_ATTRIBUTE).append(QUOTE_COL).append(OPEN_ARRAY); int len = stepArray != null ? stepArray.length : 0; @@ -351,19 +364,19 @@ private void appendSTEPArray(StringBuilder sb, StackTraceElementProxy[] stepArra sb.append(CLOSE_ARRAY); } - private void appenderMember(StringBuilder sb, String key, String value) { + protected void appenderMember(StringBuilder sb, String key, String value) { sb.append(QUOTE).append(key).append(QUOTE_COL).append(QUOTE).append(value).append(QUOTE); } - private void appenderMemberWithIntValue(StringBuilder sb, String key, int value) { + protected void appenderMemberWithIntValue(StringBuilder sb, String key, int value) { sb.append(QUOTE).append(key).append(QUOTE_COL).append(value); } - private void appenderMemberWithLongValue(StringBuilder sb, String key, long value) { + protected void appenderMemberWithLongValue(StringBuilder sb, String key, long value) { sb.append(QUOTE).append(key).append(QUOTE_COL).append(value); } - private void appendKeyValuePairs(StringBuilder sb, ILoggingEvent event) { + protected void appendKeyValuePairs(StringBuilder sb, ILoggingEvent event) { List kvpList = event.getKeyValuePairs(); if (kvpList == null || kvpList.isEmpty()) return; @@ -382,7 +395,7 @@ private void appendKeyValuePairs(StringBuilder sb, ILoggingEvent event) { sb.append(CLOSE_ARRAY); } - private void appendArgumentArray(StringBuilder sb, ILoggingEvent event) { + protected void appendArgumentArray(StringBuilder sb, ILoggingEvent event) { Object[] argumentArray = event.getArgumentArray(); if (argumentArray == null) return; @@ -399,7 +412,7 @@ private void appendArgumentArray(StringBuilder sb, ILoggingEvent event) { sb.append(CLOSE_ARRAY); } - private void appendMarkers(StringBuilder sb, ILoggingEvent event) { + protected void appendMarkers(StringBuilder sb, ILoggingEvent event) { List markerList = event.getMarkerList(); if (markerList == null) return; @@ -416,25 +429,25 @@ private void appendMarkers(StringBuilder sb, ILoggingEvent event) { sb.append(CLOSE_ARRAY); } - private String jsonEscapedToString(Object o) { + protected String jsonEscapedToString(Object o) { if (o == null) return NULL_STR; return jsonEscapeString(o.toString()); } - private String nullSafeStr(String s) { + protected String nullSafeStr(String s) { if (s == null) return NULL_STR; return s; } - private String jsonEscape(String s) { + protected String jsonEscape(String s) { if (s == null) return NULL_STR; return jsonEscapeString(s); } - private void appendMDC(StringBuilder sb, ILoggingEvent event) { + protected void appendMDC(StringBuilder sb, ILoggingEvent event) { Map map = event.getMDCPropertyMap(); sb.append(VALUE_SEPARATOR); sb.append(QUOTE).append(MDC_ATTR_NAME).append(QUOTE_COL).append(SP).append(OPEN_OBJ); @@ -452,7 +465,7 @@ private void appendMDC(StringBuilder sb, ILoggingEvent event) { sb.append(CLOSE_OBJ); } - boolean isNotEmptyMap(Map map) { + protected boolean isNotEmptyMap(Map map) { if (map == null) return false; return !map.isEmpty(); diff --git a/logback-classic/src/main/java/ch/qos/logback/classic/encoder/StructuredLoggingJsonEncoder.java b/logback-classic/src/main/java/ch/qos/logback/classic/encoder/StructuredLoggingJsonEncoder.java new file mode 100644 index 0000000000..20022b2537 --- /dev/null +++ b/logback-classic/src/main/java/ch/qos/logback/classic/encoder/StructuredLoggingJsonEncoder.java @@ -0,0 +1,106 @@ +/* + * Logback: the reliable, generic, fast and flexible logging framework. + * Copyright (C) 1999-2023, QOS.ch. All rights reserved. + * + * This program and the accompanying materials are dual-licensed under + * either the terms of the Eclipse Public License v1.0 as published by + * the Eclipse Foundation + * + * or (per the licensee's choosing) + * + * under the terms of the GNU Lesser General Public License version 2.1 + * as published by the Free Software Foundation. + */ + +package ch.qos.logback.classic.encoder; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.IThrowableProxy; +import ch.qos.logback.classic.spi.LoggerContextVO; +import ch.qos.logback.classic.spi.StackTraceElementProxy; +import ch.qos.logback.core.CoreConstants; +import ch.qos.logback.core.encoder.EncoderBase; +import org.slf4j.Marker; +import org.slf4j.event.KeyValuePair; + +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static ch.qos.logback.core.CoreConstants.COLON_CHAR; +import static ch.qos.logback.core.CoreConstants.COMMA_CHAR; +import static ch.qos.logback.core.CoreConstants.DOUBLE_QUOTE_CHAR; +import static ch.qos.logback.core.CoreConstants.UTF_8_CHARSET; +import static ch.qos.logback.core.encoder.JsonEscapeUtil.jsonEscapeString; +import static ch.qos.logback.core.model.ModelConstants.NULL_STR; + +/** + * + * + * https://jsonlines.org/ https://datatracker.ietf.org/doc/html/rfc8259 + */ +public class StructuredLoggingJsonEncoder extends JsonEncoder { + + protected boolean withTimestampSeconds = false; + protected boolean withTimestampNanos = false; + protected boolean withTime = false; + protected boolean withSeverity = true; + + public StructuredLoggingJsonEncoder() { + super(); + withArguments = false; + withLevel = false; + } + + @Override + protected void appenderTimestamp(StringBuilder sb, ILoggingEvent event) { + sb.append(QUOTE).append("timestamp").append(QUOTE_COL); + sb.append(OPEN_OBJ); + Instant timestamp = event.getInstant(); + appenderMemberWithLongValue(sb, "seconds", timestamp.getEpochSecond()); + sb.append(VALUE_SEPARATOR); + appenderMemberWithIntValue(sb, "nanos", timestamp.getNano()); + sb.append(CLOSE_OBJ); + } + + @Override + protected void appenderExtra(StringBuilder sb, ILoggingEvent event) { + Instant timestamp = event.getInstant(); + if (withTimestampSeconds) { + sb.append(VALUE_SEPARATOR); + appenderMemberWithLongValue(sb, "timestampSeconds", timestamp.getEpochSecond()); + } + if (withTimestampNanos) { + sb.append(VALUE_SEPARATOR); + appenderMemberWithIntValue(sb, "timestampNanos", timestamp.getNano()); + } + if (withTime) { + sb.append(VALUE_SEPARATOR); + appenderMember(sb, "time", java.time.format.DateTimeFormatter.ISO_INSTANT.format(timestamp)); + } + if (withSeverity) { + sb.append(VALUE_SEPARATOR); + String levelStr = event.getLevel() != null ? event.getLevel().levelStr : NULL_STR; + appenderMember(sb, "severity",levelStr); + } + } + + @Override + protected void appenderMessage(StringBuilder sb, ILoggingEvent event) { + appenderMember(sb, MESSAGE_ATTR_NAME, jsonEscapeString(event.getFormattedMessage())); + } + + public void setWithTimestampSeconds(boolean withTimestampSeconds) { + this.withTimestampSeconds = withTimestampSeconds; + } + + public void setWithTimestampNanos(boolean withTimestampNanos) { + this.withTimestampNanos = withTimestampNanos; + } + + public void setWithTime(boolean withTime) { + this.withTime = withTime; + } +} diff --git a/logback-classic/src/test/java/ch/qos/logback/classic/encoder/StructuredLoggingJsonEncoderTest.java b/logback-classic/src/test/java/ch/qos/logback/classic/encoder/StructuredLoggingJsonEncoderTest.java new file mode 100644 index 0000000000..803a9be7ad --- /dev/null +++ b/logback-classic/src/test/java/ch/qos/logback/classic/encoder/StructuredLoggingJsonEncoderTest.java @@ -0,0 +1,157 @@ +/* + * Logback: the reliable, generic, fast and flexible logging framework. + * Copyright (C) 1999-2023, QOS.ch. All rights reserved. + * + * This program and the accompanying materials are dual-licensed under + * either the terms of the Eclipse Public License v1.0 as published by + * the Eclipse Foundation + * + * or (per the licensee's choosing) + * + * under the terms of the GNU Lesser General Public License version 2.1 + * as published by the Free Software Foundation. + */ + +package ch.qos.logback.classic.encoder; + +import ch.qos.logback.classic.ClassicTestConstants; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.classic.jsonTest.JsonLoggingEvent; +import ch.qos.logback.classic.jsonTest.JsonStringToLoggingEventMapper; +import ch.qos.logback.classic.jsonTest.ThrowableProxyComparator; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.LoggingEvent; +import ch.qos.logback.classic.util.LogbackMDCAdapter; +import ch.qos.logback.core.joran.spi.JoranException; +import ch.qos.logback.core.read.ListAppender; +import ch.qos.logback.core.testUtil.RandomUtil; +import ch.qos.logback.core.util.StatusPrinter; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Marker; +import org.slf4j.event.KeyValuePair; +import org.slf4j.helpers.BasicMarkerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +// When running from an IDE, add the following on the command line +// +// --add-opens ch.qos.logback.classic/ch.qos.logback.classic.jsonTest=ALL-UNNAMED +// +class StructuredLoggingJsonEncoderTest { + + int diff = RandomUtil.getPositiveInt(); + + LoggerContext loggerContext = new LoggerContext(); + Logger logger = loggerContext.getLogger(StructuredLoggingJsonEncoderTest.class); + + JsonEncoder jsonEncoder = new StructuredLoggingJsonEncoder(); + + BasicMarkerFactory markerFactory = new BasicMarkerFactory(); + + Marker markerA = markerFactory.getMarker("A"); + + Marker markerB = markerFactory.getMarker("B"); + + ListAppender listAppender = new ListAppender(); + JsonStringToLoggingEventMapper stringToLoggingEventMapper = new JsonStringToLoggingEventMapper(markerFactory); + + LogbackMDCAdapter logbackMDCAdapter = new LogbackMDCAdapter(); + + ObjectMapper objectMapper = new ObjectMapper(); + + @BeforeEach + void setUp() { + loggerContext.setName("test_" + diff); + loggerContext.setMDCAdapter(logbackMDCAdapter); + + jsonEncoder.setContext(loggerContext); + jsonEncoder.start(); + + listAppender.setContext(loggerContext); + listAppender.start(); + } + + @AfterEach + void tearDown() { + } + @Test + void smoke() throws JsonProcessingException { + Object[] args = new String[] { "logback" }; + LoggingEvent event = new LoggingEvent("x", logger, Level.WARN, "hello {}", null, args); + + byte[] resultBytes = jsonEncoder.encode(event); + String resultString = new String(resultBytes, StandardCharsets.UTF_8); + + JsonNode json = objectMapper.readTree(resultString); + Long timestampSeconds = json.get("timestamp").get("seconds").asLong(); + assertEquals(event.getInstant().getEpochSecond(), timestampSeconds); + assertEquals(event.getNanoseconds(), json.get("timestamp").get("nanos").asInt()); + + assertNull(json.get("timestampSeconds")); + assertNull(json.get("timestampNanos")); + assertNull(json.get("time")); + assertNull(json.get("arguments")); + assertNull(json.get("level")); + assertEquals("WARN", json.get("severity").asText()); + + + + } + + @Test + void withJoranWithoutTimestampSeconds() throws JsonProcessingException, JoranException, IOException { + String configFilePathStr = ClassicTestConstants.JORAN_INPUT_PREFIX + + "json/structuredLoggingJsonEncoder.xml"; + + configure(configFilePathStr); + Logger logger = loggerContext.getLogger(this.getClass().getName()); + logger.addAppender(listAppender); + + logger.debug("hello {}", "logback"); + + Path outputFilePath = Path.of(ClassicTestConstants.OUTPUT_DIR_PREFIX + "json/test-" + diff + ".json"); + List lines = Files.readAllLines(outputFilePath); + int count = 1; + assertEquals(count, lines.size()); + JsonNode json = objectMapper.readTree(lines.get(0)); + ILoggingEvent event = listAppender.list.get(0); + + assertEquals(event.getInstant().getEpochSecond(), json.get("timestampSeconds").asLong()); + assertEquals(event.getInstant().getNano(), json.get("timestampNanos").asLong()); + + String time = json.get("time").asText(); + Instant timeAsInstant = Instant.parse(time); + assertEquals(event.getInstant(), timeAsInstant); + assertEquals("hello logback", json.get("message").asText()); + } + + void configure(String file) throws JoranException { + JoranConfigurator jc = new JoranConfigurator(); + jc.setContext(loggerContext); + loggerContext.putProperty("diff", "" + diff); + jc.doConfigure(file); + } + +} \ No newline at end of file