From 7a2b8a679f3a72aad07e21eee5ee67f8a60d5733 Mon Sep 17 00:00:00 2001 From: Jazzy Date: Mon, 10 May 2021 18:51:17 -0700 Subject: [PATCH 01/12] Use custom deserliazer and apply masking in place --- json-logger/pom.xml | 2 +- .../internal/JsonloggerExtension.java | 12 +- .../internal/JsonloggerOperations.java | 818 +++++++++--------- .../internal/datamask/JsonMasker.java | 187 ++-- .../internal/destinations/Destination.java | 27 +- .../internal/singleton/ConfigsSingleton.java | 2 +- .../internal/singleton/LogEventSingleton.java | 2 +- .../singleton/MaskingDeserializer.java | 176 ++++ .../singleton/ObjectMapperSingleton.java | 27 +- json-logger/src/test/java/JsonMaskerTest.java | 30 + json-logger/src/test/resources/payload.json | 19 + 11 files changed, 727 insertions(+), 575 deletions(-) create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/MaskingDeserializer.java create mode 100644 json-logger/src/test/java/JsonMaskerTest.java create mode 100644 json-logger/src/test/resources/payload.json diff --git a/json-logger/pom.xml b/json-logger/pom.xml index f76e48d..6831ccb 100644 --- a/json-logger/pom.xml +++ b/json-logger/pom.xml @@ -6,7 +6,7 @@ 4.0.0 ORG_ID_TOKEN json-logger - 2.0.1 + 2.1.0 mule-extension JSON Logger - Mule 4 diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java index d2bc9c6..dea2eba 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java @@ -14,12 +14,12 @@ * This is the main class of an extension, is the entry point from which configurations, connection providers, operations * and sources are going to be declared. */ -@Xml(prefix = "json-logger") -@Extension(name = "JSON Logger") -@Export(resources = {"modules/JSONLoggerModule.dwl"}) -@Configurations(JsonloggerConfiguration.class) -@SubTypeMapping(baseType = Destination.class, - subTypes = {JMSDestination.class, AMQDestination.class, AMQPDestination.class}) +//@Xml(prefix = "json-logger") +//@Extension(name = "JSON Logger") +//@Export(resources = {"modules/JSONLoggerModule.dwl"}) +//@Configurations(JsonloggerConfiguration.class) +////@SubTypeMapping(baseType = Destination.class, +//// subTypes = {JMSDestination.class, AMQDestination.class, AMQPDestination.class}) public class JsonloggerExtension { } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java index a1a2d7f..24c4643 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java @@ -9,7 +9,6 @@ import org.mule.extension.jsonlogger.api.pojos.LoggerProcessor; import org.mule.extension.jsonlogger.api.pojos.Priority; import org.mule.extension.jsonlogger.api.pojos.ScopeTracePoint; -import org.mule.extension.jsonlogger.internal.datamask.JsonMasker; import org.mule.extension.jsonlogger.internal.singleton.ConfigsSingleton; import org.mule.extension.jsonlogger.internal.singleton.LogEventSingleton; import org.mule.extension.jsonlogger.internal.singleton.ObjectMapperSingleton; @@ -46,451 +45,422 @@ * This class is a container for operations, every public method in this class will be taken as an extension operation. */ public class JsonloggerOperations { - - /** - * jsonLogger: JSON Logger output log - * log: Connector internal log - */ - protected transient Logger jsonLogger; - private static final Logger LOGGER = LoggerFactory.getLogger(JsonloggerOperations.class); - - // Void Result for NIO - private final Result VOID_RESULT = Result.builder().build(); - - // JSON Object Mapper - @Inject - ObjectMapperSingleton om; - - // Log Event for External Destination - @Inject - LogEventSingleton logEvent; - - // Global definition of logger configs so that it's available for scope processor (SDK scope doesn't support passing configurations) - @Inject - ConfigsSingleton configs; - - // Transformation Service - @Inject - private TransformationService transformationService; - + + /** + * jsonLogger: JSON Logger output log + * log: Connector internal log + */ + protected transient Logger jsonLogger; + private static final Logger LOGGER = LoggerFactory.getLogger(JsonloggerOperations.class); + + // Void Result for NIO + private final Result VOID_RESULT = Result.builder().build(); + + // JSON Object Mapper + @Inject + ObjectMapperSingleton om; + + // Log Event for External Destination + @Inject + LogEventSingleton logEvent; + + // Global definition of logger configs so that it's available for scope processor (SDK scope doesn't support passing configurations) + @Inject + ConfigsSingleton configs; + + // Transformation Service + @Inject + private TransformationService transformationService; + + /** + * Log a new entry + */ + @Execution(ExecutionType.BLOCKING) + public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPPORTED) LoggerProcessor loggerProcessor, + CorrelationInfo correlationInfo, + ComponentLocation location, + @Config JsonloggerConfiguration config, + FlowListener flowListener, + CompletionCallback callback) { + + Long initialTimestamp, loggerTimestamp; + initialTimestamp = loggerTimestamp = System.currentTimeMillis(); + + initLoggerCategory(loggerProcessor.getCategory()); + + LOGGER.debug("correlationInfo.getEventId(): " + correlationInfo.getEventId()); + LOGGER.debug("correlationInfo.getCorrelationId(): " + correlationInfo.getCorrelationId()); + + try { + // Add cache entry for initial timestamp based on unique EventId + initialTimestamp = config.getCachedTimerTimestamp(correlationInfo.getCorrelationId(), initialTimestamp); + } catch (Exception e) { + LOGGER.error("initialTimestamp could not be retrieved from the cache config. Defaulting to current System.currentTimeMillis()", e); + } + + // Calculate elapsed time based on cached initialTimestamp + Long elapsed = loggerTimestamp - initialTimestamp; + + //config.printTimersKeys(); + if (elapsed == 0) { + LOGGER.debug("configuring flowListener...."); + flowListener.onComplete(new TimerRemoverRunnable(correlationInfo.getCorrelationId(), config)); + } else { + LOGGER.debug("flowListener already configured"); + } + /** - * Log a new entry + * Avoid Logger logic execution based on log priority */ - @Execution(ExecutionType.BLOCKING) - public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPPORTED) LoggerProcessor loggerProcessor, - CorrelationInfo correlationInfo, - ComponentLocation location, - @Config JsonloggerConfiguration config, - FlowListener flowListener, - CompletionCallback callback) { - - Long initialTimestamp, loggerTimestamp; - initialTimestamp = loggerTimestamp = System.currentTimeMillis(); - - initLoggerCategory(loggerProcessor.getCategory()); - - LOGGER.debug("correlationInfo.getEventId(): " + correlationInfo.getEventId()); - LOGGER.debug("correlationInfo.getCorrelationId(): " + correlationInfo.getCorrelationId()); - - try { - // Add cache entry for initial timestamp based on unique EventId - initialTimestamp = config.getCachedTimerTimestamp(correlationInfo.getCorrelationId(), initialTimestamp); - } catch (Exception e) { - LOGGER.error("initialTimestamp could not be retrieved from the cache config. Defaulting to current System.currentTimeMillis()", e); - } - - // Calculate elapsed time based on cached initialTimestamp - Long elapsed = loggerTimestamp - initialTimestamp; - - //config.printTimersKeys(); - if (elapsed == 0) { - LOGGER.debug("configuring flowListener...."); - flowListener.onComplete(new TimerRemoverRunnable(correlationInfo.getCorrelationId(), config)); - } else { - LOGGER.debug("flowListener already configured"); - } - - /** - * Avoid Logger logic execution based on log priority - */ - if (isLogEnabled(loggerProcessor.getPriority().toString())) { - // Load disabledFields - List disabledFields = (config.getJsonOutput().getDisabledFields() != null) ? Arrays.asList(config.getJsonOutput().getDisabledFields().split(",")) : new ArrayList<>(); - LOGGER.debug("The following fields will be disabled for logging: " + disabledFields); - - // Logic to disable fields and/or parse TypedValues as String for JSON log printing - //Map typedValuesAsString = new HashMap<>(); - Map typedValuesAsString = new HashMap<>(); - Map typedValuesAsJsonNode = new HashMap<>(); + if (isLogEnabled(loggerProcessor.getPriority().toString())) { + // Load disabledFields + List disabledFields = (config.getJsonOutput().getDisabledFields() != null) ? Arrays.asList(config.getJsonOutput().getDisabledFields().split(",")) : new ArrayList<>(); + LOGGER.debug("The following fields will be disabled for logging: " + disabledFields); + + // Logic to disable fields and/or parse TypedValues as String for JSON log printing + //Map typedValuesAsString = new HashMap<>(); + Map typedValuesAsString = new HashMap<>(); + Map typedValuesAsJsonNode = new HashMap<>(); + try { + PropertyUtils.describe(loggerProcessor).forEach((k, v) -> { + if (disabledFields.stream().anyMatch(k::equals)) { try { - PropertyUtils.describe(loggerProcessor).forEach((k, v) -> { - if (disabledFields.stream().anyMatch(k::equals)) { - try { - BeanUtils.setProperty(loggerProcessor, k, null); - } catch (Exception e) { - LOGGER.error("Failed disabling field: " + k, e); + BeanUtils.setProperty(loggerProcessor, k, null); + } catch (Exception e) { + LOGGER.error("Failed disabling field: " + k, e); + } + } else { + if (v != null) { + try { + if (v instanceof ParameterResolver) { + v = ((ParameterResolver) v).resolve(); + } + if (v.getClass().getCanonicalName().equals("org.mule.runtime.api.metadata.TypedValue")) { + LOGGER.debug("org.mule.runtime.api.metadata.TypedValue type was found for field: " + k); + TypedValue typedVal = (TypedValue) v; + LOGGER.debug("Parsing TypedValue for field " + k); + + LOGGER.debug("TypedValue MediaType: " + typedVal.getDataType().getMediaType()); + LOGGER.debug("TypedValue Type: " + typedVal.getDataType().getType().getCanonicalName()); + LOGGER.debug("TypedValue Class: " + typedVal.getValue().getClass().getCanonicalName()); + + // Remove unparsed field + BeanUtils.setProperty(loggerProcessor, k, null); + + // Evaluate if typedValue is null + if (typedVal.getValue() != null) { + // Should content type field be parsed as part of JSON log? + if (config.getJsonOutput().isParseContentFieldsInJsonOutput()) { + // Is content type application/json? + if (typedVal.getDataType().getMediaType().getPrimaryType().equals("application") && typedVal.getDataType().getMediaType().getSubType().equals("json")) { + + // Apply masking if needed + List dataMaskingFields = new ArrayList<>(); + if (config.getJsonOutput().getContentFieldsDataMasking() != null) { + String[] split = config.getJsonOutput().getContentFieldsDataMasking().split(","); + for (String s : split) { + dataMaskingFields.add(s.trim()); + } } - } else { - if (v != null) { - try { - if (v instanceof ParameterResolver) { - v = ((ParameterResolver) v).resolve(); - } - if (v.getClass().getCanonicalName().equals("org.mule.runtime.api.metadata.TypedValue")) { - LOGGER.debug("org.mule.runtime.api.metadata.TypedValue type was found for field: " + k); - TypedValue typedVal = (TypedValue) v; - LOGGER.debug("Parsing TypedValue for field " + k); - - LOGGER.debug("TypedValue MediaType: " + typedVal.getDataType().getMediaType()); - LOGGER.debug("TypedValue Type: " + typedVal.getDataType().getType().getCanonicalName()); - LOGGER.debug("TypedValue Class: " + typedVal.getValue().getClass().getCanonicalName()); - - // Remove unparsed field - BeanUtils.setProperty(loggerProcessor, k, null); - - // Evaluate if typedValue is null - if (typedVal.getValue() != null) { - // Should content type field be parsed as part of JSON log? - if (config.getJsonOutput().isParseContentFieldsInJsonOutput()) { - // Is content type application/json? - if (typedVal.getDataType().getMediaType().getPrimaryType().equals("application") && typedVal.getDataType().getMediaType().getSubType().equals("json")) { - // Apply masking if needed - List dataMaskingFields = (config.getJsonOutput().getContentFieldsDataMasking() != null) ? Arrays.asList(config.getJsonOutput().getContentFieldsDataMasking().split(",")) : new ArrayList<>(); - LOGGER.debug("The following JSON keys/paths will be masked for logging: " + dataMaskingFields); - if (!dataMaskingFields.isEmpty()) { - JsonNode tempContentNode = om.getObjectMapper().readTree((InputStream)typedVal.getValue()); - JsonMasker masker = new JsonMasker(dataMaskingFields, true); - JsonNode masked = masker.mask(tempContentNode); - typedValuesAsJsonNode.put(k, masked); - } else { - typedValuesAsJsonNode.put(k, om.getObjectMapper().readTree((InputStream)typedVal.getValue())); - } - } else { - typedValuesAsString.put(k, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); - } - } else { - typedValuesAsString.put(k, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); - } - } - } - } catch (Exception e) { - LOGGER.error("Failed parsing field: " + k, e); - typedValuesAsString.put(k, "Error parsing expression. See logs for details."); - } + + LOGGER.debug("The following JSON keys/paths will be masked for logging: " + dataMaskingFields); + if (!dataMaskingFields.isEmpty()) { + JsonNode tempContentNode = om.getObjectMapper(dataMaskingFields).readTree(typedVal.getValue()); + typedValuesAsJsonNode.put(k, tempContentNode); + } else { + typedValuesAsJsonNode.put(k, om.getObjectMapper(dataMaskingFields).readTree(typedVal.getValue())); } + } else { + typedValuesAsString.put(k, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); + } + } else { + typedValuesAsString.put(k, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); } - }); - } catch (Exception e) { - LOGGER.error("Unknown error while processing the logger object", e); - } - - // Aggregate Logger data into mergedLogger - ObjectNode mergedLogger = om.getObjectMapper().createObjectNode(); - mergedLogger.setAll((ObjectNode) om.getObjectMapper().valueToTree(loggerProcessor)); - - /** - * Custom field ordering for Logger Operation - * ========================================== - * This will take place after LoggerProcessor ordering which is defined by the field sequence in loggerProcessor.json - **/ - // 1. Elapsed Time - mergedLogger.put("elapsed", elapsed); - // 2. Location Info: Logger location within Mule application - if (config.getJsonOutput().isLogLocationInfo()) { - Map locationInfo = locationInfoToMap(location); - mergedLogger.putPOJO("locationInfo", locationInfo); - } - // 3. Timestamp: Add formatted timestamp entry to the logger - mergedLogger.put("timestamp", getFormattedTimestamp(loggerTimestamp)); - // 4. Content fields: String based fields - if (!typedValuesAsString.isEmpty()) { - JsonNode typedValuesNode = om.getObjectMapper().valueToTree(typedValuesAsString); - mergedLogger.setAll((ObjectNode) typedValuesNode); - } - // 5. Content fields: JSONNode based fields - if (!typedValuesAsJsonNode.isEmpty()) { - mergedLogger.setAll(typedValuesAsJsonNode); - } - // 6. Global info from config - mergedLogger.setAll((ObjectNode) om.getObjectMapper().valueToTree(config.getGlobalSettings())); - // 7. Thread Name - mergedLogger.put("threadName", Thread.currentThread().getName()); - /** End field ordering **/ - - /** Print Logger **/ - String finalLog = printObjectToLog(mergedLogger, loggerProcessor.getPriority().toString(), config.getJsonOutput().isPrettyPrint()); - - /** Forward Log to External Destination **/ - if (config.getExternalDestination() != null) { - LOGGER.debug("config.getExternalDestination().getSupportedCategories().isEmpty(): " + config.getExternalDestination().getSupportedCategories().isEmpty()); - LOGGER.debug("config.getExternalDestination().getSupportedCategories().contains(jsonLogger.getName()): " + config.getExternalDestination().getSupportedCategories().contains(jsonLogger.getName())); - if (configs.getConfig(config.getConfigName()).getExternalDestination().getSupportedCategories().isEmpty() || config.getExternalDestination().getSupportedCategories().contains(jsonLogger.getName())) { - LOGGER.debug(jsonLogger.getName() + " is a supported category for external destination"); - logEvent.publishToExternalDestination(correlationInfo.getEventId(), finalLog, config.getConfigName()); + } } + } catch (Exception e) { + LOGGER.error("Failed parsing field: " + k, e); + typedValuesAsString.put(k, "Error parsing expression. See logs for details."); + } } - } else { - LOGGER.debug("Avoiding logger operation logic execution due to log priority not being enabled"); - } - callback.success(VOID_RESULT); + } + }); + } catch (Exception e) { + LOGGER.error("Unknown error while processing the logger object", e); + } + + // Aggregate Logger data into mergedLogger + ObjectNode mergedLogger = om.getObjectMapper(new ArrayList<>()).createObjectNode(); + mergedLogger.setAll((ObjectNode) om.getObjectMapper(new ArrayList<>()).valueToTree(loggerProcessor)); + + /** + * Custom field ordering for Logger Operation + * ========================================== + * This will take place after LoggerProcessor ordering which is defined by the field sequence in loggerProcessor.json + **/ + // 1. Elapsed Time + mergedLogger.put("elapsed", elapsed); + // 2. Location Info: Logger location within Mule application + if (config.getJsonOutput().isLogLocationInfo()) { + Map locationInfo = locationInfoToMap(location); + mergedLogger.putPOJO("locationInfo", locationInfo); + } + // 3. Timestamp: Add formatted timestamp entry to the logger + mergedLogger.put("timestamp", getFormattedTimestamp(loggerTimestamp)); + // 4. Content fields: String based fields + if (!typedValuesAsString.isEmpty()) { + JsonNode typedValuesNode = om.getObjectMapper(new ArrayList<>()).valueToTree(typedValuesAsString); + mergedLogger.setAll((ObjectNode) typedValuesNode); + } + // 5. Content fields: JSONNode based fields + if (!typedValuesAsJsonNode.isEmpty()) { + mergedLogger.setAll(typedValuesAsJsonNode); + } + // 6. Global info from config + mergedLogger.setAll((ObjectNode) om.getObjectMapper(new ArrayList<>()).valueToTree(config.getGlobalSettings())); + // 7. Thread Name + mergedLogger.put("threadName", Thread.currentThread().getName()); + + /** Print Logger **/ + printObjectToLog(mergedLogger, loggerProcessor.getPriority().toString(), config.getJsonOutput().isPrettyPrint()); + + } else { + LOGGER.debug("Avoiding logger operation logic execution due to log priority not being enabled"); } - + callback.success(VOID_RESULT); + } + + /** + * Log scope + */ + @Execution(ExecutionType.BLOCKING) + public void loggerScope(@DisplayName("Module configuration") @Example("JSON_Logger_Config") @Summary("Indicate which Global config should be associated with this Scope.") String configurationRef, + @Optional(defaultValue = "INFO") Priority priority, + @Optional(defaultValue = "OUTBOUND_REQUEST_SCOPE") ScopeTracePoint scopeTracePoint, + @Optional @Summary("If not set, by default will log to the org.mule.extension.jsonlogger.JsonLogger category") String category, + @Optional(defaultValue = "#[correlationId]") @Placement(tab = "Advanced") String correlationId, + ComponentLocation location, + CorrelationInfo correlationInfo, + FlowListener flowListener, + Chain operations, + CompletionCallback callback) { + /** - * Log scope - */ - @Execution(ExecutionType.BLOCKING) - public void loggerScope(@DisplayName("Module configuration") @Example("JSON_Logger_Config") @Summary("Indicate which Global config should be associated with this Scope.") String configurationRef, - @Optional(defaultValue="INFO") Priority priority, - @Optional(defaultValue="OUTBOUND_REQUEST_SCOPE") ScopeTracePoint scopeTracePoint, - @Optional @Summary("If not set, by default will log to the org.mule.extension.jsonlogger.JsonLogger category") String category, - @Optional(defaultValue="#[correlationId]") @Placement(tab = "Advanced") String correlationId, - ComponentLocation location, - CorrelationInfo correlationInfo, - FlowListener flowListener, - Chain operations, - CompletionCallback callback) { - - /** - * BEFORE scope logger - * =================== - **/ - - Long initialTimestamp,loggerTimestamp; - initialTimestamp = loggerTimestamp = System.currentTimeMillis(); - - initLoggerCategory(category); - - LOGGER.debug("correlationInfo.getEventId(): " + correlationInfo.getEventId()); - LOGGER.debug("correlationInfo.getCorrelationId(): " + correlationInfo.getCorrelationId()); - - try { - // Add cache entry for initial timestamp based on unique EventId - initialTimestamp = configs.getConfig(configurationRef).getCachedTimerTimestamp(correlationInfo.getCorrelationId(), initialTimestamp); - } catch (Exception e) { - LOGGER.error("initialTimestamp could not be retrieved from the cache config. Defaulting to current System.currentTimeMillis()", e); - } - - // Calculate elapsed time based on cached initialTimestamp - Long elapsed = loggerTimestamp - initialTimestamp; - - //config.printTimersKeys(); - if (elapsed == 0) { - LOGGER.debug("configuring flowListener...."); - flowListener.onComplete(new TimerRemoverRunnable(correlationInfo.getCorrelationId(), configs.getConfig(configurationRef))); - } else { - LOGGER.debug("flowListener already configured"); - } - - /** - * Avoid Logger Scope logic execution based on log priority - */ - if (isLogEnabled(priority.toString())) { - // Execute Scope Logger - ObjectNode loggerProcessor = om.getObjectMapper().createObjectNode(); - - /** - * Custom field ordering for Logger Scope - * =============================== - **/ - loggerProcessor.put("correlationId", correlationId); - loggerProcessor.put("tracePoint", scopeTracePoint.toString() + "_BEFORE"); - loggerProcessor.put("priority", priority.toString()); - loggerProcessor.put("elapsed", elapsed); - loggerProcessor.put("scopeElapsed", 0); - if (configs.getConfig(configurationRef).getJsonOutput().isLogLocationInfo()) { - Map locationInfoMap = locationInfoToMap(location); - loggerProcessor.putPOJO("locationInfo", locationInfoMap); - } - loggerProcessor.put("timestamp", getFormattedTimestamp(loggerTimestamp)); - loggerProcessor.put("applicationName", configs.getConfig(configurationRef).getGlobalSettings().getApplicationName()); - loggerProcessor.put("applicationVersion", configs.getConfig(configurationRef).getGlobalSettings().getApplicationVersion()); - loggerProcessor.put("environment", configs.getConfig(configurationRef).getGlobalSettings().getEnvironment()); - loggerProcessor.put("threadName", Thread.currentThread().getName()); - - // Define JSON output formatting - // Print Logger - String finalLogBefore = printObjectToLog(loggerProcessor, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); - - // Added temp variable to comply with lambda - Long finalInitialTimestamp = initialTimestamp; - operations.process( - result -> { - - /** - * AFTER scope logger - * =================== - **/ - - Long endScopeTimestamp = System.currentTimeMillis(); - - // Calculate elapsed time - Long scopeElapsed = endScopeTimestamp - loggerTimestamp; - Long mainElapsed = endScopeTimestamp - finalInitialTimestamp; - - loggerProcessor.put("tracePoint", scopeTracePoint.toString() + "_AFTER"); - loggerProcessor.put("priority", priority.toString()); - loggerProcessor.put("elapsed", mainElapsed); - loggerProcessor.put("scopeElapsed", scopeElapsed); - loggerProcessor.put("timestamp", getFormattedTimestamp(endScopeTimestamp)); - - // Print Logger - String finalLogAfter = printObjectToLog(loggerProcessor, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); - - /** Forward Log to External Destination **/ - if (configs.getConfig(configurationRef).getExternalDestination() != null) { - publishScopeLogEvents(configurationRef, correlationId, finalLogBefore, finalLogAfter); - } - - callback.success(result); - }, - (error, previous) -> { - - /** ERROR scope logger **/ - - Long errorScopeTimestamp = System.currentTimeMillis(); - Long mainElapsed = errorScopeTimestamp - finalInitialTimestamp; - - // Calculate elapsed time - Long scopeElapsed = errorScopeTimestamp - loggerTimestamp; - - loggerProcessor.put("message", "Error found: " + error.getMessage()); - loggerProcessor.put("tracePoint", "EXCEPTION_SCOPE"); - loggerProcessor.put("priority", "ERROR"); - loggerProcessor.put("elapsed", mainElapsed); - loggerProcessor.put("scopeElapsed", scopeElapsed); - loggerProcessor.put("timestamp", getFormattedTimestamp(errorScopeTimestamp)); - - // Print Logger - String finalLogError = printObjectToLog(loggerProcessor, "ERROR", configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); - - /** Forward Log to External Destination **/ - if (configs.getConfig(configurationRef).getExternalDestination() != null) { - publishScopeLogEvents(configurationRef, correlationId, finalLogBefore, finalLogError); - } - - callback.error(error); - }); - } else { - // Execute operations without Logger - LOGGER.debug("Avoiding logger scope logic execution due to log priority not being enabled"); - operations.process( - result -> { - callback.success(result); - }, - (error, previous) -> { - callback.error(error); - }); - } + * BEFORE scope logger + * =================== + **/ + + Long initialTimestamp, loggerTimestamp; + initialTimestamp = loggerTimestamp = System.currentTimeMillis(); + + initLoggerCategory(category); + + LOGGER.debug("correlationInfo.getEventId(): " + correlationInfo.getEventId()); + LOGGER.debug("correlationInfo.getCorrelationId(): " + correlationInfo.getCorrelationId()); + + try { + // Add cache entry for initial timestamp based on unique EventId + initialTimestamp = configs.getConfig(configurationRef).getCachedTimerTimestamp(correlationInfo.getCorrelationId(), initialTimestamp); + } catch (Exception e) { + LOGGER.error("initialTimestamp could not be retrieved from the cache config. Defaulting to current System.currentTimeMillis()", e); } - - private void publishScopeLogEvents(String configurationRef, String correlationId, String finalLogBefore, String finalLogAfter) { - LOGGER.debug("externalDestination.getDestination().getSupportedCategories().isEmpty(): " + configs.getConfig(configurationRef).getExternalDestination().getSupportedCategories().isEmpty()); - LOGGER.debug("externalDestination.getDestination().getSupportedCategories().contains(jsonLogger.getName()): " + configs.getConfig(configurationRef).getExternalDestination().getSupportedCategories().contains(jsonLogger.getName())); - if (configs.getConfig(configurationRef).getExternalDestination().getSupportedCategories().isEmpty() || configs.getConfig(configurationRef).getExternalDestination().getSupportedCategories().contains(jsonLogger.getName())) { - LOGGER.debug(jsonLogger.getName() + " is a supported category for external destination"); - // Publishing before and after logEvents for better efficiency - logEvent.publishToExternalDestination(correlationId, finalLogBefore, configurationRef); - logEvent.publishToExternalDestination(correlationId, finalLogAfter, configurationRef); - } + + // Calculate elapsed time based on cached initialTimestamp + Long elapsed = loggerTimestamp - initialTimestamp; + + if (elapsed == 0) { + LOGGER.debug("configuring flowListener...."); + flowListener.onComplete(new TimerRemoverRunnable(correlationInfo.getCorrelationId(), configs.getConfig(configurationRef))); + } else { + LOGGER.debug("flowListener already configured"); } - - private Map locationInfoToMap(ComponentLocation location) { - Map locationInfo = new HashMap(); - //locationInfo.put("location", location.getLocation()); - locationInfo.put("rootContainer", location.getRootContainerName()); - locationInfo.put("component", location.getComponentIdentifier().getIdentifier().toString()); - locationInfo.put("fileName", location.getFileName().orElse("")); - locationInfo.put("lineInFile", String.valueOf(location.getLineInFile().orElse(null))); - return locationInfo; + + /** + * Avoid Logger Scope logic execution based on log priority + */ + if (isLogEnabled(priority.toString())) { + // Execute Scope Logger + ObjectNode loggerProcessor = om.getObjectMapper(new ArrayList<>()).createObjectNode(); + + /** + * Custom field ordering for Logger Scope + * =============================== + **/ + loggerProcessor.put("correlationId", correlationId); + loggerProcessor.put("tracePoint", scopeTracePoint.toString() + "_BEFORE"); + loggerProcessor.put("priority", priority.toString()); + loggerProcessor.put("elapsed", elapsed); + loggerProcessor.put("scopeElapsed", 0); + if (configs.getConfig(configurationRef).getJsonOutput().isLogLocationInfo()) { + Map locationInfoMap = locationInfoToMap(location); + loggerProcessor.putPOJO("locationInfo", locationInfoMap); + } + loggerProcessor.put("timestamp", getFormattedTimestamp(loggerTimestamp)); + loggerProcessor.put("applicationName", configs.getConfig(configurationRef).getGlobalSettings().getApplicationName()); + loggerProcessor.put("applicationVersion", configs.getConfig(configurationRef).getGlobalSettings().getApplicationVersion()); + loggerProcessor.put("environment", configs.getConfig(configurationRef).getGlobalSettings().getEnvironment()); + loggerProcessor.put("threadName", Thread.currentThread().getName()); + + // Define JSON output formatting + // Print Logger + String finalLogBefore = printObjectToLog(loggerProcessor, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); + + // Added temp variable to comply with lambda + Long finalInitialTimestamp = initialTimestamp; + operations.process( + result -> { + + /** + * AFTER scope logger + * =================== + **/ + + Long endScopeTimestamp = System.currentTimeMillis(); + + // Calculate elapsed time + Long scopeElapsed = endScopeTimestamp - loggerTimestamp; + Long mainElapsed = endScopeTimestamp - finalInitialTimestamp; + + loggerProcessor.put("tracePoint", scopeTracePoint.toString() + "_AFTER"); + loggerProcessor.put("priority", priority.toString()); + loggerProcessor.put("elapsed", mainElapsed); + loggerProcessor.put("scopeElapsed", scopeElapsed); + loggerProcessor.put("timestamp", getFormattedTimestamp(endScopeTimestamp)); + + // Print Logger + String finalLogAfter = printObjectToLog(loggerProcessor, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); + + callback.success(result); + }, + (error, previous) -> { + + /** ERROR scope logger **/ + + Long errorScopeTimestamp = System.currentTimeMillis(); + Long mainElapsed = errorScopeTimestamp - finalInitialTimestamp; + + // Calculate elapsed time + Long scopeElapsed = errorScopeTimestamp - loggerTimestamp; + + loggerProcessor.put("message", "Error found: " + error.getMessage()); + loggerProcessor.put("tracePoint", "EXCEPTION_SCOPE"); + loggerProcessor.put("priority", "ERROR"); + loggerProcessor.put("elapsed", mainElapsed); + loggerProcessor.put("scopeElapsed", scopeElapsed); + loggerProcessor.put("timestamp", getFormattedTimestamp(errorScopeTimestamp)); + + // Print Logger + String finalLogError = printObjectToLog(loggerProcessor, "ERROR", configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); + + callback.error(error); + }); + } else { + // Execute operations without Logger + LOGGER.debug("Avoiding logger scope logic execution due to log priority not being enabled"); + operations.process( + callback::success, + (error, previous) -> { + callback.error(error); + }); } - - private String getFormattedTimestamp(Long loggerTimestamp) { + } + + private Map locationInfoToMap(ComponentLocation location) { + Map locationInfo = new HashMap(); + locationInfo.put("rootContainer", location.getRootContainerName()); + locationInfo.put("component", location.getComponentIdentifier().getIdentifier().toString()); + locationInfo.put("fileName", location.getFileName().orElse("")); + locationInfo.put("lineInFile", String.valueOf(location.getLineInFile().orElse(null))); + return locationInfo; + } + + private String getFormattedTimestamp(Long loggerTimestamp) { /* Define timestamp: - DateTime: Defaults to ISO format - TimeZone: Defaults to UTC. Refer to https://en.wikipedia.org/wiki/List_of_tz_database_time_zones for valid timezones */ - DateTime dateTime = new DateTime(loggerTimestamp).withZone(org.joda.time.DateTimeZone.forID(System.getProperty("json.logger.timezone", "UTC"))); - String timestamp = dateTime.toString(); - if (System.getProperty("json.logger.dateformat") != null && !System.getProperty("json.logger.dateformat").equals("")) { - timestamp = dateTime.toString(System.getProperty("json.logger.dateformat")); - } - return timestamp; + DateTime dateTime = new DateTime(loggerTimestamp).withZone(org.joda.time.DateTimeZone.forID(System.getProperty("json.logger.timezone", "UTC"))); + String timestamp = dateTime.toString(); + if (System.getProperty("json.logger.dateformat") != null && !System.getProperty("json.logger.dateformat").equals("")) { + timestamp = dateTime.toString(System.getProperty("json.logger.dateformat")); } - - private String printObjectToLog(ObjectNode loggerObj, String priority, boolean isPrettyPrint) { - ObjectWriter ow = (isPrettyPrint) ? om.getObjectMapper().writer().withDefaultPrettyPrinter() : om.getObjectMapper().writer(); - String logLine = ""; - try { - logLine = ow.writeValueAsString(loggerObj); - } catch (Exception e) { - LOGGER.error("Error parsing log data as a string", e); - } - doLog(priority.toString(), logLine); - - return logLine; + return timestamp; + } + + private String printObjectToLog(ObjectNode loggerObj, String priority, boolean isPrettyPrint) { + ObjectWriter ow = (isPrettyPrint) ? om.getObjectMapper(new ArrayList<>()).writer().withDefaultPrettyPrinter() : om.getObjectMapper(new ArrayList<>()).writer(); + String logLine = ""; + try { + logLine = ow.writeValueAsString(loggerObj); + } catch (Exception e) { + LOGGER.error("Error parsing log data as a string", e); } - - private void doLog(String priority, String logLine) { - switch (priority) { - case "TRACE": - jsonLogger.trace(logLine); - break; - case "DEBUG": - jsonLogger.debug(logLine); - break; - case "INFO": - jsonLogger.info(logLine); - break; - case "WARN": - jsonLogger.warn(logLine); - break; - case "ERROR": - jsonLogger.error(logLine); - break; - } + doLog(priority, logLine); + + return logLine; + } + + private void doLog(String priority, String logLine) { + switch (priority) { + case "TRACE": + jsonLogger.trace(logLine); + break; + case "DEBUG": + jsonLogger.debug(logLine); + break; + case "INFO": + jsonLogger.info(logLine); + break; + case "WARN": + jsonLogger.warn(logLine); + break; + case "ERROR": + jsonLogger.error(logLine); + break; } - - private Boolean isLogEnabled(String priority) { - switch (priority) { - case "TRACE": - return jsonLogger.isTraceEnabled(); - case "DEBUG": - return jsonLogger.isDebugEnabled(); - case "INFO": - return jsonLogger.isInfoEnabled(); - case "WARN": - return jsonLogger.isWarnEnabled(); - case "ERROR": - return jsonLogger.isErrorEnabled(); - } - return false; + } + + private Boolean isLogEnabled(String priority) { + switch (priority) { + case "TRACE": + return jsonLogger.isTraceEnabled(); + case "DEBUG": + return jsonLogger.isDebugEnabled(); + case "INFO": + return jsonLogger.isInfoEnabled(); + case "WARN": + return jsonLogger.isWarnEnabled(); + case "ERROR": + return jsonLogger.isErrorEnabled(); } - - protected void initLoggerCategory(String category) { - if (category != null) { - jsonLogger = LoggerFactory.getLogger(category); - } else { - jsonLogger = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); - } - LOGGER.debug("category set: " + jsonLogger.getName()); + return false; + } + + protected void initLoggerCategory(String category) { + if (category != null) { + jsonLogger = LoggerFactory.getLogger(category); + } else { + jsonLogger = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); } - - // Allows executing timer cleanup on flowListener onComplete events - private static class TimerRemoverRunnable implements Runnable { - - private final String key; - private final JsonloggerConfiguration config; - - public TimerRemoverRunnable(String key, JsonloggerConfiguration config) { - this.key = key; - this.config = config; - } - - @Override - public void run() { - LOGGER.debug("Removing key: " + key); - config.removeCachedTimerTimestamp(key); - } + LOGGER.debug("category set: " + jsonLogger.getName()); + } + + // Allows executing timer cleanup on flowListener onComplete events + private static class TimerRemoverRunnable implements Runnable { + + private final String key; + private final JsonloggerConfiguration config; + + public TimerRemoverRunnable(String key, JsonloggerConfiguration config) { + this.key = key; + this.config = config; + } + + @Override + public void run() { + LOGGER.debug("Removing key: " + key); + config.removeCachedTimerTimestamp(key); } + } } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/datamask/JsonMasker.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/datamask/JsonMasker.java index 38f6a77..a55b948 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/datamask/JsonMasker.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/datamask/JsonMasker.java @@ -1,133 +1,90 @@ package org.mule.extension.jsonlogger.internal.datamask; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.MapperFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; -import com.jayway.jsonpath.Configuration; -import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.Option; -import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; import java.util.*; import java.util.regex.Pattern; public class JsonMasker { - - private static final Pattern digits = Pattern.compile("\\d"); - private static final Pattern capitalLetters = Pattern.compile("[A-Z]"); - private static final Pattern nonSpecialCharacters = Pattern.compile("[^X\\s!-/:-@\\[-`{-~]"); - - private static final Configuration jsonPathConfig = Configuration.builder() - .jsonProvider(new JacksonJsonNodeJsonProvider()) - .options(Option.AS_PATH_LIST, Option.SUPPRESS_EXCEPTIONS).build(); - - private final Set blacklistedKeys; - private final Set blacklistedJsonPaths; - private final boolean enabled; - - public JsonMasker(Collection blacklist, boolean enabled) { - this.enabled = enabled; - - blacklistedKeys = new HashSet<>(); - blacklistedJsonPaths = new HashSet<>(); - - blacklist.forEach(item -> { - if (item.startsWith("$")) { - blacklistedJsonPaths.add(JsonPath.compile(item)); - } else { - blacklistedKeys.add(item.toUpperCase()); - } - }); - } - - public JsonMasker() { - this(Collections.emptySet(), true); - } - - public JsonMasker(boolean enabled) { - this(Collections.emptySet(), enabled); + + private static final Pattern digits = Pattern.compile("\\d"); + private static final Pattern capitalLetters = Pattern.compile("[A-Z]"); + private static final Pattern nonSpecialCharacters = Pattern.compile("[^X\\s!-/:-@\\[-`{-~]"); + + private final Set blacklistedKeys; + private final boolean enabled; + + public JsonMasker(Collection blacklist, boolean enabled) { + this.enabled = enabled; + + blacklistedKeys = new HashSet<>(); + + blacklist.forEach(item -> { + blacklistedKeys.add(item.toUpperCase()); + }); + } + + public JsonNode mask(JsonNode target) { + if (!enabled) + return target; + if (target == null) + return null; + + return traverseAndMask(target.deepCopy(), "$", false); + } + + private JsonNode traverseAndMask(JsonNode target, String path, Boolean isBlackListed) { + if (target.isTextual() && isBlackListed) { + return new TextNode(maskString(target.asText())); } - - public JsonMasker(Collection blacklist) { - this(blacklist, true); + if (target.isNumber() && isBlackListed) { + return new TextNode(maskNumber(target.asText())); } - - public JsonNode mask(JsonNode target) { - if (!enabled) - return target; - if (target == null) - return null; - - Set expandedBlacklistedPaths = new HashSet<>(); - for (JsonPath jsonPath : blacklistedJsonPaths) { - if (jsonPath.isDefinite()) { - expandedBlacklistedPaths.add(jsonPath.getPath()); - } else { - for (JsonNode node : jsonPath.read(target, jsonPathConfig)) { - expandedBlacklistedPaths.add(node.asText()); - } - } + + if (target.isObject()) { + Iterator> fields = target.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + String childPath = appendPath(path, field.getKey()); + if (blacklistedKeys.contains(field.getKey().toUpperCase()) || isBlackListed) { + ((ObjectNode) target).replace(field.getKey(), traverseAndMask(field.getValue(), childPath, true)); + } else { + ((ObjectNode) target).replace(field.getKey(), traverseAndMask(field.getValue(), childPath, false)); } - - return traverseAndMask(target.deepCopy(), expandedBlacklistedPaths, "$", false); + } } - - @SuppressWarnings("ConstantConditions") - private JsonNode traverseAndMask(JsonNode target, Set expandedBlacklistedPaths, String path, Boolean isBlackListed) { - if (target.isTextual() && isBlackListed) { - return new TextNode(maskString(target.asText())); - } - if (target.isNumber() && isBlackListed) { - return new TextNode(maskNumber(target.asText())); - } - - if (target.isObject()) { - Iterator> fields = target.fields(); - while (fields.hasNext()) { - Map.Entry field = fields.next(); - String childPath = appendPath(path, field.getKey()); - if (blacklistedKeys.contains(field.getKey().toUpperCase()) || expandedBlacklistedPaths.contains(childPath) || isBlackListed == true) { - ((ObjectNode) target).replace(field.getKey(), traverseAndMask(field.getValue(), expandedBlacklistedPaths, childPath, true)); - } else { - ((ObjectNode) target).replace(field.getKey(), traverseAndMask(field.getValue(), expandedBlacklistedPaths, childPath, false)); - } - } + if (target.isArray()) { + for (int i = 0; i < target.size(); i++) { + String childPath = appendPath(path, i); + if (isBlackListed) { + ((ArrayNode) target).set(i, traverseAndMask(target.get(i), childPath, true)); + } else { + ((ArrayNode) target).set(i, traverseAndMask(target.get(i), childPath, false)); } - if (target.isArray()) { - for (int i = 0; i < target.size(); i++) { - String childPath = appendPath(path, i); - if (expandedBlacklistedPaths.contains(childPath) || isBlackListed == true) { - ((ArrayNode) target).set(i, traverseAndMask(target.get(i), expandedBlacklistedPaths, childPath, true)); - } else { - ((ArrayNode) target).set(i, traverseAndMask(target.get(i), expandedBlacklistedPaths, childPath, false)); - } - } - } - return target; - } - - private static String appendPath(String path, String key) { - return path + "['" + key + "']"; + } } - - private static String appendPath(String path, int ind) { - return path + "[" + ind + "]"; - } - - private static String maskString(String value) { - String tmpMasked = digits.matcher(value).replaceAll("*"); - tmpMasked = capitalLetters.matcher(tmpMasked).replaceAll("X"); - return nonSpecialCharacters.matcher(tmpMasked).replaceAll("x"); - } - - private static String maskNumber(String value) { - return value.replaceAll("[0-9]", "*"); - } - + return target; + } + + private static String appendPath(String path, String key) { + return path + "['" + key + "']"; + } + + private static String appendPath(String path, int ind) { + return path + "[" + ind + "]"; + } + + private static String maskString(String value) { + String tmpMasked = digits.matcher(value).replaceAll("*"); + tmpMasked = capitalLetters.matcher(tmpMasked).replaceAll("X"); + return nonSpecialCharacters.matcher(tmpMasked).replaceAll("x"); + } + + private static String maskNumber(String value) { + return value.replaceAll("[0-9]", "*"); + } + } \ No newline at end of file diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/Destination.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/Destination.java index 8a29286..76a85c0 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/Destination.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/Destination.java @@ -1,21 +1,18 @@ package org.mule.extension.jsonlogger.internal.destinations; -import org.mule.runtime.extension.api.client.ExtensionsClient; - import java.util.ArrayList; -import java.util.Map; public interface Destination { - - public String getSelectedDestinationType(); - - public ArrayList getSupportedCategories(); - - public int getMaxBatchSize(); - - public void sendToExternalDestination(String finalLog); - - public void initialise(); - - public void dispose(); + + public String getSelectedDestinationType(); + + public ArrayList getSupportedCategories(); + + public int getMaxBatchSize(); + + public void sendToExternalDestination(String finalLog); + + public void initialise(); + + public void dispose(); } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ConfigsSingleton.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ConfigsSingleton.java index ef42fdb..4ae2a27 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ConfigsSingleton.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ConfigsSingleton.java @@ -7,7 +7,7 @@ public class ConfigsSingleton { - private Map configs = new HashMap(); + private final Map configs = new HashMap<>(); public Map getConfigs() { return configs; diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java index 48292b9..e1bdb81 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java @@ -31,7 +31,7 @@ public class LogEventSingleton implements Initialisable, Disposable { @Inject ConfigsSingleton configs; - private HashMap destinations = new HashMap(); + private HashMap destinations = new HashMap<>(); // Construct the Disruptor private Disruptor disruptor; diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/MaskingDeserializer.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/MaskingDeserializer.java new file mode 100644 index 0000000..346c69c --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/MaskingDeserializer.java @@ -0,0 +1,176 @@ +package org.mule.extension.jsonlogger.internal.singleton; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.JsonNodeDeserializer; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.commons.lang.StringUtils; + +import java.io.IOException; +import java.util.List; + +public class MaskingDeserializer extends JsonNodeDeserializer { + List maskedFields; + + public MaskingDeserializer(List maskedFields) { + this.maskedFields = maskedFields; + } + + @Override + public JsonNode deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + switch (p.currentTokenId()) { + case 1: + return deserializeObject(p, ctxt, ctxt.getNodeFactory(), false); + case 3: + return deserializeArray(p, ctxt, ctxt.getNodeFactory(), false); + default: + return deserializeAny(p, ctxt, ctxt.getNodeFactory(), false); + } + } + + protected final ObjectNode deserializeObject(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { + ObjectNode node = nodeFactory.objectNode(); + + for (String key = p.nextFieldName(); key != null; key = p.nextFieldName()) { + if (maskedFields.contains(key)) { + shouldMask = true; + } + JsonToken t = p.nextToken(); + if (t == null) { + t = JsonToken.NOT_AVAILABLE; + } + + JsonNode value; + switch (t.id()) { + case 1: + value = deserializeObject(p, ctxt, nodeFactory, shouldMask); + shouldMask = false; + break; + case 2: + case 4: + case 5: + case 8: + default: + value = deserializeAny(p, ctxt, nodeFactory, shouldMask); + shouldMask = false; + break; + case 3: + value = deserializeArray(p, ctxt, nodeFactory, shouldMask); + shouldMask = false; + break; + case 6: + if (shouldMask) { + value = nodeFactory.textNode(replace(p.getText())); + } else { + value = nodeFactory.textNode(p.getText()); + } + break; + case 7: + value = this._fromInt(p, ctxt, nodeFactory); + break; + case 9: + value = nodeFactory.booleanNode(true); + break; + case 10: + value = nodeFactory.booleanNode(false); + break; + case 11: + value = nodeFactory.nullNode(); + break; + case 12: + value = this._fromEmbedded(p, ctxt, nodeFactory); + } + + JsonNode old = node.replace(key, value); + if (old != null) { + this._handleDuplicateField(p, ctxt, nodeFactory, key, node, old, value); + } + } + + return node; + } + + protected final ArrayNode deserializeArray(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { + ArrayNode node = nodeFactory.arrayNode(); + + while (true) { + JsonToken t = p.nextToken(); + switch (t.id()) { + case 1: + node.add(deserializeObject(p, ctxt, nodeFactory, shouldMask)); + break; + case 2: + case 5: + case 8: + default: + node.add(deserializeAny(p, ctxt, nodeFactory, shouldMask)); + break; + case 3: + node.add(deserializeArray(p, ctxt, nodeFactory, shouldMask)); + break; + case 4: + return node; + case 6: + if (shouldMask) { + node.add(nodeFactory.textNode(replace(p.getText()))); + } else { + node.add(nodeFactory.textNode(p.getText())); + } + break; + case 7: + node.add(this._fromInt(p, ctxt, nodeFactory)); + break; + case 9: + node.add(nodeFactory.booleanNode(true)); + break; + case 10: + node.add(nodeFactory.booleanNode(false)); + break; + case 11: + node.add(nodeFactory.nullNode()); + break; + case 12: + node.add(this._fromEmbedded(p, ctxt, nodeFactory)); + } + } + } + + protected final JsonNode deserializeAny(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { + switch (p.currentTokenId()) { + case 2: + return nodeFactory.objectNode(); + case 3: + case 4: + default: + return (JsonNode) ctxt.handleUnexpectedToken(this.handledType(), p); + case 5: + return deserializeObjectAtName(p, ctxt, nodeFactory); + case 6: + if (shouldMask) { + return nodeFactory.textNode(replace(p.getText())); + } + return nodeFactory.textNode(p.getText()); + case 7: + return this._fromInt(p, ctxt, nodeFactory); + case 8: + return this._fromFloat(p, ctxt, nodeFactory); + case 9: + return nodeFactory.booleanNode(true); + case 10: + return nodeFactory.booleanNode(false); + case 11: + return nodeFactory.nullNode(); + case 12: + return this._fromEmbedded(p, ctxt, nodeFactory); + } + } + + private String replace(String input) { + return StringUtils.repeat("*", input.length()); + } + +} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java index fd370a7..96c6098 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java @@ -1,19 +1,22 @@ package org.mule.extension.jsonlogger.internal.singleton; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.module.SimpleModule; -public class ObjectMapperSingleton { - - // JSON Object Mapper - private final ObjectMapper om = new ObjectMapper() - .setSerializationInclusion(JsonInclude.Include.NON_NULL); -// .configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true) -// .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); +import java.util.List; - public ObjectMapper getObjectMapper() { - return this.om; - } +public class ObjectMapperSingleton { + + private final ObjectMapper om = new ObjectMapper() + .setSerializationInclusion(JsonInclude.Include.NON_NULL) + .setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + + public ObjectMapper getObjectMapper(List maskedFields) { + SimpleModule module = new SimpleModule(); + module.addDeserializer(JsonNode.class, new MaskingDeserializer(maskedFields)); + this.om.registerModule(module); + return this.om; + } } diff --git a/json-logger/src/test/java/JsonMaskerTest.java b/json-logger/src/test/java/JsonMaskerTest.java new file mode 100644 index 0000000..c2975b4 --- /dev/null +++ b/json-logger/src/test/java/JsonMaskerTest.java @@ -0,0 +1,30 @@ + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Assert; +import org.junit.Test; +import org.mule.extension.jsonlogger.internal.singleton.ObjectMapperSingleton; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class JsonMaskerTest { + ClassLoader classloader = Thread.currentThread().getContextClassLoader(); + InputStream is = classloader.getResourceAsStream("payload.json"); + List maskedFields = Collections.singletonList("attributes"); + private final ObjectMapper om = new ObjectMapperSingleton().getObjectMapper(maskedFields); + + @Test + public void caseInsensitve() throws IOException { + JsonNode payload = om.readTree(is); + Map masked = om.convertValue(payload, new TypeReference>(){}); + Assert.assertEquals("{name=****, phone=************, gender=****}", masked.get("attributes").toString()); + } +} \ No newline at end of file diff --git a/json-logger/src/test/resources/payload.json b/json-logger/src/test/resources/payload.json new file mode 100644 index 0000000..edb65df --- /dev/null +++ b/json-logger/src/test/resources/payload.json @@ -0,0 +1,19 @@ +{ + "data": { + "type": "articles", + "id": "1", + "relationships": { + "author": { + "data": { + "id": "42", + "type": "people" + } + } + } + }, + "attributes": { + "name": "John", + "phone": "888-900-0000", + "gender": "male" + } +} \ No newline at end of file From fe52babb0df72866be5d678c6ff1b66bf8d6a4ab Mon Sep 17 00:00:00 2001 From: Jazzy Date: Mon, 10 May 2021 19:01:11 -0700 Subject: [PATCH 02/12] Use custom deserliazer and apply masking in place --- .../jsonlogger/internal/JsonloggerExtension.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java index dea2eba..d2bc9c6 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java @@ -14,12 +14,12 @@ * This is the main class of an extension, is the entry point from which configurations, connection providers, operations * and sources are going to be declared. */ -//@Xml(prefix = "json-logger") -//@Extension(name = "JSON Logger") -//@Export(resources = {"modules/JSONLoggerModule.dwl"}) -//@Configurations(JsonloggerConfiguration.class) -////@SubTypeMapping(baseType = Destination.class, -//// subTypes = {JMSDestination.class, AMQDestination.class, AMQPDestination.class}) +@Xml(prefix = "json-logger") +@Extension(name = "JSON Logger") +@Export(resources = {"modules/JSONLoggerModule.dwl"}) +@Configurations(JsonloggerConfiguration.class) +@SubTypeMapping(baseType = Destination.class, + subTypes = {JMSDestination.class, AMQDestination.class, AMQPDestination.class}) public class JsonloggerExtension { } From cbbbcc56dec4a089e20f6b6ec732dc064ba20ae3 Mon Sep 17 00:00:00 2001 From: Jazzy Date: Wed, 2 Jun 2021 17:57:53 -0700 Subject: [PATCH 03/12] cleanup working point --- .../internal/JsonloggerConfiguration.java | 30 +-- .../internal/JsonloggerExtension.java | 9 +- .../internal/destinations/AMQDestination.java | 189 --------------- .../destinations/AMQPDestination.java | 98 -------- .../internal/destinations/Destination.java | 18 -- .../internal/destinations/JMSDestination.java | 100 -------- .../client/AsyncMuleCourierRestClient.java | 220 ------------------ .../MuleBasedAnypointMQClientFactory.java | 35 --- .../destinations/events/LogEvent.java | 27 --- .../destinations/events/LogEventFactory.java | 10 - .../destinations/events/LogEventHandler.java | 67 ------ .../LogEventProducerWithTranslator.java | 24 -- .../internal/singleton/LogEventSingleton.java | 80 +------ json-logger/src/test/java/JsonMaskerTest.java | 2 + 14 files changed, 12 insertions(+), 897 deletions(-) delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/AMQDestination.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/AMQPDestination.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/Destination.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/JMSDestination.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/amq/client/AsyncMuleCourierRestClient.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/amq/client/MuleBasedAnypointMQClientFactory.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEvent.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEventFactory.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEventHandler.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEventProducerWithTranslator.java diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConfiguration.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConfiguration.java index 86ba69a..ee2a7d2 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConfiguration.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConfiguration.java @@ -1,16 +1,12 @@ package org.mule.extension.jsonlogger.internal; import org.mule.extension.jsonlogger.api.pojos.LoggerConfig; -import org.mule.extension.jsonlogger.internal.destinations.Destination; import org.mule.extension.jsonlogger.internal.singleton.ConfigsSingleton; import org.mule.runtime.api.lifecycle.Disposable; import org.mule.runtime.api.lifecycle.Initialisable; import org.mule.runtime.api.lifecycle.InitialisationException; import org.mule.runtime.extension.api.annotation.Operations; -import org.mule.runtime.extension.api.annotation.param.Optional; -import org.mule.runtime.extension.api.annotation.param.Parameter; import org.mule.runtime.extension.api.annotation.param.RefName; -import org.mule.runtime.extension.api.annotation.param.display.Placement; import javax.inject.Inject; import java.util.concurrent.ConcurrentHashMap; @@ -28,10 +24,6 @@ public class JsonloggerConfiguration extends LoggerConfig implements Initialisab @RefName private String configName; - @Parameter - @Optional - @Placement(tab = "Destinations") - private Destination externalDestination; public String getConfigName() { return configName; @@ -43,12 +35,6 @@ public String getConfigName() { public ConcurrentHashMap getTimers() { return timers; } - public void setTimers(ConcurrentHashMap timers) { this.timers = timers; } - - public void printTimersKeys () { - System.out.println("Current timers: " + timers); - } - public Long getCachedTimerTimestamp(String key, Long initialTimeStamp) throws Exception { Long startTimestamp = timers.putIfAbsent(key, initialTimeStamp); return (startTimestamp == null) ? timers.get(key) : startTimestamp; @@ -58,27 +44,13 @@ public void removeCachedTimerTimestamp(String key) { timers.remove(key); } - /** Init configs singleton **/ - public void setExternalDestination(Destination externalDestination) { - this.externalDestination = externalDestination; - } - - public Destination getExternalDestination() { - return externalDestination; - } - + @Override public void initialise() throws InitialisationException { - if (this.externalDestination != null) { - this.externalDestination.initialise(); - } configsSingleton.addConfig(configName, this); // Should be refactored once SDK supports passing configs to Scopes } @Override public void dispose() { - if (this.externalDestination != null) { - this.externalDestination.dispose(); - } } } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java index d2bc9c6..49b10b8 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java @@ -1,13 +1,8 @@ package org.mule.extension.jsonlogger.internal; -import org.mule.extension.jsonlogger.internal.destinations.AMQDestination; -import org.mule.extension.jsonlogger.internal.destinations.AMQPDestination; -import org.mule.extension.jsonlogger.internal.destinations.Destination; -import org.mule.extension.jsonlogger.internal.destinations.JMSDestination; +import org.mule.runtime.extension.api.annotation.Configurations; import org.mule.runtime.extension.api.annotation.Export; import org.mule.runtime.extension.api.annotation.Extension; -import org.mule.runtime.extension.api.annotation.Configurations; -import org.mule.runtime.extension.api.annotation.SubTypeMapping; import org.mule.runtime.extension.api.annotation.dsl.xml.Xml; /** @@ -18,8 +13,6 @@ @Extension(name = "JSON Logger") @Export(resources = {"modules/JSONLoggerModule.dwl"}) @Configurations(JsonloggerConfiguration.class) -@SubTypeMapping(baseType = Destination.class, - subTypes = {JMSDestination.class, AMQDestination.class, AMQPDestination.class}) public class JsonloggerExtension { } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/AMQDestination.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/AMQDestination.java deleted file mode 100644 index 0c3e831..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/AMQDestination.java +++ /dev/null @@ -1,189 +0,0 @@ -package org.mule.extension.jsonlogger.internal.destinations; - -import com.mulesoft.mq.restclient.api.*; -import com.mulesoft.mq.restclient.impl.OAuthCredentials; -import org.mule.extension.jsonlogger.internal.destinations.amq.client.MuleBasedAnypointMQClientFactory; -import org.mule.runtime.api.metadata.MediaType; -import org.mule.runtime.api.scheduler.SchedulerService; -import org.mule.runtime.extension.api.annotation.param.NullSafe; -import org.mule.runtime.extension.api.annotation.param.Optional; -import org.mule.runtime.extension.api.annotation.param.Parameter; -import org.mule.runtime.extension.api.annotation.param.display.DisplayName; -import org.mule.runtime.extension.api.annotation.param.display.Example; -import org.mule.runtime.extension.api.annotation.param.display.Password; -import org.mule.runtime.extension.api.annotation.param.display.Summary; -import org.mule.runtime.http.api.HttpService; -import org.mule.runtime.http.api.client.HttpClient; -import org.mule.runtime.http.api.client.HttpClientConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; -import java.io.ByteArrayInputStream; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -public class AMQDestination implements Destination { - - private static final Logger LOGGER = LoggerFactory.getLogger(AMQDestination.class); - - @Parameter - @Optional - @Summary("Name of the target queue or exchange destination (e.g. logger-queue, logger-exchange)") - @DisplayName("Queue or Exchange Destination") - private String queueOrExchangeDestination; - - /** - * The region URL where the Queue resides. This URL can be obtained and configured from the Anypoint Platform > MQ console. - * Copy/paste the region URL into this field." - */ - @Parameter - @DisplayName("URL") - @Example("https://mq-us-east-1.anypoint.mulesoft.com/api/v1") - @Optional(defaultValue = "https://mq-us-east-1.anypoint.mulesoft.com/api/v1") - @Summary("The region URL where the Queue resides. Obtain this URL from the Anypoint Platform > MQ") - private String url; - - /** - * In Anypoint Platform > MQ > Client Apps, click an app name (or create a new app) and - * click Copy for the Client App ID field. Paste this value in the Studio Client App ID field - */ - @Parameter - @DisplayName("Client App ID") - @Summary("The Client App ID to be used. Obtain this ID from Anypoint Platform > MQ > Client Apps") - private String clientId; - - /** - * In Anypoint Platform > MQ > Client Apps, click an app name (or create a new app) and - * click Copy for the Client Secret field. Paste this value in the Studio Client Secret field. - */ - @Parameter - @DisplayName("Client Secret") - @Password - @Summary("The Client App Secret for the given Client App ID") - private String clientSecret; - - @Parameter - @Optional - @NullSafe - @Summary("Indicate which log categories should be send (e.g. [\"my.category\",\"another.category\"]). If empty, all will be send.") - @DisplayName("Log Categories") - private ArrayList logCategories; - - @Parameter - @Optional(defaultValue = "25") - @Summary("Indicate max quantity of logs entries to be send to the external destination") - @DisplayName("Max Batch Size") - private int maxBatchSize; - - @Override - public int getMaxBatchSize() { - return this.maxBatchSize; - } - - @Inject - protected HttpService httpService; - - @Inject - protected SchedulerService schedulerService; - - private final String AMQ_HTTP_CLIENT = "amqHttpClient"; - private final String USER_AGENT_VERSION = "3.1.0"; // Version of the AMQ Connector code this logic is based of - - private HttpClientConfiguration httpClientConfiguration; - private HttpClient httpClient; - private AnypointMqClient amqClient; - private DestinationLocator destinationLocator; - private DestinationLocation location; - - @Override - public String getSelectedDestinationType() { - return "AMQ"; - } - - @Override - public ArrayList getSupportedCategories() { - return logCategories; - } - - @Override - public void sendToExternalDestination(String finalLog) { - - try { - // Send message - MediaType mediaType = MediaType.parse("application/json; charset=UTF-8"); - AnypointMQMessage message = createMessage(finalLog, true, mediaType.toString(), - mediaType.getCharset(), null, new HashMap<>(), null, null); - - this.destinationLocator.getDestination(this.location) - .send(message) - .subscribe(new CourierObserver() { - @Override - public void onSuccess(MessageIdResult result) { - LOGGER.debug("AMQ Message Id: " + result.getMessageId()); - } - - @Override - public void onError(Throwable e) { - String msg = String.format("Failed to publish message to destination '%s': %s", location, e.getMessage()); - LOGGER.error(msg, e); - } - }); - } catch (Exception e) { - LOGGER.error("Error sending message to AMQ: " + e.getMessage()); - e.printStackTrace(); - } - } - - private static AnypointMQMessage createMessage(String messageBody, boolean sendContentType, String mediaType, - java.util.Optional charset, String messageId, Map properties, - java.util.Optional deliveryDelay, java.util.Optional messageGroupId) { - AnypointMQMessageBuilder messageBuilder = new AnypointMQMessageBuilder(); - messageBuilder.withBody(new ByteArrayInputStream(messageBody.getBytes())); - - String id = java.util.Optional.ofNullable(messageId).orElseGet(UUID::randomUUID).toString(); - messageBuilder.withMessageId(id); - - if (sendContentType) { - messageBuilder.addProperty(AnypointMQMessage.Properties.AMQ_MESSAGE_CONTENT_TYPE, mediaType); - charset.map(Object::toString) - .ifPresent(value -> messageBuilder.addProperty("MULE_ENCODING", value)); - } - - if (properties != null) { - messageBuilder.withProperties(properties); - } - - return messageBuilder.build(); - } - - public void initialise() { - // Start HTTP Configuration - Long startTimestamp = System.currentTimeMillis(); - this.httpClientConfiguration = new HttpClientConfiguration.Builder() - .setName(AMQ_HTTP_CLIENT) - .build(); - this.httpClient = httpService.getClientFactory().create(this.httpClientConfiguration); - httpClient.start(); - - // Start AMQ Client - this.amqClient = new MuleBasedAnypointMQClientFactory(this.httpClient, schedulerService.ioScheduler()) - .createClient(url, new OAuthCredentials(clientId, clientSecret), USER_AGENT_VERSION); - this.amqClient.init(); - - // Locate AMQ destination - this.destinationLocator = amqClient.createDestinationLocator(); - - // Destination Location - this.location = this.destinationLocator.getDestinationLocation(queueOrExchangeDestination); - } - - public void dispose() { - this.httpClient.stop(); - this.amqClient.dispose(); - } - -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/AMQPDestination.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/AMQPDestination.java deleted file mode 100644 index 8665af4..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/AMQPDestination.java +++ /dev/null @@ -1,98 +0,0 @@ -package org.mule.extension.jsonlogger.internal.destinations; - -import com.mule.extensions.amqp.api.message.AmqpMessageBuilder; -import com.mule.extensions.amqp.api.message.AmqpProperties; -import org.mule.extension.jsonlogger.api.pojos.Priority; -import org.mule.runtime.api.metadata.TypedValue; -import org.mule.runtime.extension.api.annotation.param.NullSafe; -import org.mule.runtime.extension.api.annotation.param.Optional; -import org.mule.runtime.extension.api.annotation.param.Parameter; -import org.mule.runtime.extension.api.annotation.param.display.DisplayName; -import org.mule.runtime.extension.api.annotation.param.display.Summary; -import org.mule.runtime.extension.api.annotation.param.reference.ConfigReference; -import org.mule.runtime.extension.api.client.DefaultOperationParameters; -import org.mule.runtime.extension.api.client.ExtensionsClient; -import org.mule.runtime.extension.api.client.OperationParameters; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import static org.mule.runtime.api.metadata.DataType.JSON_STRING; - -public class AMQPDestination implements Destination { - - private static final Logger LOGGER = LoggerFactory.getLogger(AMQPDestination.class); - - @Inject - ExtensionsClient extensionsClient; - - @Parameter - @Optional - @ConfigReference(namespace = "AMQP", name = "CONFIG") - @DisplayName("Configuration Ref") - private String amqpConfigurationRef; - - @Parameter - @Optional - @Summary("Name of the target exchange destination (e.g. logger-exchange)") - @DisplayName("Exchange Destination") - private String exchangeDestination; - - @Parameter - @Optional - @NullSafe - @Summary("Indicate which log categories should be send (e.g. [\"my.category\",\"another.category\"]). If empty, all will be send.") - @DisplayName("Log Categories") - private ArrayList logCategories; - - @Parameter - @Optional(defaultValue = "25") - @Summary("Indicate max quantity of logs entries to be send to the external destination") - @DisplayName("Max Batch Size") - private int maxBatchSize; - - @Override - public int getMaxBatchSize() { - return this.maxBatchSize; - } - - @Override - public String getSelectedDestinationType() { - return "AMQP"; - } - - @Override - public ArrayList getSupportedCategories() { - return logCategories; - } - - @Override - public void sendToExternalDestination(String finalLog) { - try { - OperationParameters parameters = DefaultOperationParameters.builder().configName(this.amqpConfigurationRef) - .addParameter("exchangeName", this.exchangeDestination) - .addParameter("messageBuilder", AmqpMessageBuilder.class, DefaultOperationParameters.builder() - .addParameter("body", new TypedValue<>(finalLog, JSON_STRING)) - .addParameter("properties", new AmqpProperties())) - .build(); - extensionsClient.executeAsync("AMQP", "publish", parameters); - } catch (Exception e) { - LOGGER.error("Error: " + e.getMessage()); - e.printStackTrace(); - } - } - - @Override - public void initialise() { - - } - - @Override - public void dispose() { - - } -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/Destination.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/Destination.java deleted file mode 100644 index 76a85c0..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/Destination.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.mule.extension.jsonlogger.internal.destinations; - -import java.util.ArrayList; - -public interface Destination { - - public String getSelectedDestinationType(); - - public ArrayList getSupportedCategories(); - - public int getMaxBatchSize(); - - public void sendToExternalDestination(String finalLog); - - public void initialise(); - - public void dispose(); -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/JMSDestination.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/JMSDestination.java deleted file mode 100644 index 9f33df4..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/JMSDestination.java +++ /dev/null @@ -1,100 +0,0 @@ -package org.mule.extension.jsonlogger.internal.destinations; - -import org.mule.extension.jsonlogger.api.pojos.Priority; -import org.mule.extensions.jms.api.message.JmsMessageBuilder; -import org.mule.extensions.jms.api.message.JmsxProperties; -import org.mule.runtime.api.metadata.TypedValue; -import org.mule.runtime.extension.api.annotation.param.NullSafe; -import org.mule.runtime.extension.api.annotation.param.Optional; -import org.mule.runtime.extension.api.annotation.param.Parameter; -import org.mule.runtime.extension.api.annotation.param.display.DisplayName; -import org.mule.runtime.extension.api.annotation.param.display.Summary; -import org.mule.runtime.extension.api.annotation.param.reference.ConfigReference; -import org.mule.runtime.extension.api.client.DefaultOperationParameters; -import org.mule.runtime.extension.api.client.ExtensionsClient; -import org.mule.runtime.extension.api.client.OperationParameters; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import static org.mule.runtime.api.metadata.DataType.JSON_STRING; - -public class JMSDestination implements Destination { - - private static final Logger LOGGER = LoggerFactory.getLogger(JMSDestination.class); - - @Inject - ExtensionsClient extensionsClient; - - @Parameter - @Optional - @ConfigReference(namespace = "JMS", name = "CONFIG") - @DisplayName("Configuration Ref") - private String jmsConfigurationRef; - - @Parameter - @Optional - @Summary("Name of the target queue destination (e.g. logger-queue)") - @DisplayName("Queue Destination") - private String queueDestination; - - @Parameter - @Optional - @NullSafe - @Summary("Indicate which log categories should be send (e.g. [\"my.category\",\"another.category\"]). If empty, all will be send.") - @DisplayName("Log Categories") - private ArrayList logCategories; - - @Parameter - @Optional(defaultValue = "25") - @Summary("Indicate max quantity of logs entries to be send to the external destination") - @DisplayName("Max Batch Size") - private int maxBatchSize; - - @Override - public int getMaxBatchSize() { - return this.maxBatchSize; - } - - @Override - public String getSelectedDestinationType() { - return "JMS"; - } - - @Override - public ArrayList getSupportedCategories() { - return logCategories; - } - - @Override - public void sendToExternalDestination(String finalLog) { - try { - OperationParameters parameters = DefaultOperationParameters.builder().configName(this.jmsConfigurationRef) - .addParameter("destination", this.queueDestination) - .addParameter("messageBuilder", JmsMessageBuilder.class, DefaultOperationParameters.builder() - .addParameter("body", new TypedValue<>(finalLog, JSON_STRING)) - .addParameter("jmsxProperties", new JmsxProperties()) - .addParameter("properties", new HashMap())) - .build(); - extensionsClient.executeAsync("JMS", "publish", parameters); - } catch (Exception e) { - LOGGER.error("Error: " + e.getMessage()); - e.printStackTrace(); - } - } - - @Override - public void initialise() { - - } - - @Override - public void dispose() { - - } -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/amq/client/AsyncMuleCourierRestClient.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/amq/client/AsyncMuleCourierRestClient.java deleted file mode 100644 index 817202e..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/amq/client/AsyncMuleCourierRestClient.java +++ /dev/null @@ -1,220 +0,0 @@ -/** - * (c) 2003-2018 MuleSoft, Inc. This software is protected under international copyright law. All use of this software is subject to - * MuleSoft's Master Subscription Agreement (or other Terms of Service) separately entered into between you and MuleSoft. If such an - * agreement is not in place, you may not use the software. - */ -package org.mule.extension.jsonlogger.internal.destinations.amq.client; - -import com.mulesoft.mq.restclient.impl.OAuthCredentials; -import com.mulesoft.mq.restclient.internal.Request; -import com.mulesoft.mq.restclient.internal.RequestBuilder; -import com.mulesoft.mq.restclient.internal.Response; -import com.mulesoft.mq.restclient.internal.client.AbstractCourierRestClient; -import org.apache.commons.io.IOUtils; -import org.mule.runtime.api.scheduler.Scheduler; -import org.mule.runtime.api.util.MultiMap; -import org.mule.runtime.http.api.client.HttpClient; -import org.mule.runtime.http.api.domain.entity.ByteArrayHttpEntity; -import org.mule.runtime.http.api.domain.entity.HttpEntity; -import org.mule.runtime.http.api.domain.entity.multipart.HttpPart; -import org.mule.runtime.http.api.domain.entity.multipart.MultipartHttpEntity; -import org.mule.runtime.http.api.domain.message.request.HttpRequest; -import org.mule.runtime.http.api.domain.message.request.HttpRequestBuilder; -import org.mule.runtime.http.api.domain.message.response.HttpResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import rx.Observable; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.concurrent.TimeUnit; - -public class AsyncMuleCourierRestClient extends AbstractCourierRestClient { - - private static final Logger LOGGER = LoggerFactory.getLogger(AsyncMuleCourierRestClient.class); - private static final int RESPONSE_TIMEOUT_MILLIS = 60000; - - private final HttpClient httpClient; - private final Scheduler scheduler; - - public AsyncMuleCourierRestClient(String courierApiUrl, OAuthCredentials oAuthCredentials, - String userAgentInfo, HttpClient httpClient, Scheduler scheduler) { - super(courierApiUrl, oAuthCredentials, userAgentInfo); - this.httpClient = httpClient; - this.scheduler = scheduler; - } - - @Override - protected Observable process(Request request) { - logProcessStart(request); - return Observable.create(subscriber -> httpClient.sendAsync(((MuleBasedRequest) request).getHttpRequest(), - RESPONSE_TIMEOUT_MILLIS, true, null) - .whenCompleteAsync((response, exception) -> { - if (exception != null) { - logProcessError(request, exception); - subscriber.onError(exception); - } else { - try { - Response mqResponse = convert(response); - logProcessSuccess(request, mqResponse); - subscriber.onNext(mqResponse); - } finally { - subscriber.onCompleted(); - } - } - }, command -> { - try { - scheduler.submit(command); - } catch (Exception e){ - subscriber.onError(e); - LOGGER.debug("An error occurred while processing the request: " + e.getMessage()); - } - })); - } - - protected static Response convert(HttpResponse httpResponse) { - return new Response() { - - @Override - public String getBody() { - try (InputStream stream = httpResponse.getEntity().getContent()) { - return IOUtils.toString(stream, StandardCharsets.UTF_8); - } catch (IOException e) { - throw new RuntimeException("Can not retrieve response body.", e); - } - } - - @Override - public boolean isOk() { - return getStatusCode() >= 200 && getStatusCode() < 300; - } - - @Override - public String getStatusText() { - return httpResponse.getReasonPhrase(); - } - - @Override - public int getStatusCode() { - return httpResponse.getStatusCode(); - } - - @Override - public String getHeader(String name) { - return httpResponse.getHeaderValue(name); - } - - @Override - public boolean isUnauthorized() { - return getStatusCode() == 401; - } - }; - } - - - @Override - protected RequestBuilder newRequestBuilder() { - return new RequestBuilder() { - - private HttpRequestBuilder httpRequestBuilder = HttpRequest.builder(); - private MultiMap queryParams = new MultiMap<>(); - private HttpEntity entity; - - @Override - public RequestBuilder wrap(Request request) { - MuleBasedRequest muleBasedRequest = (MuleBasedRequest) request; - use(request.getMethod()); - to(request.getUrl()); - queryParams = muleBasedRequest.httpRequest.getQueryParams(); - entity = muleBasedRequest.httpRequest.getEntity(); - return this; - } - - @Override - public RequestBuilder use(Method method) { - httpRequestBuilder.method(method.name()); - return this; - } - - @Override - public RequestBuilder to(String url) { - httpRequestBuilder.uri(url); - return this; - } - - @Override - public RequestBuilder withBody(String body) { - this.entity = new ByteArrayHttpEntity(body.getBytes()); - return this; - } - - @Override - public RequestBuilder withHeader(String name, String value) { - httpRequestBuilder.addHeader(name, value); - return this; - } - - @Override - public RequestBuilder withFormParam(String name, String value) { - if (!(entity instanceof MultipartHttpEntity)) { - this.entity = new MultipartHttpEntity(new ArrayList<>()); - } - ((MultipartHttpEntity) this.entity).getParts().add(new HttpPart(name, value.getBytes(), null, 0)); - return this; - } - - @Override - public RequestBuilder withQueryParam(String name, String value) { - queryParams.put(name, value); - return this; - } - - @Override - public RequestBuilder waitingUpTo(long duration, TimeUnit unit) { - // TODO: Manage timeouts - return this; - } - - @Override - public Request build() { - httpRequestBuilder.queryParams(queryParams); - if (entity != null) { - httpRequestBuilder.entity(entity); - } - return new MuleBasedRequest(httpRequestBuilder.build()); - } - }; - } - - - class MuleBasedRequest implements Request { - - private HttpRequest httpRequest; - - public MuleBasedRequest(HttpRequest httpRequest) { - this.httpRequest = httpRequest; - } - - @Override - public RequestBuilder.Method getMethod() { - return RequestBuilder.Method.valueOf(httpRequest.getMethod()); - } - - @Override - public String getUrl() { - return httpRequest.getUri().toString(); - } - - public HttpRequest getHttpRequest() { - return httpRequest; - } - - @Override - public String toString() { - return httpRequest.toString(); - } - } - -} \ No newline at end of file diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/amq/client/MuleBasedAnypointMQClientFactory.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/amq/client/MuleBasedAnypointMQClientFactory.java deleted file mode 100644 index 49435ed..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/amq/client/MuleBasedAnypointMQClientFactory.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * (c) 2003-2018 MuleSoft, Inc. This software is protected under international copyright law. All use of this software is subject to - * MuleSoft's Master Subscription Agreement (or other Terms of Service) separately entered into between you and MuleSoft. If such an - * agreement is not in place, you may not use the software. - */ -package org.mule.extension.jsonlogger.internal.destinations.amq.client; - -//import com.google.common.base.Preconditions; -import com.mulesoft.mq.restclient.api.AnypointMQClientFactory; -import com.mulesoft.mq.restclient.api.AnypointMqClient; -import com.mulesoft.mq.restclient.api.CourierAuthenticationCredentials; -import com.mulesoft.mq.restclient.impl.OAuthCredentials; -import com.mulesoft.mq.restclient.internal.client.DefaultAnypointMqClient; -import org.mule.runtime.api.scheduler.Scheduler; -import org.mule.runtime.http.api.client.HttpClient; - -public class MuleBasedAnypointMQClientFactory implements AnypointMQClientFactory { - - private final HttpClient httpClient; - private final Scheduler scheduler; - - public MuleBasedAnypointMQClientFactory(HttpClient httpClient, Scheduler scheduler) { - this.httpClient = httpClient; - this.scheduler = scheduler; - } - - @Override - public AnypointMqClient createClient(String courierApiUrl, CourierAuthenticationCredentials authenticationCredentials, - String userAgentInfo) { -// Preconditions.checkArgument(authenticationCredentials instanceof OAuthCredentials); - return new DefaultAnypointMqClient(new AsyncMuleCourierRestClient(courierApiUrl, OAuthCredentials.class.cast(authenticationCredentials), - userAgentInfo, httpClient, scheduler)); - } - -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEvent.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEvent.java deleted file mode 100644 index 89b318b..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEvent.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.mule.extension.jsonlogger.internal.destinations.events; - -public class LogEvent { - - private String correlationId; - private String log; - private String configName; - - public String getLog() { - return log; - } - - public String getCorrelationId() { - return correlationId; - } - - public String getConfigName() { - return configName; - } - - public void setLogEvent(String correlationId, String log, String configName) { - this.correlationId = correlationId; - this.log = log; - this.configName = configName; - } - -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEventFactory.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEventFactory.java deleted file mode 100644 index 2463d45..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEventFactory.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.mule.extension.jsonlogger.internal.destinations.events; - -import com.lmax.disruptor.EventFactory; - -public class LogEventFactory implements EventFactory { - - public LogEvent newInstance() { - return new LogEvent(); - } -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEventHandler.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEventHandler.java deleted file mode 100644 index 591495f..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEventHandler.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.mule.extension.jsonlogger.internal.destinations.events; - -import com.lmax.disruptor.EventHandler; -import org.mule.extension.jsonlogger.internal.destinations.Destination; -import org.mule.extension.jsonlogger.internal.singleton.LogEventSingleton; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class LogEventHandler implements EventHandler { - - private static final Logger LOGGER = LoggerFactory.getLogger(LogEventHandler.class); - - private Map> aggregatedLogsPerConfig = new HashMap>(); - private Map destinations = new HashMap(); - - public LogEventHandler (Map destinations) { - this.destinations = destinations; - } - - public void onEvent(LogEvent logEvent, long sequence, boolean endOfBatch) { - LOGGER.debug("Event Log received with correlationId: " + logEvent.getCorrelationId()); - if (aggregatedLogsPerConfig.get(logEvent.getConfigName()) == null){ - List aggregatedLogs = new ArrayList<>(); - aggregatedLogs.add(logEvent.getLog()); - aggregatedLogsPerConfig.put(logEvent.getConfigName(), aggregatedLogs); - } else { - aggregatedLogsPerConfig.get(logEvent.getConfigName()).add(logEvent.getLog()); - } - if (aggregatedLogsPerConfig.get(logEvent.getConfigName()).size() >= destinations.get(logEvent.getConfigName()).getMaxBatchSize()) { - LOGGER.debug("Max batch size of " + destinations.get(logEvent.getConfigName()).getMaxBatchSize() + " reached for Config: " + logEvent.getConfigName() + ". Flushing logs..."); - flushLogs(logEvent.getConfigName()); - } - if (endOfBatch) { - LOGGER.debug("End of batch reached. Flushing all config logs..."); - flushAllLogs(); - } - } - - private void flushLogs(String configName) { - LOGGER.debug("Sending " + aggregatedLogsPerConfig.get(configName).size()+ " logs to external destination: " + this.destinations.get(configName).getSelectedDestinationType()); - try { - this.destinations.get(configName).sendToExternalDestination(aggregatedLogsPerConfig.get(configName).toString()); - } catch (Exception e) { - LOGGER.error("Error flushing aggregated logs: " + e.getMessage()); - e.printStackTrace(); - } - aggregatedLogsPerConfig.get(configName).clear(); - } - - public void flushAllLogs() { - try { - for (String configName : aggregatedLogsPerConfig.keySet()) { - if (aggregatedLogsPerConfig.get(configName).size() > 0) { - flushLogs(configName); - } - } - } catch (Exception e) { - LOGGER.error("Error flushing all aggregated logs: " + e.getMessage()); - e.printStackTrace(); - } - } -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEventProducerWithTranslator.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEventProducerWithTranslator.java deleted file mode 100644 index 4df9a75..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/destinations/events/LogEventProducerWithTranslator.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.mule.extension.jsonlogger.internal.destinations.events; - -import com.lmax.disruptor.EventTranslatorThreeArg; -import com.lmax.disruptor.EventTranslatorTwoArg; -import com.lmax.disruptor.RingBuffer; - -public class LogEventProducerWithTranslator { - - private final RingBuffer ringBuffer; - - public LogEventProducerWithTranslator(RingBuffer ringBuffer) { - this.ringBuffer = ringBuffer; - } - - private static final EventTranslatorThreeArg TRANSLATOR_THREE_ARG = new EventTranslatorThreeArg() { - public void translateTo(LogEvent logEvent, long sequence, String correlationId, String log, String configName) { - logEvent.setLogEvent(correlationId, log, configName); - } - }; - - public void onData(String correlationId, String log, String configName) { - ringBuffer.publishEvent(TRANSLATOR_THREE_ARG, correlationId, log, configName); - } -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java index e1bdb81..9997a1f 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java @@ -1,80 +1,16 @@ package org.mule.extension.jsonlogger.internal.singleton; -import com.lmax.disruptor.LiteTimeoutBlockingWaitStrategy; -import com.lmax.disruptor.RingBuffer; -import com.lmax.disruptor.WaitStrategy; -import com.lmax.disruptor.dsl.Disruptor; -import com.lmax.disruptor.dsl.ProducerType; -import com.lmax.disruptor.util.DaemonThreadFactory; -import org.mule.extension.jsonlogger.internal.destinations.Destination; -import org.mule.extension.jsonlogger.internal.destinations.events.LogEvent; -import org.mule.extension.jsonlogger.internal.destinations.events.LogEventHandler; import org.mule.runtime.api.lifecycle.Disposable; import org.mule.runtime.api.lifecycle.Initialisable; import org.mule.runtime.api.lifecycle.InitialisationException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; -import java.util.HashMap; -import java.util.concurrent.TimeUnit; public class LogEventSingleton implements Initialisable, Disposable { - - private static final Logger LOGGER = LoggerFactory.getLogger(LogEventSingleton.class); - - // Specify the size of the ring buffer, must be power of 2 - private final Integer BUFFER_SIZE = Integer.valueOf(System.getProperty("json.logger.destinations.buffersize", "1024")); - // Specify the event wait timeout in milliseconds before dropping the events - private final Integer WAIT_TIMEOUT = Integer.valueOf(System.getProperty("json.logger.destinations.waittimeout", "100")); - - @Inject - ConfigsSingleton configs; - - private HashMap destinations = new HashMap<>(); - - // Construct the Disruptor - private Disruptor disruptor; - private RingBuffer ringBuffer; - private LogEventHandler logEventHandler; - - public static void translate(LogEvent logEvent, long sequence, String correlationId, String log, String configName) { - logEvent.setLogEvent(correlationId, log, configName); - } - - public void publishToExternalDestination(String correlationId, String finalLog, String configName) { - LOGGER.debug("Publishing event to ringBuffer for destination type: " + this.destinations.get(configName).getSelectedDestinationType()); - ringBuffer.publishEvent(LogEventSingleton::translate, correlationId, finalLog, configName); - } - - @Override - public void initialise() throws InitialisationException { - LOGGER.debug("Init LogEventSingleton..."); - - // Define waitStrategy: LiteTimeoutBlockingWaitStrategy avoids "blocking wait" but may cause LogEvent loss - WaitStrategy waitStrategy = new LiteTimeoutBlockingWaitStrategy(WAIT_TIMEOUT, TimeUnit.MILLISECONDS); - - // Init disruptor ring buffer - this.disruptor = new Disruptor<>(LogEvent::new, BUFFER_SIZE, DaemonThreadFactory.INSTANCE, ProducerType.SINGLE, waitStrategy); - - // Load external destinations - configs.getConfigs().forEach( - (configName, config) -> this.destinations.put(configName, config.getExternalDestination()) - ); - this.logEventHandler = new LogEventHandler(this.destinations); - // Connect the handler with initialized list of destinations - disruptor.handleEventsWith(logEventHandler); - - // Start the Disruptor, starts all threads running - this.disruptor.start(); - - // Get the ring buffer from the Disruptor to be used for publishing. - this.ringBuffer = disruptor.getRingBuffer(); - } - - @Override - public void dispose() { - this.disruptor.shutdown(); - this.logEventHandler.flushAllLogs(); - } + + @Override + public void initialise() throws InitialisationException { + } + + @Override + public void dispose() { + } } diff --git a/json-logger/src/test/java/JsonMaskerTest.java b/json-logger/src/test/java/JsonMaskerTest.java index c2975b4..c994496 100644 --- a/json-logger/src/test/java/JsonMaskerTest.java +++ b/json-logger/src/test/java/JsonMaskerTest.java @@ -5,6 +5,8 @@ import org.junit.Assert; import org.junit.Test; import org.mule.extension.jsonlogger.internal.singleton.ObjectMapperSingleton; +import org.mule.runtime.api.metadata.DataType; +import org.mule.runtime.api.metadata.TypedValue; import java.io.IOException; import java.io.InputStream; From ac03b7f66e73dc4c13c24a01ddfc7e76f8ad89c6 Mon Sep 17 00:00:00 2001 From: Jazzy Date: Mon, 14 Jun 2021 22:38:43 -0700 Subject: [PATCH 04/12] checkpoint --- json-logger/pom.xml | 42 -- json-logger/pom.xml.original | 94 ---- .../internal/JsonloggerConnection.java | 22 - .../internal/JsonloggerOperations.java | 402 +++++------------- .../internal/datamask/JsonMasker.java | 90 ---- .../internal/singleton/LogEventSingleton.java | 2 +- .../singleton/ObjectMapperSingleton.java | 4 + .../resources/schema/loggerProcessor.json | 2 +- json-logger/src/test/java/JsonMaskerTest.java | 18 +- 9 files changed, 116 insertions(+), 560 deletions(-) delete mode 100644 json-logger/pom.xml.original delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConnection.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/datamask/JsonMasker.java diff --git a/json-logger/pom.xml b/json-logger/pom.xml index 6831ccb..4bcd7cf 100644 --- a/json-logger/pom.xml +++ b/json-logger/pom.xml @@ -83,48 +83,6 @@ commons-beanutils 1.9.4 - - joda-time - joda-time - 2.10.5 - - - org.mule.connectors - mule-jms-connector - 1.6.3 - mule-plugin - provided - - - com.mulesoft.connectors - mule-amqp-connector - 1.6.0 - mule-plugin - provided - - - com.mulesoft.muleesb.modules - anypoint-mq-rest-client - 3.1.0 - - - async-http-client - com.ning - - - compile - - - com.jayway.jsonpath - json-path - 2.4.0 - - - org.slf4j - slf4j-api - - - com.lmax disruptor diff --git a/json-logger/pom.xml.original b/json-logger/pom.xml.original deleted file mode 100644 index 519c6eb..0000000 --- a/json-logger/pom.xml.original +++ /dev/null @@ -1,94 +0,0 @@ - - - - 4.0.0 - ORG_ID_TOKEN - json-logger - 1.1.0 - mule-extension - JSON Logger - Mule 4 - - - org.mule.extensions - mule-modules-parent - 1.1.1 - - - - - - org.jsonschema2pojo - jsonschema2pojo-maven-plugin - 0.4.35 - - - cc568b69-a181-4d4c-b044-90054c52897b - jsonschema2pojo-mule-annotations - 1.0.0 - - - - org.mule.custom.annotation.utils.CustomMuleAnnotator - ${basedir}/src/main/resources/schema - ${project.build.directory}/generated-sources - org.mule.extension.jsonlogger.api.pojos - - - - - generate - - - - - - - - - - - Exchange2 - Exchange2 Repository - https://maven.anypoint.mulesoft.com/api/v1/organizations/${project.groupId}/maven - default - - - - - - org.jsonschema2pojo - jsonschema2pojo-maven-plugin - 0.4.35 - - - cc568b69-a181-4d4c-b044-90054c52897b - jsonschema2pojo-mule-annotations - 1.0.0 - - - com.google.guava - guava - 28.0-jre - - - - - - Exchange2-MulesoftServices - Exchange2 Repository - https://maven.anypoint.mulesoft.com/api/v1/organizations/cc568b69-a181-4d4c-b044-90054c52897b/maven - default - - - - - - Exchange2-MulesoftServices - Exchange2 Repository - https://maven.anypoint.mulesoft.com/api/v1/organizations/cc568b69-a181-4d4c-b044-90054c52897b/maven - default - - - diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConnection.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConnection.java deleted file mode 100644 index 85e23bd..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConnection.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.mule.extension.jsonlogger.internal; - - -/** - * This class represents an extension connection just as example (there is no real connection with anything here c:). - */ -public final class JsonloggerConnection { - - private final String id; - - public JsonloggerConnection(String id) { - this.id = id; - } - - public String getId() { - return id; - } - - public void invalidate() { - // do something to invalidate this connection! - } -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java index 24c4643..81ec123 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java @@ -7,10 +7,6 @@ import org.apache.commons.beanutils.PropertyUtils; import org.joda.time.DateTime; import org.mule.extension.jsonlogger.api.pojos.LoggerProcessor; -import org.mule.extension.jsonlogger.api.pojos.Priority; -import org.mule.extension.jsonlogger.api.pojos.ScopeTracePoint; -import org.mule.extension.jsonlogger.internal.singleton.ConfigsSingleton; -import org.mule.extension.jsonlogger.internal.singleton.LogEventSingleton; import org.mule.extension.jsonlogger.internal.singleton.ObjectMapperSingleton; import org.mule.runtime.api.component.location.ComponentLocation; import org.mule.runtime.api.meta.model.operation.ExecutionType; @@ -19,354 +15,156 @@ import org.mule.runtime.extension.api.annotation.Expression; import org.mule.runtime.extension.api.annotation.execution.Execution; import org.mule.runtime.extension.api.annotation.param.Config; -import org.mule.runtime.extension.api.annotation.param.Optional; import org.mule.runtime.extension.api.annotation.param.ParameterGroup; -import org.mule.runtime.extension.api.annotation.param.display.DisplayName; -import org.mule.runtime.extension.api.annotation.param.display.Example; -import org.mule.runtime.extension.api.annotation.param.display.Placement; -import org.mule.runtime.extension.api.annotation.param.display.Summary; -import org.mule.runtime.extension.api.runtime.operation.FlowListener; import org.mule.runtime.extension.api.runtime.operation.Result; import org.mule.runtime.extension.api.runtime.parameter.CorrelationInfo; import org.mule.runtime.extension.api.runtime.parameter.ParameterResolver; import org.mule.runtime.extension.api.runtime.process.CompletionCallback; -import org.mule.runtime.extension.api.runtime.route.Chain; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.inject.Inject; import java.io.InputStream; -import java.util.*; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.mule.runtime.api.meta.ExpressionSupport.NOT_SUPPORTED; import static org.mule.runtime.api.metadata.DataType.TEXT_STRING; -/** - * This class is a container for operations, every public method in this class will be taken as an extension operation. - */ public class JsonloggerOperations { - /** - * jsonLogger: JSON Logger output log - * log: Connector internal log - */ protected transient Logger jsonLogger; private static final Logger LOGGER = LoggerFactory.getLogger(JsonloggerOperations.class); // Void Result for NIO private final Result VOID_RESULT = Result.builder().build(); - // JSON Object Mapper @Inject ObjectMapperSingleton om; - // Log Event for External Destination - @Inject - LogEventSingleton logEvent; - - // Global definition of logger configs so that it's available for scope processor (SDK scope doesn't support passing configurations) - @Inject - ConfigsSingleton configs; - - // Transformation Service @Inject private TransformationService transformationService; - /** - * Log a new entry - */ @Execution(ExecutionType.BLOCKING) public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPPORTED) LoggerProcessor loggerProcessor, CorrelationInfo correlationInfo, ComponentLocation location, @Config JsonloggerConfiguration config, - FlowListener flowListener, CompletionCallback callback) { - Long initialTimestamp, loggerTimestamp; - initialTimestamp = loggerTimestamp = System.currentTimeMillis(); - initLoggerCategory(loggerProcessor.getCategory()); - LOGGER.debug("correlationInfo.getEventId(): " + correlationInfo.getEventId()); - LOGGER.debug("correlationInfo.getCorrelationId(): " + correlationInfo.getCorrelationId()); + if (!isLogEnabled(loggerProcessor.getPriority().toString())) { + callback.success(VOID_RESULT); + return; + } + + Map typedValuesAsString = new HashMap<>(); + Map typedValuesAsJsonNode = new HashMap<>(); + Map properties = null; try { - // Add cache entry for initial timestamp based on unique EventId - initialTimestamp = config.getCachedTimerTimestamp(correlationInfo.getCorrelationId(), initialTimestamp); - } catch (Exception e) { - LOGGER.error("initialTimestamp could not be retrieved from the cache config. Defaulting to current System.currentTimeMillis()", e); + properties = PropertyUtils.describe(loggerProcessor); + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + LOGGER.error("", e); } - // Calculate elapsed time based on cached initialTimestamp - Long elapsed = loggerTimestamp - initialTimestamp; - - //config.printTimersKeys(); - if (elapsed == 0) { - LOGGER.debug("configuring flowListener...."); - flowListener.onComplete(new TimerRemoverRunnable(correlationInfo.getCorrelationId(), config)); - } else { - LOGGER.debug("flowListener already configured"); + if (properties == null) { + callback.success(VOID_RESULT); + return; } - /** - * Avoid Logger logic execution based on log priority - */ - if (isLogEnabled(loggerProcessor.getPriority().toString())) { - // Load disabledFields - List disabledFields = (config.getJsonOutput().getDisabledFields() != null) ? Arrays.asList(config.getJsonOutput().getDisabledFields().split(",")) : new ArrayList<>(); - LOGGER.debug("The following fields will be disabled for logging: " + disabledFields); + properties.forEach((k, v) -> { + if (v == null) { + return; + } - // Logic to disable fields and/or parse TypedValues as String for JSON log printing - //Map typedValuesAsString = new HashMap<>(); - Map typedValuesAsString = new HashMap<>(); - Map typedValuesAsJsonNode = new HashMap<>(); try { - PropertyUtils.describe(loggerProcessor).forEach((k, v) -> { - if (disabledFields.stream().anyMatch(k::equals)) { - try { - BeanUtils.setProperty(loggerProcessor, k, null); - } catch (Exception e) { - LOGGER.error("Failed disabling field: " + k, e); + if (v instanceof ParameterResolver) { + v = ((ParameterResolver) v).resolve(); + } + + if (!v.getClass().getCanonicalName().equals("org.mule.runtime.api.metadata.TypedValue")) { + return; + } + + TypedValue typedVal = (TypedValue) v; + if (typedVal.getValue() == null) { + return; + } + + //TODO: what is this doing + BeanUtils.setProperty(loggerProcessor, k, null); + + if (!config.getJsonOutput().isParseContentFieldsInJsonOutput()) { + typedValuesAsString.put(k, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); + return; + } + + if (typedVal.getDataType().getMediaType().getPrimaryType().equals("application") + && typedVal.getDataType().getMediaType().getSubType().equals("json")) { + + // Apply masking if needed + List dataMaskingFields = new ArrayList<>(); + if (config.getJsonOutput().getContentFieldsDataMasking() != null) { + String[] split = config.getJsonOutput().getContentFieldsDataMasking().split(","); + for (String s : split) { + dataMaskingFields.add(s.trim()); } + } + + if (dataMaskingFields.isEmpty()) { + typedValuesAsJsonNode.put(k, om.getObjectMapper().readTree(typedVal.getValue())); } else { - if (v != null) { - try { - if (v instanceof ParameterResolver) { - v = ((ParameterResolver) v).resolve(); - } - if (v.getClass().getCanonicalName().equals("org.mule.runtime.api.metadata.TypedValue")) { - LOGGER.debug("org.mule.runtime.api.metadata.TypedValue type was found for field: " + k); - TypedValue typedVal = (TypedValue) v; - LOGGER.debug("Parsing TypedValue for field " + k); - - LOGGER.debug("TypedValue MediaType: " + typedVal.getDataType().getMediaType()); - LOGGER.debug("TypedValue Type: " + typedVal.getDataType().getType().getCanonicalName()); - LOGGER.debug("TypedValue Class: " + typedVal.getValue().getClass().getCanonicalName()); - - // Remove unparsed field - BeanUtils.setProperty(loggerProcessor, k, null); - - // Evaluate if typedValue is null - if (typedVal.getValue() != null) { - // Should content type field be parsed as part of JSON log? - if (config.getJsonOutput().isParseContentFieldsInJsonOutput()) { - // Is content type application/json? - if (typedVal.getDataType().getMediaType().getPrimaryType().equals("application") && typedVal.getDataType().getMediaType().getSubType().equals("json")) { - - // Apply masking if needed - List dataMaskingFields = new ArrayList<>(); - if (config.getJsonOutput().getContentFieldsDataMasking() != null) { - String[] split = config.getJsonOutput().getContentFieldsDataMasking().split(","); - for (String s : split) { - dataMaskingFields.add(s.trim()); - } - } - - LOGGER.debug("The following JSON keys/paths will be masked for logging: " + dataMaskingFields); - if (!dataMaskingFields.isEmpty()) { - JsonNode tempContentNode = om.getObjectMapper(dataMaskingFields).readTree(typedVal.getValue()); - typedValuesAsJsonNode.put(k, tempContentNode); - } else { - typedValuesAsJsonNode.put(k, om.getObjectMapper(dataMaskingFields).readTree(typedVal.getValue())); - } - } else { - typedValuesAsString.put(k, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); - } - } else { - typedValuesAsString.put(k, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); - } - } - } - } catch (Exception e) { - LOGGER.error("Failed parsing field: " + k, e); - typedValuesAsString.put(k, "Error parsing expression. See logs for details."); - } - } + JsonNode tempContentNode = om.getObjectMapper(dataMaskingFields).readTree(typedVal.getValue()); + typedValuesAsJsonNode.put(k, tempContentNode); } - }); + } else { + typedValuesAsString.put(k, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); + } + } catch (Exception e) { - LOGGER.error("Unknown error while processing the logger object", e); - } - - // Aggregate Logger data into mergedLogger - ObjectNode mergedLogger = om.getObjectMapper(new ArrayList<>()).createObjectNode(); - mergedLogger.setAll((ObjectNode) om.getObjectMapper(new ArrayList<>()).valueToTree(loggerProcessor)); - - /** - * Custom field ordering for Logger Operation - * ========================================== - * This will take place after LoggerProcessor ordering which is defined by the field sequence in loggerProcessor.json - **/ - // 1. Elapsed Time - mergedLogger.put("elapsed", elapsed); - // 2. Location Info: Logger location within Mule application - if (config.getJsonOutput().isLogLocationInfo()) { - Map locationInfo = locationInfoToMap(location); - mergedLogger.putPOJO("locationInfo", locationInfo); + LOGGER.error("Failed to parse: " + k, e); + typedValuesAsString.put(k, "Error parsing expression. See logs for details."); } - // 3. Timestamp: Add formatted timestamp entry to the logger - mergedLogger.put("timestamp", getFormattedTimestamp(loggerTimestamp)); - // 4. Content fields: String based fields - if (!typedValuesAsString.isEmpty()) { - JsonNode typedValuesNode = om.getObjectMapper(new ArrayList<>()).valueToTree(typedValuesAsString); - mergedLogger.setAll((ObjectNode) typedValuesNode); - } - // 5. Content fields: JSONNode based fields - if (!typedValuesAsJsonNode.isEmpty()) { - mergedLogger.setAll(typedValuesAsJsonNode); - } - // 6. Global info from config - mergedLogger.setAll((ObjectNode) om.getObjectMapper(new ArrayList<>()).valueToTree(config.getGlobalSettings())); - // 7. Thread Name - mergedLogger.put("threadName", Thread.currentThread().getName()); - - /** Print Logger **/ - printObjectToLog(mergedLogger, loggerProcessor.getPriority().toString(), config.getJsonOutput().isPrettyPrint()); - - } else { - LOGGER.debug("Avoiding logger operation logic execution due to log priority not being enabled"); - } - callback.success(VOID_RESULT); - } - - /** - * Log scope - */ - @Execution(ExecutionType.BLOCKING) - public void loggerScope(@DisplayName("Module configuration") @Example("JSON_Logger_Config") @Summary("Indicate which Global config should be associated with this Scope.") String configurationRef, - @Optional(defaultValue = "INFO") Priority priority, - @Optional(defaultValue = "OUTBOUND_REQUEST_SCOPE") ScopeTracePoint scopeTracePoint, - @Optional @Summary("If not set, by default will log to the org.mule.extension.jsonlogger.JsonLogger category") String category, - @Optional(defaultValue = "#[correlationId]") @Placement(tab = "Advanced") String correlationId, - ComponentLocation location, - CorrelationInfo correlationInfo, - FlowListener flowListener, - Chain operations, - CompletionCallback callback) { - - /** - * BEFORE scope logger - * =================== - **/ + }); - Long initialTimestamp, loggerTimestamp; - initialTimestamp = loggerTimestamp = System.currentTimeMillis(); + //TODO: Create mapper once + ObjectNode mergedLogger = om.getObjectMapper().createObjectNode(); + mergedLogger.setAll((ObjectNode) om.getObjectMapper().valueToTree(loggerProcessor)); - initLoggerCategory(category); + mergedLogger.put("elapsed", elapsed); - LOGGER.debug("correlationInfo.getEventId(): " + correlationInfo.getEventId()); - LOGGER.debug("correlationInfo.getCorrelationId(): " + correlationInfo.getCorrelationId()); - - try { - // Add cache entry for initial timestamp based on unique EventId - initialTimestamp = configs.getConfig(configurationRef).getCachedTimerTimestamp(correlationInfo.getCorrelationId(), initialTimestamp); - } catch (Exception e) { - LOGGER.error("initialTimestamp could not be retrieved from the cache config. Defaulting to current System.currentTimeMillis()", e); + if (config.getJsonOutput().isLogLocationInfo()) { + Map locationInfo = locationInfoToMap(location); + mergedLogger.putPOJO("locationInfo", locationInfo); } - // Calculate elapsed time based on cached initialTimestamp - Long elapsed = loggerTimestamp - initialTimestamp; + mergedLogger.put("timestamp", getFormattedTimestamp(loggerTimestamp)); - if (elapsed == 0) { - LOGGER.debug("configuring flowListener...."); - flowListener.onComplete(new TimerRemoverRunnable(correlationInfo.getCorrelationId(), configs.getConfig(configurationRef))); - } else { - LOGGER.debug("flowListener already configured"); + if (!typedValuesAsString.isEmpty()) { + JsonNode typedValuesNode = om.getObjectMapper(new ArrayList<>()).valueToTree(typedValuesAsString); + mergedLogger.setAll((ObjectNode) typedValuesNode); } - /** - * Avoid Logger Scope logic execution based on log priority - */ - if (isLogEnabled(priority.toString())) { - // Execute Scope Logger - ObjectNode loggerProcessor = om.getObjectMapper(new ArrayList<>()).createObjectNode(); - - /** - * Custom field ordering for Logger Scope - * =============================== - **/ - loggerProcessor.put("correlationId", correlationId); - loggerProcessor.put("tracePoint", scopeTracePoint.toString() + "_BEFORE"); - loggerProcessor.put("priority", priority.toString()); - loggerProcessor.put("elapsed", elapsed); - loggerProcessor.put("scopeElapsed", 0); - if (configs.getConfig(configurationRef).getJsonOutput().isLogLocationInfo()) { - Map locationInfoMap = locationInfoToMap(location); - loggerProcessor.putPOJO("locationInfo", locationInfoMap); - } - loggerProcessor.put("timestamp", getFormattedTimestamp(loggerTimestamp)); - loggerProcessor.put("applicationName", configs.getConfig(configurationRef).getGlobalSettings().getApplicationName()); - loggerProcessor.put("applicationVersion", configs.getConfig(configurationRef).getGlobalSettings().getApplicationVersion()); - loggerProcessor.put("environment", configs.getConfig(configurationRef).getGlobalSettings().getEnvironment()); - loggerProcessor.put("threadName", Thread.currentThread().getName()); - - // Define JSON output formatting - // Print Logger - String finalLogBefore = printObjectToLog(loggerProcessor, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); - - // Added temp variable to comply with lambda - Long finalInitialTimestamp = initialTimestamp; - operations.process( - result -> { - - /** - * AFTER scope logger - * =================== - **/ - - Long endScopeTimestamp = System.currentTimeMillis(); - - // Calculate elapsed time - Long scopeElapsed = endScopeTimestamp - loggerTimestamp; - Long mainElapsed = endScopeTimestamp - finalInitialTimestamp; - - loggerProcessor.put("tracePoint", scopeTracePoint.toString() + "_AFTER"); - loggerProcessor.put("priority", priority.toString()); - loggerProcessor.put("elapsed", mainElapsed); - loggerProcessor.put("scopeElapsed", scopeElapsed); - loggerProcessor.put("timestamp", getFormattedTimestamp(endScopeTimestamp)); - - // Print Logger - String finalLogAfter = printObjectToLog(loggerProcessor, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); - - callback.success(result); - }, - (error, previous) -> { - - /** ERROR scope logger **/ - - Long errorScopeTimestamp = System.currentTimeMillis(); - Long mainElapsed = errorScopeTimestamp - finalInitialTimestamp; - - // Calculate elapsed time - Long scopeElapsed = errorScopeTimestamp - loggerTimestamp; - - loggerProcessor.put("message", "Error found: " + error.getMessage()); - loggerProcessor.put("tracePoint", "EXCEPTION_SCOPE"); - loggerProcessor.put("priority", "ERROR"); - loggerProcessor.put("elapsed", mainElapsed); - loggerProcessor.put("scopeElapsed", scopeElapsed); - loggerProcessor.put("timestamp", getFormattedTimestamp(errorScopeTimestamp)); - - // Print Logger - String finalLogError = printObjectToLog(loggerProcessor, "ERROR", configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); - - callback.error(error); - }); - } else { - // Execute operations without Logger - LOGGER.debug("Avoiding logger scope logic execution due to log priority not being enabled"); - operations.process( - callback::success, - (error, previous) -> { - callback.error(error); - }); + if (!typedValuesAsJsonNode.isEmpty()) { + mergedLogger.setAll(typedValuesAsJsonNode); } + + mergedLogger.setAll((ObjectNode) om.getObjectMapper(new ArrayList<>()).valueToTree(config.getGlobalSettings())); + + mergedLogger.put("threadName", Thread.currentThread().getName()); + + printObjectToLog(mergedLogger, loggerProcessor.getPriority().toString(), config.getJsonOutput().isPrettyPrint()); + } private Map locationInfoToMap(ComponentLocation location) { - Map locationInfo = new HashMap(); + Map locationInfo = new HashMap<>(); locationInfo.put("rootContainer", location.getRootContainerName()); locationInfo.put("component", location.getComponentIdentifier().getIdentifier().toString()); locationInfo.put("fileName", location.getFileName().orElse("")); @@ -375,30 +173,25 @@ private Map locationInfoToMap(ComponentLocation location) { } private String getFormattedTimestamp(Long loggerTimestamp) { - /* - Define timestamp: - - DateTime: Defaults to ISO format - - TimeZone: Defaults to UTC. Refer to https://en.wikipedia.org/wiki/List_of_tz_database_time_zones for valid timezones - */ - DateTime dateTime = new DateTime(loggerTimestamp).withZone(org.joda.time.DateTimeZone.forID(System.getProperty("json.logger.timezone", "UTC"))); - String timestamp = dateTime.toString(); - if (System.getProperty("json.logger.dateformat") != null && !System.getProperty("json.logger.dateformat").equals("")) { + DateTime dateTime = new DateTime(loggerTimestamp) + .withZone(org.joda.time.DateTimeZone.forID(System.getProperty("json.logger.timezone", "UTC"))); + String timestamp; + if (System.getProperty("json.logger.dateformat") != null && !System.getProperty("json.logger.dateformat").isEmpty()) { timestamp = dateTime.toString(System.getProperty("json.logger.dateformat")); + } else { + timestamp = dateTime.toString(); } return timestamp; } - private String printObjectToLog(ObjectNode loggerObj, String priority, boolean isPrettyPrint) { + private void printObjectToLog(ObjectNode loggerObj, String priority, boolean isPrettyPrint) { ObjectWriter ow = (isPrettyPrint) ? om.getObjectMapper(new ArrayList<>()).writer().withDefaultPrettyPrinter() : om.getObjectMapper(new ArrayList<>()).writer(); - String logLine = ""; try { - logLine = ow.writeValueAsString(loggerObj); + String logLine = ow.writeValueAsString(loggerObj); + doLog(priority, logLine); } catch (Exception e) { LOGGER.error("Error parsing log data as a string", e); } - doLog(priority, logLine); - - return logLine; } private void doLog(String priority, String logLine) { @@ -443,7 +236,6 @@ protected void initLoggerCategory(String category) { } else { jsonLogger = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); } - LOGGER.debug("category set: " + jsonLogger.getName()); } // Allows executing timer cleanup on flowListener onComplete events diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/datamask/JsonMasker.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/datamask/JsonMasker.java deleted file mode 100644 index a55b948..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/datamask/JsonMasker.java +++ /dev/null @@ -1,90 +0,0 @@ -package org.mule.extension.jsonlogger.internal.datamask; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; - -import java.util.*; -import java.util.regex.Pattern; - -public class JsonMasker { - - private static final Pattern digits = Pattern.compile("\\d"); - private static final Pattern capitalLetters = Pattern.compile("[A-Z]"); - private static final Pattern nonSpecialCharacters = Pattern.compile("[^X\\s!-/:-@\\[-`{-~]"); - - private final Set blacklistedKeys; - private final boolean enabled; - - public JsonMasker(Collection blacklist, boolean enabled) { - this.enabled = enabled; - - blacklistedKeys = new HashSet<>(); - - blacklist.forEach(item -> { - blacklistedKeys.add(item.toUpperCase()); - }); - } - - public JsonNode mask(JsonNode target) { - if (!enabled) - return target; - if (target == null) - return null; - - return traverseAndMask(target.deepCopy(), "$", false); - } - - private JsonNode traverseAndMask(JsonNode target, String path, Boolean isBlackListed) { - if (target.isTextual() && isBlackListed) { - return new TextNode(maskString(target.asText())); - } - if (target.isNumber() && isBlackListed) { - return new TextNode(maskNumber(target.asText())); - } - - if (target.isObject()) { - Iterator> fields = target.fields(); - while (fields.hasNext()) { - Map.Entry field = fields.next(); - String childPath = appendPath(path, field.getKey()); - if (blacklistedKeys.contains(field.getKey().toUpperCase()) || isBlackListed) { - ((ObjectNode) target).replace(field.getKey(), traverseAndMask(field.getValue(), childPath, true)); - } else { - ((ObjectNode) target).replace(field.getKey(), traverseAndMask(field.getValue(), childPath, false)); - } - } - } - if (target.isArray()) { - for (int i = 0; i < target.size(); i++) { - String childPath = appendPath(path, i); - if (isBlackListed) { - ((ArrayNode) target).set(i, traverseAndMask(target.get(i), childPath, true)); - } else { - ((ArrayNode) target).set(i, traverseAndMask(target.get(i), childPath, false)); - } - } - } - return target; - } - - private static String appendPath(String path, String key) { - return path + "['" + key + "']"; - } - - private static String appendPath(String path, int ind) { - return path + "[" + ind + "]"; - } - - private static String maskString(String value) { - String tmpMasked = digits.matcher(value).replaceAll("*"); - tmpMasked = capitalLetters.matcher(tmpMasked).replaceAll("X"); - return nonSpecialCharacters.matcher(tmpMasked).replaceAll("x"); - } - - private static String maskNumber(String value) { - return value.replaceAll("[0-9]", "*"); - } - -} \ No newline at end of file diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java index 9997a1f..82b57b0 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java @@ -7,7 +7,7 @@ public class LogEventSingleton implements Initialisable, Disposable { @Override - public void initialise() throws InitialisationException { + public void initialise() { } @Override diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java index 96c6098..a2bd5b7 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java @@ -19,4 +19,8 @@ public ObjectMapper getObjectMapper(List maskedFields) { this.om.registerModule(module); return this.om; } + + public ObjectMapper getObjectMapper() { + return this.om; + } } diff --git a/json-logger/src/main/resources/schema/loggerProcessor.json b/json-logger/src/main/resources/schema/loggerProcessor.json index 823b4e6..7f3be00 100644 --- a/json-logger/src/main/resources/schema/loggerProcessor.json +++ b/json-logger/src/main/resources/schema/loggerProcessor.json @@ -24,7 +24,7 @@ "type": "string", "javaType": "org.mule.runtime.extension.api.runtime.parameter.ParameterResolver>", "sdk": { - "default": "#[import modules::JSONLoggerModule output application/json ---\n{\n payload: JSONLoggerModule::stringifyNonJSON(payload) \n}]", + "default": "", "summary": "NOTE: Writing the entire payload every time across your application can cause serious performance issues", "required": false, "isContent": true diff --git a/json-logger/src/test/java/JsonMaskerTest.java b/json-logger/src/test/java/JsonMaskerTest.java index c994496..89af673 100644 --- a/json-logger/src/test/java/JsonMaskerTest.java +++ b/json-logger/src/test/java/JsonMaskerTest.java @@ -5,14 +5,10 @@ import org.junit.Assert; import org.junit.Test; import org.mule.extension.jsonlogger.internal.singleton.ObjectMapperSingleton; -import org.mule.runtime.api.metadata.DataType; import org.mule.runtime.api.metadata.TypedValue; import java.io.IOException; import java.io.InputStream; -import java.time.Duration; -import java.time.Instant; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -26,7 +22,19 @@ public class JsonMaskerTest { @Test public void caseInsensitve() throws IOException { JsonNode payload = om.readTree(is); - Map masked = om.convertValue(payload, new TypeReference>(){}); + Map masked = om.convertValue(payload, new TypeReference>() { + }); Assert.assertEquals("{name=****, phone=************, gender=****}", masked.get("attributes").toString()); } + + @Test + public void typedValue() { + // check number of file descriptors here + someMethod(); + // check number of file descriptors here + } + + private void someMethod() { + //open streams here + } } \ No newline at end of file From a40de01d65e7f0b99da4db89854b4ee99b3c2887 Mon Sep 17 00:00:00 2001 From: Jazzy Date: Tue, 15 Jun 2021 00:43:43 -0700 Subject: [PATCH 05/12] checkpoint --- json-logger/pom.xml | 10 +- .../internal/JsonloggerConfiguration.java | 57 ++---- .../internal/JsonloggerExtension.java | 2 +- .../internal/JsonloggerOperations.java | 171 ++++++------------ .../internal/singleton/LogEventSingleton.java | 16 -- .../core/config/registry-bootstrap.properties | 1 - .../resources/schema/loggerProcessor.json | 2 + json-logger/src/test/java/JsonMaskerTest.java | 17 +- 8 files changed, 92 insertions(+), 184 deletions(-) delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java diff --git a/json-logger/pom.xml b/json-logger/pom.xml index 4bcd7cf..bbe3388 100644 --- a/json-logger/pom.xml +++ b/json-logger/pom.xml @@ -83,11 +83,11 @@ commons-beanutils 1.9.4 - - com.lmax - disruptor - 3.4.2 - + + + + + diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConfiguration.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConfiguration.java index ee2a7d2..8d37d87 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConfiguration.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConfiguration.java @@ -4,12 +4,10 @@ import org.mule.extension.jsonlogger.internal.singleton.ConfigsSingleton; import org.mule.runtime.api.lifecycle.Disposable; import org.mule.runtime.api.lifecycle.Initialisable; -import org.mule.runtime.api.lifecycle.InitialisationException; import org.mule.runtime.extension.api.annotation.Operations; import org.mule.runtime.extension.api.annotation.param.RefName; import javax.inject.Inject; -import java.util.concurrent.ConcurrentHashMap; /** * This class represents an extension configuration, values set in this class are commonly used across multiple @@ -17,40 +15,23 @@ */ @Operations(JsonloggerOperations.class) public class JsonloggerConfiguration extends LoggerConfig implements Initialisable, Disposable { - - @Inject - ConfigsSingleton configsSingleton; - - @RefName - private String configName; - - - public String getConfigName() { - return configName; - } - - /** Timer methods for Elapsed Time **/ - - public ConcurrentHashMap timers = new ConcurrentHashMap(); - - public ConcurrentHashMap getTimers() { return timers; } - - public Long getCachedTimerTimestamp(String key, Long initialTimeStamp) throws Exception { - Long startTimestamp = timers.putIfAbsent(key, initialTimeStamp); - return (startTimestamp == null) ? timers.get(key) : startTimestamp; - } - - public void removeCachedTimerTimestamp(String key) { - timers.remove(key); - } - - - @Override - public void initialise() throws InitialisationException { - configsSingleton.addConfig(configName, this); // Should be refactored once SDK supports passing configs to Scopes - } - - @Override - public void dispose() { - } + + @Inject + ConfigsSingleton configsSingleton; + + @RefName + private String configName; + + public String getConfigName() { + return configName; + } + + @Override + public void initialise() { + configsSingleton.addConfig(configName, this); + } + + @Override + public void dispose() { + } } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java index 49b10b8..5507c67 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java @@ -13,6 +13,6 @@ @Extension(name = "JSON Logger") @Export(resources = {"modules/JSONLoggerModule.dwl"}) @Configurations(JsonloggerConfiguration.class) -public class JsonloggerExtension { +public class JsonLoggerExtension { } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java index 81ec123..f632f3a 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java @@ -1,15 +1,14 @@ package org.mule.extension.jsonlogger.internal; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.commons.beanutils.BeanUtils; -import org.apache.commons.beanutils.PropertyUtils; -import org.joda.time.DateTime; import org.mule.extension.jsonlogger.api.pojos.LoggerProcessor; import org.mule.extension.jsonlogger.internal.singleton.ObjectMapperSingleton; import org.mule.runtime.api.component.location.ComponentLocation; import org.mule.runtime.api.meta.model.operation.ExecutionType; +import org.mule.runtime.api.metadata.MediaType; import org.mule.runtime.api.metadata.TypedValue; import org.mule.runtime.api.transformation.TransformationService; import org.mule.runtime.extension.api.annotation.Expression; @@ -25,9 +24,8 @@ import javax.inject.Inject; import java.io.InputStream; -import java.lang.reflect.InvocationTargetException; +import java.time.Instant; import java.util.ArrayList; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,7 +35,7 @@ public class JsonloggerOperations { - protected transient Logger jsonLogger; + protected transient Logger jsonLogger = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); private static final Logger LOGGER = LoggerFactory.getLogger(JsonloggerOperations.class); // Void Result for NIO @@ -56,8 +54,6 @@ public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPP @Config JsonloggerConfiguration config, CompletionCallback callback) { - initLoggerCategory(loggerProcessor.getCategory()); - if (!isLogEnabled(loggerProcessor.getPriority().toString())) { callback.success(VOID_RESULT); return; @@ -66,86 +62,17 @@ public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPP Map typedValuesAsString = new HashMap<>(); Map typedValuesAsJsonNode = new HashMap<>(); - Map properties = null; - try { - properties = PropertyUtils.describe(loggerProcessor); - } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { - LOGGER.error("", e); - } - - if (properties == null) { - callback.success(VOID_RESULT); - return; - } - - properties.forEach((k, v) -> { - if (v == null) { - return; - } - - try { - if (v instanceof ParameterResolver) { - v = ((ParameterResolver) v).resolve(); - } - - if (!v.getClass().getCanonicalName().equals("org.mule.runtime.api.metadata.TypedValue")) { - return; - } - - TypedValue typedVal = (TypedValue) v; - if (typedVal.getValue() == null) { - return; - } - - //TODO: what is this doing - BeanUtils.setProperty(loggerProcessor, k, null); - - if (!config.getJsonOutput().isParseContentFieldsInJsonOutput()) { - typedValuesAsString.put(k, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); - return; - } - - if (typedVal.getDataType().getMediaType().getPrimaryType().equals("application") - && typedVal.getDataType().getMediaType().getSubType().equals("json")) { - - // Apply masking if needed - List dataMaskingFields = new ArrayList<>(); - if (config.getJsonOutput().getContentFieldsDataMasking() != null) { - String[] split = config.getJsonOutput().getContentFieldsDataMasking().split(","); - for (String s : split) { - dataMaskingFields.add(s.trim()); - } - } - - if (dataMaskingFields.isEmpty()) { - typedValuesAsJsonNode.put(k, om.getObjectMapper().readTree(typedVal.getValue())); - } else { - JsonNode tempContentNode = om.getObjectMapper(dataMaskingFields).readTree(typedVal.getValue()); - typedValuesAsJsonNode.put(k, tempContentNode); - } - } else { - typedValuesAsString.put(k, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); - } - - } catch (Exception e) { - LOGGER.error("Failed to parse: " + k, e); - typedValuesAsString.put(k, "Error parsing expression. See logs for details."); - } - }); - + parseContent(loggerProcessor.getContent(), config); //TODO: Create mapper once ObjectNode mergedLogger = om.getObjectMapper().createObjectNode(); mergedLogger.setAll((ObjectNode) om.getObjectMapper().valueToTree(loggerProcessor)); - mergedLogger.put("elapsed", elapsed); + mergedLogger.put("timestamp", getFormattedTimestamp()); if (config.getJsonOutput().isLogLocationInfo()) { - Map locationInfo = locationInfoToMap(location); - mergedLogger.putPOJO("locationInfo", locationInfo); + mergedLogger.putPOJO("locationInfo", locationInfoToMap(location)); } - mergedLogger.put("timestamp", getFormattedTimestamp(loggerTimestamp)); - if (!typedValuesAsString.isEmpty()) { JsonNode typedValuesNode = om.getObjectMapper(new ArrayList<>()).valueToTree(typedValuesAsString); mergedLogger.setAll((ObjectNode) typedValuesNode); @@ -160,7 +87,52 @@ public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPP mergedLogger.put("threadName", Thread.currentThread().getName()); printObjectToLog(mergedLogger, loggerProcessor.getPriority().toString(), config.getJsonOutput().isPrettyPrint()); + callback.success(VOID_RESULT); + } + + private void parseContent(ParameterResolver> v, JsonloggerConfiguration config) { + Map typedValuesAsString = new HashMap<>(); + Map typedValuesAsJsonNode = new HashMap<>(); + + TypedValue typedVal = v.resolve(); + if (typedVal.getValue() == null) { + return; + } + + if (!config.getJsonOutput().isParseContentFieldsInJsonOutput() || + !typedVal.getDataType().getMediaType().matches(MediaType.APPLICATION_JSON)) { + typedValuesAsString.put(Constants.CONTENT, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); + return; + } + // Apply data masking only if needed + ObjectMapper mapper; + List dataMaskingFields = null; + if (config.getJsonOutput().getContentFieldsDataMasking() != null) { + dataMaskingFields = new ArrayList<>(); + String[] split = config + .getJsonOutput() + .getContentFieldsDataMasking() + .trim() + .split(","); + + for (String s : split) { + dataMaskingFields.add(s.trim()); + } + + } + + if (dataMaskingFields == null || dataMaskingFields.isEmpty()) { + mapper = om.getObjectMapper(); + } else { + mapper = om.getObjectMapper(dataMaskingFields); + } + + try { + typedValuesAsJsonNode.put(Constants.CONTENT, mapper.readTree(typedVal.getValue())); + } catch (Exception e) { + typedValuesAsString.put(Constants.CONTENT, e.getMessage()); + } } private Map locationInfoToMap(ComponentLocation location) { @@ -172,16 +144,9 @@ private Map locationInfoToMap(ComponentLocation location) { return locationInfo; } - private String getFormattedTimestamp(Long loggerTimestamp) { - DateTime dateTime = new DateTime(loggerTimestamp) - .withZone(org.joda.time.DateTimeZone.forID(System.getProperty("json.logger.timezone", "UTC"))); - String timestamp; - if (System.getProperty("json.logger.dateformat") != null && !System.getProperty("json.logger.dateformat").isEmpty()) { - timestamp = dateTime.toString(System.getProperty("json.logger.dateformat")); - } else { - timestamp = dateTime.toString(); - } - return timestamp; + private String getFormattedTimestamp() { + Instant now = Instant.now(); + return now.toString(); } private void printObjectToLog(ObjectNode loggerObj, String priority, boolean isPrettyPrint) { @@ -229,30 +194,4 @@ private Boolean isLogEnabled(String priority) { } return false; } - - protected void initLoggerCategory(String category) { - if (category != null) { - jsonLogger = LoggerFactory.getLogger(category); - } else { - jsonLogger = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); - } - } - - // Allows executing timer cleanup on flowListener onComplete events - private static class TimerRemoverRunnable implements Runnable { - - private final String key; - private final JsonloggerConfiguration config; - - public TimerRemoverRunnable(String key, JsonloggerConfiguration config) { - this.key = key; - this.config = config; - } - - @Override - public void run() { - LOGGER.debug("Removing key: " + key); - config.removeCachedTimerTimestamp(key); - } - } } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java deleted file mode 100644 index 82b57b0..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/LogEventSingleton.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.mule.extension.jsonlogger.internal.singleton; - -import org.mule.runtime.api.lifecycle.Disposable; -import org.mule.runtime.api.lifecycle.Initialisable; -import org.mule.runtime.api.lifecycle.InitialisationException; - -public class LogEventSingleton implements Initialisable, Disposable { - - @Override - public void initialise() { - } - - @Override - public void dispose() { - } -} diff --git a/json-logger/src/main/resources/META-INF/org/mule/runtime/core/config/registry-bootstrap.properties b/json-logger/src/main/resources/META-INF/org/mule/runtime/core/config/registry-bootstrap.properties index c7b2e44..034a10c 100644 --- a/json-logger/src/main/resources/META-INF/org/mule/runtime/core/config/registry-bootstrap.properties +++ b/json-logger/src/main/resources/META-INF/org/mule/runtime/core/config/registry-bootstrap.properties @@ -1,3 +1,2 @@ logger.objectMapperSingleton=org.mule.extension.jsonlogger.internal.singleton.ObjectMapperSingleton -logger.logEventSingleton=org.mule.extension.jsonlogger.internal.singleton.LogEventSingleton logger.configsSingleton=org.mule.extension.jsonlogger.internal.singleton.ConfigsSingleton diff --git a/json-logger/src/main/resources/schema/loggerProcessor.json b/json-logger/src/main/resources/schema/loggerProcessor.json index 7f3be00..3595b2d 100644 --- a/json-logger/src/main/resources/schema/loggerProcessor.json +++ b/json-logger/src/main/resources/schema/loggerProcessor.json @@ -15,6 +15,7 @@ "message": { "type": "string", "sdk": { + "default": "", "example": "Add a log message", "required": true, "summary": "Message to be logged" @@ -67,6 +68,7 @@ "category": { "type": "string", "sdk": { + "default": "org.mule.extension.jsonlogger.JsonLogger", "required": false, "summary": "If not set, by default will log to the org.mule.extension.jsonlogger.JsonLogger category" }, diff --git a/json-logger/src/test/java/JsonMaskerTest.java b/json-logger/src/test/java/JsonMaskerTest.java index 89af673..af85c7a 100644 --- a/json-logger/src/test/java/JsonMaskerTest.java +++ b/json-logger/src/test/java/JsonMaskerTest.java @@ -2,13 +2,16 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.beanutils.PropertyUtils; import org.junit.Assert; import org.junit.Test; +import org.mule.extension.jsonlogger.api.pojos.LoggerProcessor; import org.mule.extension.jsonlogger.internal.singleton.ObjectMapperSingleton; import org.mule.runtime.api.metadata.TypedValue; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -28,13 +31,13 @@ public void caseInsensitve() throws IOException { } @Test - public void typedValue() { - // check number of file descriptors here - someMethod(); - // check number of file descriptors here + public void typedValue() throws IllegalAccessException, NoSuchMethodException, InvocationTargetException { + LoggerProcessor loggerProcessor = new LoggerProcessor(); + Map describe = PropertyUtils.describe(loggerProcessor); + describe.forEach((k,v) -> { + System.out.println(k); + }); } - private void someMethod() { - //open streams here - } + } \ No newline at end of file From ea126f80f2a897e13a42521d8a53dd9bec1bc12d Mon Sep 17 00:00:00 2001 From: Jazzy Date: Wed, 16 Jun 2021 01:01:23 -0700 Subject: [PATCH 06/12] checkpoint --- json-logger/pom.xml | 6 +- .../internal/JsonloggerOperations.java | 282 ++++++++++++------ .../singleton/ObjectMapperSingleton.java | 10 +- .../main/resources/schema/loggerConfig.json | 8 +- json-logger/src/test/java/JsonMaskerTest.java | 43 --- 5 files changed, 203 insertions(+), 146 deletions(-) delete mode 100644 json-logger/src/test/java/JsonMaskerTest.java diff --git a/json-logger/pom.xml b/json-logger/pom.xml index bbe3388..ae44887 100644 --- a/json-logger/pom.xml +++ b/json-logger/pom.xml @@ -78,11 +78,7 @@ commons-lang 2.6 - - commons-beanutils - commons-beanutils - 1.9.4 - + diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java index f632f3a..dd32688 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java @@ -1,10 +1,11 @@ package org.mule.extension.jsonlogger.internal; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.node.ObjectNode; import org.mule.extension.jsonlogger.api.pojos.LoggerProcessor; +import org.mule.extension.jsonlogger.api.pojos.Priority; +import org.mule.extension.jsonlogger.api.pojos.ScopeTracePoint; import org.mule.extension.jsonlogger.internal.singleton.ObjectMapperSingleton; import org.mule.runtime.api.component.location.ComponentLocation; import org.mule.runtime.api.meta.model.operation.ExecutionType; @@ -14,11 +15,18 @@ import org.mule.runtime.extension.api.annotation.Expression; import org.mule.runtime.extension.api.annotation.execution.Execution; import org.mule.runtime.extension.api.annotation.param.Config; +import org.mule.runtime.extension.api.annotation.param.Optional; import org.mule.runtime.extension.api.annotation.param.ParameterGroup; +import org.mule.runtime.extension.api.annotation.param.display.DisplayName; +import org.mule.runtime.extension.api.annotation.param.display.Example; +import org.mule.runtime.extension.api.annotation.param.display.Placement; +import org.mule.runtime.extension.api.annotation.param.display.Summary; +import org.mule.runtime.extension.api.runtime.operation.FlowListener; import org.mule.runtime.extension.api.runtime.operation.Result; import org.mule.runtime.extension.api.runtime.parameter.CorrelationInfo; import org.mule.runtime.extension.api.runtime.parameter.ParameterResolver; import org.mule.runtime.extension.api.runtime.process.CompletionCallback; +import org.mule.runtime.extension.api.runtime.route.Chain; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,8 +43,7 @@ public class JsonloggerOperations { - protected transient Logger jsonLogger = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); - private static final Logger LOGGER = LoggerFactory.getLogger(JsonloggerOperations.class); + protected transient Logger log = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); // Void Result for NIO private final Result VOID_RESULT = Result.builder().build(); @@ -54,144 +61,235 @@ public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPP @Config JsonloggerConfiguration config, CompletionCallback callback) { - if (!isLogEnabled(loggerProcessor.getPriority().toString())) { + String priority = loggerProcessor.getPriority().toString(); + if (!isLogEnabled(priority)) { callback.success(VOID_RESULT); return; } - Map typedValuesAsString = new HashMap<>(); - Map typedValuesAsJsonNode = new HashMap<>(); + ObjectNode logEvent = om.getObjectMapper().createObjectNode(); - parseContent(loggerProcessor.getContent(), config); - //TODO: Create mapper once - ObjectNode mergedLogger = om.getObjectMapper().createObjectNode(); - mergedLogger.setAll((ObjectNode) om.getObjectMapper().valueToTree(loggerProcessor)); + logEvent.put(Constants.CORRELATION_ID, correlationInfo.getCorrelationId()) + .put(Constants.MESSAGE, loggerProcessor.getMessage()) + .put(Constants.TIMSTAMP, Instant.now().toString()) + .putPOJO(Constants.APPLICATION, config.getGlobalSettings()) + .put(Constants.THREAD_NAME, Thread.currentThread().getName()) + .put(Constants.PRIORITY, priority) + .put(Constants.TRACE_POINT, loggerProcessor.getTracePoint().value()); - mergedLogger.put("timestamp", getFormattedTimestamp()); + addLocationInfo(logEvent, location, config.getJsonOutput().isLogLocationInfo()); - if (config.getJsonOutput().isLogLocationInfo()) { - mergedLogger.putPOJO("locationInfo", locationInfoToMap(location)); - } - - if (!typedValuesAsString.isEmpty()) { - JsonNode typedValuesNode = om.getObjectMapper(new ArrayList<>()).valueToTree(typedValuesAsString); - mergedLogger.setAll((ObjectNode) typedValuesNode); - } - - if (!typedValuesAsJsonNode.isEmpty()) { - mergedLogger.setAll(typedValuesAsJsonNode); - } - - mergedLogger.setAll((ObjectNode) om.getObjectMapper(new ArrayList<>()).valueToTree(config.getGlobalSettings())); + addContent(logEvent, loggerProcessor.getContent(), config); - mergedLogger.put("threadName", Thread.currentThread().getName()); + log(logEvent, priority, config.getJsonOutput().isPrettyPrint()); - printObjectToLog(mergedLogger, loggerProcessor.getPriority().toString(), config.getJsonOutput().isPrettyPrint()); callback.success(VOID_RESULT); } - private void parseContent(ParameterResolver> v, JsonloggerConfiguration config) { - Map typedValuesAsString = new HashMap<>(); - Map typedValuesAsJsonNode = new HashMap<>(); - - TypedValue typedVal = v.resolve(); - if (typedVal.getValue() == null) { - return; - } - - if (!config.getJsonOutput().isParseContentFieldsInJsonOutput() || - !typedVal.getDataType().getMediaType().matches(MediaType.APPLICATION_JSON)) { - typedValuesAsString.put(Constants.CONTENT, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); - return; - } - - // Apply data masking only if needed - ObjectMapper mapper; - List dataMaskingFields = null; - if (config.getJsonOutput().getContentFieldsDataMasking() != null) { - dataMaskingFields = new ArrayList<>(); - String[] split = config - .getJsonOutput() - .getContentFieldsDataMasking() - .trim() - .split(","); + private void addContent(ObjectNode logEvent, ParameterResolver> v, JsonloggerConfiguration config) { + try { + TypedValue typedVal = v.resolve(); + if (typedVal.getValue() == null) { + return; + } - for (String s : split) { - dataMaskingFields.add(s.trim()); + if (!config.getJsonOutput().isParseContentFieldsInJsonOutput() || + !typedVal.getDataType().getMediaType().matches(MediaType.APPLICATION_JSON)) { + logEvent.put(Constants.CONTENT, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); + return; } - } - - if (dataMaskingFields == null || dataMaskingFields.isEmpty()) { - mapper = om.getObjectMapper(); - } else { - mapper = om.getObjectMapper(dataMaskingFields); - } - - try { - typedValuesAsJsonNode.put(Constants.CONTENT, mapper.readTree(typedVal.getValue())); + // Apply data masking only if needed + ObjectMapper mapper; + List dataMaskingFields = null; + if (config.getJsonOutput().getContentFieldsDataMasking() != null) { + dataMaskingFields = new ArrayList<>(); + String[] split = config + .getJsonOutput() + .getContentFieldsDataMasking() + .trim() + .split(","); + + for (String s : split) { + dataMaskingFields.add(s.trim()); + } + + } + + if (dataMaskingFields == null || dataMaskingFields.isEmpty()) { + mapper = om.getObjectMapper(); + } else { + mapper = om.getObjectMapper(dataMaskingFields); + } + + logEvent.put(Constants.CONTENT, mapper.readTree(typedVal.getValue())); + + typedVal.getValue().close(); } catch (Exception e) { - typedValuesAsString.put(Constants.CONTENT, e.getMessage()); + logEvent.put(Constants.CONTENT, e.getMessage()); } } - private Map locationInfoToMap(ComponentLocation location) { + private void addLocationInfo(ObjectNode logEvent, ComponentLocation location, boolean shouldLogLocation) { + if (!shouldLogLocation) { + return; + } Map locationInfo = new HashMap<>(); - locationInfo.put("rootContainer", location.getRootContainerName()); - locationInfo.put("component", location.getComponentIdentifier().getIdentifier().toString()); - locationInfo.put("fileName", location.getFileName().orElse("")); - locationInfo.put("lineInFile", String.valueOf(location.getLineInFile().orElse(null))); - return locationInfo; - } - - private String getFormattedTimestamp() { - Instant now = Instant.now(); - return now.toString(); + locationInfo.put(Constants.ROOT_CONTAINER, location.getRootContainerName()); + locationInfo.put(Constants.FILE_NAME, location.getFileName().orElse("")); + locationInfo.put(Constants.LINE_NUMBER, String.valueOf(location.getLineInFile().orElse(null))); + logEvent.putPOJO(Constants.LOCATION_INFO, locationInfo); } - private void printObjectToLog(ObjectNode loggerObj, String priority, boolean isPrettyPrint) { - ObjectWriter ow = (isPrettyPrint) ? om.getObjectMapper(new ArrayList<>()).writer().withDefaultPrettyPrinter() : om.getObjectMapper(new ArrayList<>()).writer(); + private void log(ObjectNode logEvent, String priority, boolean isPrettyPrint) { + ObjectWriter ow = (isPrettyPrint) ? om.getObjectMapper().writer().withDefaultPrettyPrinter() : om.getObjectMapper().writer(); try { - String logLine = ow.writeValueAsString(loggerObj); - doLog(priority, logLine); + doLog(priority, ow.writeValueAsString(logEvent)); } catch (Exception e) { - LOGGER.error("Error parsing log data as a string", e); + String s = String.format("{\"%s\": \"%s\", \"message\": \"Error parsing content as a string: %s\"}", + Constants.CORRELATION_ID, + logEvent.get(Constants.CORRELATION_ID), + e.getMessage()); + log.error(s); } } private void doLog(String priority, String logLine) { switch (priority) { case "TRACE": - jsonLogger.trace(logLine); + log.trace(logLine); break; case "DEBUG": - jsonLogger.debug(logLine); + log.debug(logLine); break; case "INFO": - jsonLogger.info(logLine); + log.info(logLine); break; case "WARN": - jsonLogger.warn(logLine); + log.warn(logLine); break; case "ERROR": - jsonLogger.error(logLine); - break; + log.error(logLine); } } private Boolean isLogEnabled(String priority) { switch (priority) { case "TRACE": - return jsonLogger.isTraceEnabled(); + return log.isTraceEnabled(); case "DEBUG": - return jsonLogger.isDebugEnabled(); + return log.isDebugEnabled(); case "INFO": - return jsonLogger.isInfoEnabled(); + return log.isInfoEnabled(); case "WARN": - return jsonLogger.isWarnEnabled(); + return log.isWarnEnabled(); case "ERROR": - return jsonLogger.isErrorEnabled(); + return log.isErrorEnabled(); } return false; } + + + @Execution(ExecutionType.BLOCKING) + public void loggerScope(@DisplayName("Module configuration") @Example("JSON_Logger_Config") @Summary("Indicate which Global config should be associated with this Scope.") String configurationRef, + @Optional(defaultValue = "INFO") Priority priority, + @Optional(defaultValue = "OUTBOUND_REQUEST_SCOPE") ScopeTracePoint scopeTracePoint, + @Optional(defaultValue = "#[correlationId]") @Placement(tab = "Advanced") String correlationId, + ComponentLocation location, + CorrelationInfo correlationInfo, + FlowListener flowListener, + Chain operations, + CompletionCallback callback) { + + Long initialTimestamp, loggerTimestamp; + initialTimestamp = loggerTimestamp = System.currentTimeMillis(); + + // Calculate elapsed time based on cached initialTimestamp + long elapsed = loggerTimestamp - initialTimestamp; + + /** + * Avoid Logger Scope logic execution based on log priority + */ + if (isLogEnabled(priority.toString())) { + // Execute Scope Logger + ObjectNode logEvent = om.getObjectMapper(new ArrayList<>()).createObjectNode(); + + /** + * Custom field ordering for Logger Scope + * =============================== + **/ + logEvent.put("correlationId", correlationId); + logEvent.put("tracePoint", scopeTracePoint.toString() + "_BEFORE"); + logEvent.put("priority", priority.toString()); + logEvent.put("elapsed", elapsed); + logEvent.put("scopeElapsed", 0); + if (configs.getConfig(configurationRef).getJsonOutput().isLogLocationInfo()) { + Map locationInfoMap = locationInfoToMap(location); + logEvent.putPOJO("locationInfo", locationInfoMap); + } + logEvent.put("timestamp", getFormattedTimestamp(loggerTimestamp)); + logEvent.put("applicationName", configs.getConfig(configurationRef).getGlobalSettings().getApplicationName()); + logEvent.put("applicationVersion", configs.getConfig(configurationRef).getGlobalSettings().getApplicationVersion()); + logEvent.put("environment", configs.getConfig(configurationRef).getGlobalSettings().getEnvironment()); + logEvent.put("threadName", Thread.currentThread().getName()); + + + String finalLogBefore = log(logEvent, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); + + // Added temp variable to comply with lambda + Long finalInitialTimestamp = initialTimestamp; + operations.process( + result -> { + + Long endScopeTimestamp = System.currentTimeMillis(); + + // Calculate elapsed time + Long scopeElapsed = endScopeTimestamp - loggerTimestamp; + Long mainElapsed = endScopeTimestamp - finalInitialTimestamp; + + loggerProcessor.put("tracePoint", scopeTracePoint.toString() + "_AFTER"); + loggerProcessor.put("priority", priority.toString()); + loggerProcessor.put("elapsed", mainElapsed); + loggerProcessor.put("scopeElapsed", scopeElapsed); + loggerProcessor.put("timestamp", getFormattedTimestamp(endScopeTimestamp)); + + // Print Logger + String finalLogAfter = printObjectToLog(loggerProcessor, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); + + callback.success(result); + }, + (error, previous) -> { + + /** ERROR scope logger **/ + + Long errorScopeTimestamp = System.currentTimeMillis(); + Long mainElapsed = errorScopeTimestamp - finalInitialTimestamp; + + // Calculate elapsed time + Long scopeElapsed = errorScopeTimestamp - loggerTimestamp; + + loggerProcessor.put("message", "Error found: " + error.getMessage()); + loggerProcessor.put("tracePoint", "EXCEPTION_SCOPE"); + loggerProcessor.put("priority", "ERROR"); + loggerProcessor.put("elapsed", mainElapsed); + loggerProcessor.put("scopeElapsed", scopeElapsed); + loggerProcessor.put("timestamp", getFormattedTimestamp(errorScopeTimestamp)); + + // Print Logger + String finalLogError = printObjectToLog(loggerProcessor, "ERROR", configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); + + callback.error(error); + }); + } else { + // Execute operations without Logger + LOGGER.debug("Avoiding logger scope logic execution due to log priority not being enabled"); + operations.process( + callback::success, + (error, previous) -> { + callback.error(error); + }); + } + } + } + diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java index a2bd5b7..3c2e9ac 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.module.SimpleModule; import java.util.List; @@ -13,11 +14,16 @@ public class ObjectMapperSingleton { .setSerializationInclusion(JsonInclude.Include.NON_NULL) .setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + private final ObjectMapper maskingMapper = new ObjectMapper() + .setSerializationInclusion(JsonInclude.Include.NON_NULL) + .setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + + public ObjectMapper getObjectMapper(List maskedFields) { SimpleModule module = new SimpleModule(); module.addDeserializer(JsonNode.class, new MaskingDeserializer(maskedFields)); - this.om.registerModule(module); - return this.om; + this.maskingMapper.registerModule(module); + return this.maskingMapper; } public ObjectMapper getObjectMapper() { diff --git a/json-logger/src/main/resources/schema/loggerConfig.json b/json-logger/src/main/resources/schema/loggerConfig.json index e8929f9..12c140b 100644 --- a/json-logger/src/main/resources/schema/loggerConfig.json +++ b/json-logger/src/main/resources/schema/loggerConfig.json @@ -6,14 +6,14 @@ "globalSettings": { "type": "object", "properties": { - "applicationName": { + "name": { "type": "string", "sdk": { "default": "${json.logger.application.name}", "summary": "Name of the Mule application. Recommendation: This value should be based on pom.xml" } }, - "applicationVersion": { + "version": { "type": "string", "sdk": { "default": "${json.logger.application.version}", @@ -83,8 +83,8 @@ "contentFieldsDataMasking": { "type": "string", "sdk": { - "summary": "Indicate which fields (inside a content type with JSON output only) should be masked before from logging separated by comma. They can be JSON keys or JSON paths. If empty, no masking will be applied. Recommendation: This value should be based on external property", - "example": "client_secret,password,$.myArray[1].someField,$..path.to.a.field", + "summary": "Indicate which fields (inside a content type with JSON output only) should be masked before from logging separated by comma. If empty, no masking will be applied. Recommendation: This value should be based on external property", + "example": "client_secret,password", "required": false, "expressionSupport": "NOT_SUPPORTED" }, diff --git a/json-logger/src/test/java/JsonMaskerTest.java b/json-logger/src/test/java/JsonMaskerTest.java deleted file mode 100644 index af85c7a..0000000 --- a/json-logger/src/test/java/JsonMaskerTest.java +++ /dev/null @@ -1,43 +0,0 @@ - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.beanutils.PropertyUtils; -import org.junit.Assert; -import org.junit.Test; -import org.mule.extension.jsonlogger.api.pojos.LoggerProcessor; -import org.mule.extension.jsonlogger.internal.singleton.ObjectMapperSingleton; -import org.mule.runtime.api.metadata.TypedValue; - -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.InvocationTargetException; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class JsonMaskerTest { - ClassLoader classloader = Thread.currentThread().getContextClassLoader(); - InputStream is = classloader.getResourceAsStream("payload.json"); - List maskedFields = Collections.singletonList("attributes"); - private final ObjectMapper om = new ObjectMapperSingleton().getObjectMapper(maskedFields); - - @Test - public void caseInsensitve() throws IOException { - JsonNode payload = om.readTree(is); - Map masked = om.convertValue(payload, new TypeReference>() { - }); - Assert.assertEquals("{name=****, phone=************, gender=****}", masked.get("attributes").toString()); - } - - @Test - public void typedValue() throws IllegalAccessException, NoSuchMethodException, InvocationTargetException { - LoggerProcessor loggerProcessor = new LoggerProcessor(); - Map describe = PropertyUtils.describe(loggerProcessor); - describe.forEach((k,v) -> { - System.out.println(k); - }); - } - - -} \ No newline at end of file From 68e2422d4248ceedfaf96d8fb296676d5e3cdaec Mon Sep 17 00:00:00 2001 From: Jazzy Date: Wed, 16 Jun 2021 01:17:18 -0700 Subject: [PATCH 07/12] checkpoint --- .../internal/JsonloggerOperations.java | 142 +++++++----------- 1 file changed, 58 insertions(+), 84 deletions(-) diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java index dd32688..4ad3780 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.mule.extension.jsonlogger.api.pojos.LoggerProcessor; import org.mule.extension.jsonlogger.api.pojos.Priority; import org.mule.extension.jsonlogger.api.pojos.ScopeTracePoint; import org.mule.extension.jsonlogger.internal.singleton.ObjectMapperSingleton; @@ -55,13 +54,13 @@ public class JsonloggerOperations { private TransformationService transformationService; @Execution(ExecutionType.BLOCKING) - public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPPORTED) LoggerProcessor loggerProcessor, + public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPPORTED) logEvent logEvent, CorrelationInfo correlationInfo, ComponentLocation location, @Config JsonloggerConfiguration config, CompletionCallback callback) { - String priority = loggerProcessor.getPriority().toString(); + String priority = logEvent.getPriority().toString(); if (!isLogEnabled(priority)) { callback.success(VOID_RESULT); return; @@ -70,16 +69,16 @@ public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPP ObjectNode logEvent = om.getObjectMapper().createObjectNode(); logEvent.put(Constants.CORRELATION_ID, correlationInfo.getCorrelationId()) - .put(Constants.MESSAGE, loggerProcessor.getMessage()) + .put(Constants.MESSAGE, logEvent.getMessage()) .put(Constants.TIMSTAMP, Instant.now().toString()) .putPOJO(Constants.APPLICATION, config.getGlobalSettings()) .put(Constants.THREAD_NAME, Thread.currentThread().getName()) .put(Constants.PRIORITY, priority) - .put(Constants.TRACE_POINT, loggerProcessor.getTracePoint().value()); + .put(Constants.TRACE_POINT, logEvent.getTracePoint().value()); addLocationInfo(logEvent, location, config.getJsonOutput().isLogLocationInfo()); - addContent(logEvent, loggerProcessor.getContent(), config); + addContent(logEvent, logEvent.getContent(), config); log(logEvent, priority, config.getJsonOutput().isPrettyPrint()); @@ -189,7 +188,6 @@ private Boolean isLogEnabled(String priority) { return false; } - @Execution(ExecutionType.BLOCKING) public void loggerScope(@DisplayName("Module configuration") @Example("JSON_Logger_Config") @Summary("Indicate which Global config should be associated with this Scope.") String configurationRef, @Optional(defaultValue = "INFO") Priority priority, @@ -204,91 +202,67 @@ public void loggerScope(@DisplayName("Module configuration") @Example("JSON_Logg Long initialTimestamp, loggerTimestamp; initialTimestamp = loggerTimestamp = System.currentTimeMillis(); - // Calculate elapsed time based on cached initialTimestamp long elapsed = loggerTimestamp - initialTimestamp; - /** - * Avoid Logger Scope logic execution based on log priority - */ - if (isLogEnabled(priority.toString())) { - // Execute Scope Logger - ObjectNode logEvent = om.getObjectMapper(new ArrayList<>()).createObjectNode(); - - /** - * Custom field ordering for Logger Scope - * =============================== - **/ - logEvent.put("correlationId", correlationId); - logEvent.put("tracePoint", scopeTracePoint.toString() + "_BEFORE"); - logEvent.put("priority", priority.toString()); - logEvent.put("elapsed", elapsed); - logEvent.put("scopeElapsed", 0); - if (configs.getConfig(configurationRef).getJsonOutput().isLogLocationInfo()) { - Map locationInfoMap = locationInfoToMap(location); - logEvent.putPOJO("locationInfo", locationInfoMap); - } - logEvent.put("timestamp", getFormattedTimestamp(loggerTimestamp)); - logEvent.put("applicationName", configs.getConfig(configurationRef).getGlobalSettings().getApplicationName()); - logEvent.put("applicationVersion", configs.getConfig(configurationRef).getGlobalSettings().getApplicationVersion()); - logEvent.put("environment", configs.getConfig(configurationRef).getGlobalSettings().getEnvironment()); - logEvent.put("threadName", Thread.currentThread().getName()); - - - String finalLogBefore = log(logEvent, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); - - // Added temp variable to comply with lambda - Long finalInitialTimestamp = initialTimestamp; - operations.process( - result -> { - - Long endScopeTimestamp = System.currentTimeMillis(); - - // Calculate elapsed time - Long scopeElapsed = endScopeTimestamp - loggerTimestamp; - Long mainElapsed = endScopeTimestamp - finalInitialTimestamp; - - loggerProcessor.put("tracePoint", scopeTracePoint.toString() + "_AFTER"); - loggerProcessor.put("priority", priority.toString()); - loggerProcessor.put("elapsed", mainElapsed); - loggerProcessor.put("scopeElapsed", scopeElapsed); - loggerProcessor.put("timestamp", getFormattedTimestamp(endScopeTimestamp)); - - // Print Logger - String finalLogAfter = printObjectToLog(loggerProcessor, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); - - callback.success(result); - }, - (error, previous) -> { - - /** ERROR scope logger **/ - - Long errorScopeTimestamp = System.currentTimeMillis(); - Long mainElapsed = errorScopeTimestamp - finalInitialTimestamp; - - // Calculate elapsed time - Long scopeElapsed = errorScopeTimestamp - loggerTimestamp; - - loggerProcessor.put("message", "Error found: " + error.getMessage()); - loggerProcessor.put("tracePoint", "EXCEPTION_SCOPE"); - loggerProcessor.put("priority", "ERROR"); - loggerProcessor.put("elapsed", mainElapsed); - loggerProcessor.put("scopeElapsed", scopeElapsed); - loggerProcessor.put("timestamp", getFormattedTimestamp(errorScopeTimestamp)); - - // Print Logger - String finalLogError = printObjectToLog(loggerProcessor, "ERROR", configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); - - callback.error(error); - }); - } else { - // Execute operations without Logger - LOGGER.debug("Avoiding logger scope logic execution due to log priority not being enabled"); + if (!isLogEnabled(priority.toString())) { operations.process( callback::success, (error, previous) -> { callback.error(error); }); + return; } + + ObjectNode logEvent = om.getObjectMapper(new ArrayList<>()).createObjectNode(); + + logEvent.put(Constants.CORRELATION_ID, correlationId) + .put(Constants.TRACE_POINT, scopeTracePoint.toString() + "_BEFORE") + .put(Constants.PRIORITY, priority.toString()) + .put(Constants.ELAPSED, elapsed) + .put(Constants.SCOPE_ELAPSED, 0) + .put(Constants.TIMSTAMP, Instant.now().toString()) + .putPOJO(Constants.APPLICATION, configs.getConfig(configurationRef).getGlobalSettings()) + .put(Constants.THREAD_NAME, Thread.currentThread().getName()); + + addLocationInfo(logEvent, location, true); + + log(logEvent, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); + + Long finalInitialTimestamp = initialTimestamp; + operations.process( + result -> { + Long scopeElapsed = endScopeTimestamp - loggerTimestamp; + Long mainElapsed = endScopeTimestamp - initialTimestamp; + + logEvent.put(Constants.TRACE_POINT, scopeTracePoint.toString() + "_AFTER") + .put(Constants.PRIORITY, priority.toString()) + .put(Constants.SCOPE_ELAPSED, scopeElapsed) + .put(Constants.TIMSTAMP, Instant.now().toString()); + + // Print Logger + String finalLogAfter = printObjectToLog(logEvent, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); + + callback.success(result); + }, + (error, previous) -> { + + + Long mainElapsed = errorScopeTimestamp - finalInitialTimestamp; + + // Calculate elapsed time + Long scopeElapsed = errorScopeTimestamp - loggerTimestamp; + + logEvent.put("message", "Error found: " + error.getMessage()); + logEvent.put("tracePoint", "EXCEPTION_SCOPE"); + logEvent.put("priority", "ERROR"); + logEvent.put("elapsed", mainElapsed); + logEvent.put("scopeElapsed", scopeElapsed); + logEvent.put("timestamp", Instant.now().toString()); + + log(logEvent, "ERROR", configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); + + callback.error(error); + }); } } From d65e90e4108e1cd35a3ef30b8c18b9f41e65aa42 Mon Sep 17 00:00:00 2001 From: Jazzy Date: Thu, 17 Jun 2021 06:06:22 -0700 Subject: [PATCH 08/12] checkpoint --- .../internal/JsonloggerConfiguration.java | 37 ---- .../internal/JsonloggerExtension.java | 8 +- .../internal/JsonloggerOperations.java | 102 ++++------ .../internal/singleton/ConfigsSingleton.java | 24 --- .../singleton/MaskingDeserializer.java | 176 ------------------ .../singleton/ObjectMapperSingleton.java | 32 ---- .../core/config/registry-bootstrap.properties | 3 +- .../schema/loggerScopeProcessor.json | 25 +++ 8 files changed, 70 insertions(+), 337 deletions(-) delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConfiguration.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ConfigsSingleton.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/MaskingDeserializer.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConfiguration.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConfiguration.java deleted file mode 100644 index 8d37d87..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerConfiguration.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.mule.extension.jsonlogger.internal; - -import org.mule.extension.jsonlogger.api.pojos.LoggerConfig; -import org.mule.extension.jsonlogger.internal.singleton.ConfigsSingleton; -import org.mule.runtime.api.lifecycle.Disposable; -import org.mule.runtime.api.lifecycle.Initialisable; -import org.mule.runtime.extension.api.annotation.Operations; -import org.mule.runtime.extension.api.annotation.param.RefName; - -import javax.inject.Inject; - -/** - * This class represents an extension configuration, values set in this class are commonly used across multiple - * operations since they represent something core from the extension. - */ -@Operations(JsonloggerOperations.class) -public class JsonloggerConfiguration extends LoggerConfig implements Initialisable, Disposable { - - @Inject - ConfigsSingleton configsSingleton; - - @RefName - private String configName; - - public String getConfigName() { - return configName; - } - - @Override - public void initialise() { - configsSingleton.addConfig(configName, this); - } - - @Override - public void dispose() { - } -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java index 5507c67..6bc0060 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java @@ -1,18 +1,16 @@ package org.mule.extension.jsonlogger.internal; +import org.mule.extension.jsonlogger.api.pojos.LoggerConfig; import org.mule.runtime.extension.api.annotation.Configurations; import org.mule.runtime.extension.api.annotation.Export; import org.mule.runtime.extension.api.annotation.Extension; +import org.mule.runtime.extension.api.annotation.Operations; import org.mule.runtime.extension.api.annotation.dsl.xml.Xml; -/** - * This is the main class of an extension, is the entry point from which configurations, connection providers, operations - * and sources are going to be declared. - */ @Xml(prefix = "json-logger") @Extension(name = "JSON Logger") @Export(resources = {"modules/JSONLoggerModule.dwl"}) -@Configurations(JsonloggerConfiguration.class) +@Configurations(JsonLoggerConfig.class) public class JsonLoggerExtension { } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java index 4ad3780..1cbd32f 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java @@ -3,9 +3,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.mule.extension.jsonlogger.api.pojos.LoggerProcessor; +import org.mule.extension.jsonlogger.api.pojos.LoggerScopeProcessor; import org.mule.extension.jsonlogger.api.pojos.Priority; import org.mule.extension.jsonlogger.api.pojos.ScopeTracePoint; -import org.mule.extension.jsonlogger.internal.singleton.ObjectMapperSingleton; +import org.mule.extension.jsonlogger.internal.mapper.ObjectMapperSingleton; import org.mule.runtime.api.component.location.ComponentLocation; import org.mule.runtime.api.meta.model.operation.ExecutionType; import org.mule.runtime.api.metadata.MediaType; @@ -16,10 +18,7 @@ import org.mule.runtime.extension.api.annotation.param.Config; import org.mule.runtime.extension.api.annotation.param.Optional; import org.mule.runtime.extension.api.annotation.param.ParameterGroup; -import org.mule.runtime.extension.api.annotation.param.display.DisplayName; -import org.mule.runtime.extension.api.annotation.param.display.Example; import org.mule.runtime.extension.api.annotation.param.display.Placement; -import org.mule.runtime.extension.api.annotation.param.display.Summary; import org.mule.runtime.extension.api.runtime.operation.FlowListener; import org.mule.runtime.extension.api.runtime.operation.Result; import org.mule.runtime.extension.api.runtime.parameter.CorrelationInfo; @@ -31,6 +30,7 @@ import javax.inject.Inject; import java.io.InputStream; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; @@ -40,11 +40,9 @@ import static org.mule.runtime.api.meta.ExpressionSupport.NOT_SUPPORTED; import static org.mule.runtime.api.metadata.DataType.TEXT_STRING; -public class JsonloggerOperations { +public class JsonLoggerOperations { - protected transient Logger log = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); - - // Void Result for NIO + protected static Logger log = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); private final Result VOID_RESULT = Result.builder().build(); @Inject @@ -53,14 +51,14 @@ public class JsonloggerOperations { @Inject private TransformationService transformationService; - @Execution(ExecutionType.BLOCKING) - public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPPORTED) logEvent logEvent, + @Execution(ExecutionType.CPU_LITE) + public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPPORTED) LoggerProcessor loggerProcessor, + @Config JsonLoggerConfig config, CorrelationInfo correlationInfo, ComponentLocation location, - @Config JsonloggerConfiguration config, CompletionCallback callback) { - String priority = logEvent.getPriority().toString(); + String priority = loggerProcessor.getPriority().toString(); if (!isLogEnabled(priority)) { callback.success(VOID_RESULT); return; @@ -69,23 +67,23 @@ public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPP ObjectNode logEvent = om.getObjectMapper().createObjectNode(); logEvent.put(Constants.CORRELATION_ID, correlationInfo.getCorrelationId()) - .put(Constants.MESSAGE, logEvent.getMessage()) + .put(Constants.MESSAGE, loggerProcessor.getMessage()) .put(Constants.TIMSTAMP, Instant.now().toString()) .putPOJO(Constants.APPLICATION, config.getGlobalSettings()) .put(Constants.THREAD_NAME, Thread.currentThread().getName()) .put(Constants.PRIORITY, priority) - .put(Constants.TRACE_POINT, logEvent.getTracePoint().value()); + .put(Constants.TRACE_POINT, loggerProcessor.getTracePoint().value()); addLocationInfo(logEvent, location, config.getJsonOutput().isLogLocationInfo()); - addContent(logEvent, logEvent.getContent(), config); + addContent(logEvent, loggerProcessor.getContent(), config); log(logEvent, priority, config.getJsonOutput().isPrettyPrint()); callback.success(VOID_RESULT); } - private void addContent(ObjectNode logEvent, ParameterResolver> v, JsonloggerConfiguration config) { + private void addContent(ObjectNode logEvent, ParameterResolver> v, JsonLoggerConfig config) { try { TypedValue typedVal = v.resolve(); if (typedVal.getValue() == null) { @@ -189,22 +187,16 @@ private Boolean isLogEnabled(String priority) { } @Execution(ExecutionType.BLOCKING) - public void loggerScope(@DisplayName("Module configuration") @Example("JSON_Logger_Config") @Summary("Indicate which Global config should be associated with this Scope.") String configurationRef, - @Optional(defaultValue = "INFO") Priority priority, - @Optional(defaultValue = "OUTBOUND_REQUEST_SCOPE") ScopeTracePoint scopeTracePoint, - @Optional(defaultValue = "#[correlationId]") @Placement(tab = "Advanced") String correlationId, - ComponentLocation location, + public void loggerScope(@ParameterGroup(name = "ScopeLogger") @Expression(value = NOT_SUPPORTED) LoggerScopeProcessor loggerProcessor, + @Config JsonLoggerConfig config, CorrelationInfo correlationInfo, + ComponentLocation location, FlowListener flowListener, Chain operations, CompletionCallback callback) { - Long initialTimestamp, loggerTimestamp; - initialTimestamp = loggerTimestamp = System.currentTimeMillis(); - - long elapsed = loggerTimestamp - initialTimestamp; - - if (!isLogEnabled(priority.toString())) { + String priority = loggerProcessor.getPriority().value(); + if (!isLogEnabled(priority)) { operations.process( callback::success, (error, previous) -> { @@ -212,55 +204,43 @@ public void loggerScope(@DisplayName("Module configuration") @Example("JSON_Logg }); return; } - + ObjectNode logEvent = om.getObjectMapper(new ArrayList<>()).createObjectNode(); + + Instant startTime = Instant.now(); - logEvent.put(Constants.CORRELATION_ID, correlationId) - .put(Constants.TRACE_POINT, scopeTracePoint.toString() + "_BEFORE") - .put(Constants.PRIORITY, priority.toString()) - .put(Constants.ELAPSED, elapsed) + logEvent.put(Constants.CORRELATION_ID, correlationInfo.getCorrelationId()) + .put(Constants.TRACE_POINT, loggerProcessor.getScopeTracePoint().value() + "_BEFORE") + .put(Constants.PRIORITY, priority) .put(Constants.SCOPE_ELAPSED, 0) - .put(Constants.TIMSTAMP, Instant.now().toString()) - .putPOJO(Constants.APPLICATION, configs.getConfig(configurationRef).getGlobalSettings()) + .put(Constants.TIMSTAMP, startTime.toString()) + .putPOJO(Constants.APPLICATION, config.getGlobalSettings()) .put(Constants.THREAD_NAME, Thread.currentThread().getName()); addLocationInfo(logEvent, location, true); - log(logEvent, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); + log(logEvent, priority, config.getJsonOutput().isPrettyPrint()); - Long finalInitialTimestamp = initialTimestamp; operations.process( result -> { - Long scopeElapsed = endScopeTimestamp - loggerTimestamp; - Long mainElapsed = endScopeTimestamp - initialTimestamp; - - logEvent.put(Constants.TRACE_POINT, scopeTracePoint.toString() + "_AFTER") - .put(Constants.PRIORITY, priority.toString()) - .put(Constants.SCOPE_ELAPSED, scopeElapsed) - .put(Constants.TIMSTAMP, Instant.now().toString()); - - // Print Logger - String finalLogAfter = printObjectToLog(logEvent, priority.toString(), configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); + Instant endTime = Instant.now(); + logEvent.put(Constants.TRACE_POINT, loggerProcessor.getScopeTracePoint().value() + "_AFTER") + .put(Constants.PRIORITY, priority) + .put(Constants.SCOPE_ELAPSED, Duration.between(startTime, endTime).toMillis()) + .put(Constants.TIMSTAMP, endTime.toString()); + log(logEvent, priority, config.getJsonOutput().isPrettyPrint()); callback.success(result); }, (error, previous) -> { + Instant endTime = Instant.now(); + logEvent.put(Constants.MESSAGE, error.getMessage()) + .put(Constants.TRACE_POINT, "EXCEPTION_SCOPE") + .put(Constants.PRIORITY, "ERROR") + .put(Constants.SCOPE_ELAPSED, Duration.between(startTime, endTime).toMillis()) + .put(Constants.TIMSTAMP, endTime.toString()); - - Long mainElapsed = errorScopeTimestamp - finalInitialTimestamp; - - // Calculate elapsed time - Long scopeElapsed = errorScopeTimestamp - loggerTimestamp; - - logEvent.put("message", "Error found: " + error.getMessage()); - logEvent.put("tracePoint", "EXCEPTION_SCOPE"); - logEvent.put("priority", "ERROR"); - logEvent.put("elapsed", mainElapsed); - logEvent.put("scopeElapsed", scopeElapsed); - logEvent.put("timestamp", Instant.now().toString()); - - log(logEvent, "ERROR", configs.getConfig(configurationRef).getJsonOutput().isPrettyPrint()); - + log(logEvent, "ERROR", config.getJsonOutput().isPrettyPrint()); callback.error(error); }); } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ConfigsSingleton.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ConfigsSingleton.java deleted file mode 100644 index 4ae2a27..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ConfigsSingleton.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.mule.extension.jsonlogger.internal.singleton; - -import org.mule.extension.jsonlogger.internal.JsonloggerConfiguration; - -import java.util.HashMap; -import java.util.Map; - -public class ConfigsSingleton { - - private final Map configs = new HashMap<>(); - - public Map getConfigs() { - return configs; - } - - public JsonloggerConfiguration getConfig(String configName) { - return this.configs.get(configName); - } - - public void addConfig(String configName, JsonloggerConfiguration config) { - this.configs.put(configName, config); - } - -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/MaskingDeserializer.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/MaskingDeserializer.java deleted file mode 100644 index 346c69c..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/MaskingDeserializer.java +++ /dev/null @@ -1,176 +0,0 @@ -package org.mule.extension.jsonlogger.internal.singleton; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.deser.std.JsonNodeDeserializer; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.commons.lang.StringUtils; - -import java.io.IOException; -import java.util.List; - -public class MaskingDeserializer extends JsonNodeDeserializer { - List maskedFields; - - public MaskingDeserializer(List maskedFields) { - this.maskedFields = maskedFields; - } - - @Override - public JsonNode deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - switch (p.currentTokenId()) { - case 1: - return deserializeObject(p, ctxt, ctxt.getNodeFactory(), false); - case 3: - return deserializeArray(p, ctxt, ctxt.getNodeFactory(), false); - default: - return deserializeAny(p, ctxt, ctxt.getNodeFactory(), false); - } - } - - protected final ObjectNode deserializeObject(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { - ObjectNode node = nodeFactory.objectNode(); - - for (String key = p.nextFieldName(); key != null; key = p.nextFieldName()) { - if (maskedFields.contains(key)) { - shouldMask = true; - } - JsonToken t = p.nextToken(); - if (t == null) { - t = JsonToken.NOT_AVAILABLE; - } - - JsonNode value; - switch (t.id()) { - case 1: - value = deserializeObject(p, ctxt, nodeFactory, shouldMask); - shouldMask = false; - break; - case 2: - case 4: - case 5: - case 8: - default: - value = deserializeAny(p, ctxt, nodeFactory, shouldMask); - shouldMask = false; - break; - case 3: - value = deserializeArray(p, ctxt, nodeFactory, shouldMask); - shouldMask = false; - break; - case 6: - if (shouldMask) { - value = nodeFactory.textNode(replace(p.getText())); - } else { - value = nodeFactory.textNode(p.getText()); - } - break; - case 7: - value = this._fromInt(p, ctxt, nodeFactory); - break; - case 9: - value = nodeFactory.booleanNode(true); - break; - case 10: - value = nodeFactory.booleanNode(false); - break; - case 11: - value = nodeFactory.nullNode(); - break; - case 12: - value = this._fromEmbedded(p, ctxt, nodeFactory); - } - - JsonNode old = node.replace(key, value); - if (old != null) { - this._handleDuplicateField(p, ctxt, nodeFactory, key, node, old, value); - } - } - - return node; - } - - protected final ArrayNode deserializeArray(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { - ArrayNode node = nodeFactory.arrayNode(); - - while (true) { - JsonToken t = p.nextToken(); - switch (t.id()) { - case 1: - node.add(deserializeObject(p, ctxt, nodeFactory, shouldMask)); - break; - case 2: - case 5: - case 8: - default: - node.add(deserializeAny(p, ctxt, nodeFactory, shouldMask)); - break; - case 3: - node.add(deserializeArray(p, ctxt, nodeFactory, shouldMask)); - break; - case 4: - return node; - case 6: - if (shouldMask) { - node.add(nodeFactory.textNode(replace(p.getText()))); - } else { - node.add(nodeFactory.textNode(p.getText())); - } - break; - case 7: - node.add(this._fromInt(p, ctxt, nodeFactory)); - break; - case 9: - node.add(nodeFactory.booleanNode(true)); - break; - case 10: - node.add(nodeFactory.booleanNode(false)); - break; - case 11: - node.add(nodeFactory.nullNode()); - break; - case 12: - node.add(this._fromEmbedded(p, ctxt, nodeFactory)); - } - } - } - - protected final JsonNode deserializeAny(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { - switch (p.currentTokenId()) { - case 2: - return nodeFactory.objectNode(); - case 3: - case 4: - default: - return (JsonNode) ctxt.handleUnexpectedToken(this.handledType(), p); - case 5: - return deserializeObjectAtName(p, ctxt, nodeFactory); - case 6: - if (shouldMask) { - return nodeFactory.textNode(replace(p.getText())); - } - return nodeFactory.textNode(p.getText()); - case 7: - return this._fromInt(p, ctxt, nodeFactory); - case 8: - return this._fromFloat(p, ctxt, nodeFactory); - case 9: - return nodeFactory.booleanNode(true); - case 10: - return nodeFactory.booleanNode(false); - case 11: - return nodeFactory.nullNode(); - case 12: - return this._fromEmbedded(p, ctxt, nodeFactory); - } - } - - private String replace(String input) { - return StringUtils.repeat("*", input.length()); - } - -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java deleted file mode 100644 index 3c2e9ac..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/singleton/ObjectMapperSingleton.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.mule.extension.jsonlogger.internal.singleton; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.module.SimpleModule; - -import java.util.List; - -public class ObjectMapperSingleton { - - private final ObjectMapper om = new ObjectMapper() - .setSerializationInclusion(JsonInclude.Include.NON_NULL) - .setSerializationInclusion(JsonInclude.Include.NON_EMPTY); - - private final ObjectMapper maskingMapper = new ObjectMapper() - .setSerializationInclusion(JsonInclude.Include.NON_NULL) - .setSerializationInclusion(JsonInclude.Include.NON_EMPTY); - - - public ObjectMapper getObjectMapper(List maskedFields) { - SimpleModule module = new SimpleModule(); - module.addDeserializer(JsonNode.class, new MaskingDeserializer(maskedFields)); - this.maskingMapper.registerModule(module); - return this.maskingMapper; - } - - public ObjectMapper getObjectMapper() { - return this.om; - } -} diff --git a/json-logger/src/main/resources/META-INF/org/mule/runtime/core/config/registry-bootstrap.properties b/json-logger/src/main/resources/META-INF/org/mule/runtime/core/config/registry-bootstrap.properties index 034a10c..1626167 100644 --- a/json-logger/src/main/resources/META-INF/org/mule/runtime/core/config/registry-bootstrap.properties +++ b/json-logger/src/main/resources/META-INF/org/mule/runtime/core/config/registry-bootstrap.properties @@ -1,2 +1 @@ -logger.objectMapperSingleton=org.mule.extension.jsonlogger.internal.singleton.ObjectMapperSingleton -logger.configsSingleton=org.mule.extension.jsonlogger.internal.singleton.ConfigsSingleton +logger.objectMapperSingleton=org.mule.extension.jsonlogger.internal.mapper.ObjectMapperSingleton diff --git a/json-logger/src/main/resources/schema/loggerScopeProcessor.json b/json-logger/src/main/resources/schema/loggerScopeProcessor.json index ab52a31..59a29f9 100644 --- a/json-logger/src/main/resources/schema/loggerScopeProcessor.json +++ b/json-logger/src/main/resources/schema/loggerScopeProcessor.json @@ -15,6 +15,31 @@ "default": "OUTBOUND_REQUEST_SCOPE", "summary": "Current processing stage" } + }, + "correlationId": { + "type": "string", + "sdk": { + "default": "#[correlationId]", + "placement": { + "tab": "Advanced" + } + } + }, + "priority": { + "type": "string", + "javaType": "org.mule.extension.jsonlogger.api.pojos.Priority", + "enum": [ + "DEBUG", + "TRACE", + "INFO", + "WARN", + "ERROR" + ], + "sdk": { + "default": "INFO", + "summary": "Logger priority" + }, + "note": "This field is mandatory. DON'T REMOVE" } } } \ No newline at end of file From 753f89d96769942611f90fd39962cb816d9e3256 Mon Sep 17 00:00:00 2001 From: Jazzy Date: Thu, 17 Jun 2021 06:06:31 -0700 Subject: [PATCH 09/12] checkpoint --- .../jsonlogger/internal/Constants.java | 18 ++ .../jsonlogger/internal/JsonLoggerConfig.java | 8 + .../internal/mapper/MaskingDeserializer.java | 176 ++++++++++++++++++ .../mapper/ObjectMapperSingleton.java | 31 +++ .../jsonlogger/internal/JsonMaskerTest.java | 51 +++++ 5 files changed, 284 insertions(+) create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/Constants.java create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonLoggerConfig.java create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/MaskingDeserializer.java create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/ObjectMapperSingleton.java create mode 100644 json-logger/src/test/java/org/mule/extension/jsonlogger/internal/JsonMaskerTest.java diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/Constants.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/Constants.java new file mode 100644 index 0000000..4eab7b5 --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/Constants.java @@ -0,0 +1,18 @@ +package org.mule.extension.jsonlogger.internal; + +class Constants { + public static final String SCOPE_ELAPSED = "scopeElapsed"; + static final String APPLICATION = "application"; + static final String CONTENT = "content"; + static final String CORRELATION_ID = "correlationId"; + static final String LOCATION_INFO = "locationInfo"; + static final String ROOT_CONTAINER = "rootContainer"; + static final String FILE_NAME = "fileName"; + static final String LINE_NUMBER = "lineNumber"; + static final String TIMSTAMP = "timestamp"; + static final String THREAD_NAME = "threadName"; + static final String MESSAGE = "message"; + static final String PRIORITY = "priority"; + static final String TRACE_POINT = "tracePoint"; + static final String ELAPSED = "elapsed"; +} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonLoggerConfig.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonLoggerConfig.java new file mode 100644 index 0000000..303cea2 --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonLoggerConfig.java @@ -0,0 +1,8 @@ +package org.mule.extension.jsonlogger.internal; + +import org.mule.extension.jsonlogger.api.pojos.LoggerConfig; +import org.mule.runtime.extension.api.annotation.Operations; + +@Operations(JsonLoggerOperations.class) +public class JsonLoggerConfig extends LoggerConfig { +} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/MaskingDeserializer.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/MaskingDeserializer.java new file mode 100644 index 0000000..62f1f23 --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/MaskingDeserializer.java @@ -0,0 +1,176 @@ +package org.mule.extension.jsonlogger.internal.mapper; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.JsonNodeDeserializer; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.commons.lang.StringUtils; + +import java.io.IOException; +import java.util.List; + +class MaskingDeserializer extends JsonNodeDeserializer { + List maskedFields; + + public MaskingDeserializer(List maskedFields) { + this.maskedFields = maskedFields; + } + + @Override + public JsonNode deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + switch (p.currentTokenId()) { + case 1: + return deserializeObject(p, ctxt, ctxt.getNodeFactory(), false); + case 3: + return deserializeArray(p, ctxt, ctxt.getNodeFactory(), false); + default: + return deserializeAny(p, ctxt, ctxt.getNodeFactory(), false); + } + } + + protected final ObjectNode deserializeObject(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { + ObjectNode node = nodeFactory.objectNode(); + + for (String key = p.nextFieldName(); key != null; key = p.nextFieldName()) { + if (maskedFields.contains(key)) { + shouldMask = true; + } + JsonToken t = p.nextToken(); + if (t == null) { + t = JsonToken.NOT_AVAILABLE; + } + + JsonNode value; + switch (t.id()) { + case 1: + value = deserializeObject(p, ctxt, nodeFactory, shouldMask); + shouldMask = false; + break; + case 2: + case 4: + case 5: + case 8: + default: + value = deserializeAny(p, ctxt, nodeFactory, shouldMask); + shouldMask = false; + break; + case 3: + value = deserializeArray(p, ctxt, nodeFactory, shouldMask); + shouldMask = false; + break; + case 6: + if (shouldMask) { + value = nodeFactory.textNode(replace(p.getText())); + } else { + value = nodeFactory.textNode(p.getText()); + } + break; + case 7: + value = this._fromInt(p, ctxt, nodeFactory); + break; + case 9: + value = nodeFactory.booleanNode(true); + break; + case 10: + value = nodeFactory.booleanNode(false); + break; + case 11: + value = nodeFactory.nullNode(); + break; + case 12: + value = this._fromEmbedded(p, ctxt, nodeFactory); + } + + JsonNode old = node.replace(key, value); + if (old != null) { + this._handleDuplicateField(p, ctxt, nodeFactory, key, node, old, value); + } + } + + return node; + } + + protected final ArrayNode deserializeArray(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { + ArrayNode node = nodeFactory.arrayNode(); + + while (true) { + JsonToken t = p.nextToken(); + switch (t.id()) { + case 1: + node.add(deserializeObject(p, ctxt, nodeFactory, shouldMask)); + break; + case 2: + case 5: + case 8: + default: + node.add(deserializeAny(p, ctxt, nodeFactory, shouldMask)); + break; + case 3: + node.add(deserializeArray(p, ctxt, nodeFactory, shouldMask)); + break; + case 4: + return node; + case 6: + if (shouldMask) { + node.add(nodeFactory.textNode(replace(p.getText()))); + } else { + node.add(nodeFactory.textNode(p.getText())); + } + break; + case 7: + node.add(this._fromInt(p, ctxt, nodeFactory)); + break; + case 9: + node.add(nodeFactory.booleanNode(true)); + break; + case 10: + node.add(nodeFactory.booleanNode(false)); + break; + case 11: + node.add(nodeFactory.nullNode()); + break; + case 12: + node.add(this._fromEmbedded(p, ctxt, nodeFactory)); + } + } + } + + protected final JsonNode deserializeAny(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { + switch (p.currentTokenId()) { + case 2: + return nodeFactory.objectNode(); + case 3: + case 4: + default: + return (JsonNode) ctxt.handleUnexpectedToken(this.handledType(), p); + case 5: + return deserializeObjectAtName(p, ctxt, nodeFactory); + case 6: + if (shouldMask) { + return nodeFactory.textNode(replace(p.getText())); + } + return nodeFactory.textNode(p.getText()); + case 7: + return this._fromInt(p, ctxt, nodeFactory); + case 8: + return this._fromFloat(p, ctxt, nodeFactory); + case 9: + return nodeFactory.booleanNode(true); + case 10: + return nodeFactory.booleanNode(false); + case 11: + return nodeFactory.nullNode(); + case 12: + return this._fromEmbedded(p, ctxt, nodeFactory); + } + } + + private String replace(String input) { + return StringUtils.repeat("*", input.length()); + } + +} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/ObjectMapperSingleton.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/ObjectMapperSingleton.java new file mode 100644 index 0000000..d5eefb8 --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/ObjectMapperSingleton.java @@ -0,0 +1,31 @@ +package org.mule.extension.jsonlogger.internal.mapper; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; + +import java.util.List; + +public class ObjectMapperSingleton { + + private final ObjectMapper om = new ObjectMapper() + .setSerializationInclusion(JsonInclude.Include.NON_NULL) + .setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + + private final ObjectMapper maskingMapper = new ObjectMapper() + .setSerializationInclusion(JsonInclude.Include.NON_NULL) + .setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + + + public ObjectMapper getObjectMapper(List maskedFields) { + SimpleModule module = new SimpleModule(); + module.addDeserializer(JsonNode.class, new MaskingDeserializer(maskedFields)); + this.maskingMapper.registerModule(module); + return this.maskingMapper; + } + + public ObjectMapper getObjectMapper() { + return this.om; + } +} diff --git a/json-logger/src/test/java/org/mule/extension/jsonlogger/internal/JsonMaskerTest.java b/json-logger/src/test/java/org/mule/extension/jsonlogger/internal/JsonMaskerTest.java new file mode 100644 index 0000000..5d4edd7 --- /dev/null +++ b/json-logger/src/test/java/org/mule/extension/jsonlogger/internal/JsonMaskerTest.java @@ -0,0 +1,51 @@ +package org.mule.extension.jsonlogger.internal; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Assert; +import org.junit.Test; +import org.mule.extension.jsonlogger.internal.mapper.ObjectMapperSingleton; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public class JsonMaskerTest { + ClassLoader classloader = Thread.currentThread().getContextClassLoader(); + InputStream is = classloader.getResourceAsStream("payload.json"); + List maskedFields = Collections.singletonList("attributes"); + private final ObjectMapper om = new ObjectMapperSingleton().getObjectMapper(maskedFields); + protected Logger log = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); + + @Test + public void caseInsensitve() throws IOException { + JsonNode payload = om.readTree(is); + Map masked = om.convertValue(payload, new TypeReference>() { + }); + Assert.assertEquals("{name=****, phone=************, gender=****}", masked.get("attributes").toString()); + } + + @Test + public void finalException() { + try { + throw new RuntimeException("Invalid string"); + } catch (Exception e) { + String s = String.format("{" + + "\"%s\": \"%s\", " + + "\"message\": \"Error parsing log data as a string: %s\"" + + "}", + Constants.CORRELATION_ID, + UUID.randomUUID().toString(), + e.getMessage()); + System.out.println(s); + } + } + + +} \ No newline at end of file From 014dc09ab1da8279b7c671028014f5164f91e051 Mon Sep 17 00:00:00 2001 From: Jazzy Date: Fri, 18 Jun 2021 01:52:19 -0700 Subject: [PATCH 10/12] working point --- json-logger/pom.xml | 43 +----- .../jsonlogger/{internal => }/Constants.java | 9 +- ...xtension.java => JsonLoggerExtension.java} | 7 +- ...rations.java => JsonLoggerOperations.java} | 111 ++++++++------- .../jsonlogger/config/JsonLoggerConfig.java | 133 ++++++++++++++++++ .../jsonlogger/config/LoggerProcessor.java | 93 ++++++++++++ .../config/LoggerScopeProcessor.java | 48 +++++++ .../extension/jsonlogger/config/Priority.java | 20 +++ .../jsonlogger/config/ScopeTracePoint.java | 18 +++ .../jsonlogger/config/TracePoint.java | 23 +++ .../jsonlogger/internal/JsonLoggerConfig.java | 8 -- .../Mapper.java} | 5 +- .../mapper/MaskingDeserializer.java | 11 +- .../core/config/registry-bootstrap.properties | 1 - .../resources/modules/JSONLoggerModule.dwl | 44 ------ .../main/resources/schema/loggerConfig.json | 100 ------------- .../resources/schema/loggerProcessor.json | 78 ---------- .../schema/loggerScopeProcessor.json | 45 ------ .../jsonlogger/internal/JsonMaskerTest.java | 6 +- .../src/test/resources/json-logger.properties | 9 -- .../src/test/resources/test-mule-config.xml | 23 --- 21 files changed, 412 insertions(+), 423 deletions(-) rename json-logger/src/main/java/org/mule/extension/jsonlogger/{internal => }/Constants.java (67%) rename json-logger/src/main/java/org/mule/extension/jsonlogger/{internal/JsonloggerExtension.java => JsonLoggerExtension.java} (53%) rename json-logger/src/main/java/org/mule/extension/jsonlogger/{internal/JsonloggerOperations.java => JsonLoggerOperations.java} (66%) create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/config/JsonLoggerConfig.java create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerProcessor.java create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerScopeProcessor.java create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/config/Priority.java create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/config/ScopeTracePoint.java create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/config/TracePoint.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonLoggerConfig.java rename json-logger/src/main/java/org/mule/extension/jsonlogger/{internal/mapper/ObjectMapperSingleton.java => mapper/Mapper.java} (91%) rename json-logger/src/main/java/org/mule/extension/jsonlogger/{internal => }/mapper/MaskingDeserializer.java (96%) delete mode 100644 json-logger/src/main/resources/META-INF/org/mule/runtime/core/config/registry-bootstrap.properties delete mode 100644 json-logger/src/main/resources/modules/JSONLoggerModule.dwl delete mode 100644 json-logger/src/main/resources/schema/loggerConfig.json delete mode 100644 json-logger/src/main/resources/schema/loggerProcessor.json delete mode 100644 json-logger/src/main/resources/schema/loggerScopeProcessor.json delete mode 100644 json-logger/src/test/resources/json-logger.properties delete mode 100644 json-logger/src/test/resources/test-mule-config.xml diff --git a/json-logger/pom.xml b/json-logger/pom.xml index ae44887..48cd9b5 100644 --- a/json-logger/pom.xml +++ b/json-logger/pom.xml @@ -4,9 +4,9 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - ORG_ID_TOKEN + 7abbfaeb-3816-4907-865e-76d7cb24120f json-logger - 2.1.0 + 3.0.0 mule-extension JSON Logger - Mule 4 @@ -28,37 +28,11 @@ maven-deploy-plugin 2.8.2 - - org.jsonschema2pojo - jsonschema2pojo-maven-plugin - 0.4.35 - - - io.github.mulesoft-consulting - jsonschema2pojo-mule-annotations - 1.2.0 - compile - - - - org.mule.custom.annotation.utils.CustomMuleAnnotator - ${basedir}/src/main/resources/schema - ${project.build.directory}/generated-sources - org.mule.extension.jsonlogger.api.pojos - - - - - generate - - - - + - Exchange2 Exchange2 Repository @@ -73,17 +47,6 @@ jackson-databind 2.10.3 - - commons-lang - commons-lang - 2.6 - - - - - - - diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/Constants.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/Constants.java similarity index 67% rename from json-logger/src/main/java/org/mule/extension/jsonlogger/internal/Constants.java rename to json-logger/src/main/java/org/mule/extension/jsonlogger/Constants.java index 4eab7b5..67cd018 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/Constants.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/Constants.java @@ -1,10 +1,11 @@ -package org.mule.extension.jsonlogger.internal; +package org.mule.extension.jsonlogger; class Constants { - public static final String SCOPE_ELAPSED = "scopeElapsed"; - static final String APPLICATION = "application"; + static final String APPLICATION_NAME = "applicationName"; + static final String APPLICATION_VERSION = "applicationVersion"; static final String CONTENT = "content"; static final String CORRELATION_ID = "correlationId"; + static final String ENVIRONMENT = "environment"; static final String LOCATION_INFO = "locationInfo"; static final String ROOT_CONTAINER = "rootContainer"; static final String FILE_NAME = "fileName"; @@ -14,5 +15,5 @@ class Constants { static final String MESSAGE = "message"; static final String PRIORITY = "priority"; static final String TRACE_POINT = "tracePoint"; - static final String ELAPSED = "elapsed"; + static final String SCOPE_ELAPSED = "scopeLapsed"; } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/JsonLoggerExtension.java similarity index 53% rename from json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java rename to json-logger/src/main/java/org/mule/extension/jsonlogger/JsonLoggerExtension.java index 6bc0060..d075ff5 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerExtension.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/JsonLoggerExtension.java @@ -1,15 +1,12 @@ -package org.mule.extension.jsonlogger.internal; +package org.mule.extension.jsonlogger; -import org.mule.extension.jsonlogger.api.pojos.LoggerConfig; +import org.mule.extension.jsonlogger.config.JsonLoggerConfig; import org.mule.runtime.extension.api.annotation.Configurations; -import org.mule.runtime.extension.api.annotation.Export; import org.mule.runtime.extension.api.annotation.Extension; -import org.mule.runtime.extension.api.annotation.Operations; import org.mule.runtime.extension.api.annotation.dsl.xml.Xml; @Xml(prefix = "json-logger") @Extension(name = "JSON Logger") -@Export(resources = {"modules/JSONLoggerModule.dwl"}) @Configurations(JsonLoggerConfig.class) public class JsonLoggerExtension { diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/JsonLoggerOperations.java similarity index 66% rename from json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java rename to json-logger/src/main/java/org/mule/extension/jsonlogger/JsonLoggerOperations.java index 1cbd32f..9a8d742 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonloggerOperations.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/JsonLoggerOperations.java @@ -1,27 +1,20 @@ -package org.mule.extension.jsonlogger.internal; +package org.mule.extension.jsonlogger; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.mule.extension.jsonlogger.api.pojos.LoggerProcessor; -import org.mule.extension.jsonlogger.api.pojos.LoggerScopeProcessor; -import org.mule.extension.jsonlogger.api.pojos.Priority; -import org.mule.extension.jsonlogger.api.pojos.ScopeTracePoint; -import org.mule.extension.jsonlogger.internal.mapper.ObjectMapperSingleton; +import org.mule.extension.jsonlogger.config.JsonLoggerConfig; +import org.mule.extension.jsonlogger.config.LoggerProcessor; +import org.mule.extension.jsonlogger.config.LoggerScopeProcessor; +import org.mule.extension.jsonlogger.mapper.Mapper; import org.mule.runtime.api.component.location.ComponentLocation; -import org.mule.runtime.api.meta.model.operation.ExecutionType; import org.mule.runtime.api.metadata.MediaType; import org.mule.runtime.api.metadata.TypedValue; import org.mule.runtime.api.transformation.TransformationService; import org.mule.runtime.extension.api.annotation.Expression; -import org.mule.runtime.extension.api.annotation.execution.Execution; import org.mule.runtime.extension.api.annotation.param.Config; -import org.mule.runtime.extension.api.annotation.param.Optional; import org.mule.runtime.extension.api.annotation.param.ParameterGroup; -import org.mule.runtime.extension.api.annotation.param.display.Placement; -import org.mule.runtime.extension.api.runtime.operation.FlowListener; import org.mule.runtime.extension.api.runtime.operation.Result; -import org.mule.runtime.extension.api.runtime.parameter.CorrelationInfo; import org.mule.runtime.extension.api.runtime.parameter.ParameterResolver; import org.mule.runtime.extension.api.runtime.process.CompletionCallback; import org.mule.runtime.extension.api.runtime.route.Chain; @@ -30,13 +23,13 @@ import javax.inject.Inject; import java.io.InputStream; -import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import static java.time.Duration.between; import static org.mule.runtime.api.meta.ExpressionSupport.NOT_SUPPORTED; import static org.mule.runtime.api.metadata.DataType.TEXT_STRING; @@ -44,17 +37,16 @@ public class JsonLoggerOperations { protected static Logger log = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); private final Result VOID_RESULT = Result.builder().build(); - - @Inject - ObjectMapperSingleton om; + private final Mapper om = new Mapper(); @Inject private TransformationService transformationService; - @Execution(ExecutionType.CPU_LITE) - public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPPORTED) LoggerProcessor loggerProcessor, - @Config JsonLoggerConfig config, - CorrelationInfo correlationInfo, + public void logger(@ParameterGroup(name = "Logger") + @Expression(value = NOT_SUPPORTED) + LoggerProcessor loggerProcessor, + @Config + JsonLoggerConfig config, ComponentLocation location, CompletionCallback callback) { @@ -66,43 +58,47 @@ public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPP ObjectNode logEvent = om.getObjectMapper().createObjectNode(); - logEvent.put(Constants.CORRELATION_ID, correlationInfo.getCorrelationId()) + logEvent.put(Constants.CORRELATION_ID, loggerProcessor.getCorrelationId()) .put(Constants.MESSAGE, loggerProcessor.getMessage()) - .put(Constants.TIMSTAMP, Instant.now().toString()) - .putPOJO(Constants.APPLICATION, config.getGlobalSettings()) - .put(Constants.THREAD_NAME, Thread.currentThread().getName()) + .put(Constants.APPLICATION_NAME, config.getApplicationName()) + .put(Constants.APPLICATION_VERSION, config.getApplicationVersion()) + .put(Constants.ENVIRONMENT, config.getEnvironment()) .put(Constants.PRIORITY, priority) - .put(Constants.TRACE_POINT, loggerProcessor.getTracePoint().value()); + .put(Constants.TRACE_POINT, loggerProcessor.getTracePoint().toString()); - addLocationInfo(logEvent, location, config.getJsonOutput().isLogLocationInfo()); + addLocationInfo(logEvent, location, config.isLogLocationInfo()); addContent(logEvent, loggerProcessor.getContent(), config); - log(logEvent, priority, config.getJsonOutput().isPrettyPrint()); + log(logEvent, priority, config.isPrettyPrint()); callback.success(VOID_RESULT); } private void addContent(ObjectNode logEvent, ParameterResolver> v, JsonLoggerConfig config) { + TypedValue typedVal = null; try { - TypedValue typedVal = v.resolve(); + typedVal = v.resolve(); if (typedVal.getValue() == null) { return; } - if (!config.getJsonOutput().isParseContentFieldsInJsonOutput() || + if (!config.isParseContentFieldsInJsonOutput() || !typedVal.getDataType().getMediaType().matches(MediaType.APPLICATION_JSON)) { - logEvent.put(Constants.CONTENT, (String) transformationService.transform(typedVal.getValue(), typedVal.getDataType(), TEXT_STRING)); + logEvent.put(Constants.CONTENT, (String) + transformationService.transform(typedVal.getValue(), + typedVal.getDataType(), + TEXT_STRING)); + return; } // Apply data masking only if needed ObjectMapper mapper; List dataMaskingFields = null; - if (config.getJsonOutput().getContentFieldsDataMasking() != null) { + if (config.getContentFieldsDataMasking() != null) { dataMaskingFields = new ArrayList<>(); String[] split = config - .getJsonOutput() .getContentFieldsDataMasking() .trim() .split(","); @@ -121,9 +117,16 @@ private void addContent(ObjectNode logEvent, ParameterResolver callback) { - String priority = loggerProcessor.getPriority().value(); + String priority = loggerProcessor.getPriority().toString(); if (!isLogEnabled(priority)) { operations.process( callback::success, @@ -204,32 +208,30 @@ public void loggerScope(@ParameterGroup(name = "ScopeLogger") @Expression(value }); return; } - + ObjectNode logEvent = om.getObjectMapper(new ArrayList<>()).createObjectNode(); - + Instant startTime = Instant.now(); - logEvent.put(Constants.CORRELATION_ID, correlationInfo.getCorrelationId()) - .put(Constants.TRACE_POINT, loggerProcessor.getScopeTracePoint().value() + "_BEFORE") + logEvent.put(Constants.CORRELATION_ID, loggerProcessor.getCorrelationId()) + .put(Constants.TRACE_POINT, loggerProcessor.getScopeTracePoint().toString() + "_BEFORE") .put(Constants.PRIORITY, priority) .put(Constants.SCOPE_ELAPSED, 0) - .put(Constants.TIMSTAMP, startTime.toString()) - .putPOJO(Constants.APPLICATION, config.getGlobalSettings()) - .put(Constants.THREAD_NAME, Thread.currentThread().getName()); + .put(Constants.TIMSTAMP, startTime.toString()); addLocationInfo(logEvent, location, true); - log(logEvent, priority, config.getJsonOutput().isPrettyPrint()); + log(logEvent, priority, true); operations.process( result -> { Instant endTime = Instant.now(); - logEvent.put(Constants.TRACE_POINT, loggerProcessor.getScopeTracePoint().value() + "_AFTER") + logEvent.put(Constants.TRACE_POINT, loggerProcessor.getScopeTracePoint().toString() + "_AFTER") .put(Constants.PRIORITY, priority) - .put(Constants.SCOPE_ELAPSED, Duration.between(startTime, endTime).toMillis()) + .put(Constants.SCOPE_ELAPSED, between(startTime, endTime).toMillis()) .put(Constants.TIMSTAMP, endTime.toString()); - log(logEvent, priority, config.getJsonOutput().isPrettyPrint()); + log(logEvent, priority, true); callback.success(result); }, (error, previous) -> { @@ -237,13 +239,10 @@ public void loggerScope(@ParameterGroup(name = "ScopeLogger") @Expression(value logEvent.put(Constants.MESSAGE, error.getMessage()) .put(Constants.TRACE_POINT, "EXCEPTION_SCOPE") .put(Constants.PRIORITY, "ERROR") - .put(Constants.SCOPE_ELAPSED, Duration.between(startTime, endTime).toMillis()) - .put(Constants.TIMSTAMP, endTime.toString()); - - log(logEvent, "ERROR", config.getJsonOutput().isPrettyPrint()); + .put(Constants.SCOPE_ELAPSED, between(startTime, endTime).toMillis()); + log(logEvent, "ERROR", true); callback.error(error); }); } - } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/JsonLoggerConfig.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/JsonLoggerConfig.java new file mode 100644 index 0000000..d5388b7 --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/JsonLoggerConfig.java @@ -0,0 +1,133 @@ +package org.mule.extension.jsonlogger.config; + +import org.mule.extension.jsonlogger.JsonLoggerOperations; +import org.mule.runtime.extension.api.annotation.Configuration; +import org.mule.runtime.extension.api.annotation.Expression; +import org.mule.runtime.extension.api.annotation.Operations; +import org.mule.runtime.extension.api.annotation.param.Optional; +import org.mule.runtime.extension.api.annotation.param.Parameter; + +import static org.mule.runtime.api.meta.ExpressionSupport.NOT_SUPPORTED; + +@Configuration(name = "config") +@Operations({JsonLoggerOperations.class}) +public class JsonLoggerConfig { + + @Parameter + @Optional(defaultValue = "") + @Expression(NOT_SUPPORTED) + String applicationName; + + @Parameter + @Optional(defaultValue = "") + @Expression(NOT_SUPPORTED) + String applicationVersion; + + @Parameter + @Optional(defaultValue = "") + @Expression(NOT_SUPPORTED) + String environment; + + @Parameter + @Optional + @Expression(NOT_SUPPORTED) + String disabledFields; + + @Parameter + @Optional + @Expression(NOT_SUPPORTED) + String contentFieldsDataMasking; + + @Parameter + @Optional(defaultValue = "true") + @Expression(NOT_SUPPORTED) + boolean prettyPrint; + + @Parameter + @Optional(defaultValue = "true") + @Expression(NOT_SUPPORTED) + boolean logLocationInfo; + + @Parameter + @Optional(defaultValue = "true") + @Expression(NOT_SUPPORTED) + boolean parseContentFieldsInJsonOutput; + + public String getApplicationName() { + return applicationName; + } + + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + + public String getApplicationVersion() { + return applicationVersion; + } + + public void setApplicationVersion(String applicationVersion) { + this.applicationVersion = applicationVersion; + } + + public String getEnvironment() { + return environment; + } + + public void setEnvironment(String environment) { + this.environment = environment; + } + + public boolean isPrettyPrint() { + return prettyPrint; + } + + public void setPrettyPrint(boolean prettyPrint) { + this.prettyPrint = prettyPrint; + } + + public boolean isLogLocationInfo() { + return logLocationInfo; + } + + public void setLogLocationInfo(boolean logLocationInfo) { + this.logLocationInfo = logLocationInfo; + } + + public boolean isParseContentFieldsInJsonOutput() { + return parseContentFieldsInJsonOutput; + } + + public void setParseContentFieldsInJsonOutput(boolean parseContentFieldsInJsonOutput) { + this.parseContentFieldsInJsonOutput = parseContentFieldsInJsonOutput; + } + + public String getDisabledFields() { + return disabledFields; + } + + public void setDisabledFields(String disabledFields) { + this.disabledFields = disabledFields; + } + + public String getContentFieldsDataMasking() { + return contentFieldsDataMasking; + } + + public void setContentFieldsDataMasking(String contentFieldsDataMasking) { + this.contentFieldsDataMasking = contentFieldsDataMasking; + } + + @Override + public String toString() { + return "{" + + "applicationName='" + applicationName + '\'' + + ", applicationVersion='" + applicationVersion + '\'' + + ", environment='" + environment + '\'' + + ", disabledFields='" + disabledFields + '\'' + + ", contentFieldsDataMasking='" + contentFieldsDataMasking + '\'' + + ", prettyPrint=" + prettyPrint + + ", logLocationInfo=" + logLocationInfo + + ", parseContentFieldsInJsonOutput=" + parseContentFieldsInJsonOutput + + '}'; + } +} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerProcessor.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerProcessor.java new file mode 100644 index 0000000..442af07 --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerProcessor.java @@ -0,0 +1,93 @@ +package org.mule.extension.jsonlogger.config; + +import org.mule.runtime.api.metadata.TypedValue; +import org.mule.runtime.extension.api.annotation.param.Content; +import org.mule.runtime.extension.api.annotation.param.Optional; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.Example; +import org.mule.runtime.extension.api.annotation.param.display.Placement; +import org.mule.runtime.extension.api.annotation.param.display.Summary; +import org.mule.runtime.extension.api.runtime.parameter.ParameterResolver; + +import java.io.InputStream; + +public class LoggerProcessor { + + @Parameter + @Optional(defaultValue = "#[correlationId]") + @Placement(tab = "Advanced") + private String correlationId; + + @Parameter + @Optional(defaultValue = "") + @Summary("Message to be logged") + @Example("Add a log message") + private String message; + + @Parameter + @Optional(defaultValue = "{}") + @Summary("NOTE: Writing the entire payload every time across your application can cause serious performance issues") + @Content + private ParameterResolver> content; + + @Parameter + @Optional(defaultValue = "START") + @Summary("Current processing stage") + private TracePoint tracePoint; + + @Parameter + @Optional(defaultValue = "INFO") + @Summary("Logger priority") + private Priority priority; + + public String getCorrelationId() { + return correlationId; + } + + public void setCorrelationId(String correlationId) { + this.correlationId = correlationId; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public ParameterResolver> getContent() { + return content; + } + + public void setContent(ParameterResolver> content) { + this.content = content; + } + + public TracePoint getTracePoint() { + return tracePoint; + } + + public void setTracePoint(TracePoint tracePoint) { + this.tracePoint = tracePoint; + } + + public Priority getPriority() { + return priority; + } + + public void setPriority(Priority priority) { + this.priority = priority; + } + + @Override + public String toString() { + return "{" + + "correlationId='" + correlationId + '\'' + + ", message='" + message + '\'' + + ", content=" + content + + ", tracePoint=" + tracePoint + + ", priority=" + priority + + '}'; + } +} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerScopeProcessor.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerScopeProcessor.java new file mode 100644 index 0000000..6e8a7bc --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerScopeProcessor.java @@ -0,0 +1,48 @@ +package org.mule.extension.jsonlogger.config; + +import org.mule.runtime.extension.api.annotation.param.Optional; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.Placement; +import org.mule.runtime.extension.api.annotation.param.display.Summary; + +public class LoggerScopeProcessor { + + @Parameter + @Optional(defaultValue = "OUTBOUND_REQUEST_SCOPE") + @Summary("Current processing stage") + private ScopeTracePoint scopeTracePoint; + + @Parameter + @Optional(defaultValue = "#[correlationId]") + @Placement(tab = "Advanced") + private String correlationId; + + @Parameter + @Optional(defaultValue = "INFO") + @Summary("Logger priority") + private Priority priority; + + public ScopeTracePoint getScopeTracePoint() { + return scopeTracePoint; + } + + public void setScopeTracePoint(ScopeTracePoint scopeTracePoint) { + this.scopeTracePoint = scopeTracePoint; + } + + public String getCorrelationId() { + return correlationId; + } + + public void setCorrelationId(String correlationId) { + this.correlationId = correlationId; + } + + public Priority getPriority() { + return priority; + } + + public void setPriority(Priority priority) { + this.priority = priority; + } +} \ No newline at end of file diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/Priority.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/Priority.java new file mode 100644 index 0000000..29f2d6e --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/Priority.java @@ -0,0 +1,20 @@ +package org.mule.extension.jsonlogger.config; + +public enum Priority { + DEBUG("DEBUG"), + TRACE("TRACE"), + INFO("INFO"), + WARN("WARN"), + ERROR("ERROR"); + + private final String value; + + Priority(String value) { + this.value = value; + } + + @Override + public String toString() { + return this.value; + } +} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/ScopeTracePoint.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/ScopeTracePoint.java new file mode 100644 index 0000000..3792827 --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/ScopeTracePoint.java @@ -0,0 +1,18 @@ +package org.mule.extension.jsonlogger.config; + +public enum ScopeTracePoint { + + DATA_TRANSFORM_SCOPE("DATA_TRANSFORM_SCOPE"), + OUTBOUND_REQUEST_SCOPE("OUTBOUND_REQUEST_SCOPE"), + FLOW_LOGIC_SCOPE("FLOW_LOGIC_SCOPE"); + private final String value; + + ScopeTracePoint(String value) { + this.value = value; + } + + @Override + public String toString() { + return this.value; + } +} \ No newline at end of file diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/TracePoint.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/TracePoint.java new file mode 100644 index 0000000..b020d18 --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/TracePoint.java @@ -0,0 +1,23 @@ +package org.mule.extension.jsonlogger.config; + +public enum TracePoint { + + START("START"), + BEFORE_TRANSFORM("BEFORE_TRANSFORM"), + AFTER_TRANSFORM("AFTER_TRANSFORM"), + BEFORE_REQUEST("BEFORE_REQUEST"), + AFTER_REQUEST("AFTER_REQUEST"), + FLOW("FLOW"), + END("END"), + EXCEPTION("EXCEPTION"); + private final String value; + + TracePoint(String value) { + this.value = value; + } + + @Override + public String toString() { + return this.value; + } +} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonLoggerConfig.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonLoggerConfig.java deleted file mode 100644 index 303cea2..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/JsonLoggerConfig.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.mule.extension.jsonlogger.internal; - -import org.mule.extension.jsonlogger.api.pojos.LoggerConfig; -import org.mule.runtime.extension.api.annotation.Operations; - -@Operations(JsonLoggerOperations.class) -public class JsonLoggerConfig extends LoggerConfig { -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/ObjectMapperSingleton.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/Mapper.java similarity index 91% rename from json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/ObjectMapperSingleton.java rename to json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/Mapper.java index d5eefb8..a6e2fc9 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/ObjectMapperSingleton.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/Mapper.java @@ -1,4 +1,4 @@ -package org.mule.extension.jsonlogger.internal.mapper; +package org.mule.extension.jsonlogger.mapper; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.JsonNode; @@ -7,7 +7,7 @@ import java.util.List; -public class ObjectMapperSingleton { +public class Mapper { private final ObjectMapper om = new ObjectMapper() .setSerializationInclusion(JsonInclude.Include.NON_NULL) @@ -17,7 +17,6 @@ public class ObjectMapperSingleton { .setSerializationInclusion(JsonInclude.Include.NON_NULL) .setSerializationInclusion(JsonInclude.Include.NON_EMPTY); - public ObjectMapper getObjectMapper(List maskedFields) { SimpleModule module = new SimpleModule(); module.addDeserializer(JsonNode.class, new MaskingDeserializer(maskedFields)); diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/MaskingDeserializer.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/MaskingDeserializer.java similarity index 96% rename from json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/MaskingDeserializer.java rename to json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/MaskingDeserializer.java index 62f1f23..06b72b8 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/internal/mapper/MaskingDeserializer.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/MaskingDeserializer.java @@ -1,4 +1,4 @@ -package org.mule.extension.jsonlogger.internal.mapper; +package org.mule.extension.jsonlogger.mapper; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; @@ -8,7 +8,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.commons.lang.StringUtils; import java.io.IOException; import java.util.List; @@ -170,7 +169,11 @@ protected final JsonNode deserializeAny(JsonParser p, DeserializationContext ctx } private String replace(String input) { - return StringUtils.repeat("*", input.length()); + int len = input.length(); + StringBuilder sb = new StringBuilder(len); + for(int i = 0; i < len; i++){ + sb.append('*'); + } + return sb.toString(); } - } diff --git a/json-logger/src/main/resources/META-INF/org/mule/runtime/core/config/registry-bootstrap.properties b/json-logger/src/main/resources/META-INF/org/mule/runtime/core/config/registry-bootstrap.properties deleted file mode 100644 index 1626167..0000000 --- a/json-logger/src/main/resources/META-INF/org/mule/runtime/core/config/registry-bootstrap.properties +++ /dev/null @@ -1 +0,0 @@ -logger.objectMapperSingleton=org.mule.extension.jsonlogger.internal.mapper.ObjectMapperSingleton diff --git a/json-logger/src/main/resources/modules/JSONLoggerModule.dwl b/json-logger/src/main/resources/modules/JSONLoggerModule.dwl deleted file mode 100644 index db14ccd..0000000 --- a/json-logger/src/main/resources/modules/JSONLoggerModule.dwl +++ /dev/null @@ -1,44 +0,0 @@ -%dw 2.0 -fun stringifyAny(inputData: Any) = if (inputData.^mimeType == "application/xml" or - inputData.^mimeType == "application/dw" or - inputData.^mimeType == "application/json") - write(inputData,inputData.^mimeType,{indent:false}) - else if (inputData.^mimeType == "*/*") - inputData - else - write(inputData,inputData.^mimeType) - -fun stringifyNonJSON(inputData: Any) = if (inputData.^mimeType == "application/xml" or - inputData.^mimeType == "application/dw") - write(inputData,inputData.^mimeType,{indent:false}) - else if (inputData.^mimeType == "application/json" or inputData.^mimeType == "*/*") - inputData - else - write(inputData,inputData.^mimeType) - -fun stringifyAnyWithMetadata(inputData: Any) = { - data: if (inputData.^mimeType == "application/xml" or - inputData.^mimeType == "application/dw" or - inputData.^mimeType == "application/json") - write(inputData,inputData.^mimeType,{indent:false}) - else if (inputData.^mimeType == "*/*") - inputData - else - write(inputData,inputData.^mimeType), - (contentLength: inputData.^contentLength) if (inputData.^contentLength != null), - (dataType: inputData.^mimeType) if (inputData.^mimeType != null), - (class: inputData.^class) if (inputData.^class != null) - } - -fun stringifyNonJSONWithMetadata(inputData: Any) = { - data: if (inputData.^mimeType == "application/xml" or - inputData.^mimeType == "application/dw") - write(inputData,inputData.^mimeType,{indent:false}) - else if (inputData.^mimeType == "application/json" or inputData.^mimeType == "*/*") - inputData - else - write(inputData,inputData.^mimeType), - (contentLength: inputData.^contentLength) if (inputData.^contentLength != null), - (dataType: inputData.^mimeType) if (inputData.^mimeType != null), - (class: inputData.^class) if (inputData.^class != null) - } diff --git a/json-logger/src/main/resources/schema/loggerConfig.json b/json-logger/src/main/resources/schema/loggerConfig.json deleted file mode 100644 index 12c140b..0000000 --- a/json-logger/src/main/resources/schema/loggerConfig.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "description": "Definition for fields globally defined in the logger config", - "type": "object", - "properties": { - "globalSettings": { - "type": "object", - "properties": { - "name": { - "type": "string", - "sdk": { - "default": "${json.logger.application.name}", - "summary": "Name of the Mule application. Recommendation: This value should be based on pom.xml" - } - }, - "version": { - "type": "string", - "sdk": { - "default": "${json.logger.application.version}", - "summary": "Version of the Mule application. Recommendation: This value should be based on pom.xml" - } - }, - "environment": { - "type": "string", - "sdk": { - "required": true, - "example": "${mule.env}", - "summary": "Name of the Mule Environment where the application is running. Recommendation: This value should be based on external property" - } - } - }, - "sdk": { - "parameterGroup": "Global Settings", - "expressionSupport": "NOT_SUPPORTED", - "placement": { - "order": 1 - } - } - }, - "jsonOutput": { - "type": "object", - "properties": { - "prettyPrint": { - "type": "boolean", - "javaType": "boolean", - "sdk": { - "summary": "Indicate if log entries should be formatted or single line", - "default": true, - "expressionSupport": "SUPPORTED" - }, - "note": "This field is mandatory. DON'T REMOVE" - }, - "logLocationInfo": { - "type": "boolean", - "javaType": "boolean", - "sdk": { - "summary": "Indicate if location information should be logged", - "default": true, - "expressionSupport": "SUPPORTED" - }, - "note": "This field is mandatory. DON'T REMOVE" - }, - "parseContentFieldsInJsonOutput": { - "type": "boolean", - "javaType": "boolean", - "sdk": { - "summary": "Indicate if Content fields should be parsed as part of the JSON logger output", - "default": true, - "expressionSupport": "SUPPORTED" - }, - "note": "This field is mandatory. DON'T REMOVE" - }, - "disabledFields": { - "type": "string", - "sdk": { - "summary": "Indicate which fields (from JSON output) should be disabled from logging separated by comma. They should match the exact name given in loggerProcessor.json schema", - "example": "message,content", - "required": false, - "expressionSupport": "NOT_SUPPORTED" - }, - "note": "This field is mandatory. DON'T REMOVE" - }, - "contentFieldsDataMasking": { - "type": "string", - "sdk": { - "summary": "Indicate which fields (inside a content type with JSON output only) should be masked before from logging separated by comma. If empty, no masking will be applied. Recommendation: This value should be based on external property", - "example": "client_secret,password", - "required": false, - "expressionSupport": "NOT_SUPPORTED" - }, - "note": "This field is mandatory. DON'T REMOVE" - } - }, - "sdk": { - "parameterGroup": "JSON Output", - "expressionSupport": "NOT_SUPPORTED" - } - } - } -} \ No newline at end of file diff --git a/json-logger/src/main/resources/schema/loggerProcessor.json b/json-logger/src/main/resources/schema/loggerProcessor.json deleted file mode 100644 index 3595b2d..0000000 --- a/json-logger/src/main/resources/schema/loggerProcessor.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "description": "Definition for fields used in the logger message processor", - "type": "object", - "properties": { - "correlationId": { - "type": "string", - "sdk": { - "default": "#[correlationId]", - "placement": { - "tab": "Advanced" - } - } - }, - "message": { - "type": "string", - "sdk": { - "default": "", - "example": "Add a log message", - "required": true, - "summary": "Message to be logged" - } - }, - "content": { - "type": "string", - "javaType": "org.mule.runtime.extension.api.runtime.parameter.ParameterResolver>", - "sdk": { - "default": "", - "summary": "NOTE: Writing the entire payload every time across your application can cause serious performance issues", - "required": false, - "isContent": true - } - }, - "tracePoint": { - "type": "string", - "javaType": "org.mule.extension.jsonlogger.api.pojos.TracePoint", - "enum": [ - "START", - "BEFORE_TRANSFORM", - "AFTER_TRANSFORM", - "BEFORE_REQUEST", - "AFTER_REQUEST", - "FLOW", - "END", - "EXCEPTION" - ], - "sdk": { - "default": "START", - "summary": "Current processing stage" - } - }, - "priority": { - "type": "string", - "javaType": "org.mule.extension.jsonlogger.api.pojos.Priority", - "enum": [ - "DEBUG", - "TRACE", - "INFO", - "WARN", - "ERROR" - ], - "sdk": { - "default": "INFO", - "summary": "Logger priority" - }, - "note": "This field is mandatory. DON'T REMOVE" - }, - "category": { - "type": "string", - "sdk": { - "default": "org.mule.extension.jsonlogger.JsonLogger", - "required": false, - "summary": "If not set, by default will log to the org.mule.extension.jsonlogger.JsonLogger category" - }, - "note": "This field is mandatory. DON'T REMOVE" - } - } -} \ No newline at end of file diff --git a/json-logger/src/main/resources/schema/loggerScopeProcessor.json b/json-logger/src/main/resources/schema/loggerScopeProcessor.json deleted file mode 100644 index 59a29f9..0000000 --- a/json-logger/src/main/resources/schema/loggerScopeProcessor.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "description": "Definition for fields used in the logger message processor", - "type": "object", - "properties": { - "scopeTracePoint": { - "type": "string", - "javaType": "org.mule.extension.jsonlogger.api.pojos.ScopeTracePoint", - "enum": [ - "DATA_TRANSFORM_SCOPE", - "OUTBOUND_REQUEST_SCOPE", - "FLOW_LOGIC_SCOPE" - ], - "sdk": { - "default": "OUTBOUND_REQUEST_SCOPE", - "summary": "Current processing stage" - } - }, - "correlationId": { - "type": "string", - "sdk": { - "default": "#[correlationId]", - "placement": { - "tab": "Advanced" - } - } - }, - "priority": { - "type": "string", - "javaType": "org.mule.extension.jsonlogger.api.pojos.Priority", - "enum": [ - "DEBUG", - "TRACE", - "INFO", - "WARN", - "ERROR" - ], - "sdk": { - "default": "INFO", - "summary": "Logger priority" - }, - "note": "This field is mandatory. DON'T REMOVE" - } - } -} \ No newline at end of file diff --git a/json-logger/src/test/java/org/mule/extension/jsonlogger/internal/JsonMaskerTest.java b/json-logger/src/test/java/org/mule/extension/jsonlogger/internal/JsonMaskerTest.java index 5d4edd7..07b9398 100644 --- a/json-logger/src/test/java/org/mule/extension/jsonlogger/internal/JsonMaskerTest.java +++ b/json-logger/src/test/java/org/mule/extension/jsonlogger/internal/JsonMaskerTest.java @@ -1,11 +1,11 @@ -package org.mule.extension.jsonlogger.internal; +package org.mule.extension.jsonlogger; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Assert; import org.junit.Test; -import org.mule.extension.jsonlogger.internal.mapper.ObjectMapperSingleton; +import org.mule.extension.jsonlogger.mapper.Mapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,7 +20,7 @@ public class JsonMaskerTest { ClassLoader classloader = Thread.currentThread().getContextClassLoader(); InputStream is = classloader.getResourceAsStream("payload.json"); List maskedFields = Collections.singletonList("attributes"); - private final ObjectMapper om = new ObjectMapperSingleton().getObjectMapper(maskedFields); + private final ObjectMapper om = new Mapper().getObjectMapper(maskedFields); protected Logger log = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); @Test diff --git a/json-logger/src/test/resources/json-logger.properties b/json-logger/src/test/resources/json-logger.properties deleted file mode 100644 index 5347ec5..0000000 --- a/json-logger/src/test/resources/json-logger.properties +++ /dev/null @@ -1,9 +0,0 @@ -# JSON Logger configuration -#json.logger.timezone=US/Eastern -#json.logger.dateformat=yyyy-MM-dd HH:mm:ss.SSS - -# JSON Logger configuration: To be replaced by Maven build -json.logger.application.name=test -json.logger.application.version=test - -json.logger.disabledFields=content \ No newline at end of file diff --git a/json-logger/src/test/resources/test-mule-config.xml b/json-logger/src/test/resources/test-mule-config.xml deleted file mode 100644 index 1601597..0000000 --- a/json-logger/src/test/resources/test-mule-config.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - From 880b9a31293cbade20ef03ac14ad90d5adbdfc61 Mon Sep 17 00:00:00 2001 From: Jazzy Date: Tue, 29 Jun 2021 16:30:57 -0700 Subject: [PATCH 11/12] Whole lotta more cleanup --- json-logger/README.md | 69 +++--- json-logger/pom.xml | 104 +++++---- .../mule/extension/jsonlogger/Constants.java | 5 +- .../jsonlogger/JsonLoggerOperations.java | 213 +++++++++--------- .../jsonlogger/config/JsonLoggerConfig.java | 91 ++------ .../jsonlogger/config/LoggerProcessor.java | 93 -------- .../config/LoggerScopeProcessor.java | 48 ---- .../extension/jsonlogger/mapper/Mapper.java | 30 --- .../mapper/MaskingDeserializer.java | 179 --------------- .../jsonlogger/internal/JsonMaskerTest.java | 51 ----- json-logger/src/test/resources/payload.json | 19 -- 11 files changed, 219 insertions(+), 683 deletions(-) delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerProcessor.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerScopeProcessor.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/Mapper.java delete mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/MaskingDeserializer.java delete mode 100644 json-logger/src/test/java/org/mule/extension/jsonlogger/internal/JsonMaskerTest.java delete mode 100644 json-logger/src/test/resources/payload.json diff --git a/json-logger/README.md b/json-logger/README.md index 37abe8f..7d41542 100644 --- a/json-logger/README.md +++ b/json-logger/README.md @@ -1,31 +1,38 @@ -# Json-logger Extension - -## 2.0.1 version - Release notes - -Bug fixes: -* Added support for large payloads - -## 2.0.0 version - Release notes - -New features: -* External Destinations -* Data masking - -Improvements: -* Field ordering - -More details in the coming blog post (stay tuned!) - -## 1.1.0 version - Release notes - -New features: -* Scoped loggers to capture "scope bound elapsed time". Great for performance tracking of specific components (e.g. outbound calls) -* Added "Parse content fields in json output" flag so that content fields can become part of final JSON output rather than a "stringified version" of the content - -Improvements: -* Removed Guava and caching in general with a more efficient handling of timers (for elapsed time) -* Optimized generation of JSON output -* Code optimizations -* Minimized dependency footprint (down from ~23MB to ~13MB) -* Optimized parsing of TypedValue content fields - +# Json-logger Extension for Mule 4 runtime + +Json logger makes uses of [Mule Java SDK](https://docs.mulesoft.com/mule-sdk/1.1/getting-started) +and contains a custom deserializer for masking fields in the JSON. + +> Data masking is not possible in any other formats at this time e.g. xml, csv, etc. + +## Serialization/Deserialization performance penalty +When the payload is passed into the json logger component as `application/json`, it is first deserialized +and masking if specified is applied in place. The serialization takes place when the log is being written out. +Therefore, at the very minimum, it is a two-step process. The transformation done before or after the json logger +to convert payload to `application/java` from `application/json` and vice versa will degrade performance. + +## Operations: +There are two operations available: +1. logger: Use this for regular logging +2. loggerScope: Use this to measure response times for external calls + +## Logging structure: +```json + { + "correlationId" : "cbfb9c1f-f904-40f2-bad9-2eac6bc05e84", + "message" : "", + "content": { + "dateOfBirth": "**-**-****" + }, + "priority" : "INFO", + "tracePoint" : "START", + "locationInfo" : { + "fileName" : "api-main-router.xml", + "rootContainer" : "api-main", + "lineNumber" : "25" + }, + "applicationName" : "proc-checkin-api-v1", + "applicationVersion" : "1.2.7", + "environment" : "local" +} +``` \ No newline at end of file diff --git a/json-logger/pom.xml b/json-logger/pom.xml index 48cd9b5..2cf05f3 100644 --- a/json-logger/pom.xml +++ b/json-logger/pom.xml @@ -3,58 +3,64 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - 4.0.0 - 7abbfaeb-3816-4907-865e-76d7cb24120f - json-logger - 3.0.0 - mule-extension - JSON Logger - Mule 4 + 4.0.0 + 7abbfaeb-3816-4907-865e-76d7cb24120f + json-logger + 3.0.0 + mule-extension + JSON Logger - Mule 4 - - org.mule.extensions - mule-modules-parent - 1.1.1 - + + org.mule.extensions + mule-modules-parent + 1.1.1 + - - - - org.apache.maven.plugins - maven-install-plugin - 2.5.2 - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.2 - + + + + org.apache.maven.plugins + maven-install-plugin + 2.5.2 + + + org.apache.maven.plugins + maven-deploy-plugin + 2.8.2 + - - + + - - - Exchange2 - Exchange2 Repository - https://maven.anypoint.mulesoft.com/api/v1/organizations/${project.groupId}/maven - default - - + + + Exchange2 + Exchange2 Repository + https://maven.anypoint.mulesoft.com/api/v1/organizations/${project.groupId}/maven + default + + - - - com.fasterxml.jackson.core - jackson-databind - 2.10.3 - - - - - - Exchange2 - Exchange2 Repository - https://maven.anypoint.mulesoft.com/api/v1/organizations/${project.groupId}/maven - default - - + + + com.fasterxml.jackson.core + jackson-databind + 2.10.3 + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.14.1 + test + + + + + + Exchange2 + Exchange2 Repository + https://maven.anypoint.mulesoft.com/api/v1/organizations/${project.groupId}/maven + default + + diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/Constants.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/Constants.java index 67cd018..591885f 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/Constants.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/Constants.java @@ -11,9 +11,10 @@ class Constants { static final String FILE_NAME = "fileName"; static final String LINE_NUMBER = "lineNumber"; static final String TIMSTAMP = "timestamp"; - static final String THREAD_NAME = "threadName"; static final String MESSAGE = "message"; static final String PRIORITY = "priority"; static final String TRACE_POINT = "tracePoint"; - static final String SCOPE_ELAPSED = "scopeLapsed"; + static final String ELAPSED = "elapsed"; + static final String START_TIME = "startTime"; + static final String END_TIME = "endTime"; } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/JsonLoggerOperations.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/JsonLoggerOperations.java index 9a8d742..90d73b9 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/JsonLoggerOperations.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/JsonLoggerOperations.java @@ -4,10 +4,10 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.node.ObjectNode; import org.mule.extension.jsonlogger.config.JsonLoggerConfig; -import org.mule.extension.jsonlogger.config.LoggerProcessor; -import org.mule.extension.jsonlogger.config.LoggerScopeProcessor; -import org.mule.extension.jsonlogger.mapper.Mapper; +import org.mule.extension.jsonlogger.config.LogProcessor; +import org.mule.extension.jsonlogger.config.ScopeProcessor; import org.mule.runtime.api.component.location.ComponentLocation; +import org.mule.runtime.api.metadata.DataType; import org.mule.runtime.api.metadata.MediaType; import org.mule.runtime.api.metadata.TypedValue; import org.mule.runtime.api.transformation.TransformationService; @@ -23,10 +23,9 @@ import javax.inject.Inject; import java.io.InputStream; +import java.io.ObjectInputStream; import java.time.Instant; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import static java.time.Duration.between; @@ -35,94 +34,135 @@ public class JsonLoggerOperations { - protected static Logger log = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); + private final static Logger log = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); private final Result VOID_RESULT = Result.builder().build(); - private final Mapper om = new Mapper(); @Inject private TransformationService transformationService; public void logger(@ParameterGroup(name = "Logger") @Expression(value = NOT_SUPPORTED) - LoggerProcessor loggerProcessor, + LogProcessor logProcessor, @Config JsonLoggerConfig config, ComponentLocation location, CompletionCallback callback) { - String priority = loggerProcessor.getPriority().toString(); + String priority = logProcessor.getPriority().toString(); if (!isLogEnabled(priority)) { callback.success(VOID_RESULT); - return; + return; } + ObjectNode logEvent = Mapper.getInstance().createObjectNode(); - ObjectNode logEvent = om.getObjectMapper().createObjectNode(); - - logEvent.put(Constants.CORRELATION_ID, loggerProcessor.getCorrelationId()) - .put(Constants.MESSAGE, loggerProcessor.getMessage()) - .put(Constants.APPLICATION_NAME, config.getApplicationName()) - .put(Constants.APPLICATION_VERSION, config.getApplicationVersion()) - .put(Constants.ENVIRONMENT, config.getEnvironment()) + logEvent.put(Constants.CORRELATION_ID, logProcessor.getCorrelationId()) + .put(Constants.MESSAGE, logProcessor.getMessage()) .put(Constants.PRIORITY, priority) - .put(Constants.TRACE_POINT, loggerProcessor.getTracePoint().toString()); + .put(Constants.TRACE_POINT, logProcessor.getTracePoint().toString()); + + addLocationInfo(logEvent, location); - addLocationInfo(logEvent, location, config.isLogLocationInfo()); + addAppInfo(logEvent, config); - addContent(logEvent, loggerProcessor.getContent(), config); + addContent(logEvent, logProcessor.getContent(), config); log(logEvent, priority, config.isPrettyPrint()); callback.success(VOID_RESULT); } - private void addContent(ObjectNode logEvent, ParameterResolver> v, JsonLoggerConfig config) { + // scope operations cannot receive config at this time + public void loggerScope(@ParameterGroup(name = "LoggerScope") + @Expression(value = NOT_SUPPORTED) + ScopeProcessor scopeProcessor, + ComponentLocation location, + Chain operations, + CompletionCallback callback) { + + String priority = scopeProcessor.getPriority().toString(); + if (!isLogEnabled(priority)) { + operations.process( + callback::success, + (error, previous) -> { + callback.error(error); + }); + return; + } + + ObjectNode logEvent = Mapper.getInstance().createObjectNode(); + + Instant startTime = Instant.now(); + + logEvent.put(Constants.CORRELATION_ID, scopeProcessor.getCorrelationId()) + .put(Constants.TRACE_POINT, scopeProcessor.getScopeTracePoint().toString() + "_BEFORE") + .put(Constants.PRIORITY, priority) + .put(Constants.START_TIME, startTime.toString()); + + addLocationInfo(logEvent, location); + + operations.process( + result -> { + Instant endTime = Instant.now(); + logEvent.put(Constants.TRACE_POINT, scopeProcessor.getScopeTracePoint().toString() + "_AFTER") + .put(Constants.PRIORITY, priority) + .put(Constants.ELAPSED, between(startTime, endTime).toMillis()) + .put(Constants.END_TIME, endTime.toString()); + + log(logEvent, priority, true); + callback.success(result); + }, + (error, previous) -> { + Instant endTime = Instant.now(); + logEvent.put(Constants.MESSAGE, error.getMessage()) + .put(Constants.TRACE_POINT, "EXCEPTION_SCOPE") + .put(Constants.PRIORITY, "ERROR") + .put(Constants.ELAPSED, between(startTime, endTime).toMillis()) + .put(Constants.END_TIME, endTime.toString()); + log(logEvent, "ERROR", true); + callback.error(error); + }); + } + + void addContent(ObjectNode logEvent, ParameterResolver> v, JsonLoggerConfig config) { TypedValue typedVal = null; + InputStream inputStream = null; try { typedVal = v.resolve(); - if (typedVal.getValue() == null) { + if (typedVal == null) { return; } - - if (!config.isParseContentFieldsInJsonOutput() || - !typedVal.getDataType().getMediaType().matches(MediaType.APPLICATION_JSON)) { - logEvent.put(Constants.CONTENT, (String) - transformationService.transform(typedVal.getValue(), - typedVal.getDataType(), - TEXT_STRING)); - + inputStream = typedVal.getValue(); + if (inputStream == null ) { return; } - - // Apply data masking only if needed - ObjectMapper mapper; - List dataMaskingFields = null; - if (config.getContentFieldsDataMasking() != null) { - dataMaskingFields = new ArrayList<>(); - String[] split = config - .getContentFieldsDataMasking() - .trim() - .split(","); - - for (String s : split) { - dataMaskingFields.add(s.trim()); - } - + + DataType dataType = typedVal.getDataType(); + + //if payload is java, then use toString() + if (MediaType.APPLICATION_JAVA.matches(dataType.getMediaType())) { + ObjectInputStream ois = new ObjectInputStream(inputStream); + logEvent.put(Constants.CONTENT, ois.readObject().toString()); + return; } - if (dataMaskingFields == null || dataMaskingFields.isEmpty()) { - mapper = om.getObjectMapper(); - } else { - mapper = om.getObjectMapper(dataMaskingFields); + //if payload is anything other java and json, then use transformationService + if (!MediaType.APPLICATION_JSON.matches(dataType.getMediaType())) { + logEvent.put(Constants.CONTENT, (String) transformationService.transform(inputStream, + dataType, + TEXT_STRING)); + return; } - logEvent.put(Constants.CONTENT, mapper.readTree(typedVal.getValue())); + ObjectMapper mapper = Mapper.getInstance(config.getContentFieldsDataMasking()); + log.error(config.getContentFieldsDataMasking()); + logEvent.put(Constants.CONTENT, mapper.readTree(inputStream)); } catch (Exception e) { logEvent.put(Constants.CONTENT, e.getMessage()); } finally { - if (typedVal != null && typedVal.getValue() != null) { + if (typedVal != null && inputStream != null) { try { - typedVal.getValue().close(); + inputStream.close(); } catch (Exception e) { log.debug("Can't close stream", e); } @@ -130,10 +170,13 @@ private void addContent(ObjectNode logEvent, ParameterResolver locationInfo = new HashMap<>(); locationInfo.put(Constants.ROOT_CONTAINER, location.getRootContainerName()); locationInfo.put(Constants.FILE_NAME, location.getFileName().orElse("")); @@ -143,8 +186,8 @@ private void addLocationInfo(ObjectNode logEvent, ComponentLocation location, bo private void log(ObjectNode logEvent, String priority, boolean isPrettyPrint) { ObjectWriter ow = isPrettyPrint ? - om.getObjectMapper().writer().withDefaultPrettyPrinter() : - om.getObjectMapper().writer(); + Mapper.getInstance().writer().withDefaultPrettyPrinter() : + Mapper.getInstance().writer(); try { doLog(priority, ow.writeValueAsString(logEvent)); } catch (Exception e) { @@ -190,59 +233,5 @@ private Boolean isLogEnabled(String priority) { } return false; } - - // scope operations cannot receive config at this time - public void loggerScope(@ParameterGroup(name = "ScopeLogger") - @Expression(value = NOT_SUPPORTED) - LoggerScopeProcessor loggerProcessor, - ComponentLocation location, - Chain operations, - CompletionCallback callback) { - - String priority = loggerProcessor.getPriority().toString(); - if (!isLogEnabled(priority)) { - operations.process( - callback::success, - (error, previous) -> { - callback.error(error); - }); - return; - } - - ObjectNode logEvent = om.getObjectMapper(new ArrayList<>()).createObjectNode(); - - Instant startTime = Instant.now(); - - logEvent.put(Constants.CORRELATION_ID, loggerProcessor.getCorrelationId()) - .put(Constants.TRACE_POINT, loggerProcessor.getScopeTracePoint().toString() + "_BEFORE") - .put(Constants.PRIORITY, priority) - .put(Constants.SCOPE_ELAPSED, 0) - .put(Constants.TIMSTAMP, startTime.toString()); - - addLocationInfo(logEvent, location, true); - - log(logEvent, priority, true); - - operations.process( - result -> { - Instant endTime = Instant.now(); - logEvent.put(Constants.TRACE_POINT, loggerProcessor.getScopeTracePoint().toString() + "_AFTER") - .put(Constants.PRIORITY, priority) - .put(Constants.SCOPE_ELAPSED, between(startTime, endTime).toMillis()) - .put(Constants.TIMSTAMP, endTime.toString()); - - log(logEvent, priority, true); - callback.success(result); - }, - (error, previous) -> { - Instant endTime = Instant.now(); - logEvent.put(Constants.MESSAGE, error.getMessage()) - .put(Constants.TRACE_POINT, "EXCEPTION_SCOPE") - .put(Constants.PRIORITY, "ERROR") - .put(Constants.SCOPE_ELAPSED, between(startTime, endTime).toMillis()); - log(logEvent, "ERROR", true); - callback.error(error); - }); - } } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/JsonLoggerConfig.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/JsonLoggerConfig.java index d5388b7..298059f 100644 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/JsonLoggerConfig.java +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/JsonLoggerConfig.java @@ -6,6 +6,7 @@ import org.mule.runtime.extension.api.annotation.Operations; import org.mule.runtime.extension.api.annotation.param.Optional; import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.Summary; import static org.mule.runtime.api.meta.ExpressionSupport.NOT_SUPPORTED; @@ -13,26 +14,6 @@ @Operations({JsonLoggerOperations.class}) public class JsonLoggerConfig { - @Parameter - @Optional(defaultValue = "") - @Expression(NOT_SUPPORTED) - String applicationName; - - @Parameter - @Optional(defaultValue = "") - @Expression(NOT_SUPPORTED) - String applicationVersion; - - @Parameter - @Optional(defaultValue = "") - @Expression(NOT_SUPPORTED) - String environment; - - @Parameter - @Optional - @Expression(NOT_SUPPORTED) - String disabledFields; - @Parameter @Optional @Expression(NOT_SUPPORTED) @@ -43,91 +24,63 @@ public class JsonLoggerConfig { @Expression(NOT_SUPPORTED) boolean prettyPrint; + @Parameter - @Optional(defaultValue = "true") - @Expression(NOT_SUPPORTED) - boolean logLocationInfo; + @Optional(defaultValue = "applicationName") + @Summary("Name of the application") + private String applicationName; @Parameter - @Optional(defaultValue = "true") - @Expression(NOT_SUPPORTED) - boolean parseContentFieldsInJsonOutput; + @Optional(defaultValue = "applicationVersion") + @Summary("Version of the application") + private String applicationVersion; + + @Parameter + @Optional(defaultValue = "environment") + private String environment; public String getApplicationName() { return applicationName; } - public void setApplicationName(String applicationName) { + public JsonLoggerConfig setApplicationName(String applicationName) { this.applicationName = applicationName; + return this; } public String getApplicationVersion() { return applicationVersion; } - public void setApplicationVersion(String applicationVersion) { + public JsonLoggerConfig setApplicationVersion(String applicationVersion) { this.applicationVersion = applicationVersion; + return this; } public String getEnvironment() { return environment; } - public void setEnvironment(String environment) { + public JsonLoggerConfig setEnvironment(String environment) { this.environment = environment; + return this; } public boolean isPrettyPrint() { return prettyPrint; } - public void setPrettyPrint(boolean prettyPrint) { + public JsonLoggerConfig setPrettyPrint(boolean prettyPrint) { this.prettyPrint = prettyPrint; - } - - public boolean isLogLocationInfo() { - return logLocationInfo; - } - - public void setLogLocationInfo(boolean logLocationInfo) { - this.logLocationInfo = logLocationInfo; - } - - public boolean isParseContentFieldsInJsonOutput() { - return parseContentFieldsInJsonOutput; - } - - public void setParseContentFieldsInJsonOutput(boolean parseContentFieldsInJsonOutput) { - this.parseContentFieldsInJsonOutput = parseContentFieldsInJsonOutput; - } - - public String getDisabledFields() { - return disabledFields; - } - - public void setDisabledFields(String disabledFields) { - this.disabledFields = disabledFields; + return this; } public String getContentFieldsDataMasking() { return contentFieldsDataMasking; } - public void setContentFieldsDataMasking(String contentFieldsDataMasking) { + public JsonLoggerConfig setContentFieldsDataMasking(String contentFieldsDataMasking) { this.contentFieldsDataMasking = contentFieldsDataMasking; - } - - @Override - public String toString() { - return "{" + - "applicationName='" + applicationName + '\'' + - ", applicationVersion='" + applicationVersion + '\'' + - ", environment='" + environment + '\'' + - ", disabledFields='" + disabledFields + '\'' + - ", contentFieldsDataMasking='" + contentFieldsDataMasking + '\'' + - ", prettyPrint=" + prettyPrint + - ", logLocationInfo=" + logLocationInfo + - ", parseContentFieldsInJsonOutput=" + parseContentFieldsInJsonOutput + - '}'; + return this; } } diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerProcessor.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerProcessor.java deleted file mode 100644 index 442af07..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerProcessor.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.mule.extension.jsonlogger.config; - -import org.mule.runtime.api.metadata.TypedValue; -import org.mule.runtime.extension.api.annotation.param.Content; -import org.mule.runtime.extension.api.annotation.param.Optional; -import org.mule.runtime.extension.api.annotation.param.Parameter; -import org.mule.runtime.extension.api.annotation.param.display.Example; -import org.mule.runtime.extension.api.annotation.param.display.Placement; -import org.mule.runtime.extension.api.annotation.param.display.Summary; -import org.mule.runtime.extension.api.runtime.parameter.ParameterResolver; - -import java.io.InputStream; - -public class LoggerProcessor { - - @Parameter - @Optional(defaultValue = "#[correlationId]") - @Placement(tab = "Advanced") - private String correlationId; - - @Parameter - @Optional(defaultValue = "") - @Summary("Message to be logged") - @Example("Add a log message") - private String message; - - @Parameter - @Optional(defaultValue = "{}") - @Summary("NOTE: Writing the entire payload every time across your application can cause serious performance issues") - @Content - private ParameterResolver> content; - - @Parameter - @Optional(defaultValue = "START") - @Summary("Current processing stage") - private TracePoint tracePoint; - - @Parameter - @Optional(defaultValue = "INFO") - @Summary("Logger priority") - private Priority priority; - - public String getCorrelationId() { - return correlationId; - } - - public void setCorrelationId(String correlationId) { - this.correlationId = correlationId; - } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - - public ParameterResolver> getContent() { - return content; - } - - public void setContent(ParameterResolver> content) { - this.content = content; - } - - public TracePoint getTracePoint() { - return tracePoint; - } - - public void setTracePoint(TracePoint tracePoint) { - this.tracePoint = tracePoint; - } - - public Priority getPriority() { - return priority; - } - - public void setPriority(Priority priority) { - this.priority = priority; - } - - @Override - public String toString() { - return "{" + - "correlationId='" + correlationId + '\'' + - ", message='" + message + '\'' + - ", content=" + content + - ", tracePoint=" + tracePoint + - ", priority=" + priority + - '}'; - } -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerScopeProcessor.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerScopeProcessor.java deleted file mode 100644 index 6e8a7bc..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LoggerScopeProcessor.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.mule.extension.jsonlogger.config; - -import org.mule.runtime.extension.api.annotation.param.Optional; -import org.mule.runtime.extension.api.annotation.param.Parameter; -import org.mule.runtime.extension.api.annotation.param.display.Placement; -import org.mule.runtime.extension.api.annotation.param.display.Summary; - -public class LoggerScopeProcessor { - - @Parameter - @Optional(defaultValue = "OUTBOUND_REQUEST_SCOPE") - @Summary("Current processing stage") - private ScopeTracePoint scopeTracePoint; - - @Parameter - @Optional(defaultValue = "#[correlationId]") - @Placement(tab = "Advanced") - private String correlationId; - - @Parameter - @Optional(defaultValue = "INFO") - @Summary("Logger priority") - private Priority priority; - - public ScopeTracePoint getScopeTracePoint() { - return scopeTracePoint; - } - - public void setScopeTracePoint(ScopeTracePoint scopeTracePoint) { - this.scopeTracePoint = scopeTracePoint; - } - - public String getCorrelationId() { - return correlationId; - } - - public void setCorrelationId(String correlationId) { - this.correlationId = correlationId; - } - - public Priority getPriority() { - return priority; - } - - public void setPriority(Priority priority) { - this.priority = priority; - } -} \ No newline at end of file diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/Mapper.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/Mapper.java deleted file mode 100644 index a6e2fc9..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/Mapper.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.mule.extension.jsonlogger.mapper; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; - -import java.util.List; - -public class Mapper { - - private final ObjectMapper om = new ObjectMapper() - .setSerializationInclusion(JsonInclude.Include.NON_NULL) - .setSerializationInclusion(JsonInclude.Include.NON_EMPTY); - - private final ObjectMapper maskingMapper = new ObjectMapper() - .setSerializationInclusion(JsonInclude.Include.NON_NULL) - .setSerializationInclusion(JsonInclude.Include.NON_EMPTY); - - public ObjectMapper getObjectMapper(List maskedFields) { - SimpleModule module = new SimpleModule(); - module.addDeserializer(JsonNode.class, new MaskingDeserializer(maskedFields)); - this.maskingMapper.registerModule(module); - return this.maskingMapper; - } - - public ObjectMapper getObjectMapper() { - return this.om; - } -} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/MaskingDeserializer.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/MaskingDeserializer.java deleted file mode 100644 index 06b72b8..0000000 --- a/json-logger/src/main/java/org/mule/extension/jsonlogger/mapper/MaskingDeserializer.java +++ /dev/null @@ -1,179 +0,0 @@ -package org.mule.extension.jsonlogger.mapper; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.deser.std.JsonNodeDeserializer; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; - -import java.io.IOException; -import java.util.List; - -class MaskingDeserializer extends JsonNodeDeserializer { - List maskedFields; - - public MaskingDeserializer(List maskedFields) { - this.maskedFields = maskedFields; - } - - @Override - public JsonNode deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - switch (p.currentTokenId()) { - case 1: - return deserializeObject(p, ctxt, ctxt.getNodeFactory(), false); - case 3: - return deserializeArray(p, ctxt, ctxt.getNodeFactory(), false); - default: - return deserializeAny(p, ctxt, ctxt.getNodeFactory(), false); - } - } - - protected final ObjectNode deserializeObject(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { - ObjectNode node = nodeFactory.objectNode(); - - for (String key = p.nextFieldName(); key != null; key = p.nextFieldName()) { - if (maskedFields.contains(key)) { - shouldMask = true; - } - JsonToken t = p.nextToken(); - if (t == null) { - t = JsonToken.NOT_AVAILABLE; - } - - JsonNode value; - switch (t.id()) { - case 1: - value = deserializeObject(p, ctxt, nodeFactory, shouldMask); - shouldMask = false; - break; - case 2: - case 4: - case 5: - case 8: - default: - value = deserializeAny(p, ctxt, nodeFactory, shouldMask); - shouldMask = false; - break; - case 3: - value = deserializeArray(p, ctxt, nodeFactory, shouldMask); - shouldMask = false; - break; - case 6: - if (shouldMask) { - value = nodeFactory.textNode(replace(p.getText())); - } else { - value = nodeFactory.textNode(p.getText()); - } - break; - case 7: - value = this._fromInt(p, ctxt, nodeFactory); - break; - case 9: - value = nodeFactory.booleanNode(true); - break; - case 10: - value = nodeFactory.booleanNode(false); - break; - case 11: - value = nodeFactory.nullNode(); - break; - case 12: - value = this._fromEmbedded(p, ctxt, nodeFactory); - } - - JsonNode old = node.replace(key, value); - if (old != null) { - this._handleDuplicateField(p, ctxt, nodeFactory, key, node, old, value); - } - } - - return node; - } - - protected final ArrayNode deserializeArray(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { - ArrayNode node = nodeFactory.arrayNode(); - - while (true) { - JsonToken t = p.nextToken(); - switch (t.id()) { - case 1: - node.add(deserializeObject(p, ctxt, nodeFactory, shouldMask)); - break; - case 2: - case 5: - case 8: - default: - node.add(deserializeAny(p, ctxt, nodeFactory, shouldMask)); - break; - case 3: - node.add(deserializeArray(p, ctxt, nodeFactory, shouldMask)); - break; - case 4: - return node; - case 6: - if (shouldMask) { - node.add(nodeFactory.textNode(replace(p.getText()))); - } else { - node.add(nodeFactory.textNode(p.getText())); - } - break; - case 7: - node.add(this._fromInt(p, ctxt, nodeFactory)); - break; - case 9: - node.add(nodeFactory.booleanNode(true)); - break; - case 10: - node.add(nodeFactory.booleanNode(false)); - break; - case 11: - node.add(nodeFactory.nullNode()); - break; - case 12: - node.add(this._fromEmbedded(p, ctxt, nodeFactory)); - } - } - } - - protected final JsonNode deserializeAny(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { - switch (p.currentTokenId()) { - case 2: - return nodeFactory.objectNode(); - case 3: - case 4: - default: - return (JsonNode) ctxt.handleUnexpectedToken(this.handledType(), p); - case 5: - return deserializeObjectAtName(p, ctxt, nodeFactory); - case 6: - if (shouldMask) { - return nodeFactory.textNode(replace(p.getText())); - } - return nodeFactory.textNode(p.getText()); - case 7: - return this._fromInt(p, ctxt, nodeFactory); - case 8: - return this._fromFloat(p, ctxt, nodeFactory); - case 9: - return nodeFactory.booleanNode(true); - case 10: - return nodeFactory.booleanNode(false); - case 11: - return nodeFactory.nullNode(); - case 12: - return this._fromEmbedded(p, ctxt, nodeFactory); - } - } - - private String replace(String input) { - int len = input.length(); - StringBuilder sb = new StringBuilder(len); - for(int i = 0; i < len; i++){ - sb.append('*'); - } - return sb.toString(); - } -} diff --git a/json-logger/src/test/java/org/mule/extension/jsonlogger/internal/JsonMaskerTest.java b/json-logger/src/test/java/org/mule/extension/jsonlogger/internal/JsonMaskerTest.java deleted file mode 100644 index 07b9398..0000000 --- a/json-logger/src/test/java/org/mule/extension/jsonlogger/internal/JsonMaskerTest.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.mule.extension.jsonlogger; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.Assert; -import org.junit.Test; -import org.mule.extension.jsonlogger.mapper.Mapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -public class JsonMaskerTest { - ClassLoader classloader = Thread.currentThread().getContextClassLoader(); - InputStream is = classloader.getResourceAsStream("payload.json"); - List maskedFields = Collections.singletonList("attributes"); - private final ObjectMapper om = new Mapper().getObjectMapper(maskedFields); - protected Logger log = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); - - @Test - public void caseInsensitve() throws IOException { - JsonNode payload = om.readTree(is); - Map masked = om.convertValue(payload, new TypeReference>() { - }); - Assert.assertEquals("{name=****, phone=************, gender=****}", masked.get("attributes").toString()); - } - - @Test - public void finalException() { - try { - throw new RuntimeException("Invalid string"); - } catch (Exception e) { - String s = String.format("{" + - "\"%s\": \"%s\", " + - "\"message\": \"Error parsing log data as a string: %s\"" + - "}", - Constants.CORRELATION_ID, - UUID.randomUUID().toString(), - e.getMessage()); - System.out.println(s); - } - } - - -} \ No newline at end of file diff --git a/json-logger/src/test/resources/payload.json b/json-logger/src/test/resources/payload.json deleted file mode 100644 index edb65df..0000000 --- a/json-logger/src/test/resources/payload.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "data": { - "type": "articles", - "id": "1", - "relationships": { - "author": { - "data": { - "id": "42", - "type": "people" - } - } - } - }, - "attributes": { - "name": "John", - "phone": "888-900-0000", - "gender": "male" - } -} \ No newline at end of file From b90a38ea3dd1054a609e439d84f06f9ba592531f Mon Sep 17 00:00:00 2001 From: Jazzy Date: Tue, 29 Jun 2021 16:31:06 -0700 Subject: [PATCH 12/12] Add unit tests --- .../org/mule/extension/jsonlogger/Mapper.java | 68 +++++++ .../jsonlogger/MaskingDeserializer.java | 184 ++++++++++++++++++ .../jsonlogger/config/LogProcessor.java | 88 +++++++++ .../jsonlogger/config/ScopeProcessor.java | 48 +++++ .../jsonlogger/JsonLoggerOperationsTest.java | 182 +++++++++++++++++ .../jsonlogger/MaskingDeserializerTest.java | 58 ++++++ json-logger/src/test/resources/log4j2.xml | 14 ++ json-logger/src/test/resources/pii.json | 17 ++ 8 files changed, 659 insertions(+) create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/Mapper.java create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/MaskingDeserializer.java create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/config/LogProcessor.java create mode 100644 json-logger/src/main/java/org/mule/extension/jsonlogger/config/ScopeProcessor.java create mode 100644 json-logger/src/test/java/org/mule/extension/jsonlogger/JsonLoggerOperationsTest.java create mode 100644 json-logger/src/test/java/org/mule/extension/jsonlogger/MaskingDeserializerTest.java create mode 100644 json-logger/src/test/resources/log4j2.xml create mode 100644 json-logger/src/test/resources/pii.json diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/Mapper.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/Mapper.java new file mode 100644 index 0000000..30a46db --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/Mapper.java @@ -0,0 +1,68 @@ +package org.mule.extension.jsonlogger; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +public class Mapper { + volatile static boolean isInitalized; + + private final ObjectMapper om = new ObjectMapper() + .setSerializationInclusion(JsonInclude.Include.NON_NULL) + .setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + + private final ObjectMapper maskingMapper = new ObjectMapper() + .setSerializationInclusion(JsonInclude.Include.NON_NULL) + .setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + + private Mapper() { + } + + private static class MapHolder { + static final ObjectMapper INSTANCE = new Mapper().om; + static final ObjectMapper MASKING_INSTANCE = new Mapper().maskingMapper; + } + + public static ObjectMapper getInstance(String contentFieldsDataMasking) { + if (contentFieldsDataMasking == null || contentFieldsDataMasking.isEmpty()) { + return MapHolder.INSTANCE; + } + buildMaskedFields(contentFieldsDataMasking); + return MapHolder.MASKING_INSTANCE; + } + + public static ObjectMapper getInstance() { + return MapHolder.INSTANCE; + } + + private static void buildMaskedFields(String contentFieldsDataMasking) { + while (!isInitalized) { + Set maskedFields = new HashSet<>(); + AtomicBoolean lock = new AtomicBoolean(false); + if (lock.compareAndSet(false, true)) { + try { + if (contentFieldsDataMasking != null && !contentFieldsDataMasking.isEmpty()) { + String[] split = contentFieldsDataMasking + .trim() + .split(","); + + for (String s : split) { + maskedFields.add(s.trim()); + } + SimpleModule module = new SimpleModule() + .addDeserializer(JsonNode.class, new MaskingDeserializer(maskedFields)); + MapHolder.MASKING_INSTANCE.registerModule(module); + } + } finally { + isInitalized = true; + lock.set(false); + } + } + } + } +} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/MaskingDeserializer.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/MaskingDeserializer.java new file mode 100644 index 0000000..4cc1232 --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/MaskingDeserializer.java @@ -0,0 +1,184 @@ +package org.mule.extension.jsonlogger; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.JsonNodeDeserializer; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.io.IOException; +import java.util.Set; + +class MaskingDeserializer extends JsonNodeDeserializer { + Set maskedFields; + + public MaskingDeserializer(Set maskedFields) { + this.maskedFields = maskedFields; + } + + @Override + public JsonNode deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + switch (p.currentTokenId()) { + case 1: + return deserializeObject(p, ctxt, ctxt.getNodeFactory(), false); + case 3: + return deserializeArray(p, ctxt, ctxt.getNodeFactory(), false); + default: + return deserializeAny(p, ctxt, ctxt.getNodeFactory(), false); + } + } + + protected final ObjectNode deserializeObject(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { + ObjectNode node = nodeFactory.objectNode(); + + for (String key = p.nextFieldName(); key != null; key = p.nextFieldName()) { + if (maskedFields.contains(key)) { + shouldMask = true; + } + JsonToken t = p.nextToken(); + if (t == null) { + t = JsonToken.NOT_AVAILABLE; + } + + JsonNode value; + switch (t.id()) { + case 1: + value = deserializeObject(p, ctxt, nodeFactory, shouldMask); + shouldMask = false; + break; + case 2: + case 4: + case 5: + case 8: + default: + value = deserializeAny(p, ctxt, nodeFactory, shouldMask); + shouldMask = false; + break; + case 3: + value = deserializeArray(p, ctxt, nodeFactory, shouldMask); + shouldMask = false; + break; + case 6: + if (shouldMask) { + value = nodeFactory.textNode(replace(p.getText())); + shouldMask = false; + } else { + value = nodeFactory.textNode(p.getText()); + } + break; + case 7: + value = this._fromInt(p, ctxt, nodeFactory); + break; + case 9: + value = nodeFactory.booleanNode(true); + break; + case 10: + value = nodeFactory.booleanNode(false); + break; + case 11: + value = nodeFactory.nullNode(); + break; + case 12: + value = this._fromEmbedded(p, ctxt, nodeFactory); + } + + JsonNode old = node.replace(key, value); + if (old != null) { + this._handleDuplicateField(p, ctxt, nodeFactory, key, node, old, value); + } + } + + return node; + } + + protected final ArrayNode deserializeArray(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { + ArrayNode node = nodeFactory.arrayNode(); + + while (true) { + JsonToken t = p.nextToken(); + switch (t.id()) { + case 1: + node.add(deserializeObject(p, ctxt, nodeFactory, shouldMask)); + break; + case 2: + case 5: + case 8: + default: + node.add(deserializeAny(p, ctxt, nodeFactory, shouldMask)); + break; + case 3: + node.add(deserializeArray(p, ctxt, nodeFactory, shouldMask)); + break; + case 4: + return node; + case 6: + if (shouldMask) { + node.add(nodeFactory.textNode(replace(p.getText()))); + } else { + node.add(nodeFactory.textNode(p.getText())); + } + break; + case 7: + node.add(this._fromInt(p, ctxt, nodeFactory)); + break; + case 9: + node.add(nodeFactory.booleanNode(true)); + break; + case 10: + node.add(nodeFactory.booleanNode(false)); + break; + case 11: + node.add(nodeFactory.nullNode()); + break; + case 12: + node.add(this._fromEmbedded(p, ctxt, nodeFactory)); + } + } + } + + protected final JsonNode deserializeAny(JsonParser p, DeserializationContext ctxt, JsonNodeFactory nodeFactory, boolean shouldMask) throws IOException { + switch (p.currentTokenId()) { + case 2: + return nodeFactory.objectNode(); + case 3: + case 4: + default: + return (JsonNode) ctxt.handleUnexpectedToken(this.handledType(), p); + case 5: + return deserializeObjectAtName(p, ctxt, nodeFactory); + case 6: + if (shouldMask) { + return nodeFactory.textNode(replace(p.getText())); + } + return nodeFactory.textNode(p.getText()); + case 7: + return this._fromInt(p, ctxt, nodeFactory); + case 8: + return this._fromFloat(p, ctxt, nodeFactory); + case 9: + return nodeFactory.booleanNode(true); + case 10: + return nodeFactory.booleanNode(false); + case 11: + return nodeFactory.nullNode(); + case 12: + return this._fromEmbedded(p, ctxt, nodeFactory); + } + } + + private String replace(String input) { + int len = input.length(); + StringBuilder sb = new StringBuilder(len); + for (int i = 0; i < len; i++) { + if (Character.isLetterOrDigit(input.charAt(i))) { + sb.append('*'); + } else { + sb.append(input.charAt(i)); + } + } + return sb.toString(); + } +} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LogProcessor.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LogProcessor.java new file mode 100644 index 0000000..88f73ee --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/LogProcessor.java @@ -0,0 +1,88 @@ +package org.mule.extension.jsonlogger.config; + +import org.mule.runtime.api.metadata.TypedValue; +import org.mule.runtime.extension.api.annotation.param.Content; +import org.mule.runtime.extension.api.annotation.param.Optional; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.Example; +import org.mule.runtime.extension.api.annotation.param.display.Placement; +import org.mule.runtime.extension.api.annotation.param.display.Summary; +import org.mule.runtime.extension.api.runtime.parameter.ParameterResolver; + +import java.io.InputStream; + +public class LogProcessor { + + @Parameter + @Optional(defaultValue = "#[correlationId]") + @Placement(tab = "Advanced") + private String correlationId; + + @Parameter + @Optional(defaultValue = "") + @Summary("Message to be logged") + @Example("Add a log message") + private String message; + + @Parameter + @Optional(defaultValue = "{}") + @Summary("NOTE: Writing the entire payload every time across your application can cause serious performance issues") + @Content + private ParameterResolver> content; + + @Parameter + @Optional(defaultValue = "START") + @Summary("Current processing stage") + private TracePoint tracePoint; + + @Parameter + @Optional(defaultValue = "INFO") + @Summary("Logger priority") + private Priority priority; + + public String getCorrelationId() { + return correlationId; + } + + public LogProcessor setCorrelationId(String correlationId) { + this.correlationId = correlationId; + return this; + } + + public String getMessage() { + return message; + } + + public LogProcessor setMessage(String message) { + this.message = message; + return this; + } + + public ParameterResolver> getContent() { + return content; + } + + public LogProcessor setContent(ParameterResolver> content) { + this.content = content; + return this; + } + + public TracePoint getTracePoint() { + return tracePoint; + } + + public LogProcessor setTracePoint(TracePoint tracePoint) { + this.tracePoint = tracePoint; + return this; + } + + public Priority getPriority() { + return priority; + } + + public LogProcessor setPriority(Priority priority) { + this.priority = priority; + return this; + } + +} diff --git a/json-logger/src/main/java/org/mule/extension/jsonlogger/config/ScopeProcessor.java b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/ScopeProcessor.java new file mode 100644 index 0000000..9180882 --- /dev/null +++ b/json-logger/src/main/java/org/mule/extension/jsonlogger/config/ScopeProcessor.java @@ -0,0 +1,48 @@ +package org.mule.extension.jsonlogger.config; + +import org.mule.runtime.extension.api.annotation.param.Optional; +import org.mule.runtime.extension.api.annotation.param.Parameter; +import org.mule.runtime.extension.api.annotation.param.display.Placement; +import org.mule.runtime.extension.api.annotation.param.display.Summary; + +public class ScopeProcessor { + + @Parameter + @Optional(defaultValue = "OUTBOUND_REQUEST_SCOPE") + @Summary("Current processing stage") + private ScopeTracePoint scopeTracePoint; + + @Parameter + @Optional(defaultValue = "#[correlationId]") + @Placement(tab = "Advanced") + private String correlationId; + + @Parameter + @Optional(defaultValue = "INFO") + @Summary("Logger priority") + private Priority priority; + + public ScopeTracePoint getScopeTracePoint() { + return scopeTracePoint; + } + + public void setScopeTracePoint(ScopeTracePoint scopeTracePoint) { + this.scopeTracePoint = scopeTracePoint; + } + + public String getCorrelationId() { + return correlationId; + } + + public void setCorrelationId(String correlationId) { + this.correlationId = correlationId; + } + + public Priority getPriority() { + return priority; + } + + public void setPriority(Priority priority) { + this.priority = priority; + } +} \ No newline at end of file diff --git a/json-logger/src/test/java/org/mule/extension/jsonlogger/JsonLoggerOperationsTest.java b/json-logger/src/test/java/org/mule/extension/jsonlogger/JsonLoggerOperationsTest.java new file mode 100644 index 0000000..17dced3 --- /dev/null +++ b/json-logger/src/test/java/org/mule/extension/jsonlogger/JsonLoggerOperationsTest.java @@ -0,0 +1,182 @@ +package org.mule.extension.jsonlogger; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.Assert; +import org.junit.Test; +import org.mule.extension.jsonlogger.config.JsonLoggerConfig; +import org.mule.extension.jsonlogger.config.LogProcessor; +import org.mule.extension.jsonlogger.config.Priority; +import org.mule.runtime.api.component.TypedComponentIdentifier; +import org.mule.runtime.api.component.location.ComponentLocation; +import org.mule.runtime.api.component.location.LocationPart; +import org.mule.runtime.api.metadata.DataType; +import org.mule.runtime.api.metadata.DataTypeParamsBuilder; +import org.mule.runtime.api.metadata.MediaType; +import org.mule.runtime.api.metadata.TypedValue; +import org.mule.runtime.core.internal.metadata.SimpleDataType; +import org.mule.runtime.extension.api.runtime.operation.Result; +import org.mule.runtime.extension.api.runtime.process.CompletionCallback; +import org.mule.runtime.module.extension.internal.runtime.resolver.StaticParameterResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.activation.DataHandler; +import javax.activation.DataSource; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.lang.reflect.Type; +import java.net.URL; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +public class JsonLoggerOperationsTest { + protected static Logger log = LoggerFactory.getLogger("org.mule.extension.jsonlogger.JsonLogger"); + + JsonLoggerOperations jsonLoggerOperations = new JsonLoggerOperations(); + LogProcessor logProcessor = new LogProcessor() + .setCorrelationId(UUID.randomUUID().toString()) + .setMessage("JsonLoggerOperationsTest") + .setPriority(Priority.INFO); + + CompletionCallback completionCallback = new CompletionCallback() { + @Override + public void success(Result result) { + + } + + @Override + public void error(Throwable throwable) { + + } + }; + + JsonLoggerConfig config = new JsonLoggerConfig().setApplicationName("json-logger"); + ComponentLocation location = new ComponentLocation() { + @Override + public String getLocation() { + return System.getProperty("user.dir"); + } + + @Override + public Optional getFileName() { + return Optional.of("api.xml"); + } + + @Override + public Optional getLineInFile() { + return Optional.of(32); + } + + @Override + public List getParts() { + return null; + } + + @Override + public TypedComponentIdentifier getComponentIdentifier() { + return null; + } + + @Override + public String getRootContainerName() { + return "http"; + } + }; + + @Test + public void locationIsAdded() { + ObjectNode logEvent = Mapper.getInstance().createObjectNode(); + jsonLoggerOperations.addLocationInfo(logEvent, location); + Assert + .assertEquals("{\"locationInfo\":{\"fileName\":\"api.xml\",\"rootContainer\":\"http\",\"lineNumber\":\"32\"}}", + logEvent.toString()); + } + + @Test + public void javaPayloadReturnsToString() throws IOException { + ObjectNode logEvent = Mapper.getInstance().createObjectNode(); + + MagicBeans magicBeans = new MagicBeans(new URL("http://localhost")); + TypedValue value = new TypedValue(magicBeans.getInputStream(), magicBeans.build()); + StaticParameterResolver resolver = new StaticParameterResolver>(value); + + jsonLoggerOperations.addContent(logEvent, resolver, config); + Assert.assertEquals("", ""); + } + + @Test + public void logHasCorrelationIdIfMaskingFails() { + try { + throw new RuntimeException("Invalid string"); + } catch (Exception e) { + String s = String.format("{" + + "\"%s\": \"%s\", " + + "\"message\": \"Error parsing log data as a string: %s\"" + + "}", + "correlationId", + UUID.randomUUID().toString(), + e.getMessage()); + System.out.println(s); + } + } + + static class MagicBeans extends DataHandler implements DataTypeParamsBuilder, Serializable { + String beanstalk = "beanstalk"; + + @Override + public String getContentType() { + return "application/java"; + } + + public MagicBeans(URL url) { + super(url); + } + + @Override + public InputStream getInputStream() throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos); + oos.writeObject(this); + oos.flush(); + oos.close(); + return new ByteArrayInputStream(baos.toByteArray()); + } + + @Override + public DataTypeParamsBuilder mediaType(String s) { + return this; + } + + @Override + public DataTypeParamsBuilder mediaType(MediaType mediaType) { + return this; + } + + @Override + public DataTypeParamsBuilder charset(String s) { + return this; + } + + @Override + public DataTypeParamsBuilder charset(Charset charset) { + return this; + } + + @Override + public DataType build() { + return DataType.fromObject(this); + } + + @Override + public String toString() { + return beanstalk; + } + } +} diff --git a/json-logger/src/test/java/org/mule/extension/jsonlogger/MaskingDeserializerTest.java b/json-logger/src/test/java/org/mule/extension/jsonlogger/MaskingDeserializerTest.java new file mode 100644 index 0000000..cd22529 --- /dev/null +++ b/json-logger/src/test/java/org/mule/extension/jsonlogger/MaskingDeserializerTest.java @@ -0,0 +1,58 @@ +package org.mule.extension.jsonlogger; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; + +@SuppressWarnings("rawTypes, unchecked") +public class MaskingDeserializerTest { + static String fieldsToBeMasked = "name, phone, documents"; + static Map masked; + + @BeforeClass + public static void setup() throws IOException { + InputStream is = Thread + .currentThread() + .getContextClassLoader() + .getResourceAsStream("pii.json"); + JsonNode payload = Mapper.getInstance(fieldsToBeMasked).readTree(is); + masked = Mapper + .getInstance() + .convertValue(payload, new TypeReference>() { + }); + } + + @Test + public void fieldsAreBeingMasked() { + LinkedHashMap passenger = (LinkedHashMap) masked.get("passenger"); + Assert.assertEquals("**** ***", passenger.get("name")); + } + + @Test + public void formatIsBeingMaintained() { + LinkedHashMap passenger = (LinkedHashMap) masked.get("passenger"); + Assert.assertEquals("***-***-****", passenger.get("phone")); + } + + @Test + public void onlyRelevantFieldsGetMasked() { + LinkedHashMap destination = (LinkedHashMap) masked.get("destination"); + Assert.assertEquals("Phoenix", destination.get("city")); + LinkedHashMap passenger = (LinkedHashMap) masked.get("passenger"); + Assert.assertEquals("{city=Dallas}", passenger.get("address").toString()); + + } + + @Test + public void nestedMasking() { + Assert.assertEquals("{passport=********}", masked.get("documents").toString()); + } +} \ No newline at end of file diff --git a/json-logger/src/test/resources/log4j2.xml b/json-logger/src/test/resources/log4j2.xml new file mode 100644 index 0000000..c1288df --- /dev/null +++ b/json-logger/src/test/resources/log4j2.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/json-logger/src/test/resources/pii.json b/json-logger/src/test/resources/pii.json new file mode 100644 index 0000000..61a0275 --- /dev/null +++ b/json-logger/src/test/resources/pii.json @@ -0,0 +1,17 @@ +{ + "passenger": { + "name": "John Doe", + "phone": "888-900-0000", + "gender": "male", + "dateOfBirth": "10-10-2020", + "address": { + "city": "Dallas" + } + }, + "destination": { + "city": "Phoenix" + }, + "documents": { + "passport": "12345689" + } +} \ No newline at end of file