NIFI-14991 Added Process Group Attributes to Component Log

- Added Process Group properties as attributes to Standard Logging Context
- Updated standard Component Log implementation to include Process Group attributes as MDC attributes
- Added MDC section to Admin Guide

Signed-off-by: Pierre Villard <[email protected]>

This closes #10330.

Author: exceptionfactory

nifi-docs/src/main/asciidoc/administration-guide.adoc

@@ -197,6 +197,29 @@ The standard logback configuration includes the following appender definitions a
 | `nifi-user.log`        | User log containing authentication and authorization messages
 |=========================
 
+=== Mapped Diagnostic Context Properties
+Logging for extension components such as Processors and Controller Services include variables in the
+link:https://www.slf4j.org/api/org/slf4j/MDC.html[Mapped Diagnostic Context] that provide additional information about
+the component location. link:https://logback.qos.ch/manual/layouts.html#mdc[MDC] values can be added to log messages
+with custom layout configuration.
+
+Component logs provide the following MDC named values:
+
+- `processGroupId` contains the UUID of the Process Group
+- `processGroupIdPath` contains the hierarchy of UUIDs for Process Groups with separators
+- `processGroupName` contains the name of the Process Group
+- `processGroupNamePath` contains of the hierarchy of names for Process Groups with separators
+
+MDC named values can be added to a Logback pattern layout using the `mdc` conversion word.
+
+[source, xml]
+----
+<pattern>%date %level [%thread] %mdc{processGroupId} %logger{40} %msg%n</pattern>
+----
+
+Logs from classes other than extension components do not have MDC named values. Logs formatted using the pattern layout
+will include an empty space when an MDC named value is not found.
+
 === Deprecation Logging
 The `nifi-deprecation.log` contains warning messages describing components and features that will be removed in
 subsequent versions. Deprecation warnings should be evaluated and addressed to avoid breaking changes when upgrading to

nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java

@@ -214,6 +214,10 @@ public final class StandardProcessGroup implements ProcessGroup {
     private static final long DEFAULT_BACKPRESSURE_OBJECT = 10_000L;
     private static final String DEFAULT_BACKPRESSURE_DATA_SIZE = "1 GB";
     private static final Pattern INVALID_DIRECTORY_NAME_CHARACTERS = Pattern.compile("[\\s\\<\\>:\\'\\\"\\/\\\\\\|\\?\\*]");
+    private static final String PATH_SEPARATOR = "/";
+    private static final String STANDARD_PROCESS_GROUP_NAME = "StandardProcessGroup";
+
+    private final Map<String, String> loggingAttributes = new ConcurrentHashMap<>();
     private volatile String logFileSuffix;
 
 
@@ -286,6 +290,7 @@ public ProcessGroup getParent() {
     @Override
     public void setParent(final ProcessGroup newParent) {
         parent.set(newParent);
+        setLoggingAttributes();
     }
 
     @Override
@@ -325,6 +330,7 @@ public void setName(final String name) {
         }
 
         this.name.set(name);
+        setLoggingAttributes();
     }
 
     @Override
@@ -4352,6 +4358,16 @@ public QueueSize getQueueSize() {
         return new QueueSize(count, contentSize);
     }
 
+    /**
+     * Get Map of Attribute Names and Values to provide additional context for logging
+     *
+     * @return Map of Attribute Names and Values
+     */
+    @Override
+    public Map<String, String> getLoggingAttributes() {
+        return Collections.unmodifiableMap(loggingAttributes);
+    }
+
     @Override
     public String getLogFileSuffix() {
         return logFileSuffix;
@@ -4542,4 +4558,63 @@ public void setStatelessFlowTimeout(final String statelessFlowTimeout) {
             LOG.warn("Attempted to set Stateless Flow Timeout for {} to invalid value: {}; ignoring this value", this, statelessFlowTimeout);
         }
     }
+
+    private void setLoggingAttributes() {
+        loggingAttributes.put(LoggingAttribute.PROCESS_GROUP_ID.attribute, id);
+
+        final String processGroupName = name.get();
+        if (processGroupName == null) {
+            loggingAttributes.put(LoggingAttribute.PROCESS_GROUP_NAME.attribute, STANDARD_PROCESS_GROUP_NAME);
+        } else {
+            loggingAttributes.put(LoggingAttribute.PROCESS_GROUP_NAME.attribute, processGroupName);
+            setGroupPath();
+        }
+    }
+
+    private void setGroupPath() {
+        final StringBuilder namePathBuilder = new StringBuilder();
+        namePathBuilder.append(PATH_SEPARATOR);
+        namePathBuilder.append(name.get());
+
+        final StringBuilder idPathBuilder = new StringBuilder();
+        idPathBuilder.append(PATH_SEPARATOR);
+        idPathBuilder.append(id);
+
+        ProcessGroup parentProcessGroup = getParent();
+        while (parentProcessGroup != null) {
+            namePathBuilder.insert(0, PATH_SEPARATOR);
+            namePathBuilder.insert(1, parentProcessGroup.getName());
+
+            idPathBuilder.insert(0, PATH_SEPARATOR);
+            idPathBuilder.insert(1, parentProcessGroup.getIdentifier());
+
+            parentProcessGroup = parentProcessGroup.getParent();
+        }
+
+        final String idPath = idPathBuilder.toString();
+        loggingAttributes.put(LoggingAttribute.PROCESS_GROUP_ID_PATH.attribute, idPath);
+
+        final String namePath = namePathBuilder.toString();
+        loggingAttributes.put(LoggingAttribute.PROCESS_GROUP_NAME_PATH.attribute, namePath);
+    }
+
+    enum LoggingAttribute {
+        PROCESS_GROUP_ID("processGroupId"),
+
+        PROCESS_GROUP_ID_PATH("processGroupIdPath"),
+
+        PROCESS_GROUP_NAME("processGroupName"),
+
+        PROCESS_GROUP_NAME_PATH("processGroupNamePath");
+
+        private final String attribute;
+
+        LoggingAttribute(final String attribute) {
+            this.attribute = attribute;
+        }
+
+        String getAttribute() {
+            return attribute;
+        }
+    }
 }

nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/LoggingContext.java

@@ -16,9 +16,17 @@
  */
 package org.apache.nifi.logging;
 
+import java.util.Map;
 import java.util.Optional;
 
 public interface LoggingContext {
+    /**
+     * Get Attributes
+     *
+     * @return Map of Attribute Name to Value such as Process Group information when provided
+     */
+    Map<String, String> getAttributes();
+
     /**
      * @return the log file name suffix. This will be the discriminating value for the dedicated logging.
      */

nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/logging/StandardLoggingContext.java

@@ -18,18 +18,41 @@
 
 import org.apache.nifi.groups.ProcessGroup;
 
+import java.util.Map;
 import java.util.Optional;
 
-
-
 public class StandardLoggingContext implements LoggingContext {
     private static final String KEY = "logFileSuffix";
+
     private volatile GroupedComponent component;
 
     public StandardLoggingContext(final GroupedComponent component) {
         this.component = component;
     }
 
+    /**
+     * Get Attributes from Component Process Group
+     *
+     * @return Attributes from Process Group or empty when Component or Process Group not found
+     */
+    @Override
+    public Map<String, String> getAttributes() {
+        final Map<String, String> attributes;
+
+        if (component == null) {
+            attributes = Map.of();
+        } else {
+            final ProcessGroup processGroup = component.getProcessGroup();
+            if (processGroup == null) {
+                attributes = Map.of();
+            } else {
+                attributes = processGroup.getLoggingAttributes();
+            }
+        }
+
+        return attributes;
+    }
+
     @Override
     public Optional<String> getLogFileSuffix() {
         if (component != null) {

nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/SimpleProcessLogger.java

@@ -25,10 +25,15 @@
 import org.apache.nifi.logging.LoggingContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
 import org.slf4j.event.Level;
+import org.slf4j.spi.LoggingEventBuilder;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 
 public class SimpleProcessLogger implements ComponentLog {
 
@@ -458,17 +463,24 @@ private Throwable findLastThrowable(final Object[] arguments) {
         return lastThrowable;
     }
 
-    private String getDiscriminatorKey() {
-        return loggingContext.getDiscriminatorKey();
-    }
+    private void log(final Level level, final String message, final Object... arguments) {
+        LoggingEventBuilder builder = logger.makeLoggingEventBuilder(level);
 
-    private String getLogFileSuffix() {
-        return loggingContext.getLogFileSuffix().orElse(null);
-    }
+        final Optional<String> loggingFileSuffixFound = loggingContext.getLogFileSuffix();
+        if (loggingFileSuffixFound.isPresent()) {
+            final String logFileSuffix = loggingFileSuffixFound.get();
+            builder = builder.addKeyValue(loggingContext.getDiscriminatorKey(), logFileSuffix);
+        }
 
-    private void log(final Level level, final String message, final Object... arguments) {
-        logger.makeLoggingEventBuilder(level)
-                .addKeyValue(getDiscriminatorKey(), getLogFileSuffix())
-                .log(message, arguments);
+        // Add attributes to Mapped Diagnostic Context for logging
+        final Map<String, String> attributes = loggingContext.getAttributes();
+        final Set<String> attributeKeys = attributes.keySet();
+        try {
+            attributes.forEach(MDC::put);
+            builder.log(message, arguments);
+        } finally {
+            // Remove MDC attributes after logging
+            attributeKeys.forEach(MDC::remove);
+        }
     }
 }