Skip to content

Commit

Permalink
Implement reading of simple key-value Logstash JSON Marker attributes
Browse files Browse the repository at this point in the history 
Supported are MapEntriesAppendingMarker and SingleFieldAppendingMarker
(i.e. ObjectAppendingMarker and RawJsonAppendingMarker) only. The attribute
value is always a string retrieved by a call to toString() method.

Typically the Logstash markers are added to logs via Markers.append() and
Markers.appendEntries() methods.

Signed-off-by: Oldřich Jedlička <[email protected]>
  • Loading branch information
@oldium
oldium committed Oct 24, 2024
1 parent 3cf37dd commit 6f0d31f
Show file tree
Hide file tree
Showing 7 changed files with 268 additions and 9 deletions.
5 changes: 5 additions & 0 deletions ...a/io/opentelemetry/javaagent/instrumentation/logback/appender/v1_0/LogbackSingletons.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ public final class LogbackSingletons {
boolean captureArguments =
config.getBoolean(
"otel.instrumentation.logback-appender.experimental.capture-arguments", false);
boolean captureLogstashAttributes =
config.getBoolean(
"otel.instrumentation.logback-appender.experimental.capture-logstash-attributes",
false);
List<String> captureMdcAttributes =
config.getList(
"otel.instrumentation.logback-appender.experimental.capture-mdc-attributes",
Expand All @@ -53,6 +57,7 @@ public final class LogbackSingletons {
.setCaptureKeyValuePairAttributes(captureKeyValuePairAttributes)
.setCaptureLoggerContext(captureLoggerContext)
.setCaptureArguments(captureArguments)
.setCaptureLogstashAttributes(captureLogstashAttributes)
.build();
}

Expand Down
11 changes: 11 additions & 0 deletions instrumentation/logback/logback-appender-1.0/library/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@ dependencies {
strictly("2.0.0")
}
}
compileOnly("net.logstash.logback:logstash-logback-encoder") {
version {
strictly("3.0")
}
}

if (findProperty("testLatestDeps") as Boolean) {
testImplementation("ch.qos.logback:logback-classic:+")
Expand Down Expand Up @@ -75,6 +80,7 @@ testing {
if (latestDepTest) {
implementation("ch.qos.logback:logback-classic:+")
implementation("org.slf4j:slf4j-api:+")
implementation("net.logstash.logback:logstash-logback-encoder:+")
} else {
implementation("ch.qos.logback:logback-classic") {
version {
Expand All @@ -86,6 +92,11 @@ testing {
strictly("2.0.0")
}
}
implementation("net.logstash.logback:logstash-logback-encoder") {
version {
strictly("3.0")
}
}
}
}
}
Expand Down
11 changes: 11 additions & 0 deletions ...in/java/io/opentelemetry/instrumentation/logback/appender/v1_0/OpenTelemetryAppender.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ public class OpenTelemetryAppender extends UnsynchronizedAppenderBase<ILoggingEv
private boolean captureKeyValuePairAttributes = false;
private boolean captureLoggerContext = false;
private boolean captureArguments = false;
private boolean captureLogstashAttributes = false;
private List<String> captureMdcAttributes = emptyList();

private volatile OpenTelemetry openTelemetry;
Expand Down Expand Up @@ -81,6 +82,7 @@ public void start() {
.setCaptureKeyValuePairAttributes(captureKeyValuePairAttributes)
.setCaptureLoggerContext(captureLoggerContext)
.setCaptureArguments(captureArguments)
.setCaptureLogstashAttributes(captureLogstashAttributes)
.build();
eventsToReplay = new ArrayBlockingQueue<>(numLogsCapturedBeforeOtelInstall);
super.start();
Expand Down Expand Up @@ -175,6 +177,15 @@ public void setCaptureArguments(boolean captureArguments) {
this.captureArguments = captureArguments;
}

/**
* Sets whether the Logstash attributes should be set to logs.
*
* @param captureLogstashAttributes To enable or disable capturing Logstash attributes
*/
public void setCaptureLogstashAttributes(boolean captureLogstashAttributes) {
this.captureLogstashAttributes = captureLogstashAttributes;
}

/** Configures the {@link MDC} attributes that will be copied to logs. */
public void setCaptureMdcAttributes(String attributes) {
if (attributes != null) {
Expand Down
212 changes: 203 additions & 9 deletions ...a/io/opentelemetry/instrumentation/logback/appender/v1_0/internal/LoggingEventMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,18 @@
import io.opentelemetry.semconv.ExceptionAttributes;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import net.logstash.logback.marker.LogstashMarker;
import net.logstash.logback.marker.MapEntriesAppendingMarker;
import net.logstash.logback.marker.SingleFieldAppendingMarker;
import org.slf4j.Marker;
import org.slf4j.event.KeyValuePair;

Expand All @@ -50,6 +56,7 @@ public final class LoggingEventMapper {
private static final boolean supportsInstant = supportsInstant();
private static final boolean supportsKeyValuePairs = supportsKeyValuePairs();
private static final boolean supportsMultipleMarkers = supportsMultipleMarkers();
private static final boolean supportsLogstashMarkers = supportsLogstashMarkers();
private static final Cache<String, AttributeKey<String>> mdcAttributeKeys = Cache.bounded(100);
private static final Cache<String, AttributeKey<String>> attributeKeys = Cache.bounded(100);

Expand All @@ -60,6 +67,8 @@ public final class LoggingEventMapper {
private static final AttributeKey<List<String>> LOG_BODY_PARAMETERS =
AttributeKey.stringArrayKey("log.body.parameters");

private static final Cache<Class<?>, Field> valueField = Cache.bounded(20);

private final boolean captureExperimentalAttributes;
private final List<String> captureMdcAttributes;
private final boolean captureAllMdcAttributes;
Expand All @@ -68,6 +77,7 @@ public final class LoggingEventMapper {
private final boolean captureKeyValuePairAttributes;
private final boolean captureLoggerContext;
private final boolean captureArguments;
private final boolean captureLogstashAttributes;

private LoggingEventMapper(Builder builder) {
this.captureExperimentalAttributes = builder.captureExperimentalAttributes;
Expand All @@ -77,6 +87,7 @@ private LoggingEventMapper(Builder builder) {
this.captureKeyValuePairAttributes = builder.captureKeyValuePairAttributes;
this.captureLoggerContext = builder.captureLoggerContext;
this.captureArguments = builder.captureArguments;
this.captureLogstashAttributes = builder.captureLogstashAttributes;
this.captureAllMdcAttributes =
builder.captureMdcAttributes.size() == 1 && builder.captureMdcAttributes.get(0).equals("*");
}
Expand Down Expand Up @@ -170,7 +181,8 @@ private void mapLoggingEvent(
}

if (captureMarkerAttribute) {
captureMarkerAttribute(attributes, loggingEvent);
boolean skipLogstashMarkers = supportsLogstashMarkers && captureLogstashAttributes;
captureMarkerAttribute(attributes, loggingEvent, skipLogstashMarkers);
}

if (supportsKeyValuePairs && captureKeyValuePairAttributes) {
Expand All @@ -187,6 +199,10 @@ private void mapLoggingEvent(
captureArguments(attributes, loggingEvent.getMessage(), loggingEvent.getArgumentArray());
}

if (supportsLogstashMarkers && captureLogstashAttributes) {
captureLogstashAttributes(attributes, loggingEvent);
}

builder.setAllAttributes(attributes.build());

// span context
Expand Down Expand Up @@ -326,31 +342,35 @@ private static boolean supportsKeyValuePairs() {
}

private static void captureMarkerAttribute(
AttributesBuilder attributes, ILoggingEvent loggingEvent) {
AttributesBuilder attributes, ILoggingEvent loggingEvent, boolean skipLogstashMarkers) {
if (supportsMultipleMarkers && hasMultipleMarkers(loggingEvent)) {
captureMultipleMarkerAttributes(attributes, loggingEvent);
captureMultipleMarkerAttributes(attributes, loggingEvent, skipLogstashMarkers);
} else {
captureSingleMarkerAttribute(attributes, loggingEvent);
captureSingleMarkerAttribute(attributes, loggingEvent, skipLogstashMarkers);
}
}

@SuppressWarnings("deprecation") // getMarker is deprecate since 1.3.0
private static void captureSingleMarkerAttribute(
AttributesBuilder attributes, ILoggingEvent loggingEvent) {
AttributesBuilder attributes, ILoggingEvent loggingEvent, boolean skipLogstashMarkers) {
Marker marker = loggingEvent.getMarker();
if (marker != null) {
if (marker != null && (!skipLogstashMarkers || !isLogstashMarker(marker))) {
attributes.put(LOG_MARKER, marker.getName());
}
}

@NoMuzzle
private static void captureMultipleMarkerAttributes(
AttributesBuilder attributes, ILoggingEvent loggingEvent) {
AttributesBuilder attributes, ILoggingEvent loggingEvent, boolean skipLogstashMarkers) {
List<String> markerNames = new ArrayList<>(loggingEvent.getMarkerList().size());
for (Marker marker : loggingEvent.getMarkerList()) {
markerNames.add(marker.getName());
if (!skipLogstashMarkers || !isLogstashMarker(marker)) {
markerNames.add(marker.getName());
}
}
if (!markerNames.isEmpty()) {
attributes.put(LOG_MARKER, markerNames.toArray(new String[0]));
}
attributes.put(LOG_MARKER, markerNames.toArray(new String[0]));
}

@NoMuzzle
Expand All @@ -369,6 +389,173 @@ private static boolean supportsMultipleMarkers() {
return true;
}

private static void captureLogstashAttributes(
AttributesBuilder attributes, ILoggingEvent loggingEvent) {
try {
if (supportsMultipleMarkers && hasMultipleMarkers(loggingEvent)) {
captureMultipleLogstashAttributes(attributes, loggingEvent);
} else {
captureSingleLogstashAttribute(attributes, loggingEvent);
}
} catch (Throwable e) {
// ignore
}
}

private static boolean isLogstashMarker(Marker marker) {
return marker instanceof LogstashMarker;
}

@NoMuzzle
@SuppressWarnings("deprecation") // getMarker is deprecate since 1.3.0
private static void captureSingleLogstashAttribute(
AttributesBuilder attributes, ILoggingEvent loggingEvent) {
Marker marker = loggingEvent.getMarker();
if (isLogstashMarker(marker)) {
LogstashMarker logstashMarker = (LogstashMarker) marker;
captureLogstashMarker(attributes, logstashMarker);
}
}

@NoMuzzle
private static void captureMultipleLogstashAttributes(
AttributesBuilder attributes, ILoggingEvent loggingEvent) {
for (Marker marker : loggingEvent.getMarkerList()) {
if (isLogstashMarker(marker)) {
LogstashMarker logstashMarker = (LogstashMarker) marker;
captureLogstashMarker(attributes, logstashMarker);
}
}
}

private static void captureLogstashMarker(
AttributesBuilder attributes, LogstashMarker logstashMarker) {
captureLogstashMarkerAttributes(attributes, logstashMarker);

if (logstashMarker.hasReferences()) {
for (Iterator<Marker> it = logstashMarker.iterator(); it.hasNext(); ) {
Marker referenceMarker = it.next();
if (isLogstashMarker(referenceMarker)) {
LogstashMarker referenceLogstashMarker = (LogstashMarker) referenceMarker;
captureLogstashMarker(attributes, referenceLogstashMarker);
}
}
}
}

@NoMuzzle
private static void captureLogstashMarkerAttributes(
AttributesBuilder attributes, LogstashMarker logstashMarker) {
if (logstashMarker instanceof SingleFieldAppendingMarker) {
SingleFieldAppendingMarker singleFieldAppendingMarker =
(SingleFieldAppendingMarker) logstashMarker;
String fieldName = singleFieldAppendingMarker.getFieldName();
String fieldValue = extractFieldValue(singleFieldAppendingMarker);
if (fieldName != null) {
if (fieldValue != null) {
attributes.put(fieldName, fieldValue);
} else {
attributes.put(fieldName, "");
}
}
} else if (logstashMarker instanceof MapEntriesAppendingMarker) {
MapEntriesAppendingMarker mapEntriesAppendingMarker =
(MapEntriesAppendingMarker) logstashMarker;
Map<?, ?> map = extractMapValue(mapEntriesAppendingMarker);
if (map != null) {
for (Map.Entry<?, ?> entry : map.entrySet()) {
Object key = entry.getKey();
Object value = entry.getValue();
if (key != null) {
if (value != null) {
attributes.put(key.toString(), value.toString());
} else {
attributes.put(key.toString(), "");
}
}
}
}
}
}

@Nullable
private static String extractFieldValue(SingleFieldAppendingMarker singleFieldAppendingMarker) {
// ObjectAppendingMarker.fieldValue since v7.0
// ObjectAppendingMarker.object since v3.0
// RawJsonAppendingMarker.rawJson since v3.0
Field field =
valueField.computeIfAbsent(
singleFieldAppendingMarker.getClass(),
clazz -> findValueField(clazz, new String[] {"fieldValue", "object", "rawJson"}));
if (field != null) {
try {
Object value = field.get(singleFieldAppendingMarker);
if (value != null) {
return value.toString();
}
} catch (IllegalAccessException e) {
// ignore
}
}
return null;
}

@Nullable
private static Map<?, ?> extractMapValue(MapEntriesAppendingMarker mapEntriesAppendingMarker) {
// MapEntriesAppendingMarker.map since v3.0
Field field =
valueField.computeIfAbsent(
mapEntriesAppendingMarker.getClass(),
clazz -> findValueField(clazz, new String[] {"map"}));
if (field != null) {
try {
Object value = field.get(mapEntriesAppendingMarker);
if (value instanceof Map) {
return (Map<?, ?>) value;
}
} catch (IllegalAccessException e) {
// ignore
}
}
return null;
}

@Nullable
private static Field findValueField(Class<?> clazz, String[] fieldNames) {
for (String fieldName : fieldNames) {
try {
Field field = clazz.getDeclaredField(fieldName);
field.setAccessible(true);
return field;
} catch (NoSuchFieldException e) {
// ignore
}
}
return null;
}

private static boolean supportsLogstashMarkers() {
try {
Class.forName("net.logstash.logback.marker.LogstashMarker");
} catch (ClassNotFoundException e) {
return false;
}

try {
Class.forName("net.logstash.logback.marker.SingleFieldAppendingMarker");
} catch (ClassNotFoundException e) {
return false;
}

try {
Class.forName("net.logstash.logback.marker.MapEntriesAppendingMarker");
} catch (ClassNotFoundException e) {
return false;
}

return true;
}

/**
* This class is internal and is hence not for public use. Its APIs are unstable and can change at
* any time.
Expand All @@ -381,6 +568,7 @@ public static final class Builder {
private boolean captureKeyValuePairAttributes;
private boolean captureLoggerContext;
private boolean captureArguments;
private boolean captureLogstashAttributes;

Builder() {}

Expand Down Expand Up @@ -426,6 +614,12 @@ public Builder setCaptureArguments(boolean captureArguments) {
return this;
}

@CanIgnoreReturnValue
public Builder setCaptureLogstashAttributes(boolean captureLogstashAttributes) {
this.captureLogstashAttributes = captureLogstashAttributes;
return this;
}

public LoggingEventMapper build() {
return new LoggingEventMapper(this);
}
Expand Down
Loading

0 comments on commit 6f0d31f

Please sign in to comment.