Fix Exception Replay with Lambda proxy classes

If JVM is started with -XX:+ShowhiddenFrames lambda proxy classes
dynamically generated are shown in the stacktraces which may be used
in the fingerprinting for Exception Replay. The proxy class generated
contains an id that is different for each loading of the class.
Upon re-transformation this id is changing which led to a different
fingerprint for the same stacktrace which will trigger a new
instrumentation and a re-transformation. And again new fingerprint...
We are fixing this by filtering out lambda proxy classes if detected.

Author: jpbempel

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/ClassNameFiltering.java

@@ -6,9 +6,11 @@
 import datadog.trace.util.ClassNameTrie;
 import java.util.Collections;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 /** A class to filter out classes based on their package name. */
 public class ClassNameFiltering implements ClassNameFilter {
+  private static final Pattern LAMBDA_PROXY_CLASS_PATTERN = Pattern.compile(".*\\$\\$Lambda.*/.*");
 
   private final ClassNameTrie includeTrie;
   private final ClassNameTrie excludeTrie;
@@ -33,7 +35,12 @@ public ClassNameFiltering(Set<String> excludes, Set<String> includes) {
   }
 
   public boolean isExcluded(String className) {
-    return includeTrie.apply(className) < 0 && excludeTrie.apply(className) > 0;
+    return (includeTrie.apply(className) < 0 && excludeTrie.apply(className) > 0)
+        || isLambdaProxyClass(className);
+  }
+
+  static boolean isLambdaProxyClass(String className) {
+    return LAMBDA_PROXY_CLASS_PATTERN.matcher(className).matches();
   }
 
   public static ClassNameFiltering allowAll() {

dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/util/ClassNameFilteringTest.java

@@ -92,4 +92,25 @@ public void testExcludeDefaults(String input) {
         new ClassNameFiltering(ThirdPartyLibraries.INSTANCE.getThirdPartyLibraries(config));
     assertTrue(classNameFiltering.isExcluded(input));
   }
+
+  @Test
+  void lambdaProxyClasses() {
+    // jdk8:  at
+    // datadog.smoketest.debugger.ServerDebuggerTestApplication$$Lambda$231/1770027171.apply(<Unknown>:1000008)
+    // jdk11: at
+    // datadog.smoketest.debugger.ServerDebuggerTestApplication$$Lambda$262/0x0000000800467040.apply(Unknown Source)
+    // jdk17:	at
+    // datadog.smoketest.debugger.ServerDebuggerTestApplication$$Lambda$303/0x00000008013dd1f8.apply(Unknown Source)
+    // jdk21: at
+    // datadog.smoketest.debugger.ServerDebuggerTestApplication$$Lambda/0x000000b801392c58.apply(Unknown Source)
+    assertTrue(
+        ClassNameFiltering.isLambdaProxyClass(
+            "datadog.smoketest.debugger.ServerDebuggerTestApplication$$Lambda$231/1770027171"));
+    assertTrue(
+        ClassNameFiltering.isLambdaProxyClass(
+            "datadog.smoketest.debugger.ServerDebuggerTestApplication$$Lambda$262/0x0000000800467040"));
+    assertTrue(
+        ClassNameFiltering.isLambdaProxyClass(
+            "at datadog.smoketest.debugger.ServerDebuggerTestApplication$$Lambda/0x000000b801392c58"));
+  }
 }

dd-smoke-tests/debugger-integration-tests/src/main/java/datadog/smoketest/debugger/ServerDebuggerTestApplication.java

@@ -5,6 +5,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Consumer;
+import java.util.function.Function;
 import okhttp3.HttpUrl;
 import okhttp3.MediaType;
 import okhttp3.OkHttpClient;
@@ -138,6 +139,8 @@ private static void runTracedMethod(String arg) {
       tracedMethodWithException(42, "foobar", 3.42, map, "var1", "var2", "var3");
     } else if ("deepOops".equals(arg)) {
       tracedMethodWithDeepException1(42, "foobar", 3.42, map, "var1", "var2", "var3");
+    } else if ("lambdaOops".equals(arg)) {
+      tracedMethodWithLambdaException(42, "foobar", 3.42, map, "var1", "var2", "var3");
     } else {
       tracedMethod(42, "foobar", 3.42, map, "var1", "var2", "var3");
     }
@@ -215,6 +218,19 @@ private static void tracedMethodWithDeepException5(
     tracedMethodWithException(argInt, argStr, argDouble, argMap, argVar);
   }
 
+  private static void tracedMethodWithLambdaException(
+      int argInt, String argStr, double argDouble, Map<String, String> argMap, String... argVar) {
+    throw toRuntimeException("lambdaOops");
+  }
+
+  private static RuntimeException toRuntimeException(String msg) {
+    return toException(RuntimeException::new, msg);
+  }
+
+  private static <S extends Throwable> S toException(Function<String, S> constructor, String msg) {
+    return constructor.apply(msg);
+  }
+
   private static class AppDispatcher extends Dispatcher {
     private final ServerDebuggerTestApplication app;
 

dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ExceptionDebuggerIntegrationTest.java

@@ -237,6 +237,40 @@ void test5CapturedFrames() throws Exception {
         });
   }
 
+  @Test
+  @DisplayName("testLambdaHiddenFrames")
+  @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "HotSpot specific test")
+  void testLambdaHiddenFrames() throws Exception {
+    additionalJvmArgs.add("-XX:+UnlockDiagnosticVMOptions");
+    additionalJvmArgs.add("-XX:+ShowHiddenFrames");
+    appUrl = startAppAndAndGetUrl();
+    execute(appUrl, TRACED_METHOD_NAME, "lambdaOops"); // instrumenting first exception
+    waitForInstrumentation(appUrl);
+    execute(appUrl, TRACED_METHOD_NAME, "lambdaOops"); // collecting snapshots and sending them
+    registerTraceListener(this::receiveExceptionReplayTrace);
+    registerSnapshotListener(this::receiveSnapshot);
+    processRequests(
+        () -> {
+          if (snapshotIdTags.isEmpty()) {
+            return false;
+          }
+          String snapshotId0 = snapshotIdTags.get(0);
+          if (traceReceived && snapshotReceived && snapshots.containsKey(snapshotId0)) {
+            Snapshot snapshot = snapshots.get(snapshotId0);
+            assertNotNull(snapshot);
+            assertEquals(
+                "lambdaOops",
+                snapshot.getCaptures().getReturn().getCapturedThrowable().getMessage());
+            assertEquals(
+                "datadog.smoketest.debugger.ServerDebuggerTestApplication.tracedMethodWithLambdaException",
+                snapshot.getStack().get(0).getFunction());
+            assertFullMethodCaptureArgs(snapshot.getCaptures().getReturn());
+            return true;
+          }
+          return false;
+        });
+  }
+
   private void resetSnapshotsAndTraces() {
     resetTraceListener();
     traceReceived = false;