Skip to content

Commit ba03bdd

Browse files
gary-huanggary-huang
committed
add LLM obs configs
1 parent 2b24697 commit ba03bdd

File tree

7 files changed

+195
-1
lines changed

7 files changed

+195
-1
lines changed

‎dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,9 @@ public static Set<InstrumenterModule.TargetSystem> getEnabledSystems() {
311311
if (cfg.isUsmEnabled()) {
312312
enabledSystems.add(InstrumenterModule.TargetSystem.USM);
313313
}
314+
if (cfg.isLlmObsEnabled()) {
315+
enabledSystems.add(InstrumenterModule.TargetSystem.LLMOBS);
316+
}
314317
return enabledSystems;
315318
}
316319

‎dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterModule.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,8 @@ public enum TargetSystem {
4949
APPSEC,
5050
IAST,
5151
CIVISIBILITY,
52-
USM
52+
USM,
53+
LLMOBS,
5354
}
5455

5556
private static final Logger log = LoggerFactory.getLogger(InstrumenterModule.class);

‎dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,9 @@ public final class ConfigDefaults {
135135

136136
static final boolean DEFAULT_IAST_STACK_TRACE_ENABLED = true;
137137

138+
static final boolean DEFAULT_LLM_OBS_ENABLED = false;
139+
static final boolean DEFAULT_LLM_OBS_AGENTLESS_ENABLED = false;
140+
138141
static final boolean DEFAULT_USM_ENABLED = false;
139142

140143
static final boolean DEFAULT_CIVISIBILITY_ENABLED = false;

‎dd-trace-api/src/main/java/datadog/trace/api/config/LlmObsConfig.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package datadog.trace.api.config;
2+
3+
/**
4+
* Constant with names of configuration options for LLM Observability. (EXPERIMENTAL AND SUBJECT TO
5+
* CHANGE)
6+
*/
7+
public final class LlmObsConfig {
8+
9+
public static final String LLM_OBS_ENABLED = "llmobs.enabled";
10+
11+
public static final String LLM_OBS_ML_APP = "llmobs.ml.app";
12+
13+
public static final String LLM_OBS_AGENTLESS_ENABLED = "llmobs.agentless.enabled";
14+
15+
private LlmObsConfig() {}
16+
}

‎internal-api/src/main/java/datadog/trace/api/Config.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import static datadog.trace.api.config.GeneralConfig.SERVICE_NAME;
1414
import static datadog.trace.api.config.IastConfig.*;
1515
import static datadog.trace.api.config.JmxFetchConfig.*;
16+
import static datadog.trace.api.config.LlmObsConfig.*;
1617
import static datadog.trace.api.config.ProfilingConfig.*;
1718
import static datadog.trace.api.config.RemoteConfigConfig.*;
1819
import static datadog.trace.api.config.TraceInstrumentationConfig.*;
@@ -307,6 +308,9 @@ public static String getHostName() {
307308
private final boolean iastStackTraceEnabled;
308309
private final boolean iastExperimentalPropagationEnabled;
309310

311+
private final boolean llmObsAgentlessEnabled;
312+
private final String llmObsMlApp;
313+
310314
private final boolean ciVisibilityTraceSanitationEnabled;
311315
private final boolean ciVisibilityAgentlessEnabled;
312316
private final String ciVisibilityAgentlessUrl;
@@ -1319,6 +1323,10 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
13191323
iastExperimentalPropagationEnabled =
13201324
configProvider.getBoolean(IAST_EXPERIMENTAL_PROPAGATION_ENABLED, false);
13211325

1326+
llmObsAgentlessEnabled =
1327+
configProvider.getBoolean(LLM_OBS_AGENTLESS_ENABLED, DEFAULT_LLM_OBS_AGENTLESS_ENABLED);
1328+
llmObsMlApp = configProvider.getString(LLM_OBS_ML_APP);
1329+
13221330
ciVisibilityTraceSanitationEnabled =
13231331
configProvider.getBoolean(CIVISIBILITY_TRACE_SANITATION_ENABLED, true);
13241332

@@ -1754,6 +1762,24 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
17541762
configProvider.getLong(
17551763
TRACE_POST_PROCESSING_TIMEOUT, ConfigDefaults.DEFAULT_TRACE_POST_PROCESSING_TIMEOUT);
17561764

1765+
if (isLlmObsEnabled()) {
1766+
log.debug("Attempting to enable LLM Observability");
1767+
if (llmObsMlApp == null || llmObsMlApp.isEmpty()) {
1768+
throw new IllegalArgumentException(
1769+
"Attempt to enable LLM Observability without ML app defined."
1770+
+ "Please ensure that the name of the ML app is provided through properties or env variable");
1771+
}
1772+
if (llmObsAgentlessEnabled && (apiKey == null || apiKey.isEmpty())) {
1773+
throw new FatalAgentMisconfigurationError(
1774+
"Attempt to start LLM Observability in Agentless mode without API key. "
1775+
+ "Please ensure that either an API key is configured, or the tracer is set up to work with the Agent");
1776+
}
1777+
log.debug(
1778+
"LLM Observability enabled for ML app {}, agentless mode {}",
1779+
llmObsMlApp,
1780+
llmObsAgentlessEnabled);
1781+
}
1782+
17571783
if (isCiVisibilityEnabled()
17581784
&& ciVisibilityAgentlessEnabled
17591785
&& (apiKey == null || apiKey.isEmpty())) {
@@ -2606,6 +2632,18 @@ public boolean isIastExperimentalPropagationEnabled() {
26062632
return iastExperimentalPropagationEnabled;
26072633
}
26082634

2635+
public boolean isLlmObsEnabled() {
2636+
return instrumenterConfig.isLlmObsEnabled();
2637+
}
2638+
2639+
public boolean isLlmObsAgentlessEnabled() {
2640+
return llmObsAgentlessEnabled;
2641+
}
2642+
2643+
public String getLlmObsMlApp() {
2644+
return llmObsMlApp;
2645+
}
2646+
26092647
public boolean isCiVisibilityEnabled() {
26102648
return instrumenterConfig.isCiVisibilityEnabled();
26112649
}

‎internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import static datadog.trace.api.ConfigDefaults.DEFAULT_CODE_ORIGIN_FOR_SPANS_ENABLED;
66
import static datadog.trace.api.ConfigDefaults.DEFAULT_IAST_ENABLED;
77
import static datadog.trace.api.ConfigDefaults.DEFAULT_INTEGRATIONS_ENABLED;
8+
import static datadog.trace.api.ConfigDefaults.DEFAULT_LLM_OBS_ENABLED;
89
import static datadog.trace.api.ConfigDefaults.DEFAULT_MEASURE_METHODS;
910
import static datadog.trace.api.ConfigDefaults.DEFAULT_RESOLVER_RESET_INTERVAL;
1011
import static datadog.trace.api.ConfigDefaults.DEFAULT_RUNTIME_CONTEXT_FIELD_INJECTION;
@@ -26,6 +27,7 @@
2627
import static datadog.trace.api.config.GeneralConfig.TRACE_TRIAGE;
2728
import static datadog.trace.api.config.GeneralConfig.TRIAGE_REPORT_TRIGGER;
2829
import static datadog.trace.api.config.IastConfig.IAST_ENABLED;
30+
import static datadog.trace.api.config.LlmObsConfig.LLM_OBS_ENABLED;
2931
import static datadog.trace.api.config.ProfilingConfig.PROFILING_DIRECT_ALLOCATION_ENABLED;
3032
import static datadog.trace.api.config.ProfilingConfig.PROFILING_DIRECT_ALLOCATION_ENABLED_DEFAULT;
3133
import static datadog.trace.api.config.ProfilingConfig.PROFILING_ENABLED;
@@ -111,6 +113,7 @@ public class InstrumenterConfig {
111113
private final boolean iastFullyDisabled;
112114
private final boolean usmEnabled;
113115
private final boolean telemetryEnabled;
116+
private final boolean llmObsEnabled;
114117

115118
private final String traceExtensionsPath;
116119

@@ -199,6 +202,7 @@ private InstrumenterConfig() {
199202
iastFullyDisabled = iastEnabled != null && !iastEnabled;
200203
usmEnabled = configProvider.getBoolean(USM_ENABLED, DEFAULT_USM_ENABLED);
201204
telemetryEnabled = configProvider.getBoolean(TELEMETRY_ENABLED, DEFAULT_TELEMETRY_ENABLED);
205+
llmObsEnabled = configProvider.getBoolean(LLM_OBS_ENABLED, DEFAULT_LLM_OBS_ENABLED);
202206
} else {
203207
// disable these features in native-image
204208
ciVisibilityEnabled = false;
@@ -207,6 +211,7 @@ private InstrumenterConfig() {
207211
iastFullyDisabled = true;
208212
telemetryEnabled = false;
209213
usmEnabled = false;
214+
llmObsEnabled = false;
210215
}
211216

212217
traceExtensionsPath = configProvider.getString(TRACE_EXTENSIONS_PATH);
@@ -355,6 +360,10 @@ public boolean isIastFullyDisabled() {
355360
return iastFullyDisabled;
356361
}
357362

363+
public boolean isLlmObsEnabled() {
364+
return llmObsEnabled;
365+
}
366+
358367
public boolean isUsmEnabled() {
359368
return usmEnabled;
360369
}

‎internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,9 @@ import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_REFRESH_BEANS_PE
6363
import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_STATSD_HOST
6464
import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_STATSD_PORT
6565
import static datadog.trace.api.config.JmxFetchConfig.JMX_TAGS
66+
import static datadog.trace.api.config.LlmObsConfig.LLM_OBS_AGENTLESS_ENABLED
67+
import static datadog.trace.api.config.LlmObsConfig.LLM_OBS_ML_APP
68+
import static datadog.trace.api.config.LlmObsConfig.LLM_OBS_ENABLED
6669
import static datadog.trace.api.config.ProfilingConfig.PROFILING_AGENTLESS
6770
import static datadog.trace.api.config.ProfilingConfig.PROFILING_API_KEY_FILE_OLD
6871
import static datadog.trace.api.config.ProfilingConfig.PROFILING_API_KEY_FILE_VERY_OLD
@@ -163,6 +166,9 @@ class ConfigTest extends DDSpecification {
163166
private static final DD_PROFILING_TAGS_ENV = "DD_PROFILING_TAGS"
164167
private static final DD_PROFILING_PROXY_PASSWORD_ENV = "DD_PROFILING_PROXY_PASSWORD"
165168
private static final DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH = "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH"
169+
private static final DD_LLMOBS_ENABLED_ENV = "DD_LLMOBS_ENABLED"
170+
private static final DD_LLMOBS_ML_APP_ENV = "DD_LLMOBS_ML_APP"
171+
private static final DD_LLMOBS_AGENTLESS_ENABLED_ENV = "DD_LLMOBS_AGENTLESS_ENABLED"
166172

167173
def "specify overrides via properties"() {
168174
setup:
@@ -2208,6 +2214,124 @@ class ConfigTest extends DDSpecification {
22082214
!hostname.trim().isEmpty()
22092215
}
22102216

2217+
def "config instantiation should fail if llm obs is enabled via sys prop and ml app is not set"() {
2218+
setup:
2219+
Properties properties = new Properties()
2220+
properties.setProperty(LLM_OBS_ENABLED, "true")
2221+
2222+
when:
2223+
new Config(ConfigProvider.withPropertiesOverride(properties))
2224+
2225+
then:
2226+
thrown IllegalArgumentException
2227+
}
2228+
2229+
def "config instantiation should fail if llm obs is enabled via env var and ml app is not set"() {
2230+
setup:
2231+
environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true")
2232+
2233+
when:
2234+
new Config()
2235+
2236+
then:
2237+
thrown IllegalArgumentException
2238+
}
2239+
2240+
2241+
def "config instantiation should NOT fail if llm obs is enabled (agentless disabled) via sys prop and ml app is set"() {
2242+
setup:
2243+
Properties properties = new Properties()
2244+
properties.setProperty(LLM_OBS_ENABLED, "true")
2245+
properties.setProperty(LLM_OBS_AGENTLESS_ENABLED, "false")
2246+
properties.setProperty(LLM_OBS_ML_APP, "test-ml-app")
2247+
2248+
when:
2249+
def config = new Config(ConfigProvider.withPropertiesOverride(properties))
2250+
2251+
then:
2252+
noExceptionThrown()
2253+
config.isLlmObsEnabled()
2254+
!config.isLlmObsAgentlessEnabled()
2255+
config.llmObsMlApp == "test-ml-app"
2256+
}
2257+
2258+
def "config instantiation should NOT fail if llm obs is enabled (agentless disabled) via env var and ml app is set"() {
2259+
setup:
2260+
environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true")
2261+
environmentVariables.set(DD_LLMOBS_ML_APP_ENV, "test-ml-app")
2262+
2263+
when:
2264+
def config = new Config()
2265+
2266+
then:
2267+
noExceptionThrown()
2268+
config.isLlmObsEnabled()
2269+
!config.isLlmObsAgentlessEnabled()
2270+
config.llmObsMlApp == "test-ml-app"
2271+
}
2272+
2273+
def "config instantiation should fail if llm obs is in agentless mode via sys prop and API key is not set"() {
2274+
setup:
2275+
Properties properties = new Properties()
2276+
properties.setProperty(LLM_OBS_ENABLED, "true")
2277+
properties.setProperty(LLM_OBS_AGENTLESS_ENABLED, "true")
2278+
properties.setProperty(LLM_OBS_ML_APP, "test-ml-app")
2279+
2280+
when:
2281+
new Config(ConfigProvider.withPropertiesOverride(properties))
2282+
2283+
then:
2284+
thrown FatalAgentMisconfigurationError
2285+
}
2286+
2287+
def "config instantiation should fail if llm obs is in agentless mode via env var and API key is not set"() {
2288+
setup:
2289+
environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true")
2290+
environmentVariables.set(DD_LLMOBS_ML_APP_ENV, "a")
2291+
environmentVariables.set(DD_LLMOBS_AGENTLESS_ENABLED_ENV, "true")
2292+
2293+
when:
2294+
new Config()
2295+
2296+
then:
2297+
thrown FatalAgentMisconfigurationError
2298+
}
2299+
2300+
def "config instantiation should NOT fail if llm obs is enabled (agentless enabled) and API key & ml app are set via sys prop"() {
2301+
setup:
2302+
Properties properties = new Properties()
2303+
properties.setProperty(LLM_OBS_ENABLED, "true")
2304+
properties.setProperty(LLM_OBS_AGENTLESS_ENABLED, "true")
2305+
properties.setProperty(LLM_OBS_ML_APP, "test-ml-app")
2306+
properties.setProperty(API_KEY, "123456789")
2307+
2308+
when:
2309+
def config = new Config(ConfigProvider.withPropertiesOverride(properties))
2310+
2311+
then:
2312+
noExceptionThrown()
2313+
config.isLlmObsEnabled()
2314+
config.isLlmObsAgentlessEnabled()
2315+
config.llmObsMlApp == "test-ml-app"
2316+
}
2317+
2318+
def "config instantiation should NOT fail if llm obs is enabled (agentless enabled) and API key & ml app are set via env var"() {
2319+
setup:
2320+
environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true")
2321+
environmentVariables.set(DD_LLMOBS_ML_APP_ENV, "a")
2322+
environmentVariables.set(DD_LLMOBS_AGENTLESS_ENABLED_ENV, "true")
2323+
environmentVariables.set(DD_API_KEY_ENV, "8663294466")
2324+
2325+
when:
2326+
def config = new Config()
2327+
2328+
then:
2329+
noExceptionThrown()
2330+
config.isLlmObsEnabled()
2331+
config.isLlmObsAgentlessEnabled()
2332+
config.llmObsMlApp == "a"
2333+
}
2334+
22112335
def "config instantiation should fail if CI visibility agentless mode is enabled and API key is not set"() {
22122336
setup:
22132337
Properties properties = new Properties()

0 commit comments

Comments
 (0)