From cbd622682457282712ff382f2da38ca40419e695 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 24 Feb 2026 10:30:30 +0100 Subject: [PATCH 01/48] llmobs: set model tag even when llmobs disabled --- .../openai_java/ChatCompletionDecorator.java | 18 +++++++------ .../openai_java/CompletionDecorator.java | 27 ++++++++++--------- .../openai_java/EmbeddingDecorator.java | 19 ++++++------- .../openai_java/ResponseDecorator.java | 23 +++++++--------- 4 files changed, 44 insertions(+), 43 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index 6c9e9cad9d9..74f14a9daf2 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -31,11 +31,6 @@ public void withChatCompletionCreateParams( AgentSpan span, ChatCompletionCreateParams params, boolean stream) { span.setResourceName(CHAT_COMPLETIONS_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/chat/completions"); - if (!llmObsEnabled) { - return; - } - - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); if (params == null) { return; } @@ -45,6 +40,12 @@ public void withChatCompletionCreateParams( .asString() .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + if (!llmObsEnabled) { + return; + } + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); + span.setTag( CommonTags.INPUT, params.messages().stream() @@ -97,13 +98,14 @@ private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { } public void withChatCompletion(AgentSpan span, ChatCompletion completion) { - if (!llmObsEnabled) { - return; - } String modelName = completion.model(); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + List output = completion.choices().stream() .map(ChatCompletionDecorator::llmMessage) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java index 2291c860d00..51e7c12ef1c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java @@ -23,11 +23,6 @@ public class CompletionDecorator { public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams params) { span.setResourceName(COMPLETIONS_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/completions"); - if (!llmObsEnabled) { - return; - } - - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); if (params == null) { return; } @@ -37,6 +32,12 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa ._value() .asString() .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + + if (!llmObsEnabled) { + return; + } + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); params .prompt() .flatMap(p -> p.string()) @@ -61,14 +62,14 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa } public void withCompletion(AgentSpan span, Completion completion) { - if (!llmObsEnabled) { - return; - } - String modelName = completion.model(); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + List output = completion.choices().stream() .map(v -> LLMObs.LLMMessage.from(null, v.text())) @@ -86,10 +87,6 @@ public void withCompletion(AgentSpan span, Completion completion) { } public void withCompletions(AgentSpan span, List completions) { - if (!llmObsEnabled) { - return; - } - if (completions.isEmpty()) { return; } @@ -99,6 +96,10 @@ public void withCompletions(AgentSpan span, List completions) { span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + Map textByChoiceIndex = new HashMap<>(); for (Completion completion : completions) { completion diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java index 88098358ebf..02d4588358c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java @@ -25,11 +25,6 @@ public class EmbeddingDecorator { public void withEmbeddingCreateParams(AgentSpan span, EmbeddingCreateParams params) { span.setResourceName(EMBEDDINGS_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/embeddings"); - if (!llmObsEnabled) { - return; - } - - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_EMBEDDING_SPAN_KIND); if (params == null) { return; } @@ -39,6 +34,12 @@ public void withEmbeddingCreateParams(AgentSpan span, EmbeddingCreateParams para .asString() .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + if (!llmObsEnabled) { + return; + } + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_EMBEDDING_SPAN_KIND); + span.setTag(CommonTags.INPUT, embeddingDocuments(params.input())); Map metadata = new HashMap<>(); @@ -59,14 +60,14 @@ private List embeddingDocuments(EmbeddingCreateParams.Input inp } public void withCreateEmbeddingResponse(AgentSpan span, CreateEmbeddingResponse response) { - if (!llmObsEnabled) { - return; - } - String modelName = response.model(); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + if (!response.data().isEmpty()) { int embeddingCount = response.data().size(); Embedding firstEmbedding = response.data().get(0); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 4bc95e32934..db2258785d4 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -37,11 +37,6 @@ public class ResponseDecorator { public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params) { span.setResourceName(RESPONSES_CREATE); span.setTag(CommonTags.OPENAI_REQUEST_ENDPOINT, "/v1/responses"); - if (!llmObsEnabled) { - return; - } - - span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); if (params == null) { return; } @@ -51,6 +46,12 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params String modelName = extractResponseModel(params._model()); span.setTag(CommonTags.OPENAI_REQUEST_MODEL, modelName); + if (!llmObsEnabled) { + return; + } + + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); + List inputMessages = new ArrayList<>(); params @@ -369,10 +370,6 @@ public void withResponse(AgentSpan span, Response response) { } public void withResponseStreamEvents(AgentSpan span, List events) { - if (!llmObsEnabled) { - return; - } - for (ResponseStreamEvent event : events) { if (event.isCompleted()) { Response response = event.asCompleted().response(); @@ -388,14 +385,14 @@ public void withResponseStreamEvents(AgentSpan span, List e } private void withResponse(AgentSpan span, Response response, boolean stream) { - if (!llmObsEnabled) { - return; - } - String modelName = extractResponseModel(response._model()); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); + if (!llmObsEnabled) { + return; + } + List outputMessages = extractResponseOutputMessages(response.output()); if (!outputMessages.isEmpty()) { span.setTag(CommonTags.OUTPUT, outputMessages); From 4f2767372d0c37dae170c3f964c317295ac93b14 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 13:30:21 +0100 Subject: [PATCH 02/48] Set metadata.stream tag no matter it's true or false --- .../instrumentation/openai_java/ChatCompletionDecorator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index 74f14a9daf2..eb918ef3886 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -56,9 +56,7 @@ public void withChatCompletionCreateParams( // maxTokens is deprecated but integration tests missing to provide maxCompletionTokens params.maxTokens().ifPresent(v -> metadata.put("max_tokens", v)); params.temperature().ifPresent(v -> metadata.put("temperature", v)); - if (stream) { - metadata.put("stream", true); - } + metadata.put("stream", stream); params .streamOptions() .ifPresent( From d128d6baddaf647fa34ad6fa11c7be0672547b15 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 13:46:41 +0100 Subject: [PATCH 03/48] Set chat/completion CACHE_READ_INPUT_TOKENS tag --- .../instrumentation/openai_java/ChatCompletionDecorator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index eb918ef3886..95c746880cf 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -117,6 +117,10 @@ public void withChatCompletion(AgentSpan span, ChatCompletion completion) { span.setTag(CommonTags.INPUT_TOKENS, usage.promptTokens()); span.setTag(CommonTags.OUTPUT_TOKENS, usage.completionTokens()); span.setTag(CommonTags.TOTAL_TOKENS, usage.totalTokens()); + usage + .promptTokensDetails() + .flatMap(details -> details.cachedTokens()) + .ifPresent(v -> span.setTag(CommonTags.CACHE_READ_INPUT_TOKENS, v)); }); } From 3fc5cebce75f121245d4b850445069741ff23d52 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 18:31:44 +0100 Subject: [PATCH 04/48] Set error nad error_type tags --- .../trace/instrumentation/openai_java/CommonTags.java | 3 +++ .../trace/instrumentation/openai_java/OpenAiDecorator.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index f779662ac0d..a4335a7edfe 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -21,6 +21,9 @@ interface CommonTags { String ML_APP = TAG_PREFIX + LLMObsTags.ML_APP; String VERSION = TAG_PREFIX + "version"; + String ERROR = TAG_PREFIX + "error"; + String ERROR_TYPE = TAG_PREFIX + "error_type"; + String ENV = TAG_PREFIX + "env"; String SERVICE = TAG_PREFIX + "service"; String PARENT_ID = TAG_PREFIX + "parent_id"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 331269bad83..2332b578ebf 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -3,6 +3,7 @@ import com.openai.core.ClientOptions; import com.openai.core.http.Headers; import datadog.trace.api.Config; +import datadog.trace.api.DDTags; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObsContext; import datadog.trace.api.telemetry.LLMObsMetricCollector; @@ -12,6 +13,7 @@ import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.ClientDecorator; +import datadog.trace.core.CoreSpan; import java.util.List; public class OpenAiDecorator extends ClientDecorator { @@ -111,6 +113,9 @@ public AgentSpan afterStart(AgentSpan span) { @Override public AgentSpan beforeFinish(AgentSpan span) { if (llmObsEnabled) { + span.setTag(CommonTags.ERROR, span.isError() ? 1 : 0); + span.setTag(CommonTags.ERROR_TYPE, span.getTag(DDTags.ERROR_TYPE)); + Object spanKindTag = span.getTag(CommonTags.SPAN_KIND); if (spanKindTag != null) { String spanKind = spanKindTag.toString(); From 021a9d1c9bf9a04ac3951120cb1dea26c9c92df5 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 22:37:22 +0100 Subject: [PATCH 05/48] Use "" instead of null for the role in CompletionDecorator to comply wthTestOpenAiLlmInteractions::test_completion --- .../instrumentation/openai_java/CompletionDecorator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java index 51e7c12ef1c..1b95491b64b 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java @@ -45,7 +45,7 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa input -> span.setTag( CommonTags.INPUT, - Collections.singletonList(LLMObs.LLMMessage.from(null, input)))); + Collections.singletonList(LLMObs.LLMMessage.from("", input)))); Map metadata = new HashMap<>(); params.maxTokens().ifPresent(v -> metadata.put("max_tokens", v)); @@ -72,7 +72,7 @@ public void withCompletion(AgentSpan span, Completion completion) { List output = completion.choices().stream() - .map(v -> LLMObs.LLMMessage.from(null, v.text())) + .map(v -> LLMObs.LLMMessage.from("", v.text())) .collect(Collectors.toList()); span.setTag(CommonTags.OUTPUT, output); @@ -116,7 +116,7 @@ public void withCompletions(AgentSpan span, List completions) { List output = textByChoiceIndex.entrySet().stream() .sorted(Map.Entry.comparingByKey()) - .map(entry -> LLMObs.LLMMessage.from(null, entry.getValue().toString())) + .map(entry -> LLMObs.LLMMessage.from("", entry.getValue().toString())) .collect(Collectors.toList()); span.setTag(CommonTags.OUTPUT, output); From 0637931c5bb710c3ea29581dcdec36ccbba7d514 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 2 Mar 2026 23:12:24 +0100 Subject: [PATCH 06/48] Use "" instead of null for the content to comply with TestOpenAiLlmInteractions::test_chat_completion_tool_call --- .../instrumentation/openai_java/ChatCompletionDecorator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index 95c746880cf..c8151c00a2a 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -131,7 +131,7 @@ private static LLMObs.LLMMessage llmMessage(ChatCompletion.Choice choice) { if (roleOpt.isPresent()) { role = String.valueOf(roleOpt.get()); } - String content = msg.content().orElse(null); + String content = msg.content().orElse(""); Optional> toolCallsOpt = msg.toolCalls(); if (toolCallsOpt.isPresent() && !toolCallsOpt.get().isEmpty()) { From 0cb41e1a9522f617afed58a4dcc2e1e5ef966388 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 3 Mar 2026 13:34:57 +0100 Subject: [PATCH 07/48] Add missing metatadata.tool_choice --- .../openai_java/ChatCompletionDecorator.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index c8151c00a2a..618c7982410 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -71,6 +71,24 @@ public void withChatCompletionCreateParams( params.n().ifPresent(v -> metadata.put("n", v)); params.seed().ifPresent(v -> metadata.put("seed", v)); span.setTag(CommonTags.METADATA, metadata); + params + .toolChoice() + .ifPresent( + toolChoice -> { + String choice = null; + if (toolChoice.isAuto()) { + choice = "auto"; + } else if (toolChoice.isAllowedToolChoice()) { + choice = "allowed_tools"; + } else if (toolChoice.isNamedToolChoice()) { + choice = "function"; + } else if (toolChoice.isNamedToolChoiceCustom()) { + choice = "custom"; + } + if (choice != null) { + metadata.put("tool_choice", choice); + } + }); } private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { From a42f8aa2e3a34c8bf14e031a502ca8bb9a468e6f Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 3 Mar 2026 21:09:03 +0100 Subject: [PATCH 08/48] Add missing tool_definitions --- .../openai_java/ChatCompletionDecorator.java | 108 ++++++++++++++++++ .../openai_java/CommonTags.java | 2 + .../openai_java/OpenAiDecorator.java | 1 - .../datadog/trace/api/llmobs/LLMObsTags.java | 1 + .../writer/ddintake/LLMObsSpanMapper.java | 1 + 5 files changed, 112 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index 618c7982410..e5def4f67bb 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -1,12 +1,16 @@ package datadog.trace.instrumentation.openai_java; +import com.openai.core.JsonValue; import com.openai.helpers.ChatCompletionAccumulator; +import com.openai.models.FunctionDefinition; import com.openai.models.chat.completions.ChatCompletion; import com.openai.models.chat.completions.ChatCompletionChunk; import com.openai.models.chat.completions.ChatCompletionCreateParams; +import com.openai.models.chat.completions.ChatCompletionFunctionTool; import com.openai.models.chat.completions.ChatCompletionMessage; import com.openai.models.chat.completions.ChatCompletionMessageParam; import com.openai.models.chat.completions.ChatCompletionMessageToolCall; +import com.openai.models.chat.completions.ChatCompletionTool; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -89,6 +93,110 @@ public void withChatCompletionCreateParams( metadata.put("tool_choice", choice); } }); + + List tools = params.tools().orElse(Collections.emptyList()); + if (!tools.isEmpty()) { + span.setTag(CommonTags.TOOL_DEFINITIONS, extractToolDefinitions(tools)); + } + } + + private List> extractToolDefinitions(List tools) { + List> toolDefinitions = new ArrayList<>(); + for (ChatCompletionTool tool : tools) { + if (tool.isFunction()) { + Map toolDef = extractFunctionToolDef(tool.asFunction()); + if (toolDef != null) { + toolDefinitions.add(toolDef); + } + } + } + return toolDefinitions; + } + + private static Map extractFunctionToolDef(ChatCompletionFunctionTool funcTool) { + // Try typed access first (works when built programmatically) + Optional funcDefOpt = funcTool._function().asKnown(); + if (funcDefOpt.isPresent()) { + FunctionDefinition funcDef = funcDefOpt.get(); + Map toolDef = new HashMap<>(); + toolDef.put("name", funcDef.name()); + funcDef.description().ifPresent(desc -> toolDef.put("description", desc)); + funcDef + .parameters() + .ifPresent( + params -> + toolDef.put("schema", jsonValueMapToObject(params._additionalProperties()))); + return toolDef; + } + + // Fall back to raw JSON extraction (when deserialized from HTTP request) + Optional rawOpt = funcTool._function().asUnknown(); + if (!rawOpt.isPresent()) { + return null; + } + Optional> objOpt = rawOpt.get().asObject(); + if (!objOpt.isPresent()) { + return null; + } + Map obj = objOpt.get(); + JsonValue nameValue = obj.get("name"); + if (nameValue == null) { + return null; + } + Optional nameOpt = nameValue.asString(); + if (!nameOpt.isPresent()) { + return null; + } + Map toolDef = new HashMap<>(); + toolDef.put("name", nameOpt.get()); + JsonValue descValue = obj.get("description"); + if (descValue != null) { + descValue.asString().ifPresent(desc -> toolDef.put("description", desc)); + } + JsonValue paramsValue = obj.get("parameters"); + if (paramsValue != null) { + Object schema = jsonValueToObject(paramsValue); + if (schema != null) { + toolDef.put("schema", schema); + } + } + return toolDef; + } + + private static Map jsonValueMapToObject(Map map) { + Map result = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), jsonValueToObject(entry.getValue())); + } + return result; + } + + private static Object jsonValueToObject(JsonValue value) { + Optional str = value.asString(); + if (str.isPresent()) { + return str.get(); + } + Optional num = value.asNumber(); + if (num.isPresent()) { + return num.get(); + } + Optional bool = value.asBoolean(); + if (bool.isPresent()) { + return bool.get(); + } + Optional> obj = value.asObject(); + if (obj.isPresent()) { + return jsonValueMapToObject(obj.get()); + } + Optional> arr = value.asArray(); + if (arr.isPresent()) { + List list = new ArrayList<>(); + for (JsonValue item : arr.get()) { + list.add(jsonValueToObject(item)); + } + return list; + } + return null; } private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index a4335a7edfe..ed5b689cd9b 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -28,6 +28,8 @@ interface CommonTags { String SERVICE = TAG_PREFIX + "service"; String PARENT_ID = TAG_PREFIX + "parent_id"; + String TOOL_DEFINITIONS = TAG_PREFIX + "tool_definitions"; + String METRIC_PREFIX = "_ml_obs_metric."; String INPUT_TOKENS = METRIC_PREFIX + "input_tokens"; String OUTPUT_TOKENS = METRIC_PREFIX + "output_tokens"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 2332b578ebf..6380f981797 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -13,7 +13,6 @@ import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.ClientDecorator; -import datadog.trace.core.CoreSpan; import java.util.List; public class OpenAiDecorator extends ClientDecorator { diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java index afa4f2b241e..130cf610dc0 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsTags.java @@ -12,4 +12,5 @@ public class LLMObsTags { public static final String MODEL_NAME = "model_name"; public static final String MODEL_VERSION = "model_version"; public static final String MODEL_PROVIDER = "model_provider"; + public static final String TOOL_DEFINITIONS = "tool_definitions"; } diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 7241d469006..e0cd7db3e02 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -230,6 +230,7 @@ private static final class MetaWriter implements MetadataConsumer { LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_NAME, LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_PROVIDER, LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_VERSION, + LLMOBS_TAG_PREFIX + LLMObsTags.TOOL_DEFINITIONS, LLMOBS_TAG_PREFIX + LLMObsTags.METADATA))); MetaWriter withWritable(Writable writable, Map errorInfo) { From 6e10255898af899792665ac5e0d84a84e204fe14 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 3 Mar 2026 21:51:50 +0100 Subject: [PATCH 09/48] Add source:integration tag --- .../datadog/trace/instrumentation/openai_java/CommonTags.java | 1 + .../trace/instrumentation/openai_java/OpenAiDecorator.java | 1 + 2 files changed, 2 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index ed5b689cd9b..0e550437026 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -20,6 +20,7 @@ interface CommonTags { String ML_APP = TAG_PREFIX + LLMObsTags.ML_APP; String VERSION = TAG_PREFIX + "version"; + String SOURCE = TAG_PREFIX + "source"; String ERROR = TAG_PREFIX + "error"; String ERROR_TYPE = TAG_PREFIX + "error_type"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 6380f981797..fae2880c082 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -98,6 +98,7 @@ public AgentSpan afterStart(AgentSpan span) { span.setTag(CommonTags.VERSION, wellKnownTags.getVersion()); span.setTag(CommonTags.ML_APP, Config.get().getLlmObsMlApp()); + span.setTag(CommonTags.SOURCE, "integration"); AgentSpanContext parent = LLMObsContext.current(); String parentSpanId = LLMObsContext.ROOT_SPAN_ID; From 34f3a07ec396815b6878bcf56024d9d1dac861e8 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 11:31:52 +0100 Subject: [PATCH 10/48] Add missing _dd attribute to the llmobs span event --- .../llmobs/writer/ddintake/LLMObsSpanMapper.java | 16 ++++++++++++++-- .../writer/ddintake/LLMObsSpanMapperTest.groovy | 4 ++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index e0cd7db3e02..1dec4ad62e5 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -52,6 +52,8 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] SPAN_ID = "span_id".getBytes(StandardCharsets.UTF_8); private static final byte[] TRACE_ID = "trace_id".getBytes(StandardCharsets.UTF_8); + private static final byte[] DD = "_dd".getBytes(StandardCharsets.UTF_8); + private static final byte[] APM_TRACE_ID = "apm_trace_id".getBytes(StandardCharsets.UTF_8); private static final byte[] PARENT_ID = "parent_id".getBytes(StandardCharsets.UTF_8); private static final byte[] NAME = "name".getBytes(StandardCharsets.UTF_8); private static final byte[] DURATION = "duration".getBytes(StandardCharsets.UTF_8); @@ -120,7 +122,7 @@ public void map(List> trace, Writable writable) { } for (CoreSpan span : llmobsSpans) { - writable.startMap(11); + writable.startMap(12); // 1 writable.writeUTF8(SPAN_ID); writable.writeString(String.valueOf(span.getSpanId()), null); @@ -156,7 +158,17 @@ public void map(List> trace, Writable writable) { writable.writeUTF8(STATUS); writable.writeString(errored ? "error" : "ok", null); - /* 9 (metrics), 10 (tags), 11 meta */ + // 9 + writable.writeUTF8(DD); + writable.startMap(3); + writable.writeUTF8(SPAN_ID); + writable.writeString(String.valueOf(span.getSpanId()), null); + writable.writeUTF8(TRACE_ID); + writable.writeString(span.getTraceId().toHexString(), null); + writable.writeUTF8(APM_TRACE_ID); + writable.writeString(span.getTraceId().toHexString(), null); + + /* 10 (metrics), 11 (tags), 12 meta */ span.processTagsAndBaggage(metaWriter.withWritable(writable, getErrorsMap(span))); } diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 20656ba2a1e..74fb39fed90 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -102,6 +102,10 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData.containsKey("start_ns") spanData.containsKey("duration") spanData["error"] == 0 + spanData.containsKey("_dd") + spanData["_dd"]["span_id"] == spanData["span_id"] + spanData["_dd"]["trace_id"] == spanData["trace_id"] + spanData["_dd"]["apm_trace_id"] == spanData["trace_id"] spanData.containsKey("meta") spanData["meta"]["span.kind"] == "llm" From a0c1139404dcc77a654c3ed30ebd91c2808db995 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 11:49:52 +0100 Subject: [PATCH 11/48] Add missing error tags --- .../writer/ddintake/LLMObsSpanMapper.java | 37 +++++++++++++++---- .../ddintake/LLMObsSpanMapperTest.groovy | 11 +++++- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 1dec4ad62e5..f4c31e2c6ec 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -60,6 +60,9 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] START_NS = "start_ns".getBytes(StandardCharsets.UTF_8); private static final byte[] STATUS = "status".getBytes(StandardCharsets.UTF_8); private static final byte[] ERROR = "error".getBytes(StandardCharsets.UTF_8); + private static final byte[] ERROR_MESSAGE = "message".getBytes(StandardCharsets.UTF_8); + private static final byte[] ERROR_TYPE = "type".getBytes(StandardCharsets.UTF_8); + private static final byte[] ERROR_STACK = "stack".getBytes(StandardCharsets.UTF_8); private static final byte[] META = "meta".getBytes(StandardCharsets.UTF_8); private static final byte[] METADATA = "metadata".getBytes(StandardCharsets.UTF_8); @@ -215,15 +218,15 @@ private static Map getErrorsMap(CoreSpan span) { Map errors = new HashMap<>(); String errorMsg = span.getTag(DDTags.ERROR_MSG); if (errorMsg != null && !errorMsg.isEmpty()) { - errors.put(DDTags.ERROR_MSG, errorMsg); + errors.put("message", errorMsg); } String errorType = span.getTag(DDTags.ERROR_TYPE); if (errorType != null && !errorType.isEmpty()) { - errors.put(DDTags.ERROR_TYPE, errorType); + errors.put("type", errorType); } String errorStack = span.getTag(DDTags.ERROR_STACK); if (errorStack != null && !errorStack.isEmpty()) { - errors.put(DDTags.ERROR_STACK, errorStack); + errors.put("stack", errorStack); } return errors; } @@ -306,15 +309,35 @@ public void accept(Metadata metadata) { } // write meta (11) - int metaSize = tagsToRemapToMeta.size() + 1 + (null != errorInfo ? errorInfo.size() : 0); + int metaSize = + tagsToRemapToMeta.size() + + 1 + + (null != errorInfo && !errorInfo.isEmpty() ? 1 : 0); writable.writeUTF8(META); writable.startMap(metaSize); writable.writeUTF8(SPAN_KIND); writable.writeString(spanKind, null); - for (Map.Entry error : errorInfo.entrySet()) { - writable.writeUTF8(error.getKey().getBytes()); - writable.writeString(error.getValue(), null); + if (null != errorInfo && !errorInfo.isEmpty()) { + writable.writeUTF8(ERROR); + writable.startMap(errorInfo.size()); + for (Map.Entry error : errorInfo.entrySet()) { + switch (error.getKey()) { + case "message": + writable.writeUTF8(ERROR_MESSAGE); + break; + case "type": + writable.writeUTF8(ERROR_TYPE); + break; + case "stack": + writable.writeUTF8(ERROR_STACK); + break; + default: + writable.writeString(error.getKey(), null); + break; + } + writable.writeString(error.getValue(), null); + } } for (Map.Entry tag : tagsToRemapToMeta.entrySet()) { diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 74fb39fed90..1923c07470b 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import datadog.communication.serialization.ByteBufferConsumer import datadog.communication.serialization.FlushingBuffer import datadog.communication.serialization.msgpack.MsgPackWriter +import datadog.trace.api.DDTags import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes import datadog.trace.bootstrap.instrumentation.api.Tags @@ -44,6 +45,10 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { llmSpan.setTag("_ml_obs_tag.input", inputMessages) llmSpan.setTag("_ml_obs_tag.output", outputMessages) llmSpan.setTag("_ml_obs_tag.metadata", [temperature: 0.7, max_tokens: 100]) + llmSpan.setError(true) + llmSpan.setTag(DDTags.ERROR_MSG, "boom") + llmSpan.setTag(DDTags.ERROR_TYPE, "java.lang.IllegalStateException") + llmSpan.setTag(DDTags.ERROR_STACK, "stacktrace") llmSpan.finish() @@ -101,7 +106,7 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData.containsKey("trace_id") spanData.containsKey("start_ns") spanData.containsKey("duration") - spanData["error"] == 0 + spanData["error"] == 1 spanData.containsKey("_dd") spanData["_dd"]["span_id"] == spanData["span_id"] spanData["_dd"]["trace_id"] == spanData["trace_id"] @@ -109,6 +114,10 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData.containsKey("meta") spanData["meta"]["span.kind"] == "llm" + spanData["meta"].containsKey("error") + spanData["meta"]["error"]["message"] == "boom" + spanData["meta"]["error"]["type"] == "java.lang.IllegalStateException" + spanData["meta"]["error"]["stack"] == "stacktrace" spanData["meta"].containsKey("input") spanData["meta"]["input"].containsKey("messages") spanData["meta"]["input"]["messages"][0].containsKey("content") From effc34302592382744dfc0dc100b264c788923c0 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 12:24:40 +0100 Subject: [PATCH 12/48] Remove error from the llmobs span event. It must be part of meta block --- .../writer/ddintake/LLMObsSpanMapper.java | 18 +++++------------- .../ddintake/LLMObsSpanMapperTest.groovy | 1 - 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index f4c31e2c6ec..e0b5dce3550 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -125,7 +125,7 @@ public void map(List> trace, Writable writable) { } for (CoreSpan span : llmobsSpans) { - writable.startMap(12); + writable.startMap(11); // 1 writable.writeUTF8(SPAN_ID); writable.writeString(String.valueOf(span.getSpanId()), null); @@ -152,16 +152,10 @@ public void map(List> trace, Writable writable) { writable.writeFloat(span.getDurationNano()); // 7 - writable.writeUTF8(ERROR); - writable.writeInt(span.getError()); - - boolean errored = span.getError() == 1; - - // 8 writable.writeUTF8(STATUS); - writable.writeString(errored ? "error" : "ok", null); + writable.writeString(span.getError() == 0 ? "ok" : "error", null); - // 9 + // 8 writable.writeUTF8(DD); writable.startMap(3); writable.writeUTF8(SPAN_ID); @@ -171,7 +165,7 @@ public void map(List> trace, Writable writable) { writable.writeUTF8(APM_TRACE_ID); writable.writeString(span.getTraceId().toHexString(), null); - /* 10 (metrics), 11 (tags), 12 meta */ + /* 9 (metrics), 10 (tags), 11 meta */ span.processTagsAndBaggage(metaWriter.withWritable(writable, getErrorsMap(span))); } @@ -310,9 +304,7 @@ public void accept(Metadata metadata) { // write meta (11) int metaSize = - tagsToRemapToMeta.size() - + 1 - + (null != errorInfo && !errorInfo.isEmpty() ? 1 : 0); + tagsToRemapToMeta.size() + 1 + (null != errorInfo && !errorInfo.isEmpty() ? 1 : 0); writable.writeUTF8(META); writable.startMap(metaSize); writable.writeUTF8(SPAN_KIND); diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 1923c07470b..6140431b836 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -106,7 +106,6 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData.containsKey("trace_id") spanData.containsKey("start_ns") spanData.containsKey("duration") - spanData["error"] == 1 spanData.containsKey("_dd") spanData["_dd"]["span_id"] == spanData["span_id"] spanData["_dd"]["trace_id"] == spanData["trace_id"] From c0e38761a4f2515ec1ed43e7173c6be3096f25bd Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 13:06:00 +0100 Subject: [PATCH 13/48] Add missing meta.text.verbosity --- .../openai-java/openai-java-3.0/build.gradle | 2 +- .../openai_java/ResponseDecorator.java | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle index cd654d03334..ceda41c2804 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/build.gradle @@ -1,6 +1,6 @@ apply from: "$rootDir/gradle/java.gradle" -def minVer = '3.0.0' +def minVer = '3.0.1' muzzle { pass { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index db2258785d4..e7be85b77c6 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -424,6 +424,7 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { (Response.Truncation t) -> metadata.put("truncation", t._value().asString().orElse(null))); + Map textMap = new HashMap<>(); response .text() .ifPresent( @@ -432,7 +433,6 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { .format() .ifPresent( format -> { - Map textMap = new HashMap<>(); Map formatMap = new HashMap<>(); if (format.isText()) { formatMap.put("type", "text"); @@ -442,9 +442,17 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { formatMap.put("type", "json_object"); } textMap.put("format", formatMap); - metadata.put("text", textMap); + }); + textConfig + .verbosity() + .ifPresent( + verbosity -> { + textMap.put("verbosity", verbosity.asString()); }); }); + if (!textMap.isEmpty()) { + metadata.put("text", textMap); + } if (stream) { metadata.put("stream", true); From b00077055aec4e578bab797a76bebd3c070ffb78 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 15:42:47 +0100 Subject: [PATCH 14/48] Add summaryText and encrypted_content --- .../openai_java/ResponseDecorator.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index e7be85b77c6..320d82da76f 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -496,12 +496,30 @@ private List extractResponseOutputMessages(List writer.name("encrypted_content").value(v)); - writer.name("id").value(reasoning.id()); + + String id = reasoning.id(); + writer.name("id").value(id == null ? "" : id); + writer.endObject(); + messages.add(LLMObs.LLMMessage.from("reasoning", writer.toString())); } } From 53471a2cb6041f96d77c3f1e1ca86ae10fcaeead Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 16:24:38 +0100 Subject: [PATCH 15/48] Add missing tool_calls and tool_results for responses --- .../openai_java/ResponseDecorator.java | 18 ++++++++ .../openai_java/ToolCallExtractor.java | 42 +++++++++++++++++++ .../java/datadog/trace/api/llmobs/LLMObs.java | 5 +++ .../writer/ddintake/LLMObsSpanMapper.java | 10 +++-- 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 320d82da76f..c61c7cc712f 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -6,6 +6,7 @@ import com.openai.models.ResponsesModel; import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; +import com.openai.models.responses.ResponseCustomToolCall; import com.openai.models.responses.ResponseFunctionToolCall; import com.openai.models.responses.ResponseInputContent; import com.openai.models.responses.ResponseInputItem; @@ -486,6 +487,23 @@ private List extractResponseOutputMessages(List toolCalls = Collections.singletonList(toolCall); messages.add(LLMObs.LLMMessage.from("assistant", null, toolCalls)); } + } else if (item.isCustomToolCall()) { + ResponseCustomToolCall customToolCall = item.asCustomToolCall(); + LLMObs.ToolCall toolCall = ToolCallExtractor.getToolCall(customToolCall); + if (toolCall != null) { + messages.add( + LLMObs.LLMMessage.from("assistant", null, Collections.singletonList(toolCall))); + } + } else if (item.isMcpCall()) { + ResponseOutputItem.McpCall mcpCall = item.asMcpCall(); + LLMObs.ToolCall toolCall = ToolCallExtractor.getToolCall(mcpCall); + List toolCalls = + toolCall == null ? null : Collections.singletonList(toolCall); + String outputText = mcpCall.output().orElse(""); + LLMObs.ToolResult toolResult = + LLMObs.ToolResult.from(mcpCall.name(), "mcp_tool_result", mcpCall.id(), outputText); + List toolResults = Collections.singletonList(toolResult); + messages.add(LLMObs.LLMMessage.from("assistant", null, toolCalls, toolResults)); } else if (item.isMessage()) { ResponseOutputMessage message = item.asMessage(); String textContent = extractMessageContent(message); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java index 357c73de0aa..21d2bf6835d 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java @@ -4,7 +4,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.openai.models.chat.completions.ChatCompletionMessageFunctionToolCall; import com.openai.models.chat.completions.ChatCompletionMessageToolCall; +import com.openai.models.responses.ResponseCustomToolCall; import com.openai.models.responses.ResponseFunctionToolCall; +import com.openai.models.responses.ResponseOutputItem.McpCall; import datadog.trace.api.llmobs.LLMObs; import java.util.Collections; import java.util.Map; @@ -64,6 +66,46 @@ public static LLMObs.ToolCall getToolCall(ResponseFunctionToolCall functionCall) return null; } + public static LLMObs.ToolCall getToolCall(ResponseCustomToolCall customToolCall) { + try { + String name = customToolCall.name(); + String callId = customToolCall.callId(); + String inputJson = customToolCall.input(); + + String type = "custom_tool_call"; + Optional typeOpt = customToolCall._type().asString(); + if (typeOpt.isPresent()) { + type = typeOpt.get(); + } + + Map arguments = parseArguments(inputJson); + return LLMObs.ToolCall.from(name, type, callId, arguments); + } catch (Exception e) { + log.debug("Failed to extract custom tool call information", e); + } + return null; + } + + public static LLMObs.ToolCall getToolCall(McpCall mcpCall) { + try { + String name = mcpCall.name(); + String callId = mcpCall.id(); + String argumentsJson = mcpCall.arguments(); + + String type = "mcp_call"; + Optional typeOpt = mcpCall._type().asString(); + if (typeOpt.isPresent()) { + type = typeOpt.get(); + } + + Map arguments = parseArguments(argumentsJson); + return LLMObs.ToolCall.from(name, type, callId, arguments); + } catch (Exception e) { + log.debug("Failed to extract MCP tool call information", e); + } + return null; + } + private static Map parseArguments(String argumentsJson) { try { return MAPPER.readValue(argumentsJson, MAP_TYPE_REF); diff --git a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java index 512a3106ce6..629faa23f5a 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java @@ -217,6 +217,11 @@ public static LLMMessage from(String role, String content, List toolCa return new LLMMessage(role, content, toolCalls, null); } + public static LLMMessage from( + String role, String content, List toolCalls, List toolResults) { + return new LLMMessage(role, content, toolCalls, toolResults); + } + public static LLMMessage from(String role, String content) { return new LLMMessage(role, content, null, null); } diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index e0b5dce3550..b4dba725406 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -354,14 +354,18 @@ public void accept(Metadata metadata) { List toolResults = message.getToolResults(); boolean hasToolCalls = null != toolCalls && !toolCalls.isEmpty(); boolean hasToolResults = null != toolResults && !toolResults.isEmpty(); - int mapSize = 2; // role and content + boolean hasContent = message.getContent() != null; + int mapSize = 1; // role + if (hasContent) mapSize++; if (hasToolCalls) mapSize++; if (hasToolResults) mapSize++; writable.startMap(mapSize); writable.writeUTF8(LLM_MESSAGE_ROLE); writable.writeString(message.getRole(), null); - writable.writeUTF8(LLM_MESSAGE_CONTENT); - writable.writeString(message.getContent(), null); + if (hasContent) { + writable.writeUTF8(LLM_MESSAGE_CONTENT); + writable.writeString(message.getContent(), null); + } if (hasToolCalls) { writable.writeUTF8(LLM_MESSAGE_TOOL_CALLS); writable.startArray(toolCalls.size()); From 2207c46e77a2f1a33948f7a2fc60d566956bd266 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 4 Mar 2026 16:55:53 +0100 Subject: [PATCH 16/48] Always set stream param to produce the same request body to be aligned with python openai instrumentation and system-tests --- .../trace/instrumentation/openai_java/ResponseDecorator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index c61c7cc712f..dce941af568 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -455,9 +455,7 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { metadata.put("text", textMap); } - if (stream) { - metadata.put("stream", true); - } + metadata.put("stream", stream); span.setTag(CommonTags.METADATA, metadata); From ca6e2d13649115fe897a7c2cca65131e4c11f912 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 13:41:50 +0100 Subject: [PATCH 17/48] Add OpenAI prompt-tracking reconstruction for responses (input.prompt with variables + chat_template, longest-first overlap handling) and support map-based LLM input serialization (messages + prompt) in LLMObs mapper. Also filter empty instruction messages to match system-test expectations. --- .../openai_java/CommonTags.java | 1 + .../openai_java/ResponseDecorator.java | 365 +++++++++++++++++- .../writer/ddintake/LLMObsSpanMapper.java | 151 +++++--- 3 files changed, 451 insertions(+), 66 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index 0e550437026..228b3d52a81 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -39,4 +39,5 @@ interface CommonTags { String CACHE_READ_INPUT_TOKENS = METRIC_PREFIX + "cache_read_input_tokens"; String REQUEST_REASONING = "_ml_obs_request.reasoning"; + String REQUEST_PROMPT = "_ml_obs_request.prompt"; } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index dce941af568..3b7e4b50a2a 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -7,6 +7,7 @@ import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; import com.openai.models.responses.ResponseCustomToolCall; +import com.openai.models.responses.EasyInputMessage; import com.openai.models.responses.ResponseFunctionToolCall; import com.openai.models.responses.ResponseInputContent; import com.openai.models.responses.ResponseInputItem; @@ -15,6 +16,7 @@ import com.openai.models.responses.ResponseOutputText; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; +import com.openai.models.responses.ResponsePrompt; import datadog.json.JsonWriter; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; @@ -24,6 +26,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -32,6 +35,8 @@ public class ResponseDecorator { public static final ResponseDecorator DECORATE = new ResponseDecorator(); private static final CharSequence RESPONSES_CREATE = UTF8BytesString.create("createResponse"); + private static final String IMAGE_FALLBACK_MARKER = "[image]"; + private static final String FILE_FALLBACK_MARKER = "[file]"; private final boolean llmObsEnabled = Config.get().isLlmObsEnabled(); @@ -111,13 +116,26 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params extractReasoningFromParams(params) .ifPresent(reasoningMap -> span.setTag(CommonTags.REQUEST_REASONING, reasoningMap)); + + extractPromptFromParams(params).ifPresent(prompt -> span.setTag(CommonTags.REQUEST_PROMPT, prompt)); } private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { - if (item.isMessage()) { + if (item.isEasyInputMessage()) { + EasyInputMessage message = item.asEasyInputMessage(); + String role = message.role().asString(); + String content = extractEasyInputMessageContent(message); + if (content == null || content.isEmpty()) { + return null; + } + return LLMObs.LLMMessage.from(role, content); + } else if (item.isMessage()) { ResponseInputItem.Message message = item.asMessage(); String role = message.role().asString(); String content = extractInputMessageContent(message); + if (content == null || content.isEmpty()) { + return null; + } return LLMObs.LLMMessage.from(role, content); } else if (item.isFunctionCall()) { // Function call is mapped to assistant message with tool_calls @@ -139,6 +157,26 @@ private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { return null; } + private String extractEasyInputMessageContent(EasyInputMessage message) { + if (message.content().isTextInput()) { + String content = message.content().asTextInput(); + return content == null || content.isEmpty() ? null : content; + } + + if (message.content().isResponseInputMessageContentList()) { + StringBuilder contentBuilder = new StringBuilder(); + for (ResponseInputContent content : message.content().asResponseInputMessageContentList()) { + String contentPart = extractInputContentText(content); + if (contentPart != null) { + contentBuilder.append(contentPart); + } + } + String result = contentBuilder.toString(); + return result.isEmpty() ? null : result; + } + return null; + } + private LLMObs.LLMMessage extractMessageFromRawJson(JsonValue jsonValue) { Optional> objOpt = jsonValue.asObject(); if (!objOpt.isPresent()) { @@ -324,14 +362,35 @@ private String removeQuotes(String str) { private String extractInputMessageContent(ResponseInputItem.Message message) { StringBuilder contentBuilder = new StringBuilder(); for (ResponseInputContent content : message.content()) { - if (content.isInputText()) { - contentBuilder.append(content.asInputText().text()); + String contentPart = extractInputContentText(content); + if (contentPart != null) { + contentBuilder.append(contentPart); } } String result = contentBuilder.toString(); return result.isEmpty() ? null : result; } + private String extractInputContentText(ResponseInputContent content) { + if (content.isInputText()) { + return content.asInputText().text(); + } + if (content.isInputImage()) { + return content.asInputImage().imageUrl().orElse(content.asInputImage().fileId().orElse("")); + } + if (content.isInputFile()) { + return content + .asInputFile() + .fileUrl() + .orElse( + content + .asInputFile() + .fileId() + .orElse(content.asInputFile().filename().orElse(FILE_FALLBACK_MARKER))); + } + return null; + } + private Optional> extractReasoningFromParams(ResponseCreateParams params) { JsonField reasoningField = params._reasoning(); if (reasoningField.isMissing()) { @@ -399,6 +458,8 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { span.setTag(CommonTags.OUTPUT, outputMessages); } + enrichInputWithPromptTracking(span, response); + Map metadata = new HashMap<>(); Object reasoningTag = span.getTag(CommonTags.REQUEST_REASONING); @@ -474,6 +535,304 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { }); } + private void enrichInputWithPromptTracking(AgentSpan span, Response response) { + Object promptTag = span.getTag(CommonTags.REQUEST_PROMPT); + if (!(promptTag instanceof Map)) { + return; + } + + Map prompt = new LinkedHashMap<>((Map) promptTag); + Map variables = Collections.emptyMap(); + Object variablesTag = prompt.get("variables"); + if (variablesTag instanceof Map) { + variables = (Map) variablesTag; + } + + Map inputMap = new LinkedHashMap<>(); + Object inputTag = span.getTag(CommonTags.INPUT); + if (inputTag instanceof Map) { + inputMap.putAll((Map) inputTag); + } + + List inputMessages = extractInputMessagesForPromptTracking(span, response); + if (!inputMessages.isEmpty()) { + inputMap.put("messages", inputMessages); + } + + List> chatTemplate = extractChatTemplate(inputMessages, variables); + if (!chatTemplate.isEmpty()) { + prompt.put("chat_template", chatTemplate); + } + + inputMap.put("prompt", prompt); + + span.setTag(CommonTags.INPUT, inputMap); + } + + private List> extractChatTemplate( + List messages, Map variables) { + Map valueToPlaceholder = new LinkedHashMap<>(); + for (Map.Entry variable : variables.entrySet()) { + if (variable.getValue() == null) { + continue; + } + String valueStr = String.valueOf(variable.getValue()); + if (valueStr.isEmpty() + || IMAGE_FALLBACK_MARKER.equals(valueStr) + || FILE_FALLBACK_MARKER.equals(valueStr)) { + continue; + } + valueToPlaceholder.put(valueStr, "{{" + variable.getKey() + "}}"); + } + + List sortedValues = new ArrayList<>(valueToPlaceholder.keySet()); + sortedValues.sort((a, b) -> Integer.compare(b.length(), a.length())); + + List> chatTemplate = new ArrayList<>(); + for (LLMObs.LLMMessage message : messages) { + String role = message.getRole(); + String content = message.getContent(); + if (role == null || role.isEmpty() || content == null || content.isEmpty()) { + continue; + } + + String templateContent = content; + for (String value : sortedValues) { + templateContent = templateContent.replace(value, valueToPlaceholder.get(value)); + } + + Map messageMap = new LinkedHashMap<>(); + messageMap.put("role", role); + messageMap.put("content", templateContent); + chatTemplate.add(messageMap); + } + return chatTemplate; + } + + private List extractInputMessagesForPromptTracking( + AgentSpan span, Response response) { + List messages = new ArrayList<>(); + + Object inputTag = span.getTag(CommonTags.INPUT); + if (inputTag instanceof List) { + for (Object messageObj : (List) inputTag) { + if (messageObj instanceof LLMObs.LLMMessage) { + messages.add((LLMObs.LLMMessage) messageObj); + } + } + } else if (inputTag instanceof Map) { + Object messagesObj = ((Map) inputTag).get("messages"); + if (messagesObj instanceof List) { + for (Object messageObj : (List) messagesObj) { + if (messageObj instanceof LLMObs.LLMMessage) { + messages.add((LLMObs.LLMMessage) messageObj); + } + } + } + } + + if (!messages.isEmpty()) { + return messages; + } + + response + .instructions() + .ifPresent( + instructions -> { + if (instructions.isInputItemList()) { + for (ResponseInputItem item : instructions.asInputItemList()) { + LLMObs.LLMMessage message = extractInputItemMessage(item); + if (message != null) { + messages.add(message); + } + } + } else if (instructions.isString()) { + String text = instructions.asString(); + if (text != null && !text.isEmpty()) { + messages.add(LLMObs.LLMMessage.from("user", text)); + } + } + }); + + if (!messages.isEmpty()) { + return messages; + } + + // Fallback for SDK union parsing mismatches: parse raw instructions payload. + Optional rawInstructions = response._instructions().asUnknown(); + if (rawInstructions.isPresent()) { + Optional> rawList = rawInstructions.get().asArray(); + if (rawList.isPresent()) { + for (JsonValue item : rawList.get()) { + LLMObs.LLMMessage message = extractMessageFromRawInstruction(item); + if (message != null) { + messages.add(message); + } + } + } + } + + return messages; + } + + private LLMObs.LLMMessage extractMessageFromRawInstruction(JsonValue instructionValue) { + Optional> objOpt = instructionValue.asObject(); + if (!objOpt.isPresent()) { + return null; + } + Map obj = objOpt.get(); + String role = getJsonString(obj.get("role")); + if (role == null || role.isEmpty()) { + return null; + } + + JsonValue contentValue = obj.get("content"); + if (contentValue == null) { + return null; + } + Optional> contentList = contentValue.asArray(); + if (!contentList.isPresent()) { + return null; + } + + StringBuilder contentBuilder = new StringBuilder(); + for (JsonValue contentItem : contentList.get()) { + Optional> contentObjOpt = contentItem.asObject(); + if (!contentObjOpt.isPresent()) { + continue; + } + Map contentObj = contentObjOpt.get(); + String type = getJsonString(contentObj.get("type")); + if ("input_text".equals(type)) { + String text = getJsonString(contentObj.get("text")); + if (text != null) { + contentBuilder.append(text); + } + } else if ("input_image".equals(type)) { + String imageUrl = getJsonString(contentObj.get("image_url")); + if (imageUrl != null && !imageUrl.isEmpty()) { + contentBuilder.append(imageUrl); + } else { + String fileId = getJsonString(contentObj.get("file_id")); + contentBuilder.append(fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId); + } + } else if ("input_file".equals(type)) { + String fileUrl = getJsonString(contentObj.get("file_url")); + if (fileUrl != null && !fileUrl.isEmpty()) { + contentBuilder.append(fileUrl); + } else { + String fileId = getJsonString(contentObj.get("file_id")); + if (fileId != null && !fileId.isEmpty()) { + contentBuilder.append(fileId); + } else { + String filename = getJsonString(contentObj.get("filename")); + contentBuilder.append( + filename == null || filename.isEmpty() ? FILE_FALLBACK_MARKER : filename); + } + } + } + } + + String content = contentBuilder.toString(); + if (content.isEmpty()) { + return null; + } + return LLMObs.LLMMessage.from(role, content); + } + + private Optional> extractPromptFromParams(ResponseCreateParams params) { + Optional promptOpt = params.prompt(); + if (!promptOpt.isPresent()) { + return Optional.empty(); + } + + ResponsePrompt prompt = promptOpt.get(); + Map promptMap = new LinkedHashMap<>(); + + String id = prompt.id(); + if (id != null && !id.isEmpty()) { + promptMap.put("id", id); + } + prompt.version().ifPresent(version -> promptMap.put("version", version)); + prompt + .variables() + .ifPresent( + variables -> { + Map normalized = normalizePromptVariables(variables); + if (!normalized.isEmpty()) { + promptMap.put("variables", normalized); + } + }); + + return promptMap.isEmpty() ? Optional.empty() : Optional.of(promptMap); + } + + private Map normalizePromptVariables(ResponsePrompt.Variables variables) { + Map normalized = new LinkedHashMap<>(); + for (Map.Entry entry : variables._additionalProperties().entrySet()) { + Object value = normalizePromptVariable(entry.getValue()); + if (value != null) { + normalized.put(entry.getKey(), value); + } + } + return normalized; + } + + private Object normalizePromptVariable(JsonValue value) { + if (value == null) { + return null; + } + + Optional asString = value.asString(); + if (asString.isPresent()) { + return asString.get(); + } + + Optional> asObject = value.asObject(); + if (!asObject.isPresent()) { + return value.toString(); + } + + Map obj = asObject.get(); + String type = getJsonString(obj.get("type")); + String text = getJsonString(obj.get("text")); + if (text != null && !text.isEmpty()) { + return text; + } + + if ("input_image".equals(type)) { + String imageUrl = getJsonString(obj.get("image_url")); + if (imageUrl != null && !imageUrl.isEmpty()) { + return imageUrl; + } + String fileId = getJsonString(obj.get("file_id")); + return fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId; + } + + if ("input_file".equals(type)) { + String fileUrl = getJsonString(obj.get("file_url")); + if (fileUrl != null && !fileUrl.isEmpty()) { + return fileUrl; + } + String fileId = getJsonString(obj.get("file_id")); + if (fileId != null && !fileId.isEmpty()) { + return fileId; + } + String filename = getJsonString(obj.get("filename")); + return filename == null || filename.isEmpty() ? FILE_FALLBACK_MARKER : filename; + } + + return value.toString(); + } + + private String getJsonString(JsonValue value) { + if (value == null) { + return null; + } + Optional asString = value.asString(); + return asString.orElse(null); + } + private List extractResponseOutputMessages(List output) { List messages = new ArrayList<>(); diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index b4dba725406..7ff16cbed4e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -337,74 +337,19 @@ public void accept(Metadata metadata) { Object val = tag.getValue(); if (key.equals(INPUT) || key.equals(OUTPUT)) { if (spanKind.equals(Tags.LLMOBS_LLM_SPAN_KIND)) { - if (!(val instanceof List)) { + writable.writeString(key, null); + if (val instanceof List) { + writable.startMap(1); + writable.writeString("messages", null); + writeLlmMessages((List) val); + } else if (key.equals(INPUT) && val instanceof Map) { + writeLlmInputMap((Map) val); + } else { LOGGER.warn( "unexpectedly found incorrect type for LLM span IO {}, expecting list", val.getClass().getName()); continue; } - writable.writeString(key, null); - writable.startMap(1); - // llm span kind must have llm objects - List messages = (List) val; - writable.writeString("messages", null); - writable.startArray(messages.size()); - for (LLMObs.LLMMessage message : messages) { - List toolCalls = message.getToolCalls(); - List toolResults = message.getToolResults(); - boolean hasToolCalls = null != toolCalls && !toolCalls.isEmpty(); - boolean hasToolResults = null != toolResults && !toolResults.isEmpty(); - boolean hasContent = message.getContent() != null; - int mapSize = 1; // role - if (hasContent) mapSize++; - if (hasToolCalls) mapSize++; - if (hasToolResults) mapSize++; - writable.startMap(mapSize); - writable.writeUTF8(LLM_MESSAGE_ROLE); - writable.writeString(message.getRole(), null); - if (hasContent) { - writable.writeUTF8(LLM_MESSAGE_CONTENT); - writable.writeString(message.getContent(), null); - } - if (hasToolCalls) { - writable.writeUTF8(LLM_MESSAGE_TOOL_CALLS); - writable.startArray(toolCalls.size()); - for (LLMObs.ToolCall toolCall : toolCalls) { - Map arguments = toolCall.getArguments(); - boolean hasArguments = null != arguments && !arguments.isEmpty(); - writable.startMap(hasArguments ? 4 : 3); - writable.writeUTF8(LLM_TOOL_CALL_NAME); - writable.writeString(toolCall.getName(), null); - writable.writeUTF8(LLM_TOOL_CALL_TYPE); - writable.writeString(toolCall.getType(), null); - writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); - writable.writeString(toolCall.getToolId(), null); - if (hasArguments) { - writable.writeUTF8(LLM_TOOL_CALL_ARGUMENTS); - writable.startMap(arguments.size()); - for (Map.Entry argument : arguments.entrySet()) { - writable.writeString(argument.getKey(), null); - writable.writeObject(argument.getValue(), null); - } - } - } - } - if (hasToolResults) { - writable.writeUTF8(LLM_MESSAGE_TOOL_RESULTS); - writable.startArray(toolResults.size()); - for (LLMObs.ToolResult toolResult : toolResults) { - writable.startMap(4); - writable.writeUTF8(LLM_TOOL_CALL_NAME); - writable.writeString(toolResult.getName(), null); - writable.writeUTF8(LLM_TOOL_CALL_TYPE); - writable.writeString(toolResult.getType(), null); - writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); - writable.writeString(toolResult.getToolId(), null); - writable.writeUTF8(LLM_TOOL_RESULT_RESULT); - writable.writeString(toolResult.getResult(), null); - } - } - } } else if (spanKind.equals(Tags.LLMOBS_EMBEDDING_SPAN_KIND) && key.equals(INPUT)) { if (!(val instanceof List)) { LOGGER.warn( @@ -442,6 +387,86 @@ public void accept(Metadata metadata) { } } } + + private void writeLlmInputMap(Map inputMap) { + writable.startMap(inputMap.size()); + for (Map.Entry entry : inputMap.entrySet()) { + String inputKey = String.valueOf(entry.getKey()); + Object inputValue = entry.getValue(); + writable.writeString(inputKey, null); + if ("messages".equals(inputKey) && inputValue instanceof List) { + writeLlmMessages((List) inputValue); + } else { + writable.writeObject(inputValue, null); + } + } + } + + private void writeLlmMessages(List messages) { + writable.startArray(messages.size()); + for (Object messageObj : messages) { + if (!(messageObj instanceof LLMObs.LLMMessage)) { + writable.writeObject(messageObj, null); + continue; + } + + LLMObs.LLMMessage message = (LLMObs.LLMMessage) messageObj; + List toolCalls = message.getToolCalls(); + List toolResults = message.getToolResults(); + boolean hasToolCalls = null != toolCalls && !toolCalls.isEmpty(); + boolean hasToolResults = null != toolResults && !toolResults.isEmpty(); + boolean hasContent = message.getContent() != null; + int mapSize = 1; + if (hasContent) mapSize++; + if (hasToolCalls) mapSize++; + if (hasToolResults) mapSize++; + writable.startMap(mapSize); + writable.writeUTF8(LLM_MESSAGE_ROLE); + writable.writeString(message.getRole(), null); + if (hasContent) { + writable.writeUTF8(LLM_MESSAGE_CONTENT); + writable.writeString(message.getContent(), null); + } + if (hasToolCalls) { + writable.writeUTF8(LLM_MESSAGE_TOOL_CALLS); + writable.startArray(toolCalls.size()); + for (LLMObs.ToolCall toolCall : toolCalls) { + Map arguments = toolCall.getArguments(); + boolean hasArguments = null != arguments && !arguments.isEmpty(); + writable.startMap(hasArguments ? 4 : 3); + writable.writeUTF8(LLM_TOOL_CALL_NAME); + writable.writeString(toolCall.getName(), null); + writable.writeUTF8(LLM_TOOL_CALL_TYPE); + writable.writeString(toolCall.getType(), null); + writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); + writable.writeString(toolCall.getToolId(), null); + if (hasArguments) { + writable.writeUTF8(LLM_TOOL_CALL_ARGUMENTS); + writable.startMap(arguments.size()); + for (Map.Entry argument : arguments.entrySet()) { + writable.writeString(argument.getKey(), null); + writable.writeObject(argument.getValue(), null); + } + } + } + } + if (hasToolResults) { + writable.writeUTF8(LLM_MESSAGE_TOOL_RESULTS); + writable.startArray(toolResults.size()); + for (LLMObs.ToolResult toolResult : toolResults) { + writable.startMap(4); + writable.writeUTF8(LLM_TOOL_CALL_NAME); + writable.writeString(toolResult.getName(), null); + writable.writeUTF8(LLM_TOOL_CALL_TYPE); + writable.writeString(toolResult.getType(), null); + writable.writeUTF8(LLM_TOOL_CALL_TOOL_ID); + writable.writeString(toolResult.getToolId(), null); + writable.writeUTF8(LLM_TOOL_RESULT_RESULT); + writable.writeString(toolResult.getResult(), null); + } + } + } + } } private static class PayloadV1 extends Payload { From 7d683b6209d4c2e90e630cf9215a237b01fd7e62 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 14:03:03 +0100 Subject: [PATCH 18/48] Fix OpenAI Responses prompt tracking to use response instructions first and return [image] (not empty) when stripped input_image URLs are missing, aligning mixed-input chat_template output with expected behavior. --- .../openai_java/ResponseDecorator.java | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 3b7e4b50a2a..aa334485730 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -4,19 +4,19 @@ import com.openai.core.JsonValue; import com.openai.models.Reasoning; import com.openai.models.ResponsesModel; +import com.openai.models.responses.EasyInputMessage; import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; import com.openai.models.responses.ResponseCustomToolCall; -import com.openai.models.responses.EasyInputMessage; import com.openai.models.responses.ResponseFunctionToolCall; import com.openai.models.responses.ResponseInputContent; import com.openai.models.responses.ResponseInputItem; import com.openai.models.responses.ResponseOutputItem; import com.openai.models.responses.ResponseOutputMessage; import com.openai.models.responses.ResponseOutputText; +import com.openai.models.responses.ResponsePrompt; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; -import com.openai.models.responses.ResponsePrompt; import datadog.json.JsonWriter; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; @@ -117,7 +117,8 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params extractReasoningFromParams(params) .ifPresent(reasoningMap -> span.setTag(CommonTags.REQUEST_REASONING, reasoningMap)); - extractPromptFromParams(params).ifPresent(prompt -> span.setTag(CommonTags.REQUEST_PROMPT, prompt)); + extractPromptFromParams(params) + .ifPresent(prompt -> span.setTag(CommonTags.REQUEST_PROMPT, prompt)); } private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { @@ -376,7 +377,10 @@ private String extractInputContentText(ResponseInputContent content) { return content.asInputText().text(); } if (content.isInputImage()) { - return content.asInputImage().imageUrl().orElse(content.asInputImage().fileId().orElse("")); + return content + .asInputImage() + .imageUrl() + .orElse(content.asInputImage().fileId().orElse(IMAGE_FALLBACK_MARKER)); } if (content.isInputFile()) { return content @@ -613,28 +617,6 @@ private List extractInputMessagesForPromptTracking( AgentSpan span, Response response) { List messages = new ArrayList<>(); - Object inputTag = span.getTag(CommonTags.INPUT); - if (inputTag instanceof List) { - for (Object messageObj : (List) inputTag) { - if (messageObj instanceof LLMObs.LLMMessage) { - messages.add((LLMObs.LLMMessage) messageObj); - } - } - } else if (inputTag instanceof Map) { - Object messagesObj = ((Map) inputTag).get("messages"); - if (messagesObj instanceof List) { - for (Object messageObj : (List) messagesObj) { - if (messageObj instanceof LLMObs.LLMMessage) { - messages.add((LLMObs.LLMMessage) messageObj); - } - } - } - } - - if (!messages.isEmpty()) { - return messages; - } - response .instructions() .ifPresent( @@ -672,6 +654,24 @@ private List extractInputMessagesForPromptTracking( } } + Object inputTag = span.getTag(CommonTags.INPUT); + if (inputTag instanceof List) { + for (Object messageObj : (List) inputTag) { + if (messageObj instanceof LLMObs.LLMMessage) { + messages.add((LLMObs.LLMMessage) messageObj); + } + } + } else if (inputTag instanceof Map) { + Object messagesObj = ((Map) inputTag).get("messages"); + if (messagesObj instanceof List) { + for (Object messageObj : (List) messagesObj) { + if (messageObj instanceof LLMObs.LLMMessage) { + messages.add((LLMObs.LLMMessage) messageObj); + } + } + } + } + return messages; } @@ -714,7 +714,8 @@ private LLMObs.LLMMessage extractMessageFromRawInstruction(JsonValue instruction contentBuilder.append(imageUrl); } else { String fileId = getJsonString(contentObj.get("file_id")); - contentBuilder.append(fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId); + contentBuilder.append( + fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId); } } else if ("input_file".equals(type)) { String fileUrl = getJsonString(contentObj.get("file_url")); From 2c17ddc1a038595f7028142b214388a3d8092eeb Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 14:24:04 +0100 Subject: [PATCH 19/48] Set LLMObs error-path defaults in Java to always emit model_name and output.messages from request params so existing error-span tests pass. --- .../openai_java/ChatCompletionDecorator.java | 14 +++++++++----- .../openai_java/CompletionDecorator.java | 15 ++++++++++----- .../openai_java/EmbeddingDecorator.java | 10 +++++----- .../openai_java/ResponseDecorator.java | 7 +++++++ 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index e5def4f67bb..d5dfde5bd6b 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -38,16 +38,20 @@ public void withChatCompletionCreateParams( if (params == null) { return; } - params - .model() - ._value() - .asString() - .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + Optional modelName = params.model()._value().asString(); + modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); if (!llmObsEnabled) { return; } + // Keep model_name and output shape stable on error paths where no response is available. + modelName.ifPresent( + str -> { + span.setTag(CommonTags.MODEL_NAME, str); + span.setTag(CommonTags.OUTPUT, Collections.singletonList(LLMObs.LLMMessage.from("", ""))); + }); + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); span.setTag( diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java index 1b95491b64b..f0f29386582 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java @@ -11,6 +11,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; public class CompletionDecorator { @@ -27,16 +28,20 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa return; } - params - .model() - ._value() - .asString() - .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + Optional modelName = params.model()._value().asString(); + modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); if (!llmObsEnabled) { return; } + // Keep model_name and output shape stable on error paths where no response is available. + modelName.ifPresent( + str -> { + span.setTag(CommonTags.MODEL_NAME, str); + span.setTag(CommonTags.OUTPUT, Collections.singletonList(LLMObs.LLMMessage.from("", ""))); + }); + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); params .prompt() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java index 02d4588358c..4e986603d93 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java @@ -28,16 +28,16 @@ public void withEmbeddingCreateParams(AgentSpan span, EmbeddingCreateParams para if (params == null) { return; } - params - .model() - ._value() - .asString() - .ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); + Optional modelName = params.model()._value().asString(); + modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); if (!llmObsEnabled) { return; } + // Keep model_name stable on error paths where no response is available. + modelName.ifPresent(str -> span.setTag(CommonTags.MODEL_NAME, str)); + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_EMBEDDING_SPAN_KIND); span.setTag(CommonTags.INPUT, embeddingDocuments(params.input())); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index aa334485730..b91ccf6c954 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -56,6 +56,13 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params return; } + // Keep model_name/output/metadata shape stable on error paths where no response is available. + if (modelName != null && !modelName.isEmpty()) { + span.setTag(CommonTags.MODEL_NAME, modelName); + } + span.setTag(CommonTags.OUTPUT, Collections.singletonList(LLMObs.LLMMessage.from("", ""))); + span.setTag(CommonTags.METADATA, new HashMap()); + span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); List inputMessages = new ArrayList<>(); From ad3b782f56e40eb3ae8e4a99e42e54d7b2aca510 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 16:05:00 +0100 Subject: [PATCH 20/48] Add OpenAI Responses tool definition extraction to populate LLMObs tool_definitions tags --- .../openai_java/ResponseDecorator.java | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index b91ccf6c954..3ca06d74f10 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -17,6 +17,8 @@ import com.openai.models.responses.ResponsePrompt; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; +import com.openai.models.responses.FunctionTool; +import com.openai.models.responses.Tool; import datadog.json.JsonWriter; import datadog.trace.api.Config; import datadog.trace.api.llmobs.LLMObs; @@ -126,6 +128,123 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params extractPromptFromParams(params) .ifPresent(prompt -> span.setTag(CommonTags.REQUEST_PROMPT, prompt)); + + List> toolDefinitions = extractToolDefinitionsFromParams(params); + if (!toolDefinitions.isEmpty()) { + span.setTag(CommonTags.TOOL_DEFINITIONS, toolDefinitions); + } + } + + private List> extractToolDefinitionsFromParams(ResponseCreateParams params) { + try { + Optional> toolsOpt = params.tools(); + if (toolsOpt.isPresent()) { + List> toolDefinitions = new ArrayList<>(); + for (Tool tool : toolsOpt.get()) { + if (!tool.isFunction()) { + continue; + } + Map toolDef = extractFunctionToolDefinition(tool.asFunction()); + if (toolDef != null) { + toolDefinitions.add(toolDef); + } + } + if (!toolDefinitions.isEmpty()) { + return toolDefinitions; + } + } + } catch (Throwable ignored) { + // fall back to raw JSON if typed extraction is unavailable or fails + } + + try { + Optional rawToolsOpt = params._tools().asUnknown(); + if (!rawToolsOpt.isPresent()) { + return Collections.emptyList(); + } + Optional> rawToolListOpt = rawToolsOpt.get().asArray(); + if (!rawToolListOpt.isPresent()) { + return Collections.emptyList(); + } + + List> toolDefinitions = new ArrayList<>(); + for (JsonValue rawTool : rawToolListOpt.get()) { + Map toolDef = extractFunctionToolDefinition(rawTool); + if (toolDef != null) { + toolDefinitions.add(toolDef); + } + } + return toolDefinitions; + } catch (Throwable ignored) { + return Collections.emptyList(); + } + } + + private Map extractFunctionToolDefinition(FunctionTool functionTool) { + String name = functionTool.name(); + if (name == null || name.isEmpty()) { + return null; + } + + Map toolDef = new HashMap<>(); + toolDef.put("name", name); + functionTool.description().ifPresent(desc -> toolDef.put("description", desc)); + functionTool + .parameters() + .ifPresent(parameters -> toolDef.put("schema", jsonValueMapToObject(parameters._additionalProperties()))); + return toolDef; + } + + private Map extractFunctionToolDefinition(JsonValue rawTool) { + Optional> toolObjOpt = rawTool.asObject(); + if (!toolObjOpt.isPresent()) { + return null; + } + + Map toolObj = toolObjOpt.get(); + String type = getJsonString(toolObj.get("type")); + if (!"function".equals(type)) { + return null; + } + + JsonValue functionObjValue = toolObj.get("function"); + Map functionObj = null; + if (functionObjValue != null) { + Optional> nestedFunctionOpt = functionObjValue.asObject(); + if (nestedFunctionOpt.isPresent()) { + functionObj = nestedFunctionOpt.get(); + } + } + + String name = + functionObj == null + ? getJsonString(toolObj.get("name")) + : getJsonString(functionObj.get("name")); + if (name == null || name.isEmpty()) { + return null; + } + + Map toolDef = new HashMap<>(); + toolDef.put("name", name); + + String description = + functionObj == null + ? getJsonString(toolObj.get("description")) + : getJsonString(functionObj.get("description")); + if (description != null) { + toolDef.put("description", description); + } + + JsonValue parameters = + functionObj == null ? toolObj.get("parameters") : functionObj.get("parameters"); + if (parameters != null) { + Object schema = jsonValueToObject(parameters); + if (schema != null) { + toolDef.put("schema", schema); + } + } + + return toolDef; } private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { @@ -841,6 +960,45 @@ private String getJsonString(JsonValue value) { return asString.orElse(null); } + private Map jsonValueMapToObject(Map map) { + Map result = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), jsonValueToObject(entry.getValue())); + } + return result; + } + + private Object jsonValueToObject(JsonValue value) { + if (value == null) { + return null; + } + Optional str = value.asString(); + if (str.isPresent()) { + return str.get(); + } + Optional num = value.asNumber(); + if (num.isPresent()) { + return num.get(); + } + Optional bool = value.asBoolean(); + if (bool.isPresent()) { + return bool.get(); + } + Optional> obj = value.asObject(); + if (obj.isPresent()) { + return jsonValueMapToObject(obj.get()); + } + Optional> arr = value.asArray(); + if (arr.isPresent()) { + List list = new ArrayList<>(); + for (JsonValue item : arr.get()) { + list.add(jsonValueToObject(item)); + } + return list; + } + return null; + } + private List extractResponseOutputMessages(List output) { List messages = new ArrayList<>(); From 1810327aef053000f2a6ffefc54b2cbf39a8aee5 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 16:13:58 +0100 Subject: [PATCH 21/48] Fix ChatCompletionServiceTest --- .../src/test/groovy/ChatCompletionServiceTest.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 22e7fbbb579..580c0ecd262 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -6,7 +6,6 @@ import com.openai.core.http.HttpResponseFor import com.openai.core.http.StreamResponse import com.openai.models.chat.completions.ChatCompletion import com.openai.models.chat.completions.ChatCompletionChunk -import com.openai.models.completions.Completion import datadog.trace.api.DDSpanTypes import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.Tags @@ -94,7 +93,7 @@ class ChatCompletionServiceTest extends OpenAiTest { } def "create async chat/completion test withRawResponse"() { - CompletableFuture> completionFuture = runUnderTrace("parent") { + CompletableFuture> completionFuture = runUnderTrace("parent") { openAiClient.async().chat().completions().withRawResponse().create(params) } From 46221e411d13675090941c7fb15c38d74401eda1 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 16:25:04 +0100 Subject: [PATCH 22/48] Extract JsonValueUtils --- .../openai_java/ChatCompletionDecorator.java | 39 ++------------ .../openai_java/ChatCompletionModule.java | 1 + .../openai_java/JsonValueUtils.java | 51 +++++++++++++++++++ .../openai_java/ResponseDecorator.java | 42 ++------------- .../openai_java/ResponseModule.java | 1 + 5 files changed, 59 insertions(+), 75 deletions(-) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index d5dfde5bd6b..f35e485c90e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -1,5 +1,8 @@ package datadog.trace.instrumentation.openai_java; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueMapToObject; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueToObject; + import com.openai.core.JsonValue; import com.openai.helpers.ChatCompletionAccumulator; import com.openai.models.FunctionDefinition; @@ -167,42 +170,6 @@ private static Map extractFunctionToolDef(ChatCompletionFunction return toolDef; } - private static Map jsonValueMapToObject(Map map) { - Map result = new HashMap<>(); - for (Map.Entry entry : map.entrySet()) { - result.put(entry.getKey(), jsonValueToObject(entry.getValue())); - } - return result; - } - - private static Object jsonValueToObject(JsonValue value) { - Optional str = value.asString(); - if (str.isPresent()) { - return str.get(); - } - Optional num = value.asNumber(); - if (num.isPresent()) { - return num.get(); - } - Optional bool = value.asBoolean(); - if (bool.isPresent()) { - return bool.get(); - } - Optional> obj = value.asObject(); - if (obj.isPresent()) { - return jsonValueMapToObject(obj.get()); - } - Optional> arr = value.asArray(); - if (arr.isPresent()) { - List list = new ArrayList<>(); - for (JsonValue item : arr.get()) { - list.add(jsonValueToObject(item)); - } - return list; - } - return null; - } - private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { String role = "unknown"; String content = null; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java index f30d02f9570..2c89bf339ad 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionModule.java @@ -18,6 +18,7 @@ public String[] helperClassNames() { packageName + ".CommonTags", packageName + ".ChatCompletionDecorator", packageName + ".OpenAiDecorator", + packageName + ".JsonValueUtils", packageName + ".HttpResponseWrapper", packageName + ".HttpStreamResponseWrapper", packageName + ".HttpStreamResponseStreamWrapper", diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java new file mode 100644 index 00000000000..c251f5c17fd --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/JsonValueUtils.java @@ -0,0 +1,51 @@ +package datadog.trace.instrumentation.openai_java; + +import com.openai.core.JsonValue; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public final class JsonValueUtils { + private JsonValueUtils() {} + + public static Map jsonValueMapToObject(Map map) { + Map result = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + result.put(entry.getKey(), jsonValueToObject(entry.getValue())); + } + return result; + } + + public static Object jsonValueToObject(JsonValue value) { + if (value == null) { + return null; + } + Optional str = value.asString(); + if (str.isPresent()) { + return str.get(); + } + Optional num = value.asNumber(); + if (num.isPresent()) { + return num.get(); + } + Optional bool = value.asBoolean(); + if (bool.isPresent()) { + return bool.get(); + } + Optional> obj = value.asObject(); + if (obj.isPresent()) { + return jsonValueMapToObject(obj.get()); + } + Optional> arr = value.asArray(); + if (arr.isPresent()) { + List list = new ArrayList<>(); + for (JsonValue item : arr.get()) { + list.add(jsonValueToObject(item)); + } + return list; + } + return null; + } +} diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 3ca06d74f10..631d5d213e1 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -1,5 +1,8 @@ package datadog.trace.instrumentation.openai_java; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueMapToObject; +import static datadog.trace.instrumentation.openai_java.JsonValueUtils.jsonValueToObject; + import com.openai.core.JsonField; import com.openai.core.JsonValue; import com.openai.models.Reasoning; @@ -960,45 +963,6 @@ private String getJsonString(JsonValue value) { return asString.orElse(null); } - private Map jsonValueMapToObject(Map map) { - Map result = new HashMap<>(); - for (Map.Entry entry : map.entrySet()) { - result.put(entry.getKey(), jsonValueToObject(entry.getValue())); - } - return result; - } - - private Object jsonValueToObject(JsonValue value) { - if (value == null) { - return null; - } - Optional str = value.asString(); - if (str.isPresent()) { - return str.get(); - } - Optional num = value.asNumber(); - if (num.isPresent()) { - return num.get(); - } - Optional bool = value.asBoolean(); - if (bool.isPresent()) { - return bool.get(); - } - Optional> obj = value.asObject(); - if (obj.isPresent()) { - return jsonValueMapToObject(obj.get()); - } - Optional> arr = value.asArray(); - if (arr.isPresent()) { - List list = new ArrayList<>(); - for (JsonValue item : arr.get()) { - list.add(jsonValueToObject(item)); - } - return list; - } - return null; - } - private List extractResponseOutputMessages(List output) { List messages = new ArrayList<>(); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java index 25266504f53..b87b3910490 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseModule.java @@ -19,6 +19,7 @@ public String[] helperClassNames() { packageName + ".ResponseDecorator", packageName + ".FunctionCallOutputExtractor", packageName + ".OpenAiDecorator", + packageName + ".JsonValueUtils", packageName + ".HttpResponseWrapper", packageName + ".HttpStreamResponseWrapper", packageName + ".HttpStreamResponseStreamWrapper", From 61ad6678b8fe9d52ad56990195f56b1c0bd1243a Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 16:48:39 +0100 Subject: [PATCH 23/48] Refactor OpenAI responses instrumentation to reuse ToolCallExtractor JSON argument parsing and remove duplicate manual parsing logic from ResponseDecorator. --- .../openai_java/ResponseDecorator.java | 77 +------------------ .../openai_java/ToolCallExtractor.java | 2 +- 2 files changed, 2 insertions(+), 77 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 631d5d213e1..810e8419176 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -352,7 +352,7 @@ private LLMObs.LLMMessage extractMessageFromRawJson(JsonValue jsonValue) { } if (callId != null && name != null && argumentsStr != null) { - Map arguments = parseJsonString(argumentsStr); + Map arguments = ToolCallExtractor.parseArguments(argumentsStr); LLMObs.ToolCall toolCall = LLMObs.ToolCall.from(name, "function_call", callId, arguments); return LLMObs.LLMMessage.from("assistant", null, Collections.singletonList(toolCall)); @@ -414,81 +414,6 @@ private LLMObs.LLMMessage extractMessageFromRawJson(JsonValue jsonValue) { return null; } - private Map parseJsonString(String jsonStr) { - if (jsonStr == null || jsonStr.isEmpty()) { - return Collections.emptyMap(); - } - try { - jsonStr = jsonStr.trim(); - if (!jsonStr.startsWith("{") || !jsonStr.endsWith("}")) { - return Collections.emptyMap(); - } - - Map result = new HashMap<>(); - String content = jsonStr.substring(1, jsonStr.length() - 1).trim(); - - if (content.isEmpty()) { - return result; - } - - // Parse JSON manually, respecting quoted strings - List pairs = splitByCommaRespectingQuotes(content); - - for (String pair : pairs) { - int colonIdx = pair.indexOf(':'); - if (colonIdx > 0) { - String key = pair.substring(0, colonIdx).trim(); - String value = pair.substring(colonIdx + 1).trim(); - - // Remove quotes from key - key = removeQuotes(key); - // Remove quotes from value - value = removeQuotes(value); - - result.put(key, value); - } - } - - return result; - } catch (Exception e) { - return Collections.emptyMap(); - } - } - - private List splitByCommaRespectingQuotes(String str) { - List result = new ArrayList<>(); - StringBuilder current = new StringBuilder(); - boolean inQuotes = false; - - for (int i = 0; i < str.length(); i++) { - char c = str.charAt(i); - - if (c == '"') { - inQuotes = !inQuotes; - current.append(c); - } else if (c == ',' && !inQuotes) { - result.add(current.toString()); - current = new StringBuilder(); - } else { - current.append(c); - } - } - - if (current.length() > 0) { - result.add(current.toString()); - } - - return result; - } - - private String removeQuotes(String str) { - str = str.trim(); - if (str.startsWith("\"") && str.endsWith("\"") && str.length() >= 2) { - return str.substring(1, str.length() - 1); - } - return str; - } - private String extractInputMessageContent(ResponseInputItem.Message message) { StringBuilder contentBuilder = new StringBuilder(); for (ResponseInputContent content : message.content()) { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java index 21d2bf6835d..ffeca857a20 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ToolCallExtractor.java @@ -106,7 +106,7 @@ public static LLMObs.ToolCall getToolCall(McpCall mcpCall) { return null; } - private static Map parseArguments(String argumentsJson) { + static Map parseArguments(String argumentsJson) { try { return MAPPER.readValue(argumentsJson, MAP_TYPE_REF); } catch (Exception e) { From f0957b79844420163b73cf302fd1faa07afa6f53 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 5 Mar 2026 19:04:37 +0100 Subject: [PATCH 24/48] Fix test assertions --- .../openai_java/ResponseDecorator.java | 6 ++++-- .../groovy/ChatCompletionServiceTest.groovy | 19 ++++++++++++++++--- .../test/groovy/CompletionServiceTest.groovy | 2 ++ .../test/groovy/EmbeddingServiceTest.groovy | 2 ++ .../test/groovy/ResponseServiceTest.groovy | 2 ++ 5 files changed, 26 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 810e8419176..07359a9b326 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -8,6 +8,7 @@ import com.openai.models.Reasoning; import com.openai.models.ResponsesModel; import com.openai.models.responses.EasyInputMessage; +import com.openai.models.responses.FunctionTool; import com.openai.models.responses.Response; import com.openai.models.responses.ResponseCreateParams; import com.openai.models.responses.ResponseCustomToolCall; @@ -20,7 +21,6 @@ import com.openai.models.responses.ResponsePrompt; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; -import com.openai.models.responses.FunctionTool; import com.openai.models.responses.Tool; import datadog.json.JsonWriter; import datadog.trace.api.Config; @@ -194,7 +194,9 @@ private Map extractFunctionToolDefinition(FunctionTool functionT functionTool.description().ifPresent(desc -> toolDef.put("description", desc)); functionTool .parameters() - .ifPresent(parameters -> toolDef.put("schema", jsonValueMapToObject(parameters._additionalProperties()))); + .ifPresent( + parameters -> + toolDef.put("schema", jsonValueMapToObject(parameters._additionalProperties()))); return toolDef; } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 580c0ecd262..fad254f69c5 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -147,7 +147,7 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(false, outputTag, [:]) + assertChatCompletionTrace(false, outputTag, [:], true) and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -178,7 +178,7 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(true, outputTag, [stream: true]) + assertChatCompletionTrace(true, outputTag, [stream: true], true) and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -294,6 +294,13 @@ class ChatCompletionServiceTest extends OpenAiTest { } private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata) { + assertChatCompletionTrace(isStreaming, outputTagsOut, metadata, false) + } + + private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata, boolean expectToolDefinitions) { + def expectedMetadata = new LinkedHashMap(metadata) + expectedMetadata.putIfAbsent("stream", isStreaming) + assertTraces(1) { trace(3) { sortSpansByStart() @@ -312,7 +319,7 @@ class ChatCompletionServiceTest extends OpenAiTest { "_ml_obs_tag.span.kind" "llm" "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String - "_ml_obs_tag.metadata" metadata + "_ml_obs_tag.metadata" expectedMetadata "_ml_obs_tag.input" List "_ml_obs_tag.output" List def outputTags = tag("_ml_obs_tag.output") @@ -324,10 +331,16 @@ class ChatCompletionServiceTest extends OpenAiTest { "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.output_tokens" Long "_ml_obs_metric.total_tokens" Long + "_ml_obs_metric.cache_read_input_tokens" Long } "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + if (expectToolDefinitions) { + "$CommonTags.TOOL_DEFINITIONS" List + } + "$CommonTags.SOURCE" "integration" + "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/chat/completions" "openai.api_base" openAiBaseApi diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index 43f5e247c55..72ffaee8052 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -165,6 +165,8 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.SOURCE" "integration" + "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/completions" "openai.api_base" openAiBaseApi diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index eb14f2999de..44999641fc2 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -64,6 +64,8 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.SOURCE" "integration" + "$CommonTags.ERROR" 0 "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.total_tokens" Long "openai.request.method" "POST" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index f747005cff4..3b205d8496f 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -220,6 +220,8 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.SOURCE" "integration" + "$CommonTags.ERROR" 0 if (reasoning != null) { "_ml_obs_request.reasoning" reasoning } From f3f1f75ec26f06c157090a87f5a64f45df3b720e Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 6 Mar 2026 10:35:37 +0100 Subject: [PATCH 25/48] Add integration tag --- .../trace/instrumentation/openai_java/CommonTags.java | 1 + .../trace/instrumentation/openai_java/OpenAiDecorator.java | 1 + .../src/test/groovy/ChatCompletionServiceTest.groovy | 1 + .../src/test/groovy/CompletionServiceTest.groovy | 1 + .../src/test/groovy/EmbeddingServiceTest.groovy | 1 + .../src/test/groovy/ResponseServiceTest.groovy | 1 + .../trace/llmobs/writer/ddintake/LLMObsSpanMapper.java | 3 ++- .../llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy | 6 ++++-- 8 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index 228b3d52a81..c9917332e7e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -19,6 +19,7 @@ interface CommonTags { String MODEL_PROVIDER = TAG_PREFIX + LLMObsTags.MODEL_PROVIDER; String ML_APP = TAG_PREFIX + LLMObsTags.ML_APP; + String INTEGRATION = TAG_PREFIX + "integration"; String VERSION = TAG_PREFIX + "version"; String SOURCE = TAG_PREFIX + "source"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index fae2880c082..033381d60bb 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -99,6 +99,7 @@ public AgentSpan afterStart(AgentSpan span) { span.setTag(CommonTags.ML_APP, Config.get().getLlmObsMlApp()); span.setTag(CommonTags.SOURCE, "integration"); + span.setTag(CommonTags.INTEGRATION, INTEGRATION); AgentSpanContext parent = LLMObsContext.current(); String parentSpanId = LLMObsContext.ROOT_SPAN_ID; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index fad254f69c5..c4a066d1895 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -340,6 +340,7 @@ class ChatCompletionServiceTest extends OpenAiTest { "$CommonTags.TOOL_DEFINITIONS" List } "$CommonTags.SOURCE" "integration" + "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/chat/completions" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index 72ffaee8052..de9af838086 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -166,6 +166,7 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String "$CommonTags.SOURCE" "integration" + "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 "openai.request.method" "POST" "openai.request.endpoint" "/v1/completions" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index 44999641fc2..112b649a856 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -65,6 +65,7 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String "$CommonTags.SOURCE" "integration" + "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.total_tokens" Long diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 3b205d8496f..030f61009ac 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -219,6 +219,7 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_metric.cache_read_input_tokens" Long "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String + "$CommonTags.INTEGRATION" "openai" "_ml_obs_tag.service" String "$CommonTags.SOURCE" "integration" "$CommonTags.ERROR" 0 diff --git a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java index 7ff16cbed4e..cb8ebd3d8a1 100644 --- a/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java +++ b/dd-trace-core/src/main/java/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapper.java @@ -70,6 +70,7 @@ public class LLMObsSpanMapper implements RemoteMapper { private static final byte[] SPANS = "spans".getBytes(StandardCharsets.UTF_8); private static final byte[] METRICS = "metrics".getBytes(StandardCharsets.UTF_8); private static final byte[] TAGS = "tags".getBytes(StandardCharsets.UTF_8); + private static final String LLMOBS_LANGUAGE_TAG = "language:jvm"; private static final byte[] LLM_MESSAGE_ROLE = "role".getBytes(StandardCharsets.UTF_8); private static final byte[] LLM_MESSAGE_CONTENT = "content".getBytes(StandardCharsets.UTF_8); @@ -293,7 +294,7 @@ public void accept(Metadata metadata) { // write tags (10) writable.writeUTF8(TAGS); writable.startArray(tagsSize + 1); - writable.writeString("language:jvm", null); + writable.writeString(LLMOBS_LANGUAGE_TAG, null); for (Map.Entry tag : metadata.getTags().entrySet()) { String key = tag.getKey(); Object value = tag.getValue(); diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 6140431b836..3cf52ae0150 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -160,7 +160,8 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { def trace = [regularSpan1, regularSpan2] CapturingByteBufferConsumer sink = new CapturingByteBufferConsumer() - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1024, sink)) + // Keep all formatted spans in a single flush for this assertion. + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(16 * 1024, sink)) when: packer.format(trace, mapper) @@ -204,7 +205,8 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { def trace1 = [llmSpan1, llmSpan2] def trace2 = [llmSpan3] CapturingByteBufferConsumer sink = new CapturingByteBufferConsumer() - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1024, sink)) + // Keep all formatted spans in a single flush for this assertion. + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(16 * 1024, sink)) when: packer.format(trace1, mapper) From 668e955e352ee8306d2f9ea1ce69570e15fe837f Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 6 Mar 2026 11:01:25 +0100 Subject: [PATCH 26/48] Add ddtrace.verion --- .../java/datadog/trace/llmobs/domain/DDLLMObsSpan.java | 3 +++ .../datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy | 9 +++++++++ .../trace/instrumentation/openai_java/CommonTags.java | 1 + .../instrumentation/openai_java/OpenAiDecorator.java | 2 ++ .../src/test/groovy/ChatCompletionServiceTest.groovy | 1 + .../src/test/groovy/CompletionServiceTest.groovy | 1 + .../src/test/groovy/EmbeddingServiceTest.groovy | 1 + .../src/test/groovy/ResponseServiceTest.groovy | 1 + 8 files changed, 19 insertions(+) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 56bc1f69c88..c7636f9cd7a 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -2,6 +2,7 @@ import datadog.context.ContextScope; import datadog.trace.api.DDSpanTypes; +import datadog.trace.api.DDTraceApiInfo; import datadog.trace.api.DDTraceId; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObs; @@ -38,6 +39,7 @@ public class DDLLMObsSpan implements LLMObsSpan { private static final String SERVICE = LLMOBS_TAG_PREFIX + "service"; private static final String VERSION = LLMOBS_TAG_PREFIX + "version"; + private static final String DDTRACE_VERSION = LLMOBS_TAG_PREFIX + "ddtrace.version"; private static final String ENV = LLMOBS_TAG_PREFIX + "env"; private static final String LLM_OBS_INSTRUMENTATION_NAME = "llmobs"; @@ -74,6 +76,7 @@ public DDLLMObsSpan( this.span.setTag(ENV, wellKnownTags.getEnv()); this.span.setTag(SERVICE, wellKnownTags.getService()); this.span.setTag(VERSION, wellKnownTags.getVersion()); + this.span.setTag(DDTRACE_VERSION, DDTraceApiInfo.VERSION); this.span.setTag(SPAN_KIND, kind); this.spanKind = kind; diff --git a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy index 7595b51e82a..87123f9f473 100644 --- a/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy +++ b/dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy @@ -2,6 +2,7 @@ package datadog.trace.llmobs.domain import datadog.trace.agent.tooling.TracerInstaller import datadog.trace.api.DDTags +import datadog.trace.api.DDTraceApiInfo import datadog.trace.api.IdGenerationStrategy import datadog.trace.api.WellKnownTags import datadog.trace.api.llmobs.LLMObs @@ -131,6 +132,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } def "test span with overwrites"() { @@ -216,6 +219,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } def "test llm span string input formatted to messages"() { @@ -267,6 +272,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } def "test llm span with messages"() { @@ -323,6 +330,8 @@ class DDLLMObsSpanTest extends DDSpecification{ def tagVersion = innerSpan.getTag(LLMOBS_TAG_PREFIX + "version") tagVersion instanceof UTF8BytesString "v1" == tagVersion.toString() + + DDTraceApiInfo.VERSION == innerSpan.getTag(LLMOBS_TAG_PREFIX + "ddtrace.version") } private LLMObsSpan givenALLMObsSpan(String kind, name){ diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java index c9917332e7e..a992c85400c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CommonTags.java @@ -21,6 +21,7 @@ interface CommonTags { String ML_APP = TAG_PREFIX + LLMObsTags.ML_APP; String INTEGRATION = TAG_PREFIX + "integration"; String VERSION = TAG_PREFIX + "version"; + String DDTRACE_VERSION = TAG_PREFIX + "ddtrace.version"; String SOURCE = TAG_PREFIX + "source"; String ERROR = TAG_PREFIX + "error"; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 033381d60bb..1bb3530fc93 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -4,6 +4,7 @@ import com.openai.core.http.Headers; import datadog.trace.api.Config; import datadog.trace.api.DDTags; +import datadog.trace.api.DDTraceApiInfo; import datadog.trace.api.WellKnownTags; import datadog.trace.api.llmobs.LLMObsContext; import datadog.trace.api.telemetry.LLMObsMetricCollector; @@ -96,6 +97,7 @@ public AgentSpan afterStart(AgentSpan span) { span.setTag(CommonTags.ENV, wellKnownTags.getEnv()); span.setTag(CommonTags.SERVICE, wellKnownTags.getService()); span.setTag(CommonTags.VERSION, wellKnownTags.getVersion()); + span.setTag(CommonTags.DDTRACE_VERSION, DDTraceApiInfo.VERSION); span.setTag(CommonTags.ML_APP, Config.get().getLlmObsMlApp()); span.setTag(CommonTags.SOURCE, "integration"); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index c4a066d1895..28430fcccac 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -336,6 +336,7 @@ class ChatCompletionServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String if (expectToolDefinitions) { "$CommonTags.TOOL_DEFINITIONS" List } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index de9af838086..dcf537df854 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -165,6 +165,7 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String "$CommonTags.SOURCE" "integration" "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index 112b649a856..32468dd00df 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -64,6 +64,7 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String "$CommonTags.SOURCE" "integration" "$CommonTags.INTEGRATION" "openai" "$CommonTags.ERROR" 0 diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 030f61009ac..62f64b04223 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -221,6 +221,7 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.ml_app" String "$CommonTags.INTEGRATION" "openai" "_ml_obs_tag.service" String + "$CommonTags.DDTRACE_VERSION" String "$CommonTags.SOURCE" "integration" "$CommonTags.ERROR" 0 if (reasoning != null) { From d57402ef743e1a5cd566cf654bc0d66b8023d548 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 6 Mar 2026 12:08:53 +0100 Subject: [PATCH 27/48] Improve test assertions --- .../groovy/ChatCompletionServiceTest.groovy | 28 ++++++- .../test/groovy/CompletionServiceTest.groovy | 46 ++++++++--- .../test/groovy/EmbeddingServiceTest.groovy | 15 ++++ .../test/groovy/ResponseServiceTest.groovy | 76 ++++++++++++++++--- .../ddintake/LLMObsSpanMapperTest.groovy | 39 +++++++++- 5 files changed, 177 insertions(+), 27 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 28430fcccac..7376f73f92e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -147,7 +147,18 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(false, outputTag, [:], true) + List> toolDefinitions = [] + assertChatCompletionTrace(false, outputTag, [:], true, toolDefinitions) + and: + toolDefinitions.size() == 1 + toolDefinitions[0].name == "extract_student_info" + toolDefinitions[0].description == "Get the student information from the body of the input text" + toolDefinitions[0].schema.type == "object" + (toolDefinitions[0].schema.properties as Map).containsKey("name") + (toolDefinitions[0].schema.properties as Map).containsKey("major") + (toolDefinitions[0].schema.properties as Map).containsKey("school") + (toolDefinitions[0].schema.properties as Map).containsKey("grades") + (toolDefinitions[0].schema.properties as Map).containsKey("clubs") and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -178,7 +189,12 @@ class ChatCompletionServiceTest extends OpenAiTest { expect: List outputTag = [] - assertChatCompletionTrace(true, outputTag, [stream: true], true) + List> toolDefinitions = [] + assertChatCompletionTrace(true, outputTag, [stream: true], true, toolDefinitions) + and: + toolDefinitions.size() == 1 + toolDefinitions[0].name == "extract_student_info" + toolDefinitions[0].description == "Get the student information from the body of the input text" and: outputTag.size() == 1 LLMObs.LLMMessage outputMsg = outputTag.get(0) @@ -294,10 +310,10 @@ class ChatCompletionServiceTest extends OpenAiTest { } private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata) { - assertChatCompletionTrace(isStreaming, outputTagsOut, metadata, false) + assertChatCompletionTrace(isStreaming, outputTagsOut, metadata, false, null) } - private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata, boolean expectToolDefinitions) { + private void assertChatCompletionTrace(boolean isStreaming, List outputTagsOut, Map metadata, boolean expectToolDefinitions, List> toolDefinitionsOut) { def expectedMetadata = new LinkedHashMap(metadata) expectedMetadata.putIfAbsent("stream", isStreaming) @@ -339,6 +355,10 @@ class ChatCompletionServiceTest extends OpenAiTest { "$CommonTags.DDTRACE_VERSION" String if (expectToolDefinitions) { "$CommonTags.TOOL_DEFINITIONS" List + def toolDefinitions = tag("$CommonTags.TOOL_DEFINITIONS") + if (toolDefinitionsOut != null && toolDefinitions != null) { + toolDefinitionsOut.addAll(toolDefinitions) + } } "$CommonTags.SOURCE" "integration" "$CommonTags.INTEGRATION" "openai" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index dcf537df854..8ca98ca3677 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -8,6 +8,7 @@ import com.openai.core.http.HttpResponseFor import com.openai.core.http.StreamResponse import com.openai.models.completions.Completion import datadog.trace.api.DDSpanTypes +import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.Tags import java.util.concurrent.CompletableFuture import java.util.stream.Stream @@ -20,7 +21,7 @@ class CompletionServiceTest extends OpenAiTest { } expect: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -35,7 +36,7 @@ class CompletionServiceTest extends OpenAiTest { resp.statusCode() == 200 resp.parse().valid // force response parsing, so it sets all the tags and: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -52,7 +53,7 @@ class CompletionServiceTest extends OpenAiTest { } expect: - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] @@ -69,7 +70,7 @@ class CompletionServiceTest extends OpenAiTest { } expect: - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] @@ -83,7 +84,7 @@ class CompletionServiceTest extends OpenAiTest { completionFuture.get() expect: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -98,7 +99,7 @@ class CompletionServiceTest extends OpenAiTest { resp.parse().valid // force response parsing, so it sets all the tags expect: - assertCompletionTrace() + assertCompletionTrace(false) where: params << [completionCreateParams(true), completionCreateParams(false)] @@ -113,7 +114,7 @@ class CompletionServiceTest extends OpenAiTest { } asyncResp.onCompleteFuture().get() expect: - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] @@ -131,13 +132,17 @@ class CompletionServiceTest extends OpenAiTest { } expect: resp.statusCode() == 200 - assertCompletionTrace() + assertCompletionTrace(true) where: params << [completionCreateStreamedParams(true), completionCreateStreamedParams(false)] } - private void assertCompletionTrace() { + private void assertCompletionTrace(boolean streamRequest) { + List inputTagsOut = [] + List outputTagsOut = [] + Map metadataOut = [:] + assertTraces(1) { trace(3) { sortSpansByStart() @@ -157,8 +162,20 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String "_ml_obs_tag.metadata" Map + def metadata = tag("_ml_obs_tag.metadata") + if (metadata != null) { + metadataOut.putAll(metadata) + } "_ml_obs_tag.input" List + def inputTags = tag("_ml_obs_tag.input") + if (inputTags != null) { + inputTagsOut.addAll(inputTags) + } "_ml_obs_tag.output" List + def outputTags = tag("_ml_obs_tag.output") + if (outputTags != null) { + outputTagsOut.addAll(outputTags) + } "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.output_tokens" Long "_ml_obs_metric.total_tokens" Long @@ -193,5 +210,16 @@ class CompletionServiceTest extends OpenAiTest { } } } + + assert inputTagsOut.size() == 1 + assert inputTagsOut[0].role == "" + assert inputTagsOut[0].content == "Tell me a story about building the best SDK!" + assert outputTagsOut.size() >= 1 + assert outputTagsOut.every { it.role == "" } + if (streamRequest) { + assert metadataOut.stream_options == [include_usage: true] + } else { + assert !metadataOut.containsKey("stream_options") + } } } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index 32468dd00df..0a4f76ff47a 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -40,6 +40,9 @@ class EmbeddingServiceTest extends OpenAiTest { } private void assertEmbeddingTrace() { + List inputTagsOut = [] + Map metadataOut = [:] + assertTraces(1) { trace(3) { sortSpansByStart() @@ -59,7 +62,15 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" "text-embedding-ada-002-v2" "_ml_obs_tag.input" List + def inputTags = tag("_ml_obs_tag.input") + if (inputTags != null) { + inputTagsOut.addAll(inputTags) + } "_ml_obs_tag.metadata" Map + def metadata = tag("_ml_obs_tag.metadata") + if (metadata != null) { + metadataOut.putAll(metadata) + } "_ml_obs_tag.output" "[1 embedding(s) returned with size 1536]" "_ml_obs_tag.parent_id" "undefined" "_ml_obs_tag.ml_app" String @@ -94,5 +105,9 @@ class EmbeddingServiceTest extends OpenAiTest { } } } + + assert inputTagsOut.size() == 1 + assert inputTagsOut[0].text == "hello world" + assert metadataOut == [encoding_format: "base64"] } } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 62f64b04223..4d26098050c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -23,7 +23,10 @@ class ResponseServiceTest extends OpenAiTest { expect: resp != null and: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -38,7 +41,10 @@ class ResponseServiceTest extends OpenAiTest { resp.statusCode() == 200 resp.parse().valid // force response parsing, so it sets all the tags and: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -55,7 +61,10 @@ class ResponseServiceTest extends OpenAiTest { } expect: - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: scenario | params @@ -76,7 +85,11 @@ class ResponseServiceTest extends OpenAiTest { } expect: - assertResponseTrace(true, "o4-mini", "o4-mini-2025-04-16", [effort: "medium", summary: "detailed"]) + Map metadata = [:] + assertResponseTrace(true, "o4-mini", "o4-mini-2025-04-16", [effort: "medium", summary: "detailed"], null, null, metadata) + and: + metadata.stream == true + metadata.reasoning == [effort: "medium", summary: "detailed"] where: responseCreateParams << [responseCreateParamsWithReasoning(false), responseCreateParamsWithReasoning(true)] @@ -93,7 +106,10 @@ class ResponseServiceTest extends OpenAiTest { } expect: - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -107,7 +123,10 @@ class ResponseServiceTest extends OpenAiTest { responseFuture.get() expect: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -122,7 +141,10 @@ class ResponseServiceTest extends OpenAiTest { resp.parse().valid // force response parsing, so it sets all the tags expect: - assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(false, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == false where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -137,7 +159,10 @@ class ResponseServiceTest extends OpenAiTest { } asyncResp.onCompleteFuture().get() expect: - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -155,7 +180,10 @@ class ResponseServiceTest extends OpenAiTest { } expect: resp.statusCode() == 200 - assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null) + Map metadata = [:] + assertResponseTrace(true, "gpt-3.5-turbo", "gpt-3.5-turbo-0125", null, null, null, metadata) + and: + metadata.stream == true where: params << [responseCreateParams(false), responseCreateParams(true)] @@ -173,8 +201,17 @@ class ResponseServiceTest extends OpenAiTest { expect: List inputTags = [] - assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags) + Map metadata = [:] + assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags, null, metadata) and: + metadata.stream == true + inputTags.size() == 3 + inputTags[1].toolCalls.size() == 1 + inputTags[1].toolCalls[0].name == "get_weather" + inputTags[1].toolCalls[0].type == "function_call" + inputTags[1].toolCalls[0].arguments == [location: "San Francisco, CA"] + inputTags[2].toolResults.size() == 1 + inputTags[2].toolResults[0].type == "function_call_output" !inputTags.isEmpty() inputTags[2].toolResults[0].result == '{"temperature": "72°F", "conditions": "sunny", "humidity": "65%"}' @@ -183,10 +220,17 @@ class ResponseServiceTest extends OpenAiTest { } private void assertResponseTrace(boolean isStreaming, String reqModel, String respModel, Map reasoning) { - assertResponseTrace(isStreaming, reqModel, respModel, reasoning, null) + assertResponseTrace(isStreaming, reqModel, respModel, reasoning, null, null, null) } - private void assertResponseTrace(boolean isStreaming, String reqModel, String respModel, Map reasoning, List inputTagsOut) { + private void assertResponseTrace( + boolean isStreaming, + String reqModel, + String respModel, + Map reasoning, + List inputTagsOut, + List outputTagsOut, + Map metadataOut) { assertTraces(1) { trace(3) { sortSpansByStart() @@ -206,12 +250,20 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String "_ml_obs_tag.metadata" Map + def metadata = tag("_ml_obs_tag.metadata") + if (metadataOut != null && metadata != null) { + metadataOut.putAll(metadata) + } "_ml_obs_tag.input" List def inputTags = tag("_ml_obs_tag.input") if (inputTagsOut != null && inputTags != null) { inputTagsOut.addAll(inputTags) } "_ml_obs_tag.output" List + def outputTags = tag("_ml_obs_tag.output") + if (outputTagsOut != null && outputTags != null) { + outputTagsOut.addAll(outputTags) + } "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.output_tokens" Long "_ml_obs_metric.total_tokens" Long diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 3cf52ae0150..6df08aa39ce 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -40,11 +40,29 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { llmSpan.setSpanType(InternalSpanTypes.LLMOBS) - def inputMessages = [LLMObs.LLMMessage.from("user", "Hello, what's the weather like?")] + def toolCall = LLMObs.ToolCall.from("get_weather", "function_call", "call_123", [location: "San Francisco"]) + def toolResult = LLMObs.ToolResult.from("get_weather", "function_call_output", "call_123", '{"temperature":"72F"}') + def inputMessages = [ + LLMObs.LLMMessage.from("user", "Hello, what's the weather like?"), + LLMObs.LLMMessage.from("assistant", null, [toolCall], [toolResult]) + ] def outputMessages = [LLMObs.LLMMessage.from("assistant", "I'll help you check the weather.")] - llmSpan.setTag("_ml_obs_tag.input", inputMessages) + llmSpan.setTag("_ml_obs_tag.input", [ + messages: inputMessages, + prompt: [ + id: "prompt_123", + version: "1", + variables: [city: "San Francisco"], + chat_template: [[role: "user", content: "Hello, what's the weather like in {{city}}?"]] + ] + ]) llmSpan.setTag("_ml_obs_tag.output", outputMessages) llmSpan.setTag("_ml_obs_tag.metadata", [temperature: 0.7, max_tokens: 100]) + llmSpan.setTag("_ml_obs_tag.tool_definitions", [[ + name: "get_weather", + description: "Get weather by city", + schema: [type: "object", properties: [city: [type: "string"]]] + ]]) llmSpan.setError(true) llmSpan.setTag(DDTags.ERROR_MSG, "boom") llmSpan.setTag(DDTags.ERROR_TYPE, "java.lang.IllegalStateException") @@ -123,12 +141,29 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { spanData["meta"]["input"]["messages"][0]["content"] == "Hello, what's the weather like?" spanData["meta"]["input"]["messages"][0].containsKey("role") spanData["meta"]["input"]["messages"][0]["role"] == "user" + spanData["meta"]["input"]["messages"][1]["role"] == "assistant" + !spanData["meta"]["input"]["messages"][1].containsKey("content") + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["name"] == "get_weather" + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["type"] == "function_call" + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["tool_id"] == "call_123" + spanData["meta"]["input"]["messages"][1]["tool_calls"][0]["arguments"] == [location: "San Francisco"] + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["name"] == "get_weather" + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["type"] == "function_call_output" + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["tool_id"] == "call_123" + spanData["meta"]["input"]["messages"][1]["tool_results"][0]["result"] == '{"temperature":"72F"}' + spanData["meta"]["input"]["prompt"]["id"] == "prompt_123" + spanData["meta"]["input"]["prompt"]["version"] == "1" + spanData["meta"]["input"]["prompt"]["variables"] == [city: "San Francisco"] + spanData["meta"]["input"]["prompt"]["chat_template"] == [[role: "user", content: "Hello, what's the weather like in {{city}}?"]] spanData["meta"].containsKey("output") spanData["meta"]["output"].containsKey("messages") spanData["meta"]["output"]["messages"][0].containsKey("content") spanData["meta"]["output"]["messages"][0]["content"] == "I'll help you check the weather." spanData["meta"]["output"]["messages"][0].containsKey("role") spanData["meta"]["output"]["messages"][0]["role"] == "assistant" + spanData["meta"]["tool_definitions"][0]["name"] == "get_weather" + spanData["meta"]["tool_definitions"][0]["description"] == "Get weather by city" + spanData["meta"]["tool_definitions"][0]["schema"] == [type: "object", properties: [city: [type: "string"]]] spanData["meta"].containsKey("metadata") spanData.containsKey("metrics") From 0c879ba692386cd944b87734bcfce2285beaa37e Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 6 Mar 2026 12:53:59 +0100 Subject: [PATCH 28/48] Fix format --- .../openai_java/ResponseDecorator.java | 4 ++++ .../src/test/groovy/ResponseServiceTest.groovy | 14 +++++++------- .../writer/ddintake/LLMObsSpanMapperTest.groovy | 12 +++++++----- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 07359a9b326..9488ad0c865 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -710,6 +710,10 @@ private List extractInputMessagesForPromptTracking( } } + if (!messages.isEmpty()) { + return messages; + } + Object inputTag = span.getTag(CommonTags.INPUT); if (inputTag instanceof List) { for (Object messageObj : (List) inputTag) { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 4d26098050c..02e28237750 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -224,13 +224,13 @@ class ResponseServiceTest extends OpenAiTest { } private void assertResponseTrace( - boolean isStreaming, - String reqModel, - String respModel, - Map reasoning, - List inputTagsOut, - List outputTagsOut, - Map metadataOut) { + boolean isStreaming, + String reqModel, + String respModel, + Map reasoning, + List inputTagsOut, + List outputTagsOut, + Map metadataOut) { assertTraces(1) { trace(3) { sortSpansByStart() diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 6df08aa39ce..7d7de1180a7 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -58,11 +58,13 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { ]) llmSpan.setTag("_ml_obs_tag.output", outputMessages) llmSpan.setTag("_ml_obs_tag.metadata", [temperature: 0.7, max_tokens: 100]) - llmSpan.setTag("_ml_obs_tag.tool_definitions", [[ - name: "get_weather", - description: "Get weather by city", - schema: [type: "object", properties: [city: [type: "string"]]] - ]]) + llmSpan.setTag("_ml_obs_tag.tool_definitions", [ + [ + name: "get_weather", + description: "Get weather by city", + schema: [type: "object", properties: [city: [type: "string"]]] + ] + ]) llmSpan.setError(true) llmSpan.setTag(DDTags.ERROR_MSG, "boom") llmSpan.setTag(DDTags.ERROR_TYPE, "java.lang.IllegalStateException") From f4e3a8b50462fd00f1122324a5b699fb715b1b4b Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 17 Mar 2026 17:35:20 -0700 Subject: [PATCH 29/48] Include input messages when instructions are present in prompt tracking --- .../openai_java/ResponseDecorator.java | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 9488ad0c865..1b66e11e098 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -692,33 +692,35 @@ private List extractInputMessagesForPromptTracking( } }); - if (!messages.isEmpty()) { - return messages; - } - // Fallback for SDK union parsing mismatches: parse raw instructions payload. - Optional rawInstructions = response._instructions().asUnknown(); - if (rawInstructions.isPresent()) { - Optional> rawList = rawInstructions.get().asArray(); - if (rawList.isPresent()) { - for (JsonValue item : rawList.get()) { - LLMObs.LLMMessage message = extractMessageFromRawInstruction(item); - if (message != null) { - messages.add(message); + if (messages.isEmpty()) { + Optional rawInstructions = response._instructions().asUnknown(); + if (rawInstructions.isPresent()) { + Optional> rawList = rawInstructions.get().asArray(); + if (rawList.isPresent()) { + for (JsonValue item : rawList.get()) { + LLMObs.LLMMessage message = extractMessageFromRawInstruction(item); + if (message != null) { + messages.add(message); + } } } } } - if (!messages.isEmpty()) { - return messages; - } + boolean hasInstructions = !messages.isEmpty(); + // Always include input messages from the span tag (set by withResponseCreateParams, which + // records both instructions as "system" and input as "user"). When instructions were already + // collected above, skip "system" messages here to avoid duplicating them. Object inputTag = span.getTag(CommonTags.INPUT); if (inputTag instanceof List) { for (Object messageObj : (List) inputTag) { if (messageObj instanceof LLMObs.LLMMessage) { - messages.add((LLMObs.LLMMessage) messageObj); + LLMObs.LLMMessage msg = (LLMObs.LLMMessage) messageObj; + if (!hasInstructions || !"system".equals(msg.getRole())) { + messages.add(msg); + } } } } else if (inputTag instanceof Map) { @@ -726,7 +728,10 @@ private List extractInputMessagesForPromptTracking( if (messagesObj instanceof List) { for (Object messageObj : (List) messagesObj) { if (messageObj instanceof LLMObs.LLMMessage) { - messages.add((LLMObs.LLMMessage) messageObj); + LLMObs.LLMMessage msg = (LLMObs.LLMMessage) messageObj; + if (!hasInstructions || !"system".equals(msg.getRole())) { + messages.add(msg); + } } } } From 028d64f1f809961e51dbccc99529eb9eca453c70 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 17 Mar 2026 18:45:42 -0700 Subject: [PATCH 30/48] Fix instructions role to system in prompt tracking --- .../trace/instrumentation/openai_java/ResponseDecorator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 1b66e11e098..bba4136d768 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -687,7 +687,7 @@ private List extractInputMessagesForPromptTracking( } else if (instructions.isString()) { String text = instructions.asString(); if (text != null && !text.isEmpty()) { - messages.add(LLMObs.LLMMessage.from("user", text)); + messages.add(LLMObs.LLMMessage.from("system", text)); } } }); From 717a8f0ecf90bd61cb35fde83bde750ec5e14937 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Thu, 19 Mar 2026 17:21:13 -0700 Subject: [PATCH 31/48] fix LLMObsSpanMapperTest --- .../trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy index 7d7de1180a7..fc254458920 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/llmobs/writer/ddintake/LLMObsSpanMapperTest.groovy @@ -74,7 +74,8 @@ class LLMObsSpanMapperTest extends DDCoreSpecification { def trace = [llmSpan] CapturingByteBufferConsumer sink = new CapturingByteBufferConsumer() - MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(1024, sink)) + // Keep all formatted spans in a single flush for this assertion. + MsgPackWriter packer = new MsgPackWriter(new FlushingBuffer(16 * 1024, sink)) when: packer.format(trace, mapper) From 8420f0a1f42072d2cd8d1a013a699a11ba05aefc Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 24 Mar 2026 15:15:30 -0700 Subject: [PATCH 32/48] Catch exception not throwable --- .../trace/instrumentation/openai_java/ResponseDecorator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index bba4136d768..7018005488d 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -156,7 +156,7 @@ private List> extractToolDefinitionsFromParams(ResponseCreat return toolDefinitions; } } - } catch (Throwable ignored) { + } catch (Exception ignored) { // fall back to raw JSON if typed extraction is unavailable or fails } From 91707faa176a9e6f370321b4eadeed31e83df206 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 24 Mar 2026 16:00:31 -0700 Subject: [PATCH 33/48] Add JsonValueUtilsTest --- .../openai_java/JsonValueUtilsTest.java | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/JsonValueUtilsTest.java diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/JsonValueUtilsTest.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/JsonValueUtilsTest.java new file mode 100644 index 00000000000..21713649956 --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/JsonValueUtilsTest.java @@ -0,0 +1,158 @@ +package datadog.trace.instrumentation.openai_java; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.openai.core.JsonValue; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class JsonValueUtilsTest { + + @Test + void testNullReturnsNull() { + assertNull(JsonValueUtils.jsonValueToObject(null)); + } + + @Test + void testString() { + assertEquals("hello", JsonValueUtils.jsonValueToObject(JsonValue.from("hello"))); + } + + @Test + void testInteger() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(42)); + assertInstanceOf(Number.class, result); + assertEquals(42, ((Number) result).intValue()); + } + + @Test + void testDouble() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(3.14)); + assertInstanceOf(Number.class, result); + assertEquals(3.14, ((Number) result).doubleValue(), 0.0001); + } + + @Test + void testBooleanTrue() { + assertEquals(true, JsonValueUtils.jsonValueToObject(JsonValue.from(true))); + } + + @Test + void testBooleanFalse() { + assertEquals(false, JsonValueUtils.jsonValueToObject(JsonValue.from(false))); + } + + @Test + void testFlatObject() { + Map input = new HashMap<>(); + input.put("key1", "value1"); + input.put("key2", 123); + + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(input)); + + assertInstanceOf(Map.class, result); + @SuppressWarnings("unchecked") + Map map = (Map) result; + assertEquals("value1", map.get("key1")); + assertEquals(123, ((Number) map.get("key2")).intValue()); + } + + @Test + void testNestedObject() { + Map inner = new HashMap<>(); + inner.put("x", "nested"); + Map outer = new HashMap<>(); + outer.put("inner", inner); + + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(outer)); + + assertInstanceOf(Map.class, result); + @SuppressWarnings("unchecked") + Map outerMap = (Map) result; + assertInstanceOf(Map.class, outerMap.get("inner")); + @SuppressWarnings("unchecked") + Map innerMap = (Map) outerMap.get("inner"); + assertEquals("nested", innerMap.get("x")); + } + + @Test + void testFlatArray() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(Arrays.asList("a", "b", "c"))); + + assertInstanceOf(List.class, result); + @SuppressWarnings("unchecked") + List list = (List) result; + assertEquals(3, list.size()); + assertEquals("a", list.get(0)); + assertEquals("b", list.get(1)); + assertEquals("c", list.get(2)); + } + + @Test + void testNestedArray() { + Object result = + JsonValueUtils.jsonValueToObject( + JsonValue.from(Arrays.asList(Arrays.asList(1, 2), Arrays.asList(3, 4)))); + + assertInstanceOf(List.class, result); + @SuppressWarnings("unchecked") + List outer = (List) result; + assertInstanceOf(List.class, outer.get(0)); + @SuppressWarnings("unchecked") + List inner = (List) outer.get(0); + assertEquals(1, ((Number) inner.get(0)).intValue()); + } + + @Test + void testMixedArray() { + Object result = + JsonValueUtils.jsonValueToObject(JsonValue.from(Arrays.asList("text", 42, true))); + + assertInstanceOf(List.class, result); + @SuppressWarnings("unchecked") + List list = (List) result; + assertEquals("text", list.get(0)); + assertEquals(42, ((Number) list.get(1)).intValue()); + assertEquals(true, list.get(2)); + } + + @Test + void testEmptyObject() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(new HashMap<>())); + assertInstanceOf(Map.class, result); + assertTrue(((Map) result).isEmpty()); + } + + @Test + void testEmptyArray() { + Object result = JsonValueUtils.jsonValueToObject(JsonValue.from(Arrays.asList())); + assertInstanceOf(List.class, result); + assertTrue(((List) result).isEmpty()); + } + + @Test + void testJsonValueMapToObject() { + Map input = new HashMap<>(); + input.put("str", JsonValue.from("hello")); + input.put("num", JsonValue.from(7)); + input.put("bool", JsonValue.from(false)); + + Map result = JsonValueUtils.jsonValueMapToObject(input); + + assertEquals("hello", result.get("str")); + assertEquals(7, ((Number) result.get("num")).intValue()); + assertEquals(false, result.get("bool")); + } + + @Test + void testJsonValueMapToObjectEmpty() { + Map result = JsonValueUtils.jsonValueMapToObject(new HashMap<>()); + assertTrue(result.isEmpty()); + } +} From 3d12515bb74cd8b541658dfffc875f9375c622d5 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 24 Mar 2026 17:54:25 -0700 Subject: [PATCH 34/48] Test that on HTTP error, the OpenAI response span retains model_name and placeholder output set by withResponseCreateParams. --- .../test/groovy/ResponseServiceTest.groovy | 67 ++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 02e28237750..ff66d56cd20 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -1,11 +1,14 @@ import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace import static datadog.trace.agent.test.utils.TraceUtils.runnableUnderTrace +import com.openai.client.okhttp.OpenAIOkHttpClient import com.openai.core.http.AsyncStreamResponse import com.openai.core.http.HttpResponseFor import com.openai.core.http.StreamResponse +import com.openai.credential.BearerTokenCredential import com.openai.models.responses.Response import com.openai.models.responses.ResponseStreamEvent +import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.api.DDSpanTypes import datadog.trace.api.llmobs.LLMObs import datadog.trace.bootstrap.instrumentation.api.Tags @@ -219,10 +222,70 @@ class ResponseServiceTest extends OpenAiTest { responseCreateParams << [responseCreateParamsWithToolInput(false), responseCreateParamsWithToolInput(true)] } - private void assertResponseTrace(boolean isStreaming, String reqModel, String respModel, Map reasoning) { - assertResponseTrace(isStreaming, reqModel, respModel, reasoning, null, null, null) + def "create response error sets model_name and placeholder output"() { + setup: + def errorBackend = TestHttpServer.httpServer { + handlers { + prefix("/v1/") { + response.status(500).send('{"error":{"message":"Internal server error","type":"server_error"}}') + } + } + } + def errorClient = OpenAIOkHttpClient.builder() + .baseUrl("${errorBackend.address.toURL()}/v1") + .credential(BearerTokenCredential.create("")) + .maxRetries(0) + .build() + + when: + runUnderTrace("parent") { + try { + errorClient.responses().create(responseCreateParams(false)) + } catch (Exception ignored) {} + } + + then: + List outputMessages = [] + assertTraces(1) { + trace(3) { + sortSpansByStart() + span(0) { + operationName "parent" + parent() + errored false + } + span(1) { + operationName "openai.request" + resourceName "createResponse" + childOf span(0) + errored true + spanType DDSpanTypes.LLMOBS + tags(false) { + "_ml_obs_tag.model_name" "gpt-3.5-turbo" + "_ml_obs_tag.output" List + def out = tag("_ml_obs_tag.output") + if (out instanceof List) outputMessages.addAll(out) + } + } + span(2) { + operationName "okhttp.request" + resourceName "POST /v1/responses" + childOf span(1) + errored false + spanType "http" + } + } + } + and: + outputMessages.size() == 1 + outputMessages[0].role == "" + outputMessages[0].content == "" + + cleanup: + errorBackend.close() } + private void assertResponseTrace( boolean isStreaming, String reqModel, From 576cec73abdbb5614fda2e5c06d4a1a45d2261cf Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 25 Mar 2026 11:41:22 -0700 Subject: [PATCH 35/48] Add "create response with prompt tracking" test to improve coverage of enrichInputWithPromptTracking(), extractChatTemplate(), extractPromptFromParams(), and normalizePromptVariable() --- .../src/test/groovy/OpenAiTest.groovy | 27 ++- .../test/groovy/ResponseServiceTest.groovy | 53 +++++- ...25cc9c569bf2fdcb+f6c73c0265d6d64d.POST.rec | 178 ++++++++++++++++++ 3 files changed, 249 insertions(+), 9 deletions(-) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/25cc9c569bf2fdcb+f6c73c0265d6d64d.POST.rec diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy index 542f9e27960..34661da8954 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy @@ -21,6 +21,7 @@ import com.openai.models.responses.ResponseCreateParams import com.openai.models.responses.ResponseFunctionToolCall import com.openai.models.responses.ResponseIncludable import com.openai.models.responses.ResponseInputItem +import com.openai.models.responses.ResponsePrompt import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.api.config.LlmObsConfig @@ -310,6 +311,31 @@ He hopes to pursue a career in software engineering after graduating.""") } } + ResponseCreateParams responseCreateParamsWithPromptTracking(boolean json) { + def variables = ResponsePrompt.Variables.builder() + .putAdditionalProperty("user_message", JsonValue.from([type: "input_text", text: "Analyze these images and document"])) + .putAdditionalProperty("user_image_1", JsonValue.from([type: "input_image", image_url: "https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"])) + .putAdditionalProperty("user_file", JsonValue.from([type: "input_file", file_url: "https://www.berkshirehathaway.com/letters/2024ltr.pdf"])) + .putAdditionalProperty("user_image_2", JsonValue.from([type: "input_image", file_id: "file-BCuhT1HQ24kmtsuuzF1mh2"])) + .build() + + def prompt = ResponsePrompt.builder() + .id("pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0") + .version("2") + .variables(variables) + .build() + + if (json) { + ResponseCreateParams.builder() + .prompt(prompt) + .build() + } else { + ResponseCreateParams.builder() + .prompt(prompt) + .build() + } + } + ChatCompletionCreateParams chatCompletionCreateParamsMultiChoice(boolean json) { if (json) { ChatCompletionCreateParams.builder() @@ -328,4 +354,3 @@ He hopes to pursue a career in software engineering after graduating.""") } } } - diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index ff66d56cd20..2a5b246ff6e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -222,6 +222,29 @@ class ResponseServiceTest extends OpenAiTest { responseCreateParams << [responseCreateParamsWithToolInput(false), responseCreateParamsWithToolInput(true)] } + def "create response with prompt tracking"() { + Response resp = runUnderTrace("parent") { + openAiClient.responses().create(params) + } + + expect: + resp != null + and: + Map metadata = [:] + Map input = [:] + assertResponseTrace(false, null, String, null, input, null, metadata, true) + and: + metadata.stream == false + input.prompt instanceof Map + input.messages instanceof List + def prompt = input.prompt as Map + prompt.chat_template instanceof List + prompt.variables instanceof Map + + where: + params << [responseCreateParamsWithPromptTracking(false), responseCreateParamsWithPromptTracking(true)] + } + def "create response error sets model_name and placeholder output"() { setup: def errorBackend = TestHttpServer.httpServer { @@ -289,11 +312,12 @@ class ResponseServiceTest extends OpenAiTest { private void assertResponseTrace( boolean isStreaming, String reqModel, - String respModel, + Object respModel, Map reasoning, - List inputTagsOut, + Object inputTagsOut, List outputTagsOut, - Map metadataOut) { + Map metadataOut, + boolean expectPromptTag = false) { assertTraces(1) { trace(3) { sortSpansByStart() @@ -313,14 +337,25 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.model_provider" "openai" "_ml_obs_tag.model_name" String "_ml_obs_tag.metadata" Map + if (expectPromptTag) { + "_ml_obs_request.prompt" Map + } def metadata = tag("_ml_obs_tag.metadata") if (metadataOut != null && metadata != null) { metadataOut.putAll(metadata) } - "_ml_obs_tag.input" List + if (inputTagsOut instanceof Map) { + "_ml_obs_tag.input" Map + } else { + "_ml_obs_tag.input" List + } def inputTags = tag("_ml_obs_tag.input") if (inputTagsOut != null && inputTags != null) { - inputTagsOut.addAll(inputTags) + if (inputTagsOut instanceof List) { + inputTagsOut.addAll(inputTags as List) + } else if (inputTagsOut instanceof Map) { + inputTagsOut.putAll(inputTags as Map) + } } "_ml_obs_tag.output" List def outputTags = tag("_ml_obs_tag.output") @@ -347,13 +382,15 @@ class ResponseServiceTest extends OpenAiTest { "openai.api_base" openAiBaseApi "$CommonTags.OPENAI_RESPONSE_MODEL" respModel if (!isStreaming) { - "openai.organization.ratelimit.requests.limit" 10000 + "openai.organization.ratelimit.requests.limit" Integer "openai.organization.ratelimit.requests.remaining" Integer - "openai.organization.ratelimit.tokens.limit" 50000000 + "openai.organization.ratelimit.tokens.limit" Integer "openai.organization.ratelimit.tokens.remaining" Integer } "$CommonTags.OPENAI_ORGANIZATION" "datadog-staging" - "$CommonTags.OPENAI_REQUEST_MODEL" reqModel + if (reqModel != null) { + "$CommonTags.OPENAI_REQUEST_MODEL" reqModel + } "$Tags.COMPONENT" "openai" "$Tags.SPAN_KIND" Tags.SPAN_KIND_CLIENT defaultTags() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/25cc9c569bf2fdcb+f6c73c0265d6d64d.POST.rec b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/25cc9c569bf2fdcb+f6c73c0265d6d64d.POST.rec new file mode 100644 index 00000000000..a72f03b8a33 --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/25cc9c569bf2fdcb+f6c73c0265d6d64d.POST.rec @@ -0,0 +1,178 @@ +method: POST +path: responses +-- begin request body -- +{"prompt":{"id":"pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0","variables":{"user_message":{"type":"input_text","text":"Analyze these images and document"},"user_image_1":{"type":"input_image","image_url":"https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"},"user_file":{"type":"input_file","file_url":"https://www.berkshirehathaway.com/letters/2024ltr.pdf"},"user_image_2":{"type":"input_image","file_id":"file-BCuhT1HQ24kmtsuuzF1mh2"}},"version":"2"}} +-- end request body -- +status code: 200 +-- begin response headers -- +alt-svc: h3=":443"; ma=86400 +cf-cache-status: DYNAMIC +cf-ray: 9e1fdb67e8b3d84e-SEA +content-type: application/json +date: Wed, 25 Mar 2026 18:08:13 GMT +openai-organization: datadog-staging +openai-processing-ms: 27175 +openai-project: proj_gt6TQZPRbZfoY2J9AQlEJMpd +openai-version: 2020-10-01 +server: cloudflare +set-cookie: __cf_bm=0omhm_EDPiOwCkTnRHMTcH2143C9foKJkpkM5V98XPA-1774462065.9075637-1.0.1.1-Dr2KjfEYzM5.5hH26NB1U9ok305D3x0N1Jj2oNi9HRpE49YL0u3dMPv7FNeawtk.1duphFGgfCFpcbqn_qCR7bW1MtVt6IISK3BKO_TDbjrSg3oVneqJuZ62wmSRDDb2; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 25 Mar 2026 18:38:13 GMT +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +x-ratelimit-limit-requests: 30000 +x-ratelimit-limit-tokens: 150000000 +x-ratelimit-remaining-requests: 29999 +x-ratelimit-remaining-tokens: 149998470 +x-ratelimit-reset-requests: 2ms +x-ratelimit-reset-tokens: 0s +x-request-id: req_55ed921a6d294700bce3397e0b75e7da +-- end response headers -- +-- begin response body -- +{ + "id": "resp_00cd8b3bfcd5108a0169c42472356c8190a2f61c5a5f3b32e7", + "object": "response", + "created_at": 1774462067, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1774462093, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": [ + { + "type": "message", + "content": [], + "role": "developer" + }, + { + "type": "message", + "content": [ + { + "type": "input_text", + "text": "Analyze the following content from the user:\n\nText message: Analyze these images and document\nImage reference 1: " + }, + { + "type": "input_image", + "detail": "auto", + "file_id": null, + "image_url": null + }, + { + "type": "input_text", + "text": "\nDocument reference: " + }, + { + "type": "input_file", + "file_id": null, + "file_url": "https://www.berkshirehathaway.com/letters/2024ltr.pdf" + }, + { + "type": "input_text", + "text": "\nImage reference 2: " + }, + { + "type": "input_image", + "detail": "auto", + "file_id": "file-BCuhT1HQ24kmtsuuzF1mh2", + "image_url": null + }, + { + "type": "input_text", + "text": "\n\nPlease provide a comprehensive analysis." + } + ], + "role": "user" + } + ], + "max_output_tokens": null, + "max_tool_calls": null, + "model": "o4-mini-2025-04-16", + "output": [ + { + "id": "rs_00cd8b3bfcd5108a0169c424754a808190a4090b0188003f86", + "type": "reasoning", + "summary": [] + }, + { + "id": "msg_00cd8b3bfcd5108a0169c42480774c819082158e702c85be03", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "Here\u2019s a step-by-step breakdown of the three items you\u2019ve provided:\n\n1. Image 1 (the two-snake logo) \n \u2022 Identification: This is the official logo of the Python programming language. \n \u2022 Design elements: \n \u2013 Two interlocking \u201csnake\u201d shapes (one blue, one yellow) arranged to form a stylized \u201cP.\u201d \n \u2013 Rounded corners, minimal line-weight, flat color palette\u2014characteristics of modern \u201cflat\u201d logo design. \n \u2013 Created to evoke both the language\u2019s namesake (Monty Python aside, a snake) and its emphasis on readability and simplicity. \n \u2022 Historical/contextual notes: \n \u2013 Python was conceived by Guido van Rossum in the late 1980s and first released in 1991. \n \u2013 Over the years the logo has come to symbolize a massive ecosystem\u2014from web back-ends (Django, Flask) to data science (Pandas, NumPy) to machine learning (scikit-learn, TensorFlow). \n \u2013 Its clean, friendly shape mirrors Python\u2019s design philosophy: code that reads like plain English, easy onboarding for newcomers. \n\n2. The Berkshire Hathaway 2024 Shareholder Letter (Warren E. Buffett) \n A. Purpose & Tone \n \u2013 Beyond the mandatory financial \u201creport,\u201d Buffett uses the letter to provide candid commentary\u2014both on successes and on mistakes. \n \u2013 He stresses the obligation of a CEO to treat shareholders as he\u2019d want to be treated if the roles were reversed. \n \u2013 He repeatedly admits errors (16 uses of \u201cmistake\u201d or \u201cerror\u201d in the last five years), contrasting this transparency with what he sees as glib \u201cfeel-good\u201d letters from many large companies. \n\n B. Major Themes \n 1. The Importance of Admitting and Correcting Mistakes \n \u2013 Mistakes in acquisitions or in assessing management fidelity can be painful but must be addressed promptly (\u201cno thumb-sucking\u201d). \n \u2013 Single great decisions\u2014e.g., GEICO purchase, finding Charlie Munger\u2014can compound into decade-long outperformance. \n\n 2. Capital Allocation Philosophy \n \u2013 The core measure Buffett emphasizes is \u201coperating earnings,\u201d excluding unrealized securities gains or losses. \n \u2013 He reaffirms that Berkshire will remain overwhelmingly invested in equities (both controlled subsidiaries and minority stakes in giants like Apple, Coca-Cola, American Express). \n \u2013 He warns against over-reliance on cash or bonds, which carry currency\u2010value risk; instead, intrinsic ownership of businesses is the best inflation hedge. \n\n 3. Insurance Float as a Growth Engine \n \u2013 Property-casualty underwriting delivers \u201cfloat\u201d (premiums received before claims are paid). \n \u2013 If priced intelligently, float can be \u201ccostless capital\u201d that Buffett can then deploy elsewhere. \n \u2013 He lauds Ajit Jain for building Berkshire\u2019s P/C operation into a global leader, but stresses the business is perilously dependent on accurate long-term loss estimates. \n\n 4. Record U.S. Tax Payments \n \u2013 In 2024 Berkshire paid $26.8 billion in U.S. federal income taxes\u2014more than any other corporation in American history. \n \u2013 Buffett attributes this partly to his long-ago decision (and shareholders\u2019 endorsement) to forgo dividends, instead reinvesting all earnings back into operations. \n\n 5. Discipline in CEO and Manager Selection \n \u2013 Buffett never considers pedigree (school attended) when hiring CEOs\u2014innate talent often outweighs credentials. \n \u2013 He illustrates this with the story of Pete Liegl (founder of Forest River), whose self-set $100 k base salary plus performance bonus yielded multiples of that investment for Berkshire over 19 years. \n\n 6. International Ventures\u2014The Japan Five \n \u2013 Since 2019 Berkshire has taken minority stakes (below 10%) in five large Japanese trading houses (Itochu, Marubeni, Mitsubishi, Mitsui, Sumitomo). \n \u2013 He praises their capital\u2010allocation discipline, low executive pay, willingness to buy back shares. \n \u2013 Berkshire balances that foreign\u2010currency exposure by matching it with yen-denominated debt, achieving near currency neutrality. \n\n 7. Shareholder-Centered Culture & Annual Meeting \n \u2013 Buffett invites shareholders to the May 3 gathering in Omaha, emphasizing accessibility of management, Q&A format, volunteer-run merch booths, and a single themed book for sale. \n \u2013 He recounts lighthearted family anecdotes (his sister Bertie\u2019s cane, grandchildren) to underscore the meeting\u2019s informal, \u201cfamily reunion\u201d atmosphere. \n\n C. Performance Snapshot \n \u2013 2024 operating earnings: $47.4 billion vs. $37.4 billion in 2023. \n \u2013 Insurance underwriting profit: $9.0 billion; investment income: $13.7 billion. \n \u2013 Continued strength at BNSF and Berkshire Hathaway Energy. \n \u2013 Slide reinstates the 1965\u20132024 compound annual gain of 19.9% vs. 10.4% for the S&P 500. \n\n D. Stylistic & Structural Notes \n \u2013 Plain\u2010spoken, anecdote-rich, avoids corporate jargon. \n \u2013 Buffett interleaves big-picture economic observations (capitalism\u2019s triumphs and pitfalls) with very granular \u201cinside\u201d details (how he negotiated with Pete Liegl). \n \u2013 Occasional dry humor (e.g., comparing his and his sister\u2019s canes as \u201cmale-deterrents\u201d). \n\n3. Image 2 \n \u2022 On my end the image rendered as a blank/empty panel. There\u2019s no visible content to describe or analyze. \n \u2022 If this was meant to convey text, a chart, code or a photograph, could you please re-upload a higher-resolution copy or a different file format? \n\n\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013\u2013 \nSummary: \n\u2013 You\u2019ve supplied two corporate \u201cicons\u201d (the Python logo and the Berkshire shareholder letter), both of which epitomize transparency, community-oriented design, and long-term stewardship\u2014one in software, the other in capital allocation. \n\u2013 The Python logo signals a language built for clarity, extensibility and an open ecosystem. \n\u2013 Buffett\u2019s letter models \u201cflat,\u201d jargon-free communication, candidly reporting mistakes alongside successes, and reaffirming a shareholder-centric culture. \n\u2013 Please let me know about Image 2, and I\u2019ll be happy to complete the visual analysis." + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt": { + "id": "pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0", + "variables": { + "user_message": { + "type": "input_text", + "text": "Analyze these images and document" + }, + "user_image_1": { + "type": "input_image", + "detail": "auto", + "file_id": null, + "image_url": null + }, + "user_file": { + "type": "input_file", + "file_id": null, + "file_url": "https://www.berkshirehathaway.com/letters/2024ltr.pdf" + }, + "user_image_2": { + "type": "input_image", + "detail": "auto", + "file_id": "file-BCuhT1HQ24kmtsuuzF1mh2", + "image_url": null + } + }, + "version": "2" + }, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": "medium", + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 11236, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 2366, + "output_tokens_details": { + "reasoning_tokens": 960 + }, + "total_tokens": 13602 + }, + "user": null, + "metadata": {} +}� +-- end response body -- From ba0cb2702698f225a67b67f088eb289e48b2fb5f Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 25 Mar 2026 14:55:07 -0700 Subject: [PATCH 36/48] Add "create response with custom tool call" test to improve coverage of getToolCall --- .../src/test/groovy/OpenAiTest.groovy | 31 +++++ .../test/groovy/ResponseServiceTest.groovy | 23 +++- ...4919ef6198916e1a+cf7135b364dfeac5.POST.rec | 112 ++++++++++++++++++ 3 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/4919ef6198916e1a+cf7135b364dfeac5.POST.rec diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy index 34661da8954..c265c704b7e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy @@ -22,6 +22,8 @@ import com.openai.models.responses.ResponseFunctionToolCall import com.openai.models.responses.ResponseIncludable import com.openai.models.responses.ResponseInputItem import com.openai.models.responses.ResponsePrompt +import com.openai.models.responses.CustomTool +import com.openai.models.responses.ToolChoiceCustom import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.api.config.LlmObsConfig @@ -336,6 +338,35 @@ He hopes to pursue a career in software engineering after graduating.""") } } + ResponseCreateParams responseCreateParamsWithCustomToolCall(boolean json) { + def customTool = CustomTool.builder() + .name("custom_weather") + .description("Return weather for a location") + .formatText() + .build() + + def toolChoice = ToolChoiceCustom.builder() + .name("custom_weather") + .type(JsonValue.from("custom")) + .build() + + if (json) { + ResponseCreateParams.builder() + .model("gpt-5") + .input("Use the custom_weather tool to answer: What's the weather in Boston?") + .addTool(customTool) + .toolChoice(toolChoice) + .build() + } else { + ResponseCreateParams.builder() + .model(ChatModel.GPT_5) + .input("Use the custom_weather tool to answer: What's the weather in Boston?") + .addTool(customTool) + .toolChoice(toolChoice) + .build() + } + } + ChatCompletionCreateParams chatCompletionCreateParamsMultiChoice(boolean json) { if (json) { ChatCompletionCreateParams.builder() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 2a5b246ff6e..f704fd3cb55 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -205,7 +205,7 @@ class ResponseServiceTest extends OpenAiTest { expect: List inputTags = [] Map metadata = [:] - assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags, null, metadata) + assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags, null, metadata, false, true) and: metadata.stream == true inputTags.size() == 3 @@ -245,6 +245,24 @@ class ResponseServiceTest extends OpenAiTest { params << [responseCreateParamsWithPromptTracking(false), responseCreateParamsWithPromptTracking(true)] } + def "create response with custom tool call"() { + Response resp = runUnderTrace("parent") { + openAiClient.responses().create(params) + } + + expect: + resp != null + and: + List outputTags = [] + Map metadata = [:] + assertResponseTrace(false, "gpt-5", String, null, null, outputTags, metadata) + and: + !outputTags.isEmpty() + + where: + params << [responseCreateParamsWithCustomToolCall(false), responseCreateParamsWithCustomToolCall(true)] + } + def "create response error sets model_name and placeholder output"() { setup: def errorBackend = TestHttpServer.httpServer { @@ -317,7 +335,8 @@ class ResponseServiceTest extends OpenAiTest { Object inputTagsOut, List outputTagsOut, Map metadataOut, - boolean expectPromptTag = false) { + boolean expectPromptTag = false, + boolean expectRateLimitTags = false) { assertTraces(1) { trace(3) { sortSpansByStart() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/4919ef6198916e1a+cf7135b364dfeac5.POST.rec b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/4919ef6198916e1a+cf7135b364dfeac5.POST.rec new file mode 100644 index 00000000000..8b748b9b57a --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/4919ef6198916e1a+cf7135b364dfeac5.POST.rec @@ -0,0 +1,112 @@ +method: POST +path: responses +-- begin request body -- +{"input":"Use the custom_weather tool to answer: What's the weather in Boston?","model":"gpt-5","tool_choice":{"name":"custom_weather","type":"custom"},"tools":[{"name":"custom_weather","type":"custom","description":"Return weather for a location","format":{"type":"text"}}]} +-- end request body -- +status code: 200 +-- begin response headers -- +alt-svc: h3=":443"; ma=86400 +cf-cache-status: DYNAMIC +cf-ray: 9e2122d2ae6712d4-SEA +content-type: application/json +date: Wed, 25 Mar 2026 21:51:20 GMT +openai-organization: datadog-staging +openai-processing-ms: 3360 +openai-project: proj_gt6TQZPRbZfoY2J9AQlEJMpd +openai-version: 2020-10-01 +server: cloudflare +set-cookie: __cf_bm=SP753lo3ga6JHum4FOPTYPa6B.h7QhGpK2Tih8iqYOI-1774475476.9017334-1.0.1.1-G_WqthMsAoHTvAkAACMpAsVWmtzzeqOHTjGZ_eY4WSxZz_pND9T9O78.pCiPD52gquRIzVxrXPd9J06VIMqiuAA9z_GWrLKLGc.s2dzDvUsWxIek6wIPjN4EOBFNlgoB; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 25 Mar 2026 22:21:20 GMT +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +x-ratelimit-limit-requests: 15000 +x-ratelimit-limit-tokens: 40000000 +x-ratelimit-remaining-requests: 14999 +x-ratelimit-remaining-tokens: 40000000 +x-ratelimit-reset-requests: 4ms +x-ratelimit-reset-tokens: 0s +x-request-id: req_dcbdd1cbcd494ff793ee2e099de4554e +-- end response headers -- +-- begin response body -- +{ + "id": "resp_0f18ec8851ed77310169c458d536188190ae13c792509ad0c7", + "object": "response", + "created_at": 1774475477, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1774475480, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-5-2025-08-07", + "output": [ + { + "id": "rs_0f18ec8851ed77310169c458d5b5108190a0c49289af33d93b", + "type": "reasoning", + "summary": [] + }, + { + "id": "ctc_0f18ec8851ed77310169c458d84a7c8190b1cefc253127979c", + "type": "custom_tool_call", + "status": "completed", + "call_id": "call_sxqlRD4E0PihmL74BNt7GqVV", + "input": "Boston, MA, USA", + "name": "custom_weather" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": "medium", + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": { + "type": "custom", + "name": "custom_weather" + }, + "tools": [ + { + "type": "custom", + "description": "Return weather for a location", + "format": { + "type": "text" + }, + "name": "custom_weather" + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 67, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 238, + "output_tokens_details": { + "reasoning_tokens": 192 + }, + "total_tokens": 305 + }, + "user": null, + "metadata": {} +}� +-- end response body -- From 8be92d704db1ba783d68d7ffd49acf6892625bd7 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 25 Mar 2026 20:28:05 -0700 Subject: [PATCH 37/48] Prevent NPE when tag value is null --- .../groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy index b12984ecbee..8c39baf2851 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/asserts/TagsAssert.groovy @@ -247,7 +247,7 @@ class TagsAssert { if (expected instanceof Pattern) { assert value =~ expected: "Tag \"$name\": \"${value.toString()}\" does not match pattern \"$expected\"" } else if (expected instanceof Class) { - assert ((Class) expected).isInstance(value): "Tag \"$name\": instance check $expected failed for \"${value.toString()}\" of class \"${value.class}\"" + assert ((Class) expected).isInstance(value): "Tag \"$name\": instance check $expected failed for \"${value.toString()}\" of class \"${value?.class}\"" } else if (expected instanceof Closure) { assert ((Closure) expected).call(value): "Tag \"$name\": closure call ${expected.toString()} failed with \"$value\"" } else if (expected instanceof CharSequence) { From 1036ed40ce5f1d0772f65dfccd6f2a0891f907ca Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 25 Mar 2026 20:29:58 -0700 Subject: [PATCH 38/48] Replace catch Throwable with catch Exception --- .../trace/instrumentation/openai_java/ResponseDecorator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 7018005488d..f46b93e819e 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -178,7 +178,7 @@ private List> extractToolDefinitionsFromParams(ResponseCreat } } return toolDefinitions; - } catch (Throwable ignored) { + } catch (Exception ignored) { return Collections.emptyList(); } } From 9911c514e788cdf18c9d5f23818bf45ddb0db50b Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 25 Mar 2026 20:58:17 -0700 Subject: [PATCH 39/48] responseCreateParamsWithPromptTracking support both known and unknown format. Test cover extractPromptFromParams and related methods --- .../openai_java/ResponseDecorator.java | 91 ++++++++++++++++--- .../src/test/groovy/OpenAiTest.groovy | 39 +++++--- .../test/groovy/ResponseServiceTest.groovy | 9 +- 3 files changed, 106 insertions(+), 33 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index f46b93e819e..14d2166f6e2 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -807,35 +807,96 @@ private LLMObs.LLMMessage extractMessageFromRawInstruction(JsonValue instruction } private Optional> extractPromptFromParams(ResponseCreateParams params) { - Optional promptOpt = params.prompt(); - if (!promptOpt.isPresent()) { + Optional typedPromptOpt = params._prompt().asKnown(); + if (typedPromptOpt.isPresent()) { + Optional> extractedPrompt = extractPrompt(typedPromptOpt.get()); + if (extractedPrompt.isPresent()) { + return extractedPrompt; + } + } + + try { + Optional rawPromptOpt = params._prompt().asUnknown(); + if (!rawPromptOpt.isPresent()) { + return Optional.empty(); + } + + Optional> rawPromptObjOpt = rawPromptOpt.get().asObject(); + if (!rawPromptObjOpt.isPresent()) { + return Optional.empty(); + } + + return extractPrompt(rawPromptObjOpt.get()); + } catch (Exception ignored) { return Optional.empty(); } + } - ResponsePrompt prompt = promptOpt.get(); + private Optional> extractPrompt(ResponsePrompt prompt) { Map promptMap = new LinkedHashMap<>(); - String id = prompt.id(); + String id = prompt._id().asString().orElse(null); if (id != null && !id.isEmpty()) { promptMap.put("id", id); } - prompt.version().ifPresent(version -> promptMap.put("version", version)); - prompt - .variables() - .ifPresent( - variables -> { - Map normalized = normalizePromptVariables(variables); - if (!normalized.isEmpty()) { - promptMap.put("variables", normalized); - } - }); + prompt._version().asString().ifPresent(version -> promptMap.put("version", version)); + + Optional typedVariablesOpt = prompt._variables().asKnown(); + if (typedVariablesOpt.isPresent()) { + Map normalized = normalizePromptVariables(typedVariablesOpt.get()); + if (!normalized.isEmpty()) { + promptMap.put("variables", normalized); + } + } else { + Optional rawVariablesOpt = prompt._variables().asUnknown(); + if (rawVariablesOpt.isPresent()) { + Optional> rawVariablesObjOpt = rawVariablesOpt.get().asObject(); + if (rawVariablesObjOpt.isPresent()) { + Map normalized = normalizePromptVariables(rawVariablesObjOpt.get()); + if (!normalized.isEmpty()) { + promptMap.put("variables", normalized); + } + } + } + } + + return promptMap.isEmpty() ? Optional.empty() : Optional.of(promptMap); + } + + private Optional> extractPrompt(Map promptObj) { + Map promptMap = new LinkedHashMap<>(); + + String id = getJsonString(promptObj.get("id")); + if (id != null && !id.isEmpty()) { + promptMap.put("id", id); + } + + String version = getJsonString(promptObj.get("version")); + if (version != null && !version.isEmpty()) { + promptMap.put("version", version); + } + + JsonValue variablesValue = promptObj.get("variables"); + if (variablesValue != null) { + Optional> variablesObjOpt = variablesValue.asObject(); + if (variablesObjOpt.isPresent()) { + Map normalized = normalizePromptVariables(variablesObjOpt.get()); + if (!normalized.isEmpty()) { + promptMap.put("variables", normalized); + } + } + } return promptMap.isEmpty() ? Optional.empty() : Optional.of(promptMap); } private Map normalizePromptVariables(ResponsePrompt.Variables variables) { + return normalizePromptVariables(variables._additionalProperties()); + } + + private Map normalizePromptVariables(Map variables) { Map normalized = new LinkedHashMap<>(); - for (Map.Entry entry : variables._additionalProperties().entrySet()) { + for (Map.Entry entry : variables.entrySet()) { Object value = normalizePromptVariable(entry.getValue()); if (value != null) { normalized.put(entry.getKey(), value); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy index c265c704b7e..e1eef17eb47 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy @@ -314,24 +314,35 @@ He hopes to pursue a career in software engineering after graduating.""") } ResponseCreateParams responseCreateParamsWithPromptTracking(boolean json) { - def variables = ResponsePrompt.Variables.builder() - .putAdditionalProperty("user_message", JsonValue.from([type: "input_text", text: "Analyze these images and document"])) - .putAdditionalProperty("user_image_1", JsonValue.from([type: "input_image", image_url: "https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"])) - .putAdditionalProperty("user_file", JsonValue.from([type: "input_file", file_url: "https://www.berkshirehathaway.com/letters/2024ltr.pdf"])) - .putAdditionalProperty("user_image_2", JsonValue.from([type: "input_image", file_id: "file-BCuhT1HQ24kmtsuuzF1mh2"])) - .build() - - def prompt = ResponsePrompt.builder() - .id("pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0") - .version("2") - .variables(variables) - .build() - if (json) { + def rawPrompt = JsonValue.from([ + id: "pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0", + variables: [ + user_message: [type: "input_text", text: "Analyze these images and document"], + user_image_1: [type: "input_image", image_url: "https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"], + user_file: [type: "input_file", file_url: "https://www.berkshirehathaway.com/letters/2024ltr.pdf"], + user_image_2: [type: "input_image", file_id: "file-BCuhT1HQ24kmtsuuzF1mh2"], + ], + version: "2", + ]) + ResponseCreateParams.builder() - .prompt(prompt) + .prompt(rawPrompt) .build() } else { + def variables = ResponsePrompt.Variables.builder() + .putAdditionalProperty("user_message", JsonValue.from([type: "input_text", text: "Analyze these images and document"])) + .putAdditionalProperty("user_image_1", JsonValue.from([type: "input_image", image_url: "https://raw.githubusercontent.com/github/explore/main/topics/python/python.png"])) + .putAdditionalProperty("user_file", JsonValue.from([type: "input_file", file_url: "https://www.berkshirehathaway.com/letters/2024ltr.pdf"])) + .putAdditionalProperty("user_image_2", JsonValue.from([type: "input_image", file_id: "file-BCuhT1HQ24kmtsuuzF1mh2"])) + .build() + + def prompt = ResponsePrompt.builder() + .id("pmpt_69201db75c4c81959c01ea6987ab023c070192cd2843dec0") + .version("2") + .variables(variables) + .build() + ResponseCreateParams.builder() .prompt(prompt) .build() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index f704fd3cb55..8029ab0bdd0 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -205,7 +205,7 @@ class ResponseServiceTest extends OpenAiTest { expect: List inputTags = [] Map metadata = [:] - assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags, null, metadata, false, true) + assertResponseTrace(true, "gpt-4.1", "gpt-4.1-2025-04-14", null, inputTags, null, metadata, false) and: metadata.stream == true inputTags.size() == 3 @@ -305,7 +305,9 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.model_name" "gpt-3.5-turbo" "_ml_obs_tag.output" List def out = tag("_ml_obs_tag.output") - if (out instanceof List) outputMessages.addAll(out) + if (out instanceof List) { + outputMessages.addAll(out) + } } } span(2) { @@ -335,8 +337,7 @@ class ResponseServiceTest extends OpenAiTest { Object inputTagsOut, List outputTagsOut, Map metadataOut, - boolean expectPromptTag = false, - boolean expectRateLimitTags = false) { + boolean expectPromptTag = false) { assertTraces(1) { trace(3) { sortSpansByStart() From c8f1a02b4c9cad8b2648b91efe0dffaf0d3da888 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 27 Mar 2026 11:21:35 -0700 Subject: [PATCH 40/48] Use safe accessors throughout OpenAI Java decorators --- .../openai_java/ChatCompletionDecorator.java | 38 +- .../openai_java/CompletionDecorator.java | 74 +++- .../openai_java/EmbeddingDecorator.java | 56 ++- .../openai_java/ResponseDecorator.java | 351 ++++++++++-------- 4 files changed, 308 insertions(+), 211 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index f35e485c90e..fa39d8cd570 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -41,7 +41,7 @@ public void withChatCompletionCreateParams( if (params == null) { return; } - Optional modelName = params.model()._value().asString(); + Optional modelName = extractChatModelName(params); modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); if (!llmObsEnabled) { @@ -101,12 +101,18 @@ public void withChatCompletionCreateParams( } }); - List tools = params.tools().orElse(Collections.emptyList()); + List tools = params._tools().asKnown().orElse(Collections.emptyList()); if (!tools.isEmpty()) { span.setTag(CommonTags.TOOL_DEFINITIONS, extractToolDefinitions(tools)); } } + private Optional extractChatModelName(ChatCompletionCreateParams params) { + Optional modelName = + params._model().asKnown().flatMap(model -> model._value().asString()); + return modelName.isPresent() ? modelName : params._model().asString(); + } + private List> extractToolDefinitions(List tools) { List> toolDefinitions = new ArrayList<>(); for (ChatCompletionTool tool : tools) { @@ -193,7 +199,7 @@ private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { } public void withChatCompletion(AgentSpan span, ChatCompletion completion) { - String modelName = completion.model(); + String modelName = completion._model().asString().orElse(null); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); @@ -202,13 +208,14 @@ public void withChatCompletion(AgentSpan span, ChatCompletion completion) { } List output = - completion.choices().stream() + completion._choices().asKnown().orElse(Collections.emptyList()).stream() .map(ChatCompletionDecorator::llmMessage) .collect(Collectors.toList()); span.setTag(CommonTags.OUTPUT, output); completion - .usage() + ._usage() + .asKnown() .ifPresent( usage -> { span.setTag(CommonTags.INPUT_TOKENS, usage.promptTokens()); @@ -222,18 +229,21 @@ public void withChatCompletion(AgentSpan span, ChatCompletion completion) { } private static LLMObs.LLMMessage llmMessage(ChatCompletion.Choice choice) { - ChatCompletionMessage msg = choice.message(); - Optional roleOpt = msg._role().asString(); - String role = "unknown"; - if (roleOpt.isPresent()) { - role = String.valueOf(roleOpt.get()); + Optional msgOpt = choice._message().asKnown(); + if (!msgOpt.isPresent()) { + return LLMObs.LLMMessage.from("unknown", ""); } - String content = msg.content().orElse(""); - Optional> toolCallsOpt = msg.toolCalls(); - if (toolCallsOpt.isPresent() && !toolCallsOpt.get().isEmpty()) { + ChatCompletionMessage msg = msgOpt.get(); + Optional roleOpt = msg._role().asString(); + String role = roleOpt.isPresent() ? String.valueOf(roleOpt.get()) : "unknown"; + String content = msg._content().asString().orElse(""); + + List toolCallsOpt = + msg._toolCalls().asKnown().orElse(Collections.emptyList()); + if (!toolCallsOpt.isEmpty()) { List toolCalls = new ArrayList<>(); - for (ChatCompletionMessageToolCall toolCall : toolCallsOpt.get()) { + for (ChatCompletionMessageToolCall toolCall : toolCallsOpt) { LLMObs.ToolCall llmObsToolCall = ToolCallExtractor.getToolCall(toolCall); if (llmObsToolCall != null) { toolCalls.add(llmObsToolCall); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java index f0f29386582..f71c18a706d 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java @@ -27,8 +27,7 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa if (params == null) { return; } - - Optional modelName = params.model()._value().asString(); + Optional modelName = extractCompletionModelName(params); modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); if (!llmObsEnabled) { @@ -43,9 +42,7 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa }); span.setTag(CommonTags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); - params - .prompt() - .flatMap(p -> p.string()) + extractPromptText(params) .ifPresent( input -> span.setTag( @@ -67,7 +64,7 @@ public void withCompletionCreateParams(AgentSpan span, CompletionCreateParams pa } public void withCompletion(AgentSpan span, Completion completion) { - String modelName = completion.model(); + String modelName = completion._model().asString().orElse(null); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); @@ -76,18 +73,28 @@ public void withCompletion(AgentSpan span, Completion completion) { } List output = - completion.choices().stream() - .map(v -> LLMObs.LLMMessage.from("", v.text())) + completion._choices().asKnown().orElse(Collections.emptyList()).stream() + .map(v -> LLMObs.LLMMessage.from("", v._text().asString().orElse(null))) .collect(Collectors.toList()); span.setTag(CommonTags.OUTPUT, output); completion - .usage() + ._usage() + .asKnown() .ifPresent( usage -> { - span.setTag(CommonTags.INPUT_TOKENS, usage.promptTokens()); - span.setTag(CommonTags.OUTPUT_TOKENS, usage.completionTokens()); - span.setTag(CommonTags.TOTAL_TOKENS, usage.totalTokens()); + usage + ._promptTokens() + .asKnown() + .ifPresent(v -> span.setTag(CommonTags.INPUT_TOKENS, v)); + usage + ._completionTokens() + .asKnown() + .ifPresent(v -> span.setTag(CommonTags.OUTPUT_TOKENS, v)); + usage + ._totalTokens() + .asKnown() + .ifPresent(v -> span.setTag(CommonTags.TOTAL_TOKENS, v)); }); } @@ -97,7 +104,7 @@ public void withCompletions(AgentSpan span, List completions) { } Completion firstCompletion = completions.get(0); - String modelName = firstCompletion.model(); + String modelName = firstCompletion._model().asString().orElse(null); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); @@ -108,13 +115,15 @@ public void withCompletions(AgentSpan span, List completions) { Map textByChoiceIndex = new HashMap<>(); for (Completion completion : completions) { completion - .choices() + ._choices() + .asKnown() + .orElse(Collections.emptyList()) .forEach( choice -> { - long index = choice.index(); + long index = choice._index().asKnown().orElse(0L); textByChoiceIndex .computeIfAbsent(index, k -> new StringBuilder()) - .append(choice.text()); + .append(choice._text().asString().orElse("")); }); } @@ -127,12 +136,37 @@ public void withCompletions(AgentSpan span, List completions) { Completion lastCompletion = completions.get(completions.size() - 1); lastCompletion - .usage() + ._usage() + .asKnown() .ifPresent( usage -> { - span.setTag(CommonTags.INPUT_TOKENS, usage.promptTokens()); - span.setTag(CommonTags.OUTPUT_TOKENS, usage.completionTokens()); - span.setTag(CommonTags.TOTAL_TOKENS, usage.totalTokens()); + usage + ._promptTokens() + .asKnown() + .ifPresent(v -> span.setTag(CommonTags.INPUT_TOKENS, v)); + usage + ._completionTokens() + .asKnown() + .ifPresent(v -> span.setTag(CommonTags.OUTPUT_TOKENS, v)); + usage + ._totalTokens() + .asKnown() + .ifPresent(v -> span.setTag(CommonTags.TOTAL_TOKENS, v)); }); } + + private Optional extractCompletionModelName(CompletionCreateParams params) { + Optional modelName = + params._model().asKnown().flatMap(model -> model._value().asString()); + return modelName.isPresent() ? modelName : params._model().asString(); + } + + private Optional extractPromptText(CompletionCreateParams params) { + Optional promptText = + params._prompt().asKnown().flatMap(CompletionCreateParams.Prompt::string); + if (promptText.isPresent()) { + return promptText; + } + return params._prompt().asUnknown().flatMap(v -> v.asString()); + } } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java index 4e986603d93..9d648fed32c 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/EmbeddingDecorator.java @@ -28,7 +28,7 @@ public void withEmbeddingCreateParams(AgentSpan span, EmbeddingCreateParams para if (params == null) { return; } - Optional modelName = params.model()._value().asString(); + Optional modelName = extractEmbeddingModelName(params); modelName.ifPresent(str -> span.setTag(CommonTags.OPENAI_REQUEST_MODEL, str)); if (!llmObsEnabled) { @@ -43,24 +43,23 @@ public void withEmbeddingCreateParams(AgentSpan span, EmbeddingCreateParams para span.setTag(CommonTags.INPUT, embeddingDocuments(params.input())); Map metadata = new HashMap<>(); - Optional encodingFormat = params.encodingFormat().flatMap(v -> v._value().asString()); + Optional encodingFormat = extractEncodingFormat(params); encodingFormat.ifPresent(v -> metadata.put("encoding_format", v)); params.dimensions().ifPresent(v -> metadata.put("dimensions", v)); span.setTag(CommonTags.METADATA, metadata); } private List embeddingDocuments(EmbeddingCreateParams.Input input) { - List inputs = Collections.emptyList(); - if (input.isString()) { - inputs = Collections.singletonList(input.asString()); - } else if (input.isArrayOfStrings()) { - inputs = input.asArrayOfStrings(); - } + List inputs = + input + .string() + .map(Collections::singletonList) + .orElseGet(() -> input.arrayOfStrings().orElse(Collections.emptyList())); return inputs.stream().map(LLMObs.Document::from).collect(Collectors.toList()); } public void withCreateEmbeddingResponse(AgentSpan span, CreateEmbeddingResponse response) { - String modelName = response.model(); + String modelName = response._model().asString().orElse(null); span.setTag(CommonTags.OPENAI_RESPONSE_MODEL, modelName); span.setTag(CommonTags.MODEL_NAME, modelName); @@ -68,17 +67,42 @@ public void withCreateEmbeddingResponse(AgentSpan span, CreateEmbeddingResponse return; } - if (!response.data().isEmpty()) { - int embeddingCount = response.data().size(); - Embedding firstEmbedding = response.data().get(0); - int embeddingSize = firstEmbedding.embedding().size(); + List data = response._data().asKnown().orElse(Collections.emptyList()); + if (!data.isEmpty()) { + int embeddingCount = data.size(); + Embedding firstEmbedding = data.get(0); + int embeddingSize = + firstEmbedding._embedding().asKnown().orElse(Collections.emptyList()).size(); span.setTag( CommonTags.OUTPUT, String.format("[%d embedding(s) returned with size %d]", embeddingCount, embeddingSize)); } - CreateEmbeddingResponse.Usage usage = response.usage(); - span.setTag(CommonTags.INPUT_TOKENS, usage.promptTokens()); - span.setTag(CommonTags.TOTAL_TOKENS, usage.totalTokens()); + response + ._usage() + .asKnown() + .ifPresent( + usage -> { + usage + ._promptTokens() + .asKnown() + .ifPresent(v -> span.setTag(CommonTags.INPUT_TOKENS, v)); + usage + ._totalTokens() + .asKnown() + .ifPresent(v -> span.setTag(CommonTags.TOTAL_TOKENS, v)); + }); + } + + private Optional extractEmbeddingModelName(EmbeddingCreateParams params) { + Optional modelName = + params._model().asKnown().flatMap(model -> model._value().asString()); + return modelName.isPresent() ? modelName : params._model().asString(); + } + + private Optional extractEncodingFormat(EmbeddingCreateParams params) { + Optional encodingFormat = + params._encodingFormat().asKnown().flatMap(format -> format._value().asString()); + return encodingFormat.isPresent() ? encodingFormat : params._encodingFormat().asString(); } } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 14d2166f6e2..176e016af40 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -17,7 +17,6 @@ import com.openai.models.responses.ResponseInputItem; import com.openai.models.responses.ResponseOutputItem; import com.openai.models.responses.ResponseOutputMessage; -import com.openai.models.responses.ResponseOutputText; import com.openai.models.responses.ResponsePrompt; import com.openai.models.responses.ResponseReasoningItem; import com.openai.models.responses.ResponseStreamEvent; @@ -73,7 +72,8 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params List inputMessages = new ArrayList<>(); params - .instructions() + ._instructions() + .asString() .ifPresent( instructions -> { inputMessages.add(LLMObs.LLMMessage.from("system", instructions)); @@ -87,11 +87,11 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params Optional inputOpt = params._input().asKnown(); if (inputOpt.isPresent()) { ResponseCreateParams.Input input = inputOpt.get(); - if (input.isText()) { - inputMessages.add(LLMObs.LLMMessage.from("user", input.asText())); - } else if (input.isResponse()) { - List inputItems = input.asResponse(); - for (ResponseInputItem item : inputItems) { + Optional inputText = input.text(); + if (inputText.isPresent()) { + inputMessages.add(LLMObs.LLMMessage.from("user", inputText.get())); + } else { + for (ResponseInputItem item : input.response().orElse(Collections.emptyList())) { LLMObs.LLMMessage message = extractInputItemMessage(item); if (message != null) { inputMessages.add(message); @@ -103,22 +103,17 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params // Handle raw list input (when SDK can't parse into known types) // This path is tested by "create streaming response with raw json tool input test" if (inputMessages.isEmpty()) { - try { - Optional rawValueOpt = params._input().asUnknown(); - if (rawValueOpt.isPresent()) { - JsonValue rawValue = rawValueOpt.get(); - Optional> rawListOpt = rawValue.asArray(); - if (rawListOpt.isPresent()) { - for (JsonValue item : rawListOpt.get()) { - LLMObs.LLMMessage message = extractMessageFromRawJson(item); - if (message != null) { - inputMessages.add(message); - } + Optional rawValueOpt = params._input().asUnknown(); + if (rawValueOpt.isPresent()) { + Optional> rawListOpt = rawValueOpt.get().asArray(); + if (rawListOpt.isPresent()) { + for (JsonValue item : rawListOpt.get()) { + LLMObs.LLMMessage message = extractMessageFromRawJson(item); + if (message != null) { + inputMessages.add(message); } } } - } catch (Exception e) { - // Ignore parsing errors for raw input } } @@ -139,48 +134,40 @@ public void withResponseCreateParams(AgentSpan span, ResponseCreateParams params } private List> extractToolDefinitionsFromParams(ResponseCreateParams params) { - try { - Optional> toolsOpt = params.tools(); - if (toolsOpt.isPresent()) { - List> toolDefinitions = new ArrayList<>(); - for (Tool tool : toolsOpt.get()) { - if (!tool.isFunction()) { - continue; - } - Map toolDef = extractFunctionToolDefinition(tool.asFunction()); - if (toolDef != null) { - toolDefinitions.add(toolDef); - } + Optional> toolsOpt = params._tools().asKnown(); + if (toolsOpt.isPresent()) { + List> toolDefinitions = new ArrayList<>(); + for (Tool tool : toolsOpt.get()) { + if (!tool.isFunction()) { + continue; } - if (!toolDefinitions.isEmpty()) { - return toolDefinitions; + Map toolDef = extractFunctionToolDefinition(tool.asFunction()); + if (toolDef != null) { + toolDefinitions.add(toolDef); } } - } catch (Exception ignored) { - // fall back to raw JSON if typed extraction is unavailable or fails + if (!toolDefinitions.isEmpty()) { + return toolDefinitions; + } } - try { - Optional rawToolsOpt = params._tools().asUnknown(); - if (!rawToolsOpt.isPresent()) { - return Collections.emptyList(); - } - Optional> rawToolListOpt = rawToolsOpt.get().asArray(); - if (!rawToolListOpt.isPresent()) { - return Collections.emptyList(); - } + Optional rawToolsOpt = params._tools().asUnknown(); + if (!rawToolsOpt.isPresent()) { + return Collections.emptyList(); + } + Optional> rawToolListOpt = rawToolsOpt.get().asArray(); + if (!rawToolListOpt.isPresent()) { + return Collections.emptyList(); + } - List> toolDefinitions = new ArrayList<>(); - for (JsonValue rawTool : rawToolListOpt.get()) { - Map toolDef = extractFunctionToolDefinition(rawTool); - if (toolDef != null) { - toolDefinitions.add(toolDef); - } + List> toolDefinitions = new ArrayList<>(); + for (JsonValue rawTool : rawToolListOpt.get()) { + Map toolDef = extractFunctionToolDefinition(rawTool); + if (toolDef != null) { + toolDefinitions.add(toolDef); } - return toolDefinitions; - } catch (Exception ignored) { - return Collections.emptyList(); } + return toolDefinitions; } private Map extractFunctionToolDefinition(FunctionTool functionTool) { @@ -255,7 +242,8 @@ private Map extractFunctionToolDefinition(JsonValue rawTool) { private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { if (item.isEasyInputMessage()) { EasyInputMessage message = item.asEasyInputMessage(); - String role = message.role().asString(); + String role = + message._role().asKnown().map(EasyInputMessage.Role::asString).orElse("unknown"); String content = extractEasyInputMessageContent(message); if (content == null || content.isEmpty()) { return null; @@ -263,22 +251,29 @@ private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { return LLMObs.LLMMessage.from(role, content); } else if (item.isMessage()) { ResponseInputItem.Message message = item.asMessage(); - String role = message.role().asString(); + String role = + message._role().asKnown().map(ResponseInputItem.Message.Role::asString).orElse("unknown"); String content = extractInputMessageContent(message); if (content == null || content.isEmpty()) { return null; } return LLMObs.LLMMessage.from(role, content); - } else if (item.isFunctionCall()) { + } + + Optional functionCallOpt = item.functionCall(); + if (functionCallOpt.isPresent()) { // Function call is mapped to assistant message with tool_calls - ResponseFunctionToolCall functionCall = item.asFunctionCall(); + ResponseFunctionToolCall functionCall = functionCallOpt.get(); LLMObs.ToolCall toolCall = ToolCallExtractor.getToolCall(functionCall); if (toolCall != null) { List toolCalls = Collections.singletonList(toolCall); return LLMObs.LLMMessage.from("assistant", null, toolCalls); } - } else if (item.isFunctionCallOutput()) { - ResponseInputItem.FunctionCallOutput output = item.asFunctionCallOutput(); + } + + Optional functionCallOutput = item.functionCallOutput(); + if (functionCallOutput.isPresent()) { + ResponseInputItem.FunctionCallOutput output = functionCallOutput.get(); String callId = output.callId(); String result = FunctionCallOutputExtractor.getOutputAsString(output); LLMObs.ToolResult toolResult = @@ -290,23 +285,28 @@ private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { } private String extractEasyInputMessageContent(EasyInputMessage message) { - if (message.content().isTextInput()) { - String content = message.content().asTextInput(); + Optional contentValue = message._content().asKnown(); + if (!contentValue.isPresent()) { + return null; + } + + EasyInputMessage.Content contentValueTyped = contentValue.get(); + Optional textInput = contentValueTyped.textInput(); + if (textInput.isPresent()) { + String content = textInput.get(); return content == null || content.isEmpty() ? null : content; } - if (message.content().isResponseInputMessageContentList()) { - StringBuilder contentBuilder = new StringBuilder(); - for (ResponseInputContent content : message.content().asResponseInputMessageContentList()) { - String contentPart = extractInputContentText(content); - if (contentPart != null) { - contentBuilder.append(contentPart); - } + StringBuilder contentBuilder = new StringBuilder(); + for (ResponseInputContent content : + contentValueTyped.responseInputMessageContentList().orElse(Collections.emptyList())) { + String contentPart = extractInputContentText(content); + if (contentPart != null) { + contentBuilder.append(contentPart); } - String result = contentBuilder.toString(); - return result.isEmpty() ? null : result; } - return null; + String result = contentBuilder.toString(); + return result.isEmpty() ? null : result; } private LLMObs.LLMMessage extractMessageFromRawJson(JsonValue jsonValue) { @@ -429,26 +429,23 @@ private String extractInputMessageContent(ResponseInputItem.Message message) { } private String extractInputContentText(ResponseInputContent content) { - if (content.isInputText()) { - return content.asInputText().text(); - } - if (content.isInputImage()) { - return content - .asInputImage() - .imageUrl() - .orElse(content.asInputImage().fileId().orElse(IMAGE_FALLBACK_MARKER)); - } - if (content.isInputFile()) { - return content - .asInputFile() - .fileUrl() - .orElse( - content - .asInputFile() - .fileId() - .orElse(content.asInputFile().filename().orElse(FILE_FALLBACK_MARKER))); + Optional inputText = content.inputText().map(v -> v.text()); + if (inputText.isPresent()) { + return inputText.get(); } - return null; + + Optional inputImage = + content + .inputImage() + .map(v -> v.imageUrl().orElse(v.fileId().orElse(IMAGE_FALLBACK_MARKER))); + if (inputImage.isPresent()) { + return inputImage.get(); + } + + return content + .inputFile() + .map(v -> v.fileUrl().orElse(v.fileId().orElse(v.filename().orElse(FILE_FALLBACK_MARKER)))) + .orElse(null); } private Optional> extractReasoningFromParams(ResponseCreateParams params) { @@ -513,7 +510,8 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { return; } - List outputMessages = extractResponseOutputMessages(response.output()); + List outputMessages = + extractResponseOutputMessages(response._output().asKnown().orElse(Collections.emptyList())); if (!outputMessages.isEmpty()) { span.setTag(CommonTags.OUTPUT, outputMessages); } @@ -531,46 +529,58 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { response.temperature().ifPresent(v -> metadata.put("temperature", v)); response.topP().ifPresent(v -> metadata.put("top_p", v)); - Response.ToolChoice toolChoice = response.toolChoice(); - if (toolChoice.isOptions()) { - metadata.put("tool_choice", toolChoice.asOptions()._value().asString().orElse(null)); - } else if (toolChoice.isTypes()) { - metadata.put("tool_choice", toolChoice.asTypes().type().toString().toLowerCase()); - } else if (toolChoice.isFunction()) { - metadata.put("tool_choice", "function"); - } - response - .truncation() + ._toolChoice() + .asKnown() .ifPresent( - (Response.Truncation t) -> - metadata.put("truncation", t._value().asString().orElse(null))); + toolChoice -> { + toolChoice + .options() + .flatMap(v -> v._value().asString()) + .ifPresent(v -> metadata.put("tool_choice", v)); + if (!metadata.containsKey("tool_choice")) { + toolChoice + .types() + .map(v -> v.type().toString().toLowerCase()) + .ifPresent(v -> metadata.put("tool_choice", v)); + } + if (!metadata.containsKey("tool_choice") && toolChoice.function().isPresent()) { + metadata.put("tool_choice", "function"); + } + }); + + response + ._truncation() + .asKnown() + .flatMap(t -> t._value().asString()) + .ifPresent(v -> metadata.put("truncation", v)); Map textMap = new HashMap<>(); response - .text() + ._text() + .asKnown() .ifPresent( textConfig -> { textConfig - .format() + ._format() + .asKnown() .ifPresent( format -> { Map formatMap = new HashMap<>(); - if (format.isText()) { + if (format.text().isPresent()) { formatMap.put("type", "text"); - } else if (format.isJsonSchema()) { + } else if (format.jsonSchema().isPresent()) { formatMap.put("type", "json_schema"); - } else if (format.isJsonObject()) { + } else if (format.jsonObject().isPresent()) { formatMap.put("type", "json_object"); } textMap.put("format", formatMap); }); textConfig - .verbosity() - .ifPresent( - verbosity -> { - textMap.put("verbosity", verbosity.asString()); - }); + ._verbosity() + .asKnown() + .flatMap(verbosity -> verbosity._value().asString()) + .ifPresent(verbosity -> textMap.put("verbosity", verbosity)); }); if (!textMap.isEmpty()) { metadata.put("text", textMap); @@ -581,7 +591,8 @@ private void withResponse(AgentSpan span, Response response, boolean stream) { span.setTag(CommonTags.METADATA, metadata); response - .usage() + ._usage() + .asKnown() .ifPresent( usage -> { span.setTag(CommonTags.INPUT_TOKENS, usage.inputTokens()); @@ -674,22 +685,25 @@ private List extractInputMessagesForPromptTracking( List messages = new ArrayList<>(); response - .instructions() + ._instructions() + .asKnown() .ifPresent( instructions -> { - if (instructions.isInputItemList()) { - for (ResponseInputItem item : instructions.asInputItemList()) { - LLMObs.LLMMessage message = extractInputItemMessage(item); - if (message != null) { - messages.add(message); - } - } - } else if (instructions.isString()) { - String text = instructions.asString(); - if (text != null && !text.isEmpty()) { - messages.add(LLMObs.LLMMessage.from("system", text)); + for (ResponseInputItem item : + instructions.inputItemList().orElse(Collections.emptyList())) { + LLMObs.LLMMessage message = extractInputItemMessage(item); + if (message != null) { + messages.add(message); } } + instructions + .string() + .ifPresent( + text -> { + if (text != null && !text.isEmpty()) { + messages.add(LLMObs.LLMMessage.from("system", text)); + } + }); }); // Fallback for SDK union parsing mismatches: parse raw instructions payload. @@ -815,21 +829,17 @@ private Optional> extractPromptFromParams(ResponseCreatePara } } - try { - Optional rawPromptOpt = params._prompt().asUnknown(); - if (!rawPromptOpt.isPresent()) { - return Optional.empty(); - } - - Optional> rawPromptObjOpt = rawPromptOpt.get().asObject(); - if (!rawPromptObjOpt.isPresent()) { - return Optional.empty(); - } + Optional rawPromptOpt = params._prompt().asUnknown(); + if (!rawPromptOpt.isPresent()) { + return Optional.empty(); + } - return extractPrompt(rawPromptObjOpt.get()); - } catch (Exception ignored) { + Optional> rawPromptObjOpt = rawPromptOpt.get().asObject(); + if (!rawPromptObjOpt.isPresent()) { return Optional.empty(); } + + return extractPrompt(rawPromptObjOpt.get()); } private Optional> extractPrompt(ResponsePrompt prompt) { @@ -964,22 +974,31 @@ private List extractResponseOutputMessages(List messages = new ArrayList<>(); for (ResponseOutputItem item : output) { - if (item.isFunctionCall()) { - ResponseFunctionToolCall functionCall = item.asFunctionCall(); + Optional functionCallOpt = item.functionCall(); + if (functionCallOpt.isPresent()) { + ResponseFunctionToolCall functionCall = functionCallOpt.get(); LLMObs.ToolCall toolCall = ToolCallExtractor.getToolCall(functionCall); if (toolCall != null) { List toolCalls = Collections.singletonList(toolCall); messages.add(LLMObs.LLMMessage.from("assistant", null, toolCalls)); } - } else if (item.isCustomToolCall()) { - ResponseCustomToolCall customToolCall = item.asCustomToolCall(); + continue; + } + + Optional customToolCallOpt = item.customToolCall(); + if (customToolCallOpt.isPresent()) { + ResponseCustomToolCall customToolCall = customToolCallOpt.get(); LLMObs.ToolCall toolCall = ToolCallExtractor.getToolCall(customToolCall); if (toolCall != null) { messages.add( LLMObs.LLMMessage.from("assistant", null, Collections.singletonList(toolCall))); } - } else if (item.isMcpCall()) { - ResponseOutputItem.McpCall mcpCall = item.asMcpCall(); + continue; + } + + Optional mcpCallOpt = item.mcpCall(); + if (mcpCallOpt.isPresent()) { + ResponseOutputItem.McpCall mcpCall = mcpCallOpt.get(); LLMObs.ToolCall toolCall = ToolCallExtractor.getToolCall(mcpCall); List toolCalls = toolCall == null ? null : Collections.singletonList(toolCall); @@ -988,21 +1007,32 @@ private List extractResponseOutputMessages(List toolResults = Collections.singletonList(toolResult); messages.add(LLMObs.LLMMessage.from("assistant", null, toolCalls, toolResults)); - } else if (item.isMessage()) { - ResponseOutputMessage message = item.asMessage(); + continue; + } + + Optional messageOpt = item.message(); + if (messageOpt.isPresent()) { + ResponseOutputMessage message = messageOpt.get(); String textContent = extractMessageContent(message); Optional roleOpt = message._role().asString(); String role = roleOpt.orElse("assistant"); messages.add(LLMObs.LLMMessage.from(role, textContent)); - } else if (item.isReasoning()) { - ResponseReasoningItem reasoning = item.asReasoning(); + continue; + } + + Optional reasoningOpt = item.reasoning(); + if (reasoningOpt.isPresent()) { + ResponseReasoningItem reasoning = reasoningOpt.get(); try (JsonWriter writer = new JsonWriter()) { writer.beginObject(); - String summaryText = null; - if (!reasoning.summary().isEmpty()) { - summaryText = reasoning.summary().get(0).text(); - } + String summaryText = + reasoning + ._summary() + .asKnown() + .filter(summary -> !summary.isEmpty()) + .flatMap(summary -> summary.get(0)._text().asString()) + .orElse(null); writer.name("summary"); if (summaryText != null && !summaryText.isEmpty()) { writer.value(summaryText); @@ -1011,13 +1041,14 @@ private List extractResponseOutputMessages(List encryptedContent = reasoning._encryptedContent().asString(); + if (encryptedContent.isPresent()) { + writer.value(encryptedContent.get()); } else { writer.nullValue(); } - String id = reasoning.id(); + String id = reasoning._id().asString().orElse(null); writer.name("id").value(id == null ? "" : id); writer.endObject(); @@ -1031,11 +1062,9 @@ private List extractResponseOutputMessages(List contentBuilder.append(outputText.text())); } String result = contentBuilder.toString(); return result.isEmpty() ? null : result; From 61de1c035858d5ae1fdbd8386adee4b81d89e5b0 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 27 Mar 2026 12:34:17 -0700 Subject: [PATCH 41/48] Simplify OpenAI decorator fallbacks without dropping raw JSON support --- .../openai_java/ChatCompletionDecorator.java | 32 +++++++++---------- .../openai_java/CompletionDecorator.java | 7 +--- .../openai_java/ResponseDecorator.java | 17 ++++++---- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java index fa39d8cd570..bbb24f9194b 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ChatCompletionDecorator.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -61,6 +62,7 @@ public void withChatCompletionCreateParams( CommonTags.INPUT, params.messages().stream() .map(ChatCompletionDecorator::llmMessage) + .filter(Objects::nonNull) .collect(Collectors.toList())); Map metadata = new HashMap<>(); @@ -177,25 +179,19 @@ private static Map extractFunctionToolDef(ChatCompletionFunction } private static LLMObs.LLMMessage llmMessage(ChatCompletionMessageParam m) { - String role = "unknown"; - String content = null; if (m.isAssistant()) { - role = "assistant"; - content = m.asAssistant().content().map(v -> v.text().orElse(null)).orElse(null); + return LLMObs.LLMMessage.from( + "assistant", m.asAssistant().content().map(v -> v.text().orElse(null)).orElse(null)); } else if (m.isDeveloper()) { - role = "developer"; - content = m.asDeveloper().content().text().orElse(null); + return LLMObs.LLMMessage.from("developer", m.asDeveloper().content().text().orElse(null)); } else if (m.isSystem()) { - role = "system"; - content = m.asSystem().content().text().orElse(null); + return LLMObs.LLMMessage.from("system", m.asSystem().content().text().orElse(null)); } else if (m.isTool()) { - role = "tool"; - content = m.asTool().content().text().orElse(null); + return LLMObs.LLMMessage.from("tool", m.asTool().content().text().orElse(null)); } else if (m.isUser()) { - role = "user"; - content = m.asUser().content().text().orElse(null); + return LLMObs.LLMMessage.from("user", m.asUser().content().text().orElse(null)); } - return LLMObs.LLMMessage.from(role, content); + return null; } public void withChatCompletion(AgentSpan span, ChatCompletion completion) { @@ -210,6 +206,7 @@ public void withChatCompletion(AgentSpan span, ChatCompletion completion) { List output = completion._choices().asKnown().orElse(Collections.emptyList()).stream() .map(ChatCompletionDecorator::llmMessage) + .filter(Objects::nonNull) .collect(Collectors.toList()); span.setTag(CommonTags.OUTPUT, output); @@ -231,12 +228,15 @@ public void withChatCompletion(AgentSpan span, ChatCompletion completion) { private static LLMObs.LLMMessage llmMessage(ChatCompletion.Choice choice) { Optional msgOpt = choice._message().asKnown(); if (!msgOpt.isPresent()) { - return LLMObs.LLMMessage.from("unknown", ""); + return null; } ChatCompletionMessage msg = msgOpt.get(); - Optional roleOpt = msg._role().asString(); - String role = roleOpt.isPresent() ? String.valueOf(roleOpt.get()) : "unknown"; + Optional roleOpt = msg._role().asString(); + if (!roleOpt.isPresent()) { + return null; + } + String role = roleOpt.get(); String content = msg._content().asString().orElse(""); List toolCallsOpt = diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java index f71c18a706d..341f4e2b138 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/CompletionDecorator.java @@ -162,11 +162,6 @@ private Optional extractCompletionModelName(CompletionCreateParams param } private Optional extractPromptText(CompletionCreateParams params) { - Optional promptText = - params._prompt().asKnown().flatMap(CompletionCreateParams.Prompt::string); - if (promptText.isPresent()) { - return promptText; - } - return params._prompt().asUnknown().flatMap(v -> v.asString()); + return params._prompt().asKnown().flatMap(CompletionCreateParams.Prompt::string); } } diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 176e016af40..f223d401e02 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -242,22 +242,27 @@ private Map extractFunctionToolDefinition(JsonValue rawTool) { private LLMObs.LLMMessage extractInputItemMessage(ResponseInputItem item) { if (item.isEasyInputMessage()) { EasyInputMessage message = item.asEasyInputMessage(); - String role = - message._role().asKnown().map(EasyInputMessage.Role::asString).orElse("unknown"); + Optional role = message._role().asKnown().map(EasyInputMessage.Role::asString); + if (!role.isPresent()) { + return null; + } String content = extractEasyInputMessageContent(message); if (content == null || content.isEmpty()) { return null; } - return LLMObs.LLMMessage.from(role, content); + return LLMObs.LLMMessage.from(role.get(), content); } else if (item.isMessage()) { ResponseInputItem.Message message = item.asMessage(); - String role = - message._role().asKnown().map(ResponseInputItem.Message.Role::asString).orElse("unknown"); + Optional role = + message._role().asKnown().map(ResponseInputItem.Message.Role::asString); + if (!role.isPresent()) { + return null; + } String content = extractInputMessageContent(message); if (content == null || content.isEmpty()) { return null; } - return LLMObs.LLMMessage.from(role, content); + return LLMObs.LLMMessage.from(role.get(), content); } Optional functionCallOpt = item.functionCall(); From 1b6615ae12b647b1b74bfad2e3e4c65253196d84 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 30 Mar 2026 16:38:33 -0700 Subject: [PATCH 42/48] clean up dead code --- .../openai_java/ResponseDecorator.java | 120 ------------------ 1 file changed, 120 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index f223d401e02..43ec663d412 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -701,130 +701,10 @@ private List extractInputMessagesForPromptTracking( messages.add(message); } } - instructions - .string() - .ifPresent( - text -> { - if (text != null && !text.isEmpty()) { - messages.add(LLMObs.LLMMessage.from("system", text)); - } - }); }); - - // Fallback for SDK union parsing mismatches: parse raw instructions payload. - if (messages.isEmpty()) { - Optional rawInstructions = response._instructions().asUnknown(); - if (rawInstructions.isPresent()) { - Optional> rawList = rawInstructions.get().asArray(); - if (rawList.isPresent()) { - for (JsonValue item : rawList.get()) { - LLMObs.LLMMessage message = extractMessageFromRawInstruction(item); - if (message != null) { - messages.add(message); - } - } - } - } - } - - boolean hasInstructions = !messages.isEmpty(); - - // Always include input messages from the span tag (set by withResponseCreateParams, which - // records both instructions as "system" and input as "user"). When instructions were already - // collected above, skip "system" messages here to avoid duplicating them. - Object inputTag = span.getTag(CommonTags.INPUT); - if (inputTag instanceof List) { - for (Object messageObj : (List) inputTag) { - if (messageObj instanceof LLMObs.LLMMessage) { - LLMObs.LLMMessage msg = (LLMObs.LLMMessage) messageObj; - if (!hasInstructions || !"system".equals(msg.getRole())) { - messages.add(msg); - } - } - } - } else if (inputTag instanceof Map) { - Object messagesObj = ((Map) inputTag).get("messages"); - if (messagesObj instanceof List) { - for (Object messageObj : (List) messagesObj) { - if (messageObj instanceof LLMObs.LLMMessage) { - LLMObs.LLMMessage msg = (LLMObs.LLMMessage) messageObj; - if (!hasInstructions || !"system".equals(msg.getRole())) { - messages.add(msg); - } - } - } - } - } - return messages; } - private LLMObs.LLMMessage extractMessageFromRawInstruction(JsonValue instructionValue) { - Optional> objOpt = instructionValue.asObject(); - if (!objOpt.isPresent()) { - return null; - } - Map obj = objOpt.get(); - String role = getJsonString(obj.get("role")); - if (role == null || role.isEmpty()) { - return null; - } - - JsonValue contentValue = obj.get("content"); - if (contentValue == null) { - return null; - } - Optional> contentList = contentValue.asArray(); - if (!contentList.isPresent()) { - return null; - } - - StringBuilder contentBuilder = new StringBuilder(); - for (JsonValue contentItem : contentList.get()) { - Optional> contentObjOpt = contentItem.asObject(); - if (!contentObjOpt.isPresent()) { - continue; - } - Map contentObj = contentObjOpt.get(); - String type = getJsonString(contentObj.get("type")); - if ("input_text".equals(type)) { - String text = getJsonString(contentObj.get("text")); - if (text != null) { - contentBuilder.append(text); - } - } else if ("input_image".equals(type)) { - String imageUrl = getJsonString(contentObj.get("image_url")); - if (imageUrl != null && !imageUrl.isEmpty()) { - contentBuilder.append(imageUrl); - } else { - String fileId = getJsonString(contentObj.get("file_id")); - contentBuilder.append( - fileId == null || fileId.isEmpty() ? IMAGE_FALLBACK_MARKER : fileId); - } - } else if ("input_file".equals(type)) { - String fileUrl = getJsonString(contentObj.get("file_url")); - if (fileUrl != null && !fileUrl.isEmpty()) { - contentBuilder.append(fileUrl); - } else { - String fileId = getJsonString(contentObj.get("file_id")); - if (fileId != null && !fileId.isEmpty()) { - contentBuilder.append(fileId); - } else { - String filename = getJsonString(contentObj.get("filename")); - contentBuilder.append( - filename == null || filename.isEmpty() ? FILE_FALLBACK_MARKER : filename); - } - } - } - } - - String content = contentBuilder.toString(); - if (content.isEmpty()) { - return null; - } - return LLMObs.LLMMessage.from(role, content); - } - private Optional> extractPromptFromParams(ResponseCreateParams params) { Optional typedPromptOpt = params._prompt().asKnown(); if (typedPromptOpt.isPresent()) { From d7d48663588146845ef7cc5786864d934b467f5e Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 30 Mar 2026 20:56:13 -0700 Subject: [PATCH 43/48] improve coverage for chat/completion test with raw tool definition --- .../groovy/ChatCompletionServiceTest.groovy | 18 +++++ .../src/test/groovy/OpenAiTest.groovy | 25 ++++++ ...6e0d5646f707a719+a3dada95fd9bf5b8.POST.rec | 79 +++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/chat/completions/6e0d5646f707a719+a3dada95fd9bf5b8.POST.rec diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 7376f73f92e..b7a8533bed8 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -176,6 +176,24 @@ class ChatCompletionServiceTest extends OpenAiTest { ] } + def "create chat/completion test with raw tool definition"() { + runUnderTrace("parent") { + openAiClient.chat().completions().create(chatCompletionCreateParamsWithRawTools()) + } + + expect: + List> toolDefinitions = [] + assertChatCompletionTrace(false, null, [:], true, toolDefinitions) + and: + toolDefinitions.size() == 1 + toolDefinitions[0].name == "extract_student_info_raw" + toolDefinitions[0].description == "Extract student information from the input text" + toolDefinitions[0].schema.type == "object" + (toolDefinitions[0].schema.properties as Map).containsKey("name") + (toolDefinitions[0].schema.properties as Map).containsKey("major") + toolDefinitions[0].schema.required == ["name"] + } + def "create streaming chat/completion test with tool calls"() { runnableUnderTrace("parent") { StreamResponse streamCompletion = openAiClient.chat().completions().createStreaming(chatCompletionCreateParamsWithTools()) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy index e1eef17eb47..55a0aede92f 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy @@ -259,6 +259,31 @@ He hopes to pursue a career in software engineering after graduating.""") .build() } + ChatCompletionCreateParams chatCompletionCreateParamsWithRawTools() { + def functionMap = [ + name: "extract_student_info_raw", + description: "Extract student information from the input text", + parameters: [ + type: "object", + properties: [ + name: [type: "string", description: "Name of the student"], + major: [type: "string", description: "Major subject"], + ], + required: ["name"], + ] + ] + + ChatCompletionCreateParams.builder() + .model("gpt-4o-mini") + .addUserMessage("""Extract the student's name and major. +Alice Johnson majors in mathematics at UCLA.""") + .addTool(ChatCompletionFunctionTool.builder() + .type(JsonValue.from("function")) + .function(JsonValue.from(functionMap)) + .build()) + .build() + } + ResponseCreateParams responseCreateParamsWithToolInput(boolean json) { if (json) { def rawInputJson = [ diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/chat/completions/6e0d5646f707a719+a3dada95fd9bf5b8.POST.rec b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/chat/completions/6e0d5646f707a719+a3dada95fd9bf5b8.POST.rec new file mode 100644 index 00000000000..b1a391a669d --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/chat/completions/6e0d5646f707a719+a3dada95fd9bf5b8.POST.rec @@ -0,0 +1,79 @@ +method: POST +path: chat/completions +-- begin request body -- +{"messages":[{"content":"Extract the student's name and major.\nAlice Johnson majors in mathematics at UCLA.","role":"user"}],"model":"gpt-4o-mini","tools":[{"function":{"name":"extract_student_info_raw","description":"Extract student information from the input text","parameters":{"type":"object","properties":{"name":{"type":"string","description":"Name of the student"},"major":{"type":"string","description":"Major subject"}},"required":["name"]}},"type":"function"}]} +-- end request body -- +status code: 200 +-- begin response headers -- +access-control-expose-headers: X-Request-ID +alt-svc: h3=":443"; ma=86400 +cf-cache-status: DYNAMIC +cf-ray: 9e4b0789acd790db-SEA +content-type: application/json +date: Mon, 30 Mar 2026 23:52:42 GMT +openai-organization: datadog-staging +openai-processing-ms: 761 +openai-project: proj_gt6TQZPRbZfoY2J9AQlEJMpd +openai-version: 2020-10-01 +server: cloudflare +set-cookie: __cf_bm=YbazZcnGinaCKEY7xcsvMbSgAfFK4OnyNb0buowhHmQ-1774914761.223589-1.0.1.1-pyISndsOHIqSgXLbmn13qxLtW5sn5oUHjFLZEO1IOU92PC5ZlkczCYuuZ264hSrRUwrtIs70rZksPcWnEX52pek9SpLKcLSAuMKodNi1kEQLgkJ3SjlsS25Cs36xS_Uj; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 00:22:42 GMT +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +x-openai-proxy-wasm: v0.1 +x-ratelimit-limit-requests: 30000 +x-ratelimit-limit-tokens: 150000000 +x-ratelimit-remaining-requests: 29999 +x-ratelimit-remaining-tokens: 149999977 +x-ratelimit-reset-requests: 2ms +x-ratelimit-reset-tokens: 0s +x-request-id: req_faf1e3c90613473f8c64913b1fec229f +-- end response headers -- +-- begin response body -- +{ + "id": "chatcmpl-DPGhNPfOzsEQ6wHbUZQGAh3eytTQ3", + "object": "chat.completion", + "created": 1774914761, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_Lc1TdPTzoAg9uVJnloAbFNzC", + "type": "function", + "function": { + "name": "extract_student_info_raw", + "arguments": "{\"name\":\"Alice Johnson\",\"major\":\"mathematics\"}" + } + } + ], + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 75, + "completion_tokens": 23, + "total_tokens": 98, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_e738e3044b" +} +� +-- end response body -- From 173db8aa54dfedfa5a4aa8ae59389b4add3a5633 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 31 Mar 2026 12:04:22 -0700 Subject: [PATCH 44/48] improve coverage for extractFunctionToolDefinition --- .../src/test/groovy/OpenAiTest.groovy | 31 +++++ .../test/groovy/ResponseServiceTest.groovy | 35 +++++- ...88547372cbe505bb+77909386d4b47615.POST.rec | 118 ++++++++++++++++++ 3 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/88547372cbe505bb+77909386d4b47615.POST.rec diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy index 55a0aede92f..f7bb1719ced 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy @@ -22,6 +22,7 @@ import com.openai.models.responses.ResponseFunctionToolCall import com.openai.models.responses.ResponseIncludable import com.openai.models.responses.ResponseInputItem import com.openai.models.responses.ResponsePrompt +import com.openai.models.responses.FunctionTool import com.openai.models.responses.CustomTool import com.openai.models.responses.ToolChoiceCustom import datadog.trace.agent.test.InstrumentationSpecification @@ -403,6 +404,36 @@ Alice Johnson majors in mathematics at UCLA.""") } } + ResponseCreateParams responseCreateParamsWithFunctionTool(boolean json) { + def functionTool = FunctionTool.builder() + .name("extract_student_info") + .description("Extract student information from the input text") + .strict(false) + .parameters(FunctionTool.Parameters.builder() + .putAdditionalProperty("type", JsonValue.from("object")) + .putAdditionalProperty("properties", JsonValue.from([ + name: [type: "string", description: "Name of the student"], + major: [type: "string", description: "Major subject"], + ])) + .putAdditionalProperty("required", JsonValue.from(["name"])) + .build()) + .build() + + if (json) { + ResponseCreateParams.builder() + .model("gpt-4.1") + .input("Extract the student's name and major from: Alice Johnson majors in mathematics at UCLA.") + .addTool(functionTool) + .build() + } else { + ResponseCreateParams.builder() + .model(ChatModel.GPT_4_1) + .input("Extract the student's name and major from: Alice Johnson majors in mathematics at UCLA.") + .addTool(functionTool) + .build() + } + } + ChatCompletionCreateParams chatCompletionCreateParamsMultiChoice(boolean json) { if (json) { ChatCompletionCreateParams.builder() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 8029ab0bdd0..eace1fbd2a0 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -263,6 +263,31 @@ class ResponseServiceTest extends OpenAiTest { params << [responseCreateParamsWithCustomToolCall(false), responseCreateParamsWithCustomToolCall(true)] } + def "create response with function tool definition"() { + Response resp = runUnderTrace("parent") { + openAiClient.responses().create(params) + } + + expect: + resp != null + and: + List> toolDefinitions = [] + Map metadata = [:] + assertResponseTrace(false, "gpt-4.1", String, null, null, null, metadata, false, toolDefinitions) + and: + metadata.stream == false + toolDefinitions.size() == 1 + toolDefinitions[0].name == "extract_student_info" + toolDefinitions[0].description == "Extract student information from the input text" + toolDefinitions[0].schema.type == "object" + (toolDefinitions[0].schema.properties as Map).containsKey("name") + (toolDefinitions[0].schema.properties as Map).containsKey("major") + toolDefinitions[0].schema.required == ["name"] + + where: + params << [responseCreateParamsWithFunctionTool(false), responseCreateParamsWithFunctionTool(true)] + } + def "create response error sets model_name and placeholder output"() { setup: def errorBackend = TestHttpServer.httpServer { @@ -337,7 +362,8 @@ class ResponseServiceTest extends OpenAiTest { Object inputTagsOut, List outputTagsOut, Map metadataOut, - boolean expectPromptTag = false) { + boolean expectPromptTag = false, + List> toolDefinitionsOut = null) { assertTraces(1) { trace(3) { sortSpansByStart() @@ -382,6 +408,13 @@ class ResponseServiceTest extends OpenAiTest { if (outputTagsOut != null && outputTags != null) { outputTagsOut.addAll(outputTags) } + if (toolDefinitionsOut != null) { + "$CommonTags.TOOL_DEFINITIONS" List + def toolDefinitions = tag("$CommonTags.TOOL_DEFINITIONS") + if (toolDefinitions != null) { + toolDefinitionsOut.addAll(toolDefinitions) + } + } "_ml_obs_metric.input_tokens" Long "_ml_obs_metric.output_tokens" Long "_ml_obs_metric.total_tokens" Long diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/88547372cbe505bb+77909386d4b47615.POST.rec b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/88547372cbe505bb+77909386d4b47615.POST.rec new file mode 100644 index 00000000000..122e6b723ed --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/88547372cbe505bb+77909386d4b47615.POST.rec @@ -0,0 +1,118 @@ +method: POST +path: responses +-- begin request body -- +{"input":"Extract the student's name and major from: Alice Johnson majors in mathematics at UCLA.","model":"gpt-4.1","tools":[{"name":"extract_student_info","parameters":{"type":"object","properties":{"name":{"type":"string","description":"Name of the student"},"major":{"type":"string","description":"Major subject"}},"required":["name"]},"strict":false,"type":"function","description":"Extract student information from the input text"}]} +-- end request body -- +status code: 200 +-- begin response headers -- +alt-svc: h3=":443"; ma=86400 +cf-cache-status: DYNAMIC +cf-ray: 9e4c76ebab89c643-SEA +content-type: application/json +date: Tue, 31 Mar 2026 04:03:29 GMT +openai-organization: datadog-staging +openai-processing-ms: 386 +openai-project: proj_gt6TQZPRbZfoY2J9AQlEJMpd +openai-version: 2020-10-01 +server: cloudflare +set-cookie: __cf_bm=v7wJG8beY0rR0zAeGSbrjUylMKwdoU1o2JC1RJxWQ2Q-1774929809.2301216-1.0.1.1-itjWK4_8cHGqON.6qDgVcjPIeJvd4kdlY6_jvZMWmnq2qhXXAFBRxuVe6Zdd1xqC.5iXN2h1XYnTqT1HYWKxTOs1cwhfrZIJSCZBxwMASDPr_zkGEfNv1ZED4IjX5YTT; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 04:33:29 GMT +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +x-ratelimit-limit-requests: 10000 +x-ratelimit-limit-tokens: 30000000 +x-ratelimit-remaining-requests: 9999 +x-ratelimit-remaining-tokens: 30000000 +x-ratelimit-reset-requests: 6ms +x-ratelimit-reset-tokens: 0s +x-request-id: req_db65016ae7c043a99aaa6bffb6d7de02 +-- end response headers -- +-- begin response body -- +{ + "id": "resp_0d4bdbca404a2d0e0169cb47918f788193b059be56688eb242", + "object": "response", + "created_at": 1774929809, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1774929809, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "id": "fc_0d4bdbca404a2d0e0169cb4791d5f48193a45d5d7599a40650", + "type": "function_call", + "status": "completed", + "arguments": "{\"name\":\"Alice Johnson\",\"major\":\"mathematics\"}", + "call_id": "call_mTLcdnmGX1CFmMkQRTZ3SsxR", + "name": "extract_student_info" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [ + { + "type": "function", + "description": "Extract student information from the input text", + "name": "extract_student_info", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of the student" + }, + "major": { + "type": "string", + "description": "Major subject" + } + }, + "required": [ + "name" + ] + }, + "strict": false + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 69, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 23, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 92 + }, + "user": null, + "metadata": {} +}� +-- end response body -- From c25c3e9474ace805ff438ce0c8fdd4cce9274743 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 31 Mar 2026 12:48:04 -0700 Subject: [PATCH 45/48] improve coverage for extractFunctionToolDefinition --- .../openai_java/ResponseDecorator.java | 11 -- .../src/test/groovy/OpenAiTest.groovy | 32 +++++ .../test/groovy/ResponseServiceTest.groovy | 30 +++++ ...b50cfc5e45c46579+2db979cef81b04c5.POST.rec | 120 ++++++++++++++++++ 4 files changed, 182 insertions(+), 11 deletions(-) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/b50cfc5e45c46579+2db979cef81b04c5.POST.rec diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java index 43ec663d412..3017fc8b203 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/ResponseDecorator.java @@ -742,17 +742,6 @@ private Optional> extractPrompt(ResponsePrompt prompt) { if (!normalized.isEmpty()) { promptMap.put("variables", normalized); } - } else { - Optional rawVariablesOpt = prompt._variables().asUnknown(); - if (rawVariablesOpt.isPresent()) { - Optional> rawVariablesObjOpt = rawVariablesOpt.get().asObject(); - if (rawVariablesObjOpt.isPresent()) { - Map normalized = normalizePromptVariables(rawVariablesObjOpt.get()); - if (!normalized.isEmpty()) { - promptMap.put("variables", normalized); - } - } - } } return promptMap.isEmpty() ? Optional.empty() : Optional.of(promptMap); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy index f7bb1719ced..aaf24fb2cd3 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy @@ -434,6 +434,38 @@ Alice Johnson majors in mathematics at UCLA.""") } } + ResponseCreateParams responseCreateParamsWithRawFunctionTool() { + def rawTools = ObjectMappers.jsonMapper().readValue( + """[ + { + "type": "function", + "name": "extract_student_info_raw", + "description": "Extract student information from the input text", + "parameters": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "Name of the student"}, + "major": {"type": "string", "description": "Major subject"} + }, + "required": ["name"] + } + } + ]""", + com.openai.core.JsonField + ) + + ResponseCreateParams params = ResponseCreateParams.builder() + .model(ChatModel.GPT_4_1) + .input("Extract the student's name and major from: Alice Johnson majors in mathematics at UCLA.") + .build() + + def toolsField = params._body().class.getDeclaredField("tools") + toolsField.accessible = true + toolsField.set(params._body(), rawTools) + + params + } + ChatCompletionCreateParams chatCompletionCreateParamsMultiChoice(boolean json) { if (json) { ChatCompletionCreateParams.builder() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index eace1fbd2a0..8296204946a 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -7,6 +7,7 @@ import com.openai.core.http.HttpResponseFor import com.openai.core.http.StreamResponse import com.openai.credential.BearerTokenCredential import com.openai.models.responses.Response +import com.openai.models.responses.ResponseCreateParams import com.openai.models.responses.ResponseStreamEvent import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.api.DDSpanTypes @@ -288,6 +289,35 @@ class ResponseServiceTest extends OpenAiTest { params << [responseCreateParamsWithFunctionTool(false), responseCreateParamsWithFunctionTool(true)] } + def "create response with raw function tool definition"() { + ResponseCreateParams params = responseCreateParamsWithRawFunctionTool() + + expect: + params._tools().asKnown().empty + params._tools().asUnknown().present + + when: + Response resp = runUnderTrace("parent") { + openAiClient.responses().create(params) + } + + then: + resp != null + and: + List> toolDefinitions = [] + Map metadata = [:] + assertResponseTrace(false, "gpt-4.1", String, null, null, null, metadata, false, toolDefinitions) + and: + metadata.stream == false + toolDefinitions.size() == 1 + toolDefinitions[0].name == "extract_student_info_raw" + toolDefinitions[0].description == "Extract student information from the input text" + toolDefinitions[0].schema.type == "object" + (toolDefinitions[0].schema.properties as Map).containsKey("name") + (toolDefinitions[0].schema.properties as Map).containsKey("major") + toolDefinitions[0].schema.required == ["name"] + } + def "create response error sets model_name and placeholder output"() { setup: def errorBackend = TestHttpServer.httpServer { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/b50cfc5e45c46579+2db979cef81b04c5.POST.rec b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/b50cfc5e45c46579+2db979cef81b04c5.POST.rec new file mode 100644 index 00000000000..655b101f52a --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/b50cfc5e45c46579+2db979cef81b04c5.POST.rec @@ -0,0 +1,120 @@ +method: POST +path: responses +-- begin request body -- +{"input":"Extract the student's name and major from: Alice Johnson majors in mathematics at UCLA.","model":"gpt-4.1","tools":[{"type":"function","name":"extract_student_info_raw","description":"Extract student information from the input text","parameters":{"type":"object","properties":{"name":{"type":"string","description":"Name of the student"},"major":{"type":"string","description":"Major subject"}},"required":["name"]}}]} +-- end request body -- +status code: 200 +-- begin response headers -- +alt-svc: h3=":443"; ma=86400 +cf-cache-status: DYNAMIC +cf-ray: 9e51b962ca5855dd-SEA +content-type: application/json +date: Tue, 31 Mar 2026 19:22:41 GMT +openai-organization: datadog-staging +openai-processing-ms: 603 +openai-project: proj_gt6TQZPRbZfoY2J9AQlEJMpd +openai-version: 2020-10-01 +server: cloudflare +set-cookie: __cf_bm=R9BwJrSxwTzGoTGS6fXKcV8eDh1BX04veUMDBYwS15U-1774984960.4454625-1.0.1.1-uz3nOd8CenO4BXUzwOLsVhYY9tSMN8QsxbSSmNNauy.HfizmaxKuo.qcNXXpDXriny8fkQsNY9RazpAkA0lO982O0MLRFe4DwnuuTV6PjtsinRv6EOE99E1TmXwynZMx; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 19:52:41 GMT +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +x-ratelimit-limit-requests: 10000 +x-ratelimit-limit-tokens: 30000000 +x-ratelimit-remaining-requests: 9999 +x-ratelimit-remaining-tokens: 30000000 +x-ratelimit-reset-requests: 6ms +x-ratelimit-reset-tokens: 0s +x-request-id: req_40a85f5080bf4950a7a3f0767d0f5487 +-- end response headers -- +-- begin response body -- +{ + "id": "resp_087b549c7dee59590169cc1f0081c88190b42f2bb8e56cbdbf", + "object": "response", + "created_at": 1774984960, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1774984961, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "id": "fc_087b549c7dee59590169cc1f00ebb88190aa809d7470cb514b", + "type": "function_call", + "status": "completed", + "arguments": "{\"name\":\"Alice Johnson\",\"major\":\"mathematics\"}", + "call_id": "call_VXETAWtdjoslNAwxpS8pJsJR", + "name": "extract_student_info_raw" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [ + { + "type": "function", + "description": "Extract student information from the input text", + "name": "extract_student_info_raw", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of the student" + }, + "major": { + "type": "string", + "description": "Major subject" + } + }, + "required": [ + "name", + "major" + ], + "additionalProperties": false + }, + "strict": true + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 70, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 24, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 94 + }, + "user": null, + "metadata": {} +}� +-- end response body -- From 9e1afcf19669da13dd80fe79c0f036506c60fbf7 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 31 Mar 2026 13:03:15 -0700 Subject: [PATCH 46/48] improve coverage for mcp tool call --- .../src/test/groovy/OpenAiTest.groovy | 17 + .../test/groovy/ResponseServiceTest.groovy | 25 ++ ...39590e9b44463dcf+68f287a23bcd52c3.POST.rec | 313 ++++++++++++++++++ 3 files changed, 355 insertions(+) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/39590e9b44463dcf+68f287a23bcd52c3.POST.rec diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy index aaf24fb2cd3..30c7026c1a0 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy @@ -4,6 +4,7 @@ import com.openai.client.OpenAIClient import com.openai.client.okhttp.OkHttpClient import com.openai.client.okhttp.OpenAIOkHttpClient import com.openai.core.ClientOptions +import com.openai.core.ObjectMappers import com.openai.credential.BearerTokenCredential import com.openai.core.JsonValue import com.openai.models.ChatModel @@ -24,6 +25,7 @@ import com.openai.models.responses.ResponseInputItem import com.openai.models.responses.ResponsePrompt import com.openai.models.responses.FunctionTool import com.openai.models.responses.CustomTool +import com.openai.models.responses.Tool import com.openai.models.responses.ToolChoiceCustom import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.server.http.TestHttpServer @@ -466,6 +468,21 @@ Alice Johnson majors in mathematics at UCLA.""") params } + ResponseCreateParams responseCreateParamsWithMcpToolCall() { + def mcpTool = Tool.Mcp.builder() + .serverLabel("openai_docs") + .serverDescription("OpenAI documentation MCP server.") + .serverUrl("https://developers.openai.com/mcp") + .requireApproval(Tool.Mcp.RequireApproval.McpToolApprovalSetting.NEVER) + .build() + + ResponseCreateParams.builder() + .model(ChatModel.GPT_5) + .input("Use the OpenAI docs MCP server to find the Responses API MCP tool documentation and summarize the require_approval option.") + .addTool(mcpTool) + .build() + } + ChatCompletionCreateParams chatCompletionCreateParamsMultiChoice(boolean json) { if (json) { ChatCompletionCreateParams.builder() diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 8296204946a..63c39256349 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -318,6 +318,31 @@ class ResponseServiceTest extends OpenAiTest { toolDefinitions[0].schema.required == ["name"] } + def "create response with mcp tool call"() { + Response resp = runUnderTrace("parent") { + openAiClient.responses().create(responseCreateParamsWithMcpToolCall()) + } + + expect: + resp != null + and: + List outputTags = [] + Map metadata = [:] + assertResponseTrace(false, "gpt-5", String, null, null, outputTags, metadata) + and: + metadata.stream == false + outputTags.any { + it.toolCalls?.any { toolCall -> + toolCall.type == "mcp_call" && toolCall.name + } + } + outputTags.any { + it.toolResults?.any { toolResult -> + toolResult.type == "mcp_tool_result" && toolResult.name + } + } + } + def "create response error sets model_name and placeholder output"() { setup: def errorBackend = TestHttpServer.httpServer { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/39590e9b44463dcf+68f287a23bcd52c3.POST.rec b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/39590e9b44463dcf+68f287a23bcd52c3.POST.rec new file mode 100644 index 00000000000..2bbcfb374d8 --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/39590e9b44463dcf+68f287a23bcd52c3.POST.rec @@ -0,0 +1,313 @@ +method: POST +path: responses +-- begin request body -- +{"input":"Use the OpenAI docs MCP server to find the Responses API MCP tool documentation and summarize the require_approval option.","model":"gpt-5","tools":[{"server_label":"openai_docs","server_url":"https://developers.openai.com/mcp","type":"mcp","require_approval":"never","server_description":"OpenAI documentation MCP server."}]} +-- end request body -- +status code: 200 +-- begin response headers -- +alt-svc: h3=":443"; ma=86400 +cf-cache-status: DYNAMIC +cf-ray: 9e51ee6ffd53714e-SEA +content-type: application/json +date: Tue, 31 Mar 2026 20:01:24 GMT +openai-organization: datadog-staging +openai-processing-ms: 150568 +openai-project: proj_gt6TQZPRbZfoY2J9AQlEJMpd +openai-version: 2020-10-01 +server: cloudflare +set-cookie: __cf_bm=LebYQQZq7br0DBp6zmZwd49fMf3p85YCoNwEAXmzhZU-1774987133.4334397-1.0.1.1-O8RQ33RyMOKJ70k5G9rUQrZSR7lt.RKHx7xF77GUEkUlzRZQMYCVG0r3MJwuxFS7c7YoRo6Y6r4Gdp1VcVMYLNpJFR5PNKlBN2sEhEHAYPjDCVoVrAPND45jCmR_9BsT; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 20:31:24 GMT +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +x-ratelimit-limit-requests: 15000 +x-ratelimit-limit-tokens: 40000000 +x-ratelimit-remaining-requests: 14999 +x-ratelimit-remaining-tokens: 39998356 +x-ratelimit-reset-requests: 4ms +x-ratelimit-reset-tokens: 2ms +x-request-id: req_6d5873ea010c45b3b89f63b5d8515e0c +-- end response headers -- +-- begin response body -- +{ + "id": "resp_0485e7c463a292010169cc277d7ffc8190ac68ebe78ecdfb96", + "object": "response", + "created_at": 1774987133, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1774987284, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-5-2025-08-07", + "output": [ + { + "id": "mcpl_0485e7c463a292010169cc277d8e5c8190a48dbb87e55aaefc", + "type": "mcp_list_tools", + "server_label": "openai_docs", + "tools": [ + { + "annotations": { + "read_only": false + }, + "description": "Search across `platform.openai.com` + `developers.openai.com` docs. Use this whenever you are working with the OpenAI API (including the Responses API), OpenAI API SDKs, ChatGPT Apps SDK, or ChatGPT Codex. Results include URLs\u2014**after `search`, use `fetch_openai_doc`** to read/quote the exact markdown.", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "minLength": 1 + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 50 + }, + "cursor": { + "type": "string" + } + }, + "required": [ + "query" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "name": "search_openai_docs" + }, + { + "annotations": { + "read_only": false + }, + "description": "List/browse pages from `platform.openai.com` + `developers.openai.com` that this server crawls (useful when you don\u2019t know the right query yet or you\u2019re paging through results). Search across `platform.openai.com` + `developers.openai.com` docs. Use this whenever you are working with the OpenAI API (including the Responses API), OpenAI API SDKs, ChatGPT Apps SDK, or ChatGPT Codex. Results include URLs\u2014**after `list`, use `fetch_openai_doc`** on a result URL to get the full markdown.", + "input_schema": { + "type": "object", + "properties": { + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 50 + }, + "cursor": { + "type": "string" + } + }, + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "name": "list_openai_docs" + }, + { + "annotations": { + "read_only": false + }, + "description": "Fetch the markdown for a specific doc page (from `developers.openai.com` or `platform.openai.com`) so you can quote/summarize exact, up-to-date guidance (schemas, examples, limits, edge cases). Prefer to **`search_openai_docs` first** (or `list_openai_docs` if you\u2019re browsing) to find the best URL, then `fetch_openai_doc` to pull the exact text; you can pass `anchor` (e.g. `#streaming`) to fetch just that section.", + "input_schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "minLength": 1 + }, + "anchor": { + "type": "string" + } + }, + "required": [ + "url" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "name": "fetch_openai_doc" + }, + { + "annotations": { + "read_only": false + }, + "description": "List all OpenAI API endpoint URLs available in the OpenAPI spec.", + "input_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "name": "list_api_endpoints" + }, + { + "annotations": { + "read_only": false + }, + "description": "Return the OpenAPI spec for a specific API endpoint URL. Optionally filter code samples by language, or return only code samples.", + "input_schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "minLength": 1 + }, + "languages": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "codeExamplesOnly": { + "type": "boolean" + } + }, + "required": [ + "url" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "name": "get_openapi_spec" + } + ] + }, + { + "id": "rs_0485e7c463a292010169cc277efbf88190a1a4dbdcafdc27e8", + "type": "reasoning", + "summary": [] + }, + { + "id": "mcp_0485e7c463a292010169cc27802684819094ad3e0380a0ba44", + "type": "mcp_call", + "status": "completed", + "approval_request_id": null, + "arguments": "{\"query\":\"Responses API MCP tool require_approval\", \"limit\": 10}", + "error": null, + "name": "search_openai_docs", + "output": "{\"hits\": [{\"url\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/#configure-an-mcp-tool\", \"url_without_anchor\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"anchor\": \"configure-an-mcp-tool\", \"content\": \"Add MCP tools in one of two places:\\r\\nAt the session level with session.tools in session.update, if you want the server available for the full session.\\r\\nAt the response level with response.tools in response.create, if you only need MCP for one turn.\\r\\nIn Realtime, the MCP tool shape is:\\r\\ntype: \\\"mcp\\\"\\r\\nserver_label\\r\\nOne of server_url or connector_id\\r\\nOptional authorization and headers\\r\\nOptional allowed_tools\\r\\nOptional require_approval\\r\\nOptional server_description\\r\\nThis example makes a docs MCP server available for the full session:\\r\\nBuilt-in connectors use the same MCP tool shape, but pass connector_id\\ninstead of server_url. For example, Google Calendar uses\\nconnector_googlecalendar. In Realtime, use these built-in connectors for read\\nactions, such as searching or reading events or emails. Pass the user\\u2019s OAuth\\naccess token in authorization, and narrow the tool surface with\\nallowed_tools when possible:\\r\\nRemote MCP servers do not automatically receive the full conversation context,\\nbut they can see any data the model sends in a tool call.\\nKeep the tool surface narrow with allowed_tools,\\nand require approval for any action you would not auto-run.\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Realtime API with MCP\", \"lvl2\": \"Configure an MCP tool\", \"lvl3\": null, \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"4-https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 turn.\\r\\nIn Realtime, the MCP tool shape is:\\r\\ntype: \\\"mcp \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"Add MCP tools in one of two places:\\r\\nAt the session level with session.tools in session.update, if you want the server available for the full session.\\r\\nAt the response level with response.tools in response.create, if you only need MCP for one turn.\\r\\nIn Realtime, the MCP tool shape is:\\r\\ntype: \\\"mcp\\\"\\r\\nserver_label\\r\\nOne of server_url or connector_id\\r\\nOptional authorization and headers\\r\\nOptional allowed_tools\\r\\nOptional require_approval\\r\\nOptional server_description\\r\\nThis example makes a docs MCP server available for the full session:\\r\\nBuilt-in connectors use the same MCP tool shape, but pass connector_id\\ninstead of server_url. For example, Google Calendar uses\\nconnector_googlecalendar. In Realtime, use these built-in connectors for read\\nactions, such as searching or reading events or emails. Pass the user\\u2019s OAuth\\naccess token in authorization, and narrow the tool surface with\\nallowed_tools when possible:\\r\\nRemote MCP servers do not automatically receive the full conversation context,\\nbut they can see any data the model sends in a tool call.\\nKeep the tool surface narrow with allowed_tools,\\nand require approval for any action you would not auto-run.\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"responses\", \"mcp\", \"tool\", \"require\", \"approval\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Realtime API with MCP\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\", \"mcp\"]}, \"lvl2\": {\"value\": \"Configure an MCP tool\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"tool\"]}}}}, {\"url\": \"https://platform.openai.com/settings/organization/api-keys\", \"url_without_anchor\": \"https://platform.openai.com/settings/organization/api-keys\", \"content\": \"View and manage your API keys.\", \"hierarchy\": {\"lvl0\": \"Settings\", \"lvl1\": \"API keys\"}, \"type\": \"lvl1\", \"access_level\": \"authenticated\", \"objectID\": \"custom-record-settings-api-keys\", \"_snippetResult\": {\"content\": {\"value\": \"View and manage your API keys.\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"View and manage your API keys.\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Settings\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"API keys\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\"]}}}}, {\"url\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/#common-failures\", \"url_without_anchor\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"anchor\": \"common-failures\", \"content\": \"mcp_list_tools.failed: the Realtime API could not import tools from the remote server or connector. Check server_url or connector_id, authentication, server reachability, and any allowed_tools names you specified.\\r\\nresponse.mcp_call.failed: the model selected a tool, but the tool call did not complete. Inspect the event payload and the later mcp_call item for MCP protocol, execution, or transport errors.\\r\\nmcp_approval_request with no matching mcp_approval_response: the tool call cannot continue until your client explicitly approves or rejects it.\\r\\nA turn starts while mcp_list_tools.in_progress is still active: only tools that have already finished loading are eligible for that turn.\\r\\nA response uses tool_choice: \\\"required\\\" but no tools are currently available: the model has nothing eligible to call. Wait for mcp_list_tools.completed, confirm that at least one tool was imported, or use a different tool_choice for turns that do not require a tool.\\r\\nMCP tool definition validation fails before import starts: common causes are a duplicate server_label in the same tools array, setting both server_url and connector_id, omitting both of them on the initial session creation request, using an invalid connector_id, or sending both authorization and headers.Authorization. For connectors, do not send headers.Authorization at all.\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Realtime API with MCP\", \"lvl2\": \"Common failures\", \"lvl3\": null, \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"8-https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 approval_request with no matching mcp_approval_response: the tool \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"mcp_list_tools.failed: the Realtime API could not import tools from the remote server or connector. Check server_url or connector_id, authentication, server reachability, and any allowed_tools names you specified.\\r\\nresponse.mcp_call.failed: the model selected a tool, but the tool call did not complete. Inspect the event payload and the later mcp_call item for MCP protocol, execution, or transport errors.\\r\\nmcp_approval_request with no matching mcp_approval_response: the tool call cannot continue until your client explicitly approves or rejects it.\\r\\nA turn starts while mcp_list_tools.in_progress is still active: only tools that have already finished loading are eligible for that turn.\\r\\nA response uses tool_choice: \\\"required\\\" but no tools are currently available: the model has nothing eligible to call. Wait for mcp_list_tools.completed, confirm that at least one tool was imported, or use a different tool_choice for turns that do not require a tool.\\r\\nMCP tool definition validation fails before import starts: common causes are a duplicate server_label in the same tools array, setting both server_url and connector_id, omitting both of them on the initial session creation request, using an invalid connector_id, or sending both authorization and headers.Authorization. For connectors, do not send headers.Authorization at all.\", \"matchLevel\": \"full\", \"fullyHighlighted\": false, \"matchedWords\": [\"responses\", \"api\", \"mcp\", \"tool\", \"require\", \"approval\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Realtime API with MCP\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\", \"mcp\"]}, \"lvl2\": {\"value\": \"Common failures\", \"matchLevel\": \"none\", \"matchedWords\": []}}}}, {\"url\": \"https://developers.openai.com/api/docs/guides/tools-connectors-mcp/#quickstart\", \"url_without_anchor\": \"https://developers.openai.com/api/docs/guides/tools-connectors-mcp/\", \"anchor\": \"quickstart\", \"content\": \"Check out the examples below to see how remote MCP servers and connectors work through the Responses API. Both connectors and remote MCP servers can be used with the mcp built-in tool type.\\r\\n\\r\\nRemote MCP servers require a server_url. Depending on the server,\\nyou may also need an OAuth authorization parameter containing an\\naccess token.\\r\\n\\r\\nIt is very important that developers trust any remote MCP server they use with\\nthe Responses API. A malicious server can exfiltrate sensitive data from\\nanything that enters the model\\u2019s context. Carefully review the Risks and Safety section below before using this tool.\\r\\n\\r\\nConnectors require a connector_id parameter, and an OAuth access\\ntoken provided by your application in the authorization parameter.\\r\\n\\r\\nThe API will return new items in the output array of the model response. If the model decides to use a Connector or MCP server, it will first make a request to list available tools from the server, which will create a mcp_list_tools output item. From the simple remote MCP server example above, it contains only one tool definition:\\r\\nIf the model decides to call one of the available tools from the MCP server, you will also find a mcp_call output which will show what the model sent to the MCP tool, and what the MCP tool sent back as output.\\r\\nRead on in the guide below to learn more about how the MCP tool works, how to filter available tools, and how to handle tool call approval requests.\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"MCP and Connectors\", \"lvl2\": \"Quickstart\", \"lvl3\": null, \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"4-https://developers.openai.com/api/docs/guides/tools-connectors-mcp/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 built-in tool type.\\r\\n\\r\\nRemote MCP servers require a server \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"Check out the examples below to see how remote MCP servers and connectors work through the Responses API. Both connectors and remote MCP servers can be used with the mcp built-in tool type.\\r\\n\\r\\nRemote MCP servers require a server_url. Depending on the server,\\nyou may also need an OAuth authorization parameter containing an\\naccess token.\\r\\n\\r\\nIt is very important that developers trust any remote MCP server they use with\\nthe Responses API. A malicious server can exfiltrate sensitive data from\\nanything that enters the model\\u2019s context. Carefully review the Risks and Safety section below before using this tool.\\r\\n\\r\\nConnectors require a connector_id parameter, and an OAuth access\\ntoken provided by your application in the authorization parameter.\\r\\n\\r\\nThe API will return new items in the output array of the model response. If the model decides to use a Connector or MCP server, it will first make a request to list available tools from the server, which will create a mcp_list_tools output item. From the simple remote MCP server example above, it contains only one tool definition:\\r\\nIf the model decides to call one of the available tools from the MCP server, you will also find a mcp_call output which will show what the model sent to the MCP tool, and what the MCP tool sent back as output.\\r\\nRead on in the guide below to learn more about how the MCP tool works, how to filter available tools, and how to handle tool call approval requests.\", \"matchLevel\": \"full\", \"fullyHighlighted\": false, \"matchedWords\": [\"responses\", \"api\", \"mcp\", \"tool\", \"require\", \"approval\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"MCP and Connectors\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\"]}, \"lvl2\": {\"value\": \"Quickstart\", \"matchLevel\": \"none\", \"matchedWords\": []}}}}, {\"url\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/#approve-or-reject-mcp-tool-calls\", \"url_without_anchor\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"anchor\": \"approve-or-reject-mcp-tool-calls\", \"content\": \"If a tool requires approval, the Realtime API inserts an mcp_approval_request item into the conversation. To continue, send a new conversation.item.create event whose item.type is mcp_approval_response.\\r\\nIf you reject the request, set approve to false and optionally include a reason.\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Realtime API with MCP\", \"lvl2\": \"Approve or reject MCP tool calls\", \"lvl3\": null, \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"10-https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 tool requires approval, the Realtime API inserts an mcp_approval \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"If a tool requires approval, the Realtime API inserts an mcp_approval_request item into the conversation. To continue, send a new conversation.item.create event whose item.type is mcp_approval_response.\\r\\nIf you reject the request, set approve to false and optionally include a reason.\", \"matchLevel\": \"full\", \"fullyHighlighted\": false, \"matchedWords\": [\"responses\", \"api\", \"mcp\", \"tool\", \"require\", \"approval\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Realtime API with MCP\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\", \"mcp\"]}, \"lvl2\": {\"value\": \"Approve or reject MCP tool calls\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"tool\"]}}}}, {\"url\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/#realtime-mcp-flow\", \"url_without_anchor\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"anchor\": \"realtime-mcp-flow\", \"content\": \"Unlike Realtime function tools, remote MCP tools are executed by the Realtime API itself. Your client does not run the remote tool and return a function_call_output. Instead, your client configures access, listens for MCP lifecycle events, and optionally sends an approval response if the server asks for one.\\r\\nA typical flow looks like this:\\r\\nYou send session.update or response.create with a tools entry whose type is mcp.\\r\\nThe server begins importing tools and emits mcp_list_tools.in_progress.\\r\\nWhile listing is still in progress, the model cannot call a tool that has not been loaded yet. If you want to wait before starting a turn that depends on those tools, listen for mcp_list_tools.completed. The conversation.item.done event whose item.type is mcp_list_tools shows which tool names were actually imported. If import fails, you will receive mcp_list_tools.failed.\\r\\nThe user speaks or sends text, and a response is created, either by your client or automatically by the session configuration.\\r\\nIf the model chooses an MCP tool, you will see response.mcp_call_arguments.delta and response.mcp_call_arguments.done.\\r\\nIf approval is required, the server adds a conversation item whose item.type is mcp_approval_request. Your client must answer it with an mcp_approval_response item.\\r\\nOnce the tool runs, you will see response.mcp_call.in_progress. On success, you will later receive a response.output_item.done event whose item.type is mcp_call; on failure, you will receive response.mcp_call.failed. The assistant message item and response.done complete the turn.\\r\\nThis event handler covers the main checkpoints:\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Realtime API with MCP\", \"lvl2\": \"Realtime MCP flow\", \"lvl3\": null, \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"6-https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 it with an mcp_approval_response item.\\r\\nOnce the tool \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"Unlike Realtime function tools, remote MCP tools are executed by the Realtime API itself. Your client does not run the remote tool and return a function_call_output. Instead, your client configures access, listens for MCP lifecycle events, and optionally sends an approval response if the server asks for one.\\r\\nA typical flow looks like this:\\r\\nYou send session.update or response.create with a tools entry whose type is mcp.\\r\\nThe server begins importing tools and emits mcp_list_tools.in_progress.\\r\\nWhile listing is still in progress, the model cannot call a tool that has not been loaded yet. If you want to wait before starting a turn that depends on those tools, listen for mcp_list_tools.completed. The conversation.item.done event whose item.type is mcp_list_tools shows which tool names were actually imported. If import fails, you will receive mcp_list_tools.failed.\\r\\nThe user speaks or sends text, and a response is created, either by your client or automatically by the session configuration.\\r\\nIf the model chooses an MCP tool, you will see response.mcp_call_arguments.delta and response.mcp_call_arguments.done.\\r\\nIf approval is required, the server adds a conversation item whose item.type is mcp_approval_request. Your client must answer it with an mcp_approval_response item.\\r\\nOnce the tool runs, you will see response.mcp_call.in_progress. On success, you will later receive a response.output_item.done event whose item.type is mcp_call; on failure, you will receive response.mcp_call.failed. The assistant message item and response.done complete the turn.\\r\\nThis event handler covers the main checkpoints:\", \"matchLevel\": \"full\", \"fullyHighlighted\": false, \"matchedWords\": [\"responses\", \"api\", \"mcp\", \"tool\", \"require\", \"approval\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Realtime API with MCP\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\", \"mcp\"]}, \"lvl2\": {\"value\": \"Realtime MCP flow\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\"]}}}}, {\"url\": \"https://developers.openai.com/apps-sdk/deploy/submission/#app-review--approval-faqs\", \"url_without_anchor\": \"https://developers.openai.com/apps-sdk/deploy/submission/\", \"anchor\": \"app-review--approval-faqs\", \"content\": \"How long does app review take?\\r\\nThe app directory and Apps SDK are currently in beta, and review timelines may vary as we continue to build and scale our processes. Please do not contact support to request expedited review, as these requests cannot be accommodated.\\r\\nWhat are common rejection reasons and how can I resolve them?\\r\\nWe\\u2019re unable to connect to your MCP server using the MCP URL and/or test credentials we were given. For servers requiring authentication, our review team must be able to log into a demo account with no further configuration required.\\nEnsure that the provided URL and credentials are correct, do not feature MFA (including requiring SMS codes, login through systems that require SMS, email or other verification schemes).\\nEnsure that the provided credentials can be used to log in successfully (test them outside any company networks or LANs, or other internal networks).\\nConfirm that the credentials have not expired.\\r\\nFor servers requiring authentication, our review team must be able to log into a demo account with no further configuration required.\\r\\nEnsure that the provided URL and credentials are correct, do not feature MFA (including requiring SMS codes, login through systems that require SMS, email or other verification schemes).\\r\\nEnsure that the provided credentials can be used to log in successfully (test them outside any company networks or LANs, or other internal networks).\\r\\nConfirm that the credentials have not expired.\\r\\nOne or more of your test cases did not produce correct results. Review all test cases carefully and rerun each one. Ensure that outputs match the expected results. Verify that there are no errors in the UI (if applicable) - for example, issues with loading content, images, or other UI issues.\\nEnsure that the returned textual output closely adheres to the user\\u2019s request, and does not offer extraneous information that is irrelevant to the request, including personal identifiers.\\nEnsure that all test cases pass on both ChatGPT web and mobile apps.\\nCompare actual outputs to clearly defined expected behavior for each tool and fix any mismatch so results are relevant to the user\\u2019s input and the app \\u201creliably does what it promises\\u201d.\\nIf required, in your resubmission, modify your test cases and expected responses to be clear and unambiguous.\\r\\nReview all test cases carefully and rerun each one. Ensure that outputs match the expected results. Verify that there are no errors in the UI (if applicable) - for example, issues with loading content, images, or other UI issues.\\r\\nEnsure that the returned textual output closely adheres to the user\\u2019s request, and does not offer extraneous information that is irrelevant to the request, including personal identifiers.\\r\\nEnsure that all test cases pass on both ChatGPT web and mobile apps.\\r\\nCompare actual outputs to clearly defined expected behavior for each tool and fix any mismatch so results are relevant to the user\\u2019s input and the app \\u201creliably does what it promises\\u201d.\\r\\nIf required, in your resubmission, modify your test cases and expected responses to be clear and unambiguous.\\r\\nYour app returns user-related data types that are not disclosed in your privacy policy. Audit your MCP tool responses in developer mode by running a few realistic example requests and listing every user-related field your app returns (including nested fields and \\u201cdebug\\u201d payloads). Ensure tools return only what\\u2019s strictly necessary for the user\\u2019s request and remove any unnecessary PII, telemetry/internal identifiers (e.g., session/trace/request IDs, timestamps, internal account IDs, logs) and/or any auth secrets (tokens/keys/passwords).\\nYou may also consider updating your published privacy policy so it clearly discloses all categories of personal data you collect/process/return and why\\u2014if a field isn\\u2019t truly needed, remove it rather than disclose it.\\nIf a user identifier is truly necessary, make it explicitly requested and clearly tied to the user\\u2019s intent (not \\u201clooked up and echoed\\u201d by default)\\r\\nAudit your MCP tool responses in developer mode by running a few realistic example requests and listing every user-related field your app returns (including nested fields and \\u201cdebug\\u201d payloads). Ensure tools return only what\\u2019s strictly necessary for the user\\u2019s request and remove any unnecessary PII, telemetry/internal identifiers (e.g., session/trace/request IDs, timestamps, internal account IDs, logs) and/or any auth secrets (tokens/keys/passwords).\\r\\nYou may also consider updating your published privacy policy so it clearly discloses all categories of personal data you collect/process/return and why\\u2014if a field isn\\u2019t truly needed, remove it rather than disclose it.\\r\\nIf a user identifier is truly necessary, make it explicitly requested and clearly tied to the user\\u2019s intent (not \\u201clooked up and echoed\\u201d by default)\\r\\nTool hint annotations do not appear to match the tool\\u2019s behavior: readOnlyHint: Set to true if it strictly fetches/looks up/lists/retrieves data and does not modify anything. Set to false if the tool can create/update/delete anything, trigger actions (send emails/messages, run jobs, enqueue tasks, write logs, start workflows), or otherwise change state.\\ndestructiveHint: Set to true if it can cause irreversible outcomes (deleting, overwriting, sending messages/transactions you can\\u2019t undo, revoking access, destructive admin actions, etc.), even in only select modes, via default parameters, or through indirect side effects. Ensure the justification provided clearly describes what is irreversible and under what conditions, including any safeguards like confirmation steps, dry-run options, or scoping constraints. Otherwise, set to false.\\nopenWorldHint: Set to true if it can write to or change publicly visible internet state (e.g., posting to social media/blogs/forums, sending emails/SMS/messages to external recipients, creating public tickets/issues, publishing pages, pushing code/content to public endpoints, submitting forms to third parties, or otherwise affecting systems outside a private/first-party context). Set to false only if it operates entirely within closed/private systems (including internal writes) and cannot change the state of the publicly visible internet.\\r\\nreadOnlyHint: Set to true if it strictly fetches/looks up/lists/retrieves data and does not modify anything. Set to false if the tool can create/update/delete anything, trigger actions (send emails/messages, run jobs, enqueue tasks, write logs, start workflows), or otherwise change state.\\r\\ndestructiveHint: Set to true if it can cause irreversible outcomes (deleting, overwriting, sending messages/transactions you can\\u2019t undo, revoking access, destructive admin actions, etc.), even in only select modes, via default parameters, or through indirect side effects. Ensure the justification provided clearly describes what is irreversible and under what conditions, including any safeguards like confirmation steps, dry-run options, or scoping constraints. Otherwise, set to false.\\r\\nopenWorldHint: Set to true if it can write to or change publicly visible internet state (e.g., posting to social media/blogs/forums, sending emails/SMS/messages to external recipients, creating public tickets/issues, publishing pages, pushing code/content to public endpoints, submitting forms to third parties, or otherwise affecting systems outside a private/first-party context). Set to false only if it operates entirely within closed/private systems (including internal writes) and cannot change the state of the publicly visible internet.\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Submit and maintain your app\", \"lvl2\": \"App review & approval\", \"lvl3\": \"App review & approval FAQs\", \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"25-https://developers.openai.com/apps-sdk/deploy/submission/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 your privacy policy. Audit your MCP tool responses in developer \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"How long does app review take?\\r\\nThe app directory and Apps SDK are currently in beta, and review timelines may vary as we continue to build and scale our processes. Please do not contact support to request expedited review, as these requests cannot be accommodated.\\r\\nWhat are common rejection reasons and how can I resolve them?\\r\\nWe\\u2019re unable to connect to your MCP server using the MCP URL and/or test credentials we were given. For servers requiring authentication, our review team must be able to log into a demo account with no further configuration required.\\nEnsure that the provided URL and credentials are correct, do not feature MFA (including requiring SMS codes, login through systems that require SMS, email or other verification schemes).\\nEnsure that the provided credentials can be used to log in successfully (test them outside any company networks or LANs, or other internal networks).\\nConfirm that the credentials have not expired.\\r\\nFor servers requiring authentication, our review team must be able to log into a demo account with no further configuration required.\\r\\nEnsure that the provided URL and credentials are correct, do not feature MFA (including requiring SMS codes, login through systems that require SMS, email or other verification schemes).\\r\\nEnsure that the provided credentials can be used to log in successfully (test them outside any company networks or LANs, or other internal networks).\\r\\nConfirm that the credentials have not expired.\\r\\nOne or more of your test cases did not produce correct results. Review all test cases carefully and rerun each one. Ensure that outputs match the expected results. Verify that there are no errors in the UI (if applicable) - for example, issues with loading content, images, or other UI issues.\\nEnsure that the returned textual output closely adheres to the user\\u2019s request, and does not offer extraneous information that is irrelevant to the request, including personal identifiers.\\nEnsure that all test cases pass on both ChatGPT web and mobile apps.\\nCompare actual outputs to clearly defined expected behavior for each tool and fix any mismatch so results are relevant to the user\\u2019s input and the app \\u201creliably does what it promises\\u201d.\\nIf required, in your resubmission, modify your test cases and expected responses to be clear and unambiguous.\\r\\nReview all test cases carefully and rerun each one. Ensure that outputs match the expected results. Verify that there are no errors in the UI (if applicable) - for example, issues with loading content, images, or other UI issues.\\r\\nEnsure that the returned textual output closely adheres to the user\\u2019s request, and does not offer extraneous information that is irrelevant to the request, including personal identifiers.\\r\\nEnsure that all test cases pass on both ChatGPT web and mobile apps.\\r\\nCompare actual outputs to clearly defined expected behavior for each tool and fix any mismatch so results are relevant to the user\\u2019s input and the app \\u201creliably does what it promises\\u201d.\\r\\nIf required, in your resubmission, modify your test cases and expected responses to be clear and unambiguous.\\r\\nYour app returns user-related data types that are not disclosed in your privacy policy. Audit your MCP tool responses in developer mode by running a few realistic example requests and listing every user-related field your app returns (including nested fields and \\u201cdebug\\u201d payloads). Ensure tools return only what\\u2019s strictly necessary for the user\\u2019s request and remove any unnecessary PII, telemetry/internal identifiers (e.g., session/trace/request IDs, timestamps, internal account IDs, logs) and/or any auth secrets (tokens/keys/passwords).\\nYou may also consider updating your published privacy policy so it clearly discloses all categories of personal data you collect/process/return and why\\u2014if a field isn\\u2019t truly needed, remove it rather than disclose it.\\nIf a user identifier is truly necessary, make it explicitly requested and clearly tied to the user\\u2019s intent (not \\u201clooked up and echoed\\u201d by default)\\r\\nAudit your MCP tool responses in developer mode by running a few realistic example requests and listing every user-related field your app returns (including nested fields and \\u201cdebug\\u201d payloads). Ensure tools return only what\\u2019s strictly necessary for the user\\u2019s request and remove any unnecessary PII, telemetry/internal identifiers (e.g., session/trace/request IDs, timestamps, internal account IDs, logs) and/or any auth secrets (tokens/keys/passwords).\\r\\nYou may also consider updating your published privacy policy so it clearly discloses all categories of personal data you collect/process/return and why\\u2014if a field isn\\u2019t truly needed, remove it rather than disclose it.\\r\\nIf a user identifier is truly necessary, make it explicitly requested and clearly tied to the user\\u2019s intent (not \\u201clooked up and echoed\\u201d by default)\\r\\nTool hint annotations do not appear to match the tool\\u2019s behavior: readOnlyHint: Set to true if it strictly fetches/looks up/lists/retrieves data and does not modify anything. Set to false if the tool can create/update/delete anything, trigger actions (send emails/messages, run jobs, enqueue tasks, write logs, start workflows), or otherwise change state.\\ndestructiveHint: Set to true if it can cause irreversible outcomes (deleting, overwriting, sending messages/transactions you can\\u2019t undo, revoking access, destructive admin actions, etc.), even in only select modes, via default parameters, or through indirect side effects. Ensure the justification provided clearly describes what is irreversible and under what conditions, including any safeguards like confirmation steps, dry-run options, or scoping constraints. Otherwise, set to false.\\nopenWorldHint: Set to true if it can write to or change publicly visible internet state (e.g., posting to social media/blogs/forums, sending emails/SMS/messages to external recipients, creating public tickets/issues, publishing pages, pushing code/content to public endpoints, submitting forms to third parties, or otherwise affecting systems outside a private/first-party context). Set to false only if it operates entirely within closed/private systems (including internal writes) and cannot change the state of the publicly visible internet.\\r\\nreadOnlyHint: Set to true if it strictly fetches/looks up/lists/retrieves data and does not modify anything. Set to false if the tool can create/update/delete anything, trigger actions (send emails/messages, run jobs, enqueue tasks, write logs, start workflows), or otherwise change state.\\r\\ndestructiveHint: Set to true if it can cause irreversible outcomes (deleting, overwriting, sending messages/transactions you can\\u2019t undo, revoking access, destructive admin actions, etc.), even in only select modes, via default parameters, or through indirect side effects. Ensure the justification provided clearly describes what is irreversible and under what conditions, including any safeguards like confirmation steps, dry-run options, or scoping constraints. Otherwise, set to false.\\r\\nopenWorldHint: Set to true if it can write to or change publicly visible internet state (e.g., posting to social media/blogs/forums, sending emails/SMS/messages to external recipients, creating public tickets/issues, publishing pages, pushing code/content to public endpoints, submitting forms to third parties, or otherwise affecting systems outside a private/first-party context). Set to false only if it operates entirely within closed/private systems (including internal writes) and cannot change the state of the publicly visible internet.\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"responses\", \"api\", \"mcp\", \"tool\", \"require\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Submit and maintain your app\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\"]}, \"lvl2\": {\"value\": \"App review & approval\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\", \"approval\"]}, \"lvl3\": {\"value\": \"App review & approval FAQs\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\", \"approval\"]}}}}, {\"url\": \"https://developers.openai.com/api/docs/guides/tools-connectors-mcp/#approvals\", \"url_without_anchor\": \"https://developers.openai.com/api/docs/guides/tools-connectors-mcp/\", \"anchor\": \"approvals\", \"content\": \"By default, OpenAI will request your approval before any data is shared with a connector or remote MCP server. Approvals help you maintain control and visibility over what data is being sent to an MCP server. We highly recommend that you carefully review (and optionally log) all data being shared with a remote MCP server. A request for an approval to make an MCP tool call creates a mcp_approval_request item in the Response\\u2019s output that looks like this:\\r\\nYou can then respond to this by creating a new Response object and appending an mcp_approval_response item to it.\\r\\nHere we\\u2019re using the previous_response_id parameter to chain this new Response, with the previous Response that generated the approval request. But you can also pass back the outputs from one response, as inputs into another for maximum control over what enter\\u2019s the model\\u2019s context.\\r\\nIf and when you feel comfortable trusting a remote MCP server, you can choose to skip the approvals for reduced latency. To do this, you can set the require_approval parameter of the MCP tool to an object listing just the tools you\\u2019d like to skip approvals for like shown below, or set it to the value 'never' to skip approvals for all tools in that remote MCP server.\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"MCP and Connectors\", \"lvl2\": \"How it works\", \"lvl3\": \"Step 2: Calling tools\", \"lvl4\": \"Approvals\", \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"14-https://developers.openai.com/api/docs/guides/tools-connectors-mcp/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 require_approval parameter of the MCP tool to an object \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"By default, OpenAI will request your approval before any data is shared with a connector or remote MCP server. Approvals help you maintain control and visibility over what data is being sent to an MCP server. We highly recommend that you carefully review (and optionally log) all data being shared with a remote MCP server. A request for an approval to make an MCP tool call creates a mcp_approval_request item in the Response\\u2019s output that looks like this:\\r\\nYou can then respond to this by creating a new Response object and appending an mcp_approval_response item to it.\\r\\nHere we\\u2019re using the previous_response_id parameter to chain this new Response, with the previous Response that generated the approval request. But you can also pass back the outputs from one response, as inputs into another for maximum control over what enter\\u2019s the model\\u2019s context.\\r\\nIf and when you feel comfortable trusting a remote MCP server, you can choose to skip the approvals for reduced latency. To do this, you can set the require_approval parameter of the MCP tool to an object listing just the tools you\\u2019d like to skip approvals for like shown below, or set it to the value 'never' to skip approvals for all tools in that remote MCP server.\", \"matchLevel\": \"full\", \"fullyHighlighted\": false, \"matchedWords\": [\"responses\", \"api\", \"mcp\", \"tool\", \"require\", \"approval\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"MCP and Connectors\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\"]}, \"lvl2\": {\"value\": \"How it works\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl3\": {\"value\": \"Step 2: Calling tools\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"tool\"]}, \"lvl4\": {\"value\": \"Approvals\", \"matchLevel\": \"partial\", \"fullyHighlighted\": true, \"matchedWords\": [\"approval\"]}}}}, {\"url\": \"https://developers.openai.com/api/docs/guides/your-data/#which-models-and-features-are-eligible-for-data-residency\", \"url_without_anchor\": \"https://developers.openai.com/api/docs/guides/your-data/\", \"anchor\": \"which-models-and-features-are-eligible-for-data-residency\", \"content\": \"The following models and API services are eligible for data residency today for the regions specified below.\\r\\nTable 1: Regional data residency capabilities\\r\\nUS\\r\\n\\u2705\\r\\n\\u2705\\r\\n\\u274c\\r\\nText, Audio, Voice, Image\\r\\nus.api.openai.com (required)\\r\\nEurope (EEA + Switzerland)\\r\\n\\u2705\\r\\n\\u2705\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\neu.api.openai.com (required)\\r\\nAustralia\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nau.api.openai.com (optional)\\r\\nCanada\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nca.api.openai.com (optional)\\r\\nJapan\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\njp.api.openai.com (optional)\\r\\nIndia\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nin.api.openai.com (optional)\\r\\nSingapore\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nsg.api.openai.com (optional)\\r\\nSouth Korea\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nkr.api.openai.com (optional)\\r\\nUnited Kingdom\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\ngb.api.openai.com (required)\\r\\nUnited Arab Emirates\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nae.api.openai.com (required)\\r\\n* Image support in these regions requires approval for enhanced Zero Data Retention or enhanced Modified Abuse Monitoring.\\r\\nTable 2: API endpoint and tool support\\r\\n/v1/audio/transcriptions /v1/audio/translations /v1/audio/speech\\r\\ntts-1whisper-1gpt-4o-ttsgpt-4o-transcribegpt-4o-mini-transcribe\\r\\nAll\\r\\n/v1/batches\\r\\ngpt-5.4-pro-2026-03-05gpt-5.2-pro-2025-12-11gpt-5-pro-2025-10-06gpt-5-2025-08-07gpt-5.4-2026-03-05gpt-5.2-2025-12-11gpt-5.1-2025-11-13gpt-5-mini-2025-08-07gpt-5-nano-2025-08-07gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14gpt-4.1-nano-2025-04-14o3-2025-04-16o4-mini-2025-04-16o1-proo1-pro-2025-03-19o3-mini-2025-01-31o1-2024-12-17o1-mini-2024-09-12o1-previewgpt-4o-2024-11-20gpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4-turbo-2024-04-09gpt-4-0613gpt-3.5-turbo-0125\\r\\nAll\\r\\n/v1/chat/completions\\r\\ngpt-5-2025-08-07gpt-5.4-2026-03-05gpt-5.2-2025-12-11gpt-5.1-2025-11-13gpt-5-mini-2025-08-07gpt-5-nano-2025-08-07gpt-5-chat-latest-2025-08-07gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14gpt-4.1-nano-2025-04-14o3-mini-2025-01-31o3-2025-04-16o4-mini-2025-04-16o1-2024-12-17o1-mini-2024-09-12o1-previewgpt-4o-2024-11-20gpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4-turbo-2024-04-09gpt-4-0613gpt-3.5-turbo-0125\\r\\nAll\\r\\n/v1/embeddings\\r\\ntext-embedding-3-smalltext-embedding-3-largetext-embedding-ada-002\\r\\nAll\\r\\n/v1/evals\\r\\n\\r\\nUS and EU\\r\\n/v1/files\\r\\n\\r\\nAll\\r\\n/v1/fine_tuning/jobs\\r\\ngpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14\\r\\nAll\\r\\n/v1/images/edits\\r\\ngpt-image-1gpt-image-1.5gpt-image-1-mini\\r\\nAll\\r\\n/v1/images/generations\\r\\ndall-e-3gpt-image-1gpt-image-1.5gpt-image-1-mini\\r\\nAll\\r\\n/v1/moderations\\r\\ntext-moderation-latest*omni-moderation-latest\\r\\nAll\\r\\n/v1/realtime\\r\\ngpt-4o-realtime-preview-2025-06-03gpt-realtimegpt-realtime-1.5gpt-realtime-mini\\r\\nUS and EU\\r\\n/v1/realtime\\r\\ngpt-4o-realtime-preview-2024-12-17gpt-4o-realtime-preview-2024-10-01gpt-4o-mini-realtime-preview-2024-12-17\\r\\nUS only\\r\\n/v1/responses\\r\\ngpt-5.4-pro-2026-03-05gpt-5.2-pro-2025-12-11gpt-5-pro-2025-10-06gpt-5-2025-08-07gpt-5.4-2026-03-05gpt-5.2-2025-12-11gpt-5.1-2025-11-13gpt-5-mini-2025-08-07gpt-5-nano-2025-08-07gpt-5-chat-latest-2025-08-07gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14gpt-4.1-nano-2025-04-14o3-2025-04-16o4-mini-2025-04-16o1-proo1-pro-2025-03-19computer-use-preview*o3-mini-2025-01-31o1-2024-12-17o1-mini-2024-09-12o1-previewgpt-4o-2024-11-20gpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4-turbo-2024-04-09gpt-4-0613gpt-3.5-turbo-0125\\r\\nAll\\r\\n/v1/responses File Search\\r\\n\\r\\nAll\\r\\n/v1/responses Web Search\\r\\n\\r\\nAll\\r\\n/v1/vector_stores\\r\\n\\r\\nAll\\r\\nCode Interpreter tool\\r\\n\\r\\nAll\\r\\nFile Search\\r\\n\\r\\nAll\\r\\nFile Uploads\\r\\n\\r\\nAll, when used with base64 file uploads\\r\\nRemote MCP server tool\\r\\n\\r\\nAll, but MCP servers are third-party services, and data sent to an MCP server is subject to their data residency policies.\\r\\nScale Tier\\r\\n\\r\\nAll\\r\\nStructured Outputs (excluding schema)\\r\\n\\r\\nAll\\r\\nSupported Input Modalities\\r\\n\\r\\nText Image Audio/Voice\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Data controls in the OpenAI platform\", \"lvl2\": \"Data residency controls\", \"lvl3\": \"Which models and features are eligible for data residency?\", \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"42-https://developers.openai.com/api/docs/guides/your-data/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 file uploads\\r\\nRemote MCP server tool\\r\\n\\r\\nAll, but MCP servers \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"The following models and API services are eligible for data residency today for the regions specified below.\\r\\nTable 1: Regional data residency capabilities\\r\\nUS\\r\\n\\u2705\\r\\n\\u2705\\r\\n\\u274c\\r\\nText, Audio, Voice, Image\\r\\nus.api.openai.com (required)\\r\\nEurope (EEA + Switzerland)\\r\\n\\u2705\\r\\n\\u2705\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\neu.api.openai.com (required)\\r\\nAustralia\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nau.api.openai.com (optional)\\r\\nCanada\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nca.api.openai.com (optional)\\r\\nJapan\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\njp.api.openai.com (optional)\\r\\nIndia\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nin.api.openai.com (optional)\\r\\nSingapore\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nsg.api.openai.com (optional)\\r\\nSouth Korea\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nkr.api.openai.com (optional)\\r\\nUnited Kingdom\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\ngb.api.openai.com (required)\\r\\nUnited Arab Emirates\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nae.api.openai.com (required)\\r\\n* Image support in these regions requires approval for enhanced Zero Data Retention or enhanced Modified Abuse Monitoring.\\r\\nTable 2: API endpoint and tool support\\r\\n/v1/audio/transcriptions /v1/audio/translations /v1/audio/speech\\r\\ntts-1whisper-1gpt-4o-ttsgpt-4o-transcribegpt-4o-mini-transcribe\\r\\nAll\\r\\n/v1/batches\\r\\ngpt-5.4-pro-2026-03-05gpt-5.2-pro-2025-12-11gpt-5-pro-2025-10-06gpt-5-2025-08-07gpt-5.4-2026-03-05gpt-5.2-2025-12-11gpt-5.1-2025-11-13gpt-5-mini-2025-08-07gpt-5-nano-2025-08-07gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14gpt-4.1-nano-2025-04-14o3-2025-04-16o4-mini-2025-04-16o1-proo1-pro-2025-03-19o3-mini-2025-01-31o1-2024-12-17o1-mini-2024-09-12o1-previewgpt-4o-2024-11-20gpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4-turbo-2024-04-09gpt-4-0613gpt-3.5-turbo-0125\\r\\nAll\\r\\n/v1/chat/completions\\r\\ngpt-5-2025-08-07gpt-5.4-2026-03-05gpt-5.2-2025-12-11gpt-5.1-2025-11-13gpt-5-mini-2025-08-07gpt-5-nano-2025-08-07gpt-5-chat-latest-2025-08-07gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14gpt-4.1-nano-2025-04-14o3-mini-2025-01-31o3-2025-04-16o4-mini-2025-04-16o1-2024-12-17o1-mini-2024-09-12o1-previewgpt-4o-2024-11-20gpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4-turbo-2024-04-09gpt-4-0613gpt-3.5-turbo-0125\\r\\nAll\\r\\n/v1/embeddings\\r\\ntext-embedding-3-smalltext-embedding-3-largetext-embedding-ada-002\\r\\nAll\\r\\n/v1/evals\\r\\n\\r\\nUS and EU\\r\\n/v1/files\\r\\n\\r\\nAll\\r\\n/v1/fine_tuning/jobs\\r\\ngpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14\\r\\nAll\\r\\n/v1/images/edits\\r\\ngpt-image-1gpt-image-1.5gpt-image-1-mini\\r\\nAll\\r\\n/v1/images/generations\\r\\ndall-e-3gpt-image-1gpt-image-1.5gpt-image-1-mini\\r\\nAll\\r\\n/v1/moderations\\r\\ntext-moderation-latest*omni-moderation-latest\\r\\nAll\\r\\n/v1/realtime\\r\\ngpt-4o-realtime-preview-2025-06-03gpt-realtimegpt-realtime-1.5gpt-realtime-mini\\r\\nUS and EU\\r\\n/v1/realtime\\r\\ngpt-4o-realtime-preview-2024-12-17gpt-4o-realtime-preview-2024-10-01gpt-4o-mini-realtime-preview-2024-12-17\\r\\nUS only\\r\\n/v1/responses\\r\\ngpt-5.4-pro-2026-03-05gpt-5.2-pro-2025-12-11gpt-5-pro-2025-10-06gpt-5-2025-08-07gpt-5.4-2026-03-05gpt-5.2-2025-12-11gpt-5.1-2025-11-13gpt-5-mini-2025-08-07gpt-5-nano-2025-08-07gpt-5-chat-latest-2025-08-07gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14gpt-4.1-nano-2025-04-14o3-2025-04-16o4-mini-2025-04-16o1-proo1-pro-2025-03-19computer-use-preview*o3-mini-2025-01-31o1-2024-12-17o1-mini-2024-09-12o1-previewgpt-4o-2024-11-20gpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4-turbo-2024-04-09gpt-4-0613gpt-3.5-turbo-0125\\r\\nAll\\r\\n/v1/responses File Search\\r\\n\\r\\nAll\\r\\n/v1/responses Web Search\\r\\n\\r\\nAll\\r\\n/v1/vector_stores\\r\\n\\r\\nAll\\r\\nCode Interpreter tool\\r\\n\\r\\nAll\\r\\nFile Search\\r\\n\\r\\nAll\\r\\nFile Uploads\\r\\n\\r\\nAll, when used with base64 file uploads\\r\\nRemote MCP server tool\\r\\n\\r\\nAll, but MCP servers are third-party services, and data sent to an MCP server is subject to their data residency policies.\\r\\nScale Tier\\r\\n\\r\\nAll\\r\\nStructured Outputs (excluding schema)\\r\\n\\r\\nAll\\r\\nSupported Input Modalities\\r\\n\\r\\nText Image Audio/Voice\", \"matchLevel\": \"full\", \"fullyHighlighted\": false, \"matchedWords\": [\"responses\", \"api\", \"mcp\", \"tool\", \"require\", \"approval\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Data controls in the OpenAI platform\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl2\": {\"value\": \"Data residency controls\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl3\": {\"value\": \"Which models and features are eligible for data residency?\", \"matchLevel\": \"none\", \"matchedWords\": []}}}}, {\"url\": \"https://developers.openai.com/api/docs/mcp/#test-and-connect-your-mcp-server\", \"url_without_anchor\": \"https://developers.openai.com/api/docs/mcp/\", \"anchor\": \"test-and-connect-your-mcp-server\", \"content\": \"You can test your MCP server with a deep research model in the prompts dashboard. Create a new prompt, or edit an existing one, and add a new MCP tool to the prompt configuration. Remember that MCP servers used via API for deep research have to be configured with no approval required.\\r\\nIf you are testing this server in ChatGPT as an app, follow Connect from ChatGPT.\\r\\n\\r\\nOnce you have configured your MCP server, you can chat with a model using it via the Prompts UI.\\r\\n\\r\\nYou can test the MCP server using the Responses API directly with a request like this one:\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Building MCP servers for ChatGPT Apps and API integrations\", \"lvl2\": \"Test and connect your MCP server\", \"lvl3\": null, \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"14-https://developers.openai.com/api/docs/mcp/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 the MCP server using the Responses API directly with a \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"You can test your MCP server with a deep research model in the prompts dashboard. Create a new prompt, or edit an existing one, and add a new MCP tool to the prompt configuration. Remember that MCP servers used via API for deep research have to be configured with no approval required.\\r\\nIf you are testing this server in ChatGPT as an app, follow Connect from ChatGPT.\\r\\n\\r\\nOnce you have configured your MCP server, you can chat with a model using it via the Prompts UI.\\r\\n\\r\\nYou can test the MCP server using the Responses API directly with a request like this one:\", \"matchLevel\": \"full\", \"fullyHighlighted\": false, \"matchedWords\": [\"responses\", \"api\", \"mcp\", \"tool\", \"require\", \"approval\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Building MCP servers for ChatGPT Apps and API integrations\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\", \"mcp\"]}, \"lvl2\": {\"value\": \"Test and connect your MCP server\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\"]}}}}], \"nbHits\": 12, \"page\": 0, \"nextCursor\": \"eyJwYWdlIjowLCJvZmZzZXRzIjpbOSwxXSwiaGl0c1BlckluZGV4IjoxMH0=\"}", + "server_label": "openai_docs" + }, + { + "id": "rs_0485e7c463a292010169cc2781f3708190b3f99dbcd2fffc96", + "type": "reasoning", + "summary": [] + }, + { + "id": "mcp_0485e7c463a292010169cc2784646c819085469f491037fcfc", + "type": "mcp_call", + "status": "completed", + "approval_request_id": null, + "arguments": "{\"url\": \"https://developers.openai.com/api/docs/guides/tools-connectors-mcp/\", \"anchor\": \"approvals\"}", + "error": null, + "name": "fetch_openai_doc", + "output": "#### Approvals\n\nBy default, OpenAI will request your approval before any data is shared with a connector or remote MCP server. Approvals help you maintain control and visibility over what data is being sent to an MCP server. We highly recommend that you carefully review (and optionally log) all data being shared with a remote MCP server. A request for an approval to make an MCP tool call creates a `mcp_approval_request` item in the Response's output that looks like this:\n\n```json\n{\n \"id\": \"mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339\",\n \"type\": \"mcp_approval_request\",\n \"arguments\": \"{\\\"diceRollExpression\\\":\\\"2d4 + 1\\\"}\",\n \"name\": \"roll\",\n \"server_label\": \"dmcp\"\n}\n```\n\nYou can then respond to this by creating a new Response object and appending an `mcp_approval_response` item to it.\n\nApproving the use of tools in an API request\n\n```bash\ncurl https://api.openai.com/v1/responses \\\\\n-H \"Content-Type: application/json\" \\\\\n-H \"Authorization: Bearer $OPENAI_API_KEY\" \\\\\n-d '{\n \"model\": \"gpt-5\",\n \"tools\": [\n {\n \"type\": \"mcp\",\n \"server_label\": \"dmcp\",\n \"server_description\": \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n \"server_url\": \"https://dmcp-server.deno.dev/sse\",\n \"require_approval\": \"always\",\n }\n ],\n \"previous_response_id\": \"resp_682d498bdefc81918b4a6aa477bfafd904ad1e533afccbfa\",\n \"input\": [{\n \"type\": \"mcp_approval_response\",\n \"approve\": true,\n \"approval_request_id\": \"mcpr_682d498e3bd4819196a0ce1664f8e77b04ad1e533afccbfa\"\n }]\n }'\n```\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst resp = await client.responses.create({\n model: \"gpt-5\",\n tools: [{\n type: \"mcp\",\n server_label: \"dmcp\",\n server_description: \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n server_url: \"https://dmcp-server.deno.dev/sse\",\n require_approval: \"always\",\n }],\n previous_response_id: \"resp_682d498bdefc81918b4a6aa477bfafd904ad1e533afccbfa\",\n input: [{\n type: \"mcp_approval_response\",\n approve: true,\n approval_request_id: \"mcpr_682d498e3bd4819196a0ce1664f8e77b04ad1e533afccbfa\"\n }],\n});\n\nconsole.log(resp.output_text);\n```\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI()\n\nresp = client.responses.create(\n model=\"gpt-5\",\n tools=[{\n \"type\": \"mcp\",\n \"server_label\": \"dmcp\",\n \"server_description\": \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n \"server_url\": \"https://dmcp-server.deno.dev/sse\",\n \"require_approval\": \"always\",\n }],\n previous_response_id=\"resp_682d498bdefc81918b4a6aa477bfafd904ad1e533afccbfa\",\n input=[{\n \"type\": \"mcp_approval_response\",\n \"approve\": True,\n \"approval_request_id\": \"mcpr_682d498e3bd4819196a0ce1664f8e77b04ad1e533afccbfa\"\n }],\n)\n\nprint(resp.output_text)\n```\n\n```csharp\nusing OpenAI.Responses;\n\nstring key = Environment.GetEnvironmentVariable(\"OPENAI_API_KEY\")!;\nOpenAIResponseClient client = new(model: \"gpt-5\", apiKey: key);\n\nResponseCreationOptions options = new();\noptions.Tools.Add(ResponseTool.CreateMcpTool(\n serverLabel: \"dmcp\",\n serverUri: new Uri(\"https://dmcp-server.deno.dev/sse\"),\n toolCallApprovalPolicy: new McpToolCallApprovalPolicy(GlobalMcpToolCallApprovalPolicy.AlwaysRequireApproval)\n));\n\n// STEP 1: Create response that requests tool call approval\nOpenAIResponse response1 = (OpenAIResponse)client.CreateResponse([\n ResponseItem.CreateUserMessageItem([\n ResponseContentPart.CreateInputTextPart(\"Roll 2d4+1\")\n ])\n], options);\n\nMcpToolCallApprovalRequestItem? approvalRequestItem = response1.OutputItems.Last() as McpToolCallApprovalRequestItem;\n\n// STEP 2: Approve the tool call request and get final response\noptions.PreviousResponseId = response1.Id;\nOpenAIResponse response2 = (OpenAIResponse)client.CreateResponse([\n ResponseItem.CreateMcpApprovalResponseItem(approvalRequestItem!.Id, approved: true),\n], options);\n\nConsole.WriteLine(response2.GetOutputText());\n```\n\n\nHere we're using the `previous_response_id` parameter to chain this new Response, with the previous Response that generated the approval request. But you can also pass back the [outputs from one response, as inputs into another](https://developers.openai.com/api/docs/guides/conversation-state#manually-manage-conversation-state) for maximum control over what enter's the model's context.\n\nIf and when you feel comfortable trusting a remote MCP server, you can choose to skip the approvals for reduced latency. To do this, you can set the `require_approval` parameter of the MCP tool to an object listing just the tools you'd like to skip approvals for like shown below, or set it to the value `'never'` to skip approvals for all tools in that remote MCP server.\n\nNever require approval for some tools\n\n```bash\ncurl https://api.openai.com/v1/responses \\\\\n-H \"Content-Type: application/json\" \\\\\n-H \"Authorization: Bearer $OPENAI_API_KEY\" \\\\\n-d '{\n \"model\": \"gpt-5\",\n \"tools\": [\n {\n \"type\": \"mcp\",\n \"server_label\": \"deepwiki\",\n \"server_url\": \"https://mcp.deepwiki.com/mcp\",\n \"require_approval\": {\n \"never\": {\n \"tool_names\": [\"ask_question\", \"read_wiki_structure\"]\n }\n }\n }\n ],\n \"input\": \"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?\"\n }'\n```\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst resp = await client.responses.create({\n model: \"gpt-5\",\n tools: [\n {\n type: \"mcp\",\n server_label: \"deepwiki\",\n server_url: \"https://mcp.deepwiki.com/mcp\",\n require_approval: {\n never: {\n tool_names: [\"ask_question\", \"read_wiki_structure\"]\n }\n }\n },\n ],\n input: \"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?\",\n});\n\nconsole.log(resp.output_text);\n```\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI()\n\nresp = client.responses.create(\n model=\"gpt-5\",\n tools=[\n {\n \"type\": \"mcp\",\n \"server_label\": \"deepwiki\",\n \"server_url\": \"https://mcp.deepwiki.com/mcp\",\n \"require_approval\": {\n \"never\": {\n \"tool_names\": [\"ask_question\", \"read_wiki_structure\"]\n }\n }\n },\n ],\n input=\"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?\",\n)\n\nprint(resp.output_text)\n```\n\n```csharp\nusing OpenAI.Responses;\n\nstring key = Environment.GetEnvironmentVariable(\"OPENAI_API_KEY\")!;\nOpenAIResponseClient client = new(model: \"gpt-5\", apiKey: key);\n\nResponseCreationOptions options = new();\noptions.Tools.Add(ResponseTool.CreateMcpTool(\n serverLabel: \"deepwiki\",\n serverUri: new Uri(\"https://mcp.deepwiki.com/mcp\"),\n allowedTools: new McpToolFilter() { ToolNames = { \"ask_question\", \"read_wiki_structure\" } },\n toolCallApprovalPolicy: new McpToolCallApprovalPolicy(GlobalMcpToolCallApprovalPolicy.NeverRequireApproval)\n));\n\nOpenAIResponse response = (OpenAIResponse)client.CreateResponse([\n ResponseItem.CreateUserMessageItem([\n ResponseContentPart.CreateInputTextPart(\"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?\")\n ])\n], options);\n\nConsole.WriteLine(response.GetOutputText());\n```", + "server_label": "openai_docs" + }, + { + "id": "rs_0485e7c463a292010169cc2785a2988190b14ed2ee3022f11e", + "type": "reasoning", + "summary": [] + }, + { + "id": "mcp_0485e7c463a292010169cc278d6ff88190a306fb2b9012a966", + "type": "mcp_call", + "status": "failed", + "approval_request_id": null, + "arguments": "{\"query\":\"site:developers.openai.com require_approval MCP tool Responses API require_approval object tool_names\", \"limit\": 10}", + "error": { + "type": "http_error", + "code": 424, + "message": "424: " + }, + "name": "search_openai_docs", + "output": null, + "server_label": "openai_docs" + }, + { + "id": "rs_0485e7c463a292010169cc27c9f0788190b8a793918b939e29", + "type": "reasoning", + "summary": [] + }, + { + "id": "mcp_0485e7c463a292010169cc27caea7c81908ea1919de9bdfa41", + "type": "mcp_call", + "status": "failed", + "approval_request_id": null, + "arguments": "{\"query\":\"require_approval MCP tool responses api\", \"limit\": 10}", + "error": { + "type": "http_error", + "code": 424, + "message": "424: " + }, + "name": "search_openai_docs", + "output": null, + "server_label": "openai_docs" + }, + { + "id": "rs_0485e7c463a292010169cc28075fb08190a2104545116f8eb7", + "type": "reasoning", + "summary": [] + }, + { + "id": "msg_0485e7c463a292010169cc280e2e748190958d28893f4d00da", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "Here\u2019s the relevant OpenAI docs page and section, then a concise summary of require_approval:\n\nSource\n- MCP and Connectors \u2192 Approvals (Responses API): https://developers.openai.com/api/docs/guides/tools-connectors-mcp/#approvals\n\nSummary of require_approval for the MCP tool (Responses API)\n- What it controls: Whether the platform must request your explicit approval before a model can call a tool on a remote MCP server or connector (and therefore send data to it).\n- Default behavior: Approval is required by default before any data is shared with a connector or remote MCP server. When required, the Responses API emits an mcp_approval_request item; you must reply with a new Response containing an mcp_approval_response (approve: true/false, approval_request_id).\n- Configuration options:\n - \"always\": Always require approval for tool calls to that MCP server (explicitly enforcing the default).\n - \"never\": Never require approval for any tools on that MCP server (reduced latency; only do this if you trust the server).\n - Object form (per-tool skipping): Set require_approval to an object specifying tool_names under never to skip approvals only for certain tools, while keeping approvals required for others. Example:\n - require_approval: { \"never\": { \"tool_names\": [\"ask_question\", \"read_wiki_structure\"] } }\n- Recommended practice: Keep approvals on unless you fully trust the remote MCP server; approvals provide visibility and control over what data is sent. Optionally log what\u2019s being shared.\n\nSupporting snippets from the docs\n- \u201cBy default, OpenAI will request your approval before any data is shared with a connector or remote MCP server\u2026 A request for an approval to make an MCP tool call creates a mcp_approval_request item\u2026 You can then respond \u2026 by appending an mcp_approval_response item\u2026\u201d\n- \u201cIf and when you feel comfortable trusting a remote MCP server, you can choose to skip the approvals for reduced latency\u2026 set the require_approval parameter \u2026 to an object listing just the tools you\u2019d like to skip approvals for\u2026 or set it to the value \u2018never\u2019 to skip approvals for all tools\u2026\u201d (See: Approvals section linked above)" + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": "medium", + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [ + { + "type": "mcp", + "allowed_tools": null, + "headers": null, + "require_approval": "never", + "server_description": "OpenAI documentation MCP server.", + "server_label": "openai_docs", + "server_url": "https://developers.openai.com/mcp" + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 21629, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 1760, + "output_tokens_details": { + "reasoning_tokens": 960 + }, + "total_tokens": 23389 + }, + "user": null, + "metadata": {} +}� +-- end response body -- From a17b7cd52f5df3a576baa02c122ab5daa6025e27 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 31 Mar 2026 13:40:05 -0700 Subject: [PATCH 47/48] improve coverage for tool choice --- .../groovy/ChatCompletionServiceTest.groovy | 30 +++++++ .../src/test/groovy/OpenAiTest.groovy | 12 +++ ...44176131e15bf8c0+81d6f6852e8115be.POST.rec | 79 +++++++++++++++++++ 3 files changed, 121 insertions(+) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/chat/completions/44176131e15bf8c0+81d6f6852e8115be.POST.rec diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index b7a8533bed8..a16920209dc 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -194,6 +194,36 @@ class ChatCompletionServiceTest extends OpenAiTest { toolDefinitions[0].schema.required == ["name"] } + def "create chat/completion test with tool choice"() { + runUnderTrace("parent") { + openAiClient.chat().completions().create(chatCompletionCreateParamsWithToolChoice()) + } + + expect: + List outputTag = [] + List> toolDefinitions = [] + assertChatCompletionTrace(false, outputTag, [tool_choice: "function"], true, toolDefinitions) + and: + toolDefinitions.size() == 1 + toolDefinitions[0].name == "extract_student_info" + toolDefinitions[0].description == "Get the student information from the body of the input text" + and: + outputTag.size() == 1 + LLMObs.LLMMessage outputMsg = outputTag.get(0) + outputMsg.toolCalls.size() == 1 + def toolcall = outputMsg.toolCalls.get(0) + toolcall.name == "extract_student_info" + toolcall.toolId instanceof String + toolcall.type == "function" + toolcall.arguments == [ + name: 'David Nguyen', + major: 'computer science', + school: 'Stanford University', + grades: 3.8, + clubs: ['Chess Club', 'South Asian Student Association'] + ] + } + def "create streaming chat/completion test with tool calls"() { runnableUnderTrace("parent") { StreamResponse streamCompletion = openAiClient.chat().completions().createStreaming(chatCompletionCreateParamsWithTools()) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy index 30c7026c1a0..b23ef80aeb3 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/OpenAiTest.groovy @@ -14,6 +14,7 @@ import com.openai.models.Reasoning import com.openai.models.ReasoningEffort import com.openai.models.chat.completions.ChatCompletionCreateParams import com.openai.models.chat.completions.ChatCompletionFunctionTool +import com.openai.models.chat.completions.ChatCompletionNamedToolChoice import com.openai.models.chat.completions.ChatCompletionStreamOptions import com.openai.models.completions.CompletionCreateParams import com.openai.models.embeddings.EmbeddingCreateParams @@ -262,6 +263,17 @@ He hopes to pursue a career in software engineering after graduating.""") .build() } + ChatCompletionCreateParams chatCompletionCreateParamsWithToolChoice() { + chatCompletionCreateParamsWithTools().toBuilder() + .toolChoice(ChatCompletionNamedToolChoice.builder() + .type(JsonValue.from("function")) + .function(ChatCompletionNamedToolChoice.Function.builder() + .name("extract_student_info") + .build()) + .build()) + .build() + } + ChatCompletionCreateParams chatCompletionCreateParamsWithRawTools() { def functionMap = [ name: "extract_student_info_raw", diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/chat/completions/44176131e15bf8c0+81d6f6852e8115be.POST.rec b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/chat/completions/44176131e15bf8c0+81d6f6852e8115be.POST.rec new file mode 100644 index 00000000000..115836eb003 --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/chat/completions/44176131e15bf8c0+81d6f6852e8115be.POST.rec @@ -0,0 +1,79 @@ +method: POST +path: chat/completions +-- begin request body -- +{"messages":[{"content":"David Nguyen is a sophomore majoring in computer science at Stanford University and has a GPA of 3.8.\nDavid is an active member of the university's Chess Club and the South Asian Student Association.\nHe hopes to pursue a career in software engineering after graduating.","role":"user"}],"model":"gpt-4o-mini","tool_choice":{"function":{"name":"extract_student_info"},"type":"function"},"tools":[{"function":{"name":"extract_student_info","description":"Get the student information from the body of the input text","parameters":{"type":"object","properties":{"name":{"type":"string","description":"Name of the person"},"major":{"type":"string","description":"Major subject."},"school":{"type":"string","description":"The university name."},"grades":{"type":"integer","description":"GPA of the student."},"clubs":{"type":"array","description":"School clubs for extracurricular activities. ","items":{"type":"string","description":"Name of School Club"}}}}},"type":"function"}]} +-- end request body -- +status code: 200 +-- begin response headers -- +access-control-expose-headers: X-Request-ID +alt-svc: h3=":443"; ma=86400 +cf-cache-status: DYNAMIC +cf-ray: 9e52289bcf4b6811-SEA +content-type: application/json +date: Tue, 31 Mar 2026 20:38:37 GMT +openai-organization: datadog-staging +openai-processing-ms: 975 +openai-project: proj_gt6TQZPRbZfoY2J9AQlEJMpd +openai-version: 2020-10-01 +server: cloudflare +set-cookie: __cf_bm=8UpsxSUpYP5PmHi3NoNcvUAPHElwZCix7hG5h.Jprr8-1774989516.1296408-1.0.1.1-2GEVgpgzsaA8PIC.4pr9thdXAw7zglpkJZF_LD08k1itVMbyhtxUjskkov5EnFuRLGkMaXiKrl5sngZSQF3mFA_h_Cfh.bCgd.k3ovK4p9VcbEmkfWy7MStLsa5i3LFh; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 21:08:37 GMT +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +x-openai-proxy-wasm: v0.1 +x-ratelimit-limit-requests: 30000 +x-ratelimit-limit-tokens: 150000000 +x-ratelimit-remaining-requests: 29999 +x-ratelimit-remaining-tokens: 149999927 +x-ratelimit-reset-requests: 2ms +x-ratelimit-reset-tokens: 0s +x-request-id: req_6de1d82c867f4dc3a10eb169eda01c8f +-- end response headers -- +-- begin response body -- +{ + "id": "chatcmpl-DPa96rNBMqoTGuLjKfIw3huHXDuai", + "object": "chat.completion", + "created": 1774989516, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_FPVMvOQnA8cKnmYI83lMmQ75", + "type": "function", + "function": { + "name": "extract_student_info", + "arguments": "{\"name\":\"David Nguyen\",\"major\":\"computer science\",\"school\":\"Stanford University\",\"grades\":3.8,\"clubs\":[\"Chess Club\",\"South Asian Student Association\"]}" + } + } + ], + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 162, + "completion_tokens": 34, + "total_tokens": 196, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_1bf41b89ce" +} +� +-- end response body -- From 408d335bb9ecaf5515ed6ca190b61c0be344cf65 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 31 Mar 2026 15:37:01 -0700 Subject: [PATCH 48/48] Fix latestDepTest --- .../test/groovy/ResponseServiceTest.groovy | 1 - ...43357cca927dd5fd+de853cdc24b16f2a.POST.rec | 289 ++++++++++++++++++ 2 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/43357cca927dd5fd+de853cdc24b16f2a.POST.rec diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index 63c39256349..1c11808cd33 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -293,7 +293,6 @@ class ResponseServiceTest extends OpenAiTest { ResponseCreateParams params = responseCreateParamsWithRawFunctionTool() expect: - params._tools().asKnown().empty params._tools().asUnknown().present when: diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/43357cca927dd5fd+de853cdc24b16f2a.POST.rec b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/43357cca927dd5fd+de853cdc24b16f2a.POST.rec new file mode 100644 index 00000000000..f641344c63b --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/resources/http-records/responses/43357cca927dd5fd+de853cdc24b16f2a.POST.rec @@ -0,0 +1,289 @@ +method: POST +path: responses +-- begin request body -- +{"input":"Use the OpenAI docs MCP server to find the Responses API MCP tool documentation and summarize the require_approval option.","model":"gpt-5","tools":[{"server_label":"openai_docs","type":"mcp","require_approval":"never","server_description":"OpenAI documentation MCP server.","server_url":"https://developers.openai.com/mcp"}]} +-- end request body -- +status code: 200 +-- begin response headers -- +alt-svc: h3=":443"; ma=86400 +cf-cache-status: DYNAMIC +cf-ray: 9e52af54ea23b35f-SEA +content-type: application/json +date: Tue, 31 Mar 2026 22:11:20 GMT +openai-organization: datadog-staging +openai-processing-ms: 45954 +openai-project: proj_gt6TQZPRbZfoY2J9AQlEJMpd +openai-version: 2020-10-01 +server: cloudflare +set-cookie: __cf_bm=QvFC8n_6x8f5hTHUiIpQWdm7cqzvFsgxH8UP.1v3k10-1774995034.386376-1.0.1.1-OFn4tI5LhCuC_iXs55kdgHcGwA_N4zea.adueG2aIkxP0enoARrqgJOp4zkaM7rTuwC2fSiJYHvAmaqHqALT0iyPmMVA6_foEpVLfs3m7eceKq022LWFYlT2lh.imYiu; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 22:41:20 GMT +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +x-ratelimit-limit-requests: 15000 +x-ratelimit-limit-tokens: 40000000 +x-ratelimit-remaining-requests: 14999 +x-ratelimit-remaining-tokens: 39996226 +x-ratelimit-reset-requests: 4ms +x-ratelimit-reset-tokens: 5ms +x-request-id: req_c3af1069e8d144a08bafcf9940bde90a +-- end response headers -- +-- begin response body -- +{ + "id": "resp_07637789c765520d0169cc465a70488195b30d4aae353d277d", + "object": "response", + "created_at": 1774995034, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1774995080, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-5-2025-08-07", + "output": [ + { + "id": "mcpl_07637789c765520d0169cc465a7bec81959c9744f3ad7a6016", + "type": "mcp_list_tools", + "server_label": "openai_docs", + "tools": [ + { + "annotations": { + "read_only": false + }, + "description": "Search across `platform.openai.com` + `developers.openai.com` docs. Use this whenever you are working with the OpenAI API (including the Responses API), OpenAI API SDKs, ChatGPT Apps SDK, or ChatGPT Codex. Results include URLs\u2014**after `search`, use `fetch_openai_doc`** to read/quote the exact markdown.", + "input_schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "minLength": 1 + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 50 + }, + "cursor": { + "type": "string" + } + }, + "required": [ + "query" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "name": "search_openai_docs" + }, + { + "annotations": { + "read_only": false + }, + "description": "List/browse pages from `platform.openai.com` + `developers.openai.com` that this server crawls (useful when you don\u2019t know the right query yet or you\u2019re paging through results). Search across `platform.openai.com` + `developers.openai.com` docs. Use this whenever you are working with the OpenAI API (including the Responses API), OpenAI API SDKs, ChatGPT Apps SDK, or ChatGPT Codex. Results include URLs\u2014**after `list`, use `fetch_openai_doc`** on a result URL to get the full markdown.", + "input_schema": { + "type": "object", + "properties": { + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 50 + }, + "cursor": { + "type": "string" + } + }, + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "name": "list_openai_docs" + }, + { + "annotations": { + "read_only": false + }, + "description": "Fetch the markdown for a specific doc page (from `developers.openai.com` or `platform.openai.com`) so you can quote/summarize exact, up-to-date guidance (schemas, examples, limits, edge cases). Prefer to **`search_openai_docs` first** (or `list_openai_docs` if you\u2019re browsing) to find the best URL, then `fetch_openai_doc` to pull the exact text; you can pass `anchor` (e.g. `#streaming`) to fetch just that section.", + "input_schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "minLength": 1 + }, + "anchor": { + "type": "string" + } + }, + "required": [ + "url" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "name": "fetch_openai_doc" + }, + { + "annotations": { + "read_only": false + }, + "description": "List all OpenAI API endpoint URLs available in the OpenAPI spec.", + "input_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "name": "list_api_endpoints" + }, + { + "annotations": { + "read_only": false + }, + "description": "Return the OpenAPI spec for a specific API endpoint URL. Optionally filter code samples by language, or return only code samples.", + "input_schema": { + "type": "object", + "properties": { + "url": { + "type": "string", + "minLength": 1 + }, + "languages": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "codeExamplesOnly": { + "type": "boolean" + } + }, + "required": [ + "url" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "name": "get_openapi_spec" + } + ] + }, + { + "id": "rs_07637789c765520d0169cc465bb1fc819597c79f87f7a85a8c", + "type": "reasoning", + "summary": [] + }, + { + "id": "mcp_07637789c765520d0169cc46604d348195b5c594a38ad50f74", + "type": "mcp_call", + "status": "completed", + "approval_request_id": null, + "arguments": "{\"query\":\"MCP tool Responses API require_approval option\", \"limit\": 10}", + "error": null, + "name": "search_openai_docs", + "output": "{\"hits\": [{\"url\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/#configure-an-mcp-tool\", \"url_without_anchor\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"anchor\": \"configure-an-mcp-tool\", \"content\": \"Add MCP tools in one of two places:\\r\\nAt the session level with session.tools in session.update, if you want the server available for the full session.\\r\\nAt the response level with response.tools in response.create, if you only need MCP for one turn.\\r\\nIn Realtime, the MCP tool shape is:\\r\\ntype: \\\"mcp\\\"\\r\\nserver_label\\r\\nOne of server_url or connector_id\\r\\nOptional authorization and headers\\r\\nOptional allowed_tools\\r\\nOptional require_approval\\r\\nOptional server_description\\r\\nThis example makes a docs MCP server available for the full session:\\r\\nBuilt-in connectors use the same MCP tool shape, but pass connector_id\\ninstead of server_url. For example, Google Calendar uses\\nconnector_googlecalendar. In Realtime, use these built-in connectors for read\\nactions, such as searching or reading events or emails. Pass the user\\u2019s OAuth\\naccess token in authorization, and narrow the tool surface with\\nallowed_tools when possible:\\r\\nRemote MCP servers do not automatically receive the full conversation context,\\nbut they can see any data the model sends in a tool call.\\nKeep the tool surface narrow with allowed_tools,\\nand require approval for any action you would not auto-run.\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Realtime API with MCP\", \"lvl2\": \"Configure an MCP tool\", \"lvl3\": null, \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"4-https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 and headers\\r\\nOptional allowed_tools\\r\\nOptional require_approval\\r\\nOptional server \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"Add MCP tools in one of two places:\\r\\nAt the session level with session.tools in session.update, if you want the server available for the full session.\\r\\nAt the response level with response.tools in response.create, if you only need MCP for one turn.\\r\\nIn Realtime, the MCP tool shape is:\\r\\ntype: \\\"mcp\\\"\\r\\nserver_label\\r\\nOne of server_url or connector_id\\r\\nOptional authorization and headers\\r\\nOptional allowed_tools\\r\\nOptional require_approval\\r\\nOptional server_description\\r\\nThis example makes a docs MCP server available for the full session:\\r\\nBuilt-in connectors use the same MCP tool shape, but pass connector_id\\ninstead of server_url. For example, Google Calendar uses\\nconnector_googlecalendar. In Realtime, use these built-in connectors for read\\nactions, such as searching or reading events or emails. Pass the user\\u2019s OAuth\\naccess token in authorization, and narrow the tool surface with\\nallowed_tools when possible:\\r\\nRemote MCP servers do not automatically receive the full conversation context,\\nbut they can see any data the model sends in a tool call.\\nKeep the tool surface narrow with allowed_tools,\\nand require approval for any action you would not auto-run.\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"tool\", \"responses\", \"require\", \"approval\", \"option\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Realtime API with MCP\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"api\"]}, \"lvl2\": {\"value\": \"Configure an MCP tool\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"tool\"]}}}}, {\"url\": \"https://platform.openai.com/settings/organization/api-keys\", \"url_without_anchor\": \"https://platform.openai.com/settings/organization/api-keys\", \"content\": \"View and manage your API keys.\", \"hierarchy\": {\"lvl0\": \"Settings\", \"lvl1\": \"API keys\"}, \"type\": \"lvl1\", \"access_level\": \"authenticated\", \"objectID\": \"custom-record-settings-api-keys\", \"_snippetResult\": {\"content\": {\"value\": \"View and manage your API keys.\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"View and manage your API keys.\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Settings\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"API keys\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\"]}}}}, {\"url\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/#realtime-mcp-flow\", \"url_without_anchor\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"anchor\": \"realtime-mcp-flow\", \"content\": \"Unlike Realtime function tools, remote MCP tools are executed by the Realtime API itself. Your client does not run the remote tool and return a function_call_output. Instead, your client configures access, listens for MCP lifecycle events, and optionally sends an approval response if the server asks for one.\\r\\nA typical flow looks like this:\\r\\nYou send session.update or response.create with a tools entry whose type is mcp.\\r\\nThe server begins importing tools and emits mcp_list_tools.in_progress.\\r\\nWhile listing is still in progress, the model cannot call a tool that has not been loaded yet. If you want to wait before starting a turn that depends on those tools, listen for mcp_list_tools.completed. The conversation.item.done event whose item.type is mcp_list_tools shows which tool names were actually imported. If import fails, you will receive mcp_list_tools.failed.\\r\\nThe user speaks or sends text, and a response is created, either by your client or automatically by the session configuration.\\r\\nIf the model chooses an MCP tool, you will see response.mcp_call_arguments.delta and response.mcp_call_arguments.done.\\r\\nIf approval is required, the server adds a conversation item whose item.type is mcp_approval_request. Your client must answer it with an mcp_approval_response item.\\r\\nOnce the tool runs, you will see response.mcp_call.in_progress. On success, you will later receive a response.output_item.done event whose item.type is mcp_call; on failure, you will receive response.mcp_call.failed. The assistant message item and response.done complete the turn.\\r\\nThis event handler covers the main checkpoints:\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Realtime API with MCP\", \"lvl2\": \"Realtime MCP flow\", \"lvl3\": null, \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"6-https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 for MCP lifecycle events, and optionally sends an approval response \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"Unlike Realtime function tools, remote MCP tools are executed by the Realtime API itself. Your client does not run the remote tool and return a function_call_output. Instead, your client configures access, listens for MCP lifecycle events, and optionally sends an approval response if the server asks for one.\\r\\nA typical flow looks like this:\\r\\nYou send session.update or response.create with a tools entry whose type is mcp.\\r\\nThe server begins importing tools and emits mcp_list_tools.in_progress.\\r\\nWhile listing is still in progress, the model cannot call a tool that has not been loaded yet. If you want to wait before starting a turn that depends on those tools, listen for mcp_list_tools.completed. The conversation.item.done event whose item.type is mcp_list_tools shows which tool names were actually imported. If import fails, you will receive mcp_list_tools.failed.\\r\\nThe user speaks or sends text, and a response is created, either by your client or automatically by the session configuration.\\r\\nIf the model chooses an MCP tool, you will see response.mcp_call_arguments.delta and response.mcp_call_arguments.done.\\r\\nIf approval is required, the server adds a conversation item whose item.type is mcp_approval_request. Your client must answer it with an mcp_approval_response item.\\r\\nOnce the tool runs, you will see response.mcp_call.in_progress. On success, you will later receive a response.output_item.done event whose item.type is mcp_call; on failure, you will receive response.mcp_call.failed. The assistant message item and response.done complete the turn.\\r\\nThis event handler covers the main checkpoints:\", \"matchLevel\": \"full\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"tool\", \"responses\", \"api\", \"require\", \"approval\", \"option\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Realtime API with MCP\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"api\"]}, \"lvl2\": {\"value\": \"Realtime MCP flow\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\"]}}}}, {\"url\": \"https://developers.openai.com/apps-sdk/deploy/submission/#app-review--approval-faqs\", \"url_without_anchor\": \"https://developers.openai.com/apps-sdk/deploy/submission/\", \"anchor\": \"app-review--approval-faqs\", \"content\": \"How long does app review take?\\r\\nThe app directory and Apps SDK are currently in beta, and review timelines may vary as we continue to build and scale our processes. Please do not contact support to request expedited review, as these requests cannot be accommodated.\\r\\nWhat are common rejection reasons and how can I resolve them?\\r\\nWe\\u2019re unable to connect to your MCP server using the MCP URL and/or test credentials we were given. For servers requiring authentication, our review team must be able to log into a demo account with no further configuration required.\\nEnsure that the provided URL and credentials are correct, do not feature MFA (including requiring SMS codes, login through systems that require SMS, email or other verification schemes).\\nEnsure that the provided credentials can be used to log in successfully (test them outside any company networks or LANs, or other internal networks).\\nConfirm that the credentials have not expired.\\r\\nFor servers requiring authentication, our review team must be able to log into a demo account with no further configuration required.\\r\\nEnsure that the provided URL and credentials are correct, do not feature MFA (including requiring SMS codes, login through systems that require SMS, email or other verification schemes).\\r\\nEnsure that the provided credentials can be used to log in successfully (test them outside any company networks or LANs, or other internal networks).\\r\\nConfirm that the credentials have not expired.\\r\\nOne or more of your test cases did not produce correct results. Review all test cases carefully and rerun each one. Ensure that outputs match the expected results. Verify that there are no errors in the UI (if applicable) - for example, issues with loading content, images, or other UI issues.\\nEnsure that the returned textual output closely adheres to the user\\u2019s request, and does not offer extraneous information that is irrelevant to the request, including personal identifiers.\\nEnsure that all test cases pass on both ChatGPT web and mobile apps.\\nCompare actual outputs to clearly defined expected behavior for each tool and fix any mismatch so results are relevant to the user\\u2019s input and the app \\u201creliably does what it promises\\u201d.\\nIf required, in your resubmission, modify your test cases and expected responses to be clear and unambiguous.\\r\\nReview all test cases carefully and rerun each one. Ensure that outputs match the expected results. Verify that there are no errors in the UI (if applicable) - for example, issues with loading content, images, or other UI issues.\\r\\nEnsure that the returned textual output closely adheres to the user\\u2019s request, and does not offer extraneous information that is irrelevant to the request, including personal identifiers.\\r\\nEnsure that all test cases pass on both ChatGPT web and mobile apps.\\r\\nCompare actual outputs to clearly defined expected behavior for each tool and fix any mismatch so results are relevant to the user\\u2019s input and the app \\u201creliably does what it promises\\u201d.\\r\\nIf required, in your resubmission, modify your test cases and expected responses to be clear and unambiguous.\\r\\nYour app returns user-related data types that are not disclosed in your privacy policy. Audit your MCP tool responses in developer mode by running a few realistic example requests and listing every user-related field your app returns (including nested fields and \\u201cdebug\\u201d payloads). Ensure tools return only what\\u2019s strictly necessary for the user\\u2019s request and remove any unnecessary PII, telemetry/internal identifiers (e.g., session/trace/request IDs, timestamps, internal account IDs, logs) and/or any auth secrets (tokens/keys/passwords).\\nYou may also consider updating your published privacy policy so it clearly discloses all categories of personal data you collect/process/return and why\\u2014if a field isn\\u2019t truly needed, remove it rather than disclose it.\\nIf a user identifier is truly necessary, make it explicitly requested and clearly tied to the user\\u2019s intent (not \\u201clooked up and echoed\\u201d by default)\\r\\nAudit your MCP tool responses in developer mode by running a few realistic example requests and listing every user-related field your app returns (including nested fields and \\u201cdebug\\u201d payloads). Ensure tools return only what\\u2019s strictly necessary for the user\\u2019s request and remove any unnecessary PII, telemetry/internal identifiers (e.g., session/trace/request IDs, timestamps, internal account IDs, logs) and/or any auth secrets (tokens/keys/passwords).\\r\\nYou may also consider updating your published privacy policy so it clearly discloses all categories of personal data you collect/process/return and why\\u2014if a field isn\\u2019t truly needed, remove it rather than disclose it.\\r\\nIf a user identifier is truly necessary, make it explicitly requested and clearly tied to the user\\u2019s intent (not \\u201clooked up and echoed\\u201d by default)\\r\\nTool hint annotations do not appear to match the tool\\u2019s behavior: readOnlyHint: Set to true if it strictly fetches/looks up/lists/retrieves data and does not modify anything. Set to false if the tool can create/update/delete anything, trigger actions (send emails/messages, run jobs, enqueue tasks, write logs, start workflows), or otherwise change state.\\ndestructiveHint: Set to true if it can cause irreversible outcomes (deleting, overwriting, sending messages/transactions you can\\u2019t undo, revoking access, destructive admin actions, etc.), even in only select modes, via default parameters, or through indirect side effects. Ensure the justification provided clearly describes what is irreversible and under what conditions, including any safeguards like confirmation steps, dry-run options, or scoping constraints. Otherwise, set to false.\\nopenWorldHint: Set to true if it can write to or change publicly visible internet state (e.g., posting to social media/blogs/forums, sending emails/SMS/messages to external recipients, creating public tickets/issues, publishing pages, pushing code/content to public endpoints, submitting forms to third parties, or otherwise affecting systems outside a private/first-party context). Set to false only if it operates entirely within closed/private systems (including internal writes) and cannot change the state of the publicly visible internet.\\r\\nreadOnlyHint: Set to true if it strictly fetches/looks up/lists/retrieves data and does not modify anything. Set to false if the tool can create/update/delete anything, trigger actions (send emails/messages, run jobs, enqueue tasks, write logs, start workflows), or otherwise change state.\\r\\ndestructiveHint: Set to true if it can cause irreversible outcomes (deleting, overwriting, sending messages/transactions you can\\u2019t undo, revoking access, destructive admin actions, etc.), even in only select modes, via default parameters, or through indirect side effects. Ensure the justification provided clearly describes what is irreversible and under what conditions, including any safeguards like confirmation steps, dry-run options, or scoping constraints. Otherwise, set to false.\\r\\nopenWorldHint: Set to true if it can write to or change publicly visible internet state (e.g., posting to social media/blogs/forums, sending emails/SMS/messages to external recipients, creating public tickets/issues, publishing pages, pushing code/content to public endpoints, submitting forms to third parties, or otherwise affecting systems outside a private/first-party context). Set to false only if it operates entirely within closed/private systems (including internal writes) and cannot change the state of the publicly visible internet.\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Submit and maintain your app\", \"lvl2\": \"App review & approval\", \"lvl3\": \"App review & approval FAQs\", \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"25-https://developers.openai.com/apps-sdk/deploy/submission/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 your privacy policy. Audit your MCP tool responses in developer \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"How long does app review take?\\r\\nThe app directory and Apps SDK are currently in beta, and review timelines may vary as we continue to build and scale our processes. Please do not contact support to request expedited review, as these requests cannot be accommodated.\\r\\nWhat are common rejection reasons and how can I resolve them?\\r\\nWe\\u2019re unable to connect to your MCP server using the MCP URL and/or test credentials we were given. For servers requiring authentication, our review team must be able to log into a demo account with no further configuration required.\\nEnsure that the provided URL and credentials are correct, do not feature MFA (including requiring SMS codes, login through systems that require SMS, email or other verification schemes).\\nEnsure that the provided credentials can be used to log in successfully (test them outside any company networks or LANs, or other internal networks).\\nConfirm that the credentials have not expired.\\r\\nFor servers requiring authentication, our review team must be able to log into a demo account with no further configuration required.\\r\\nEnsure that the provided URL and credentials are correct, do not feature MFA (including requiring SMS codes, login through systems that require SMS, email or other verification schemes).\\r\\nEnsure that the provided credentials can be used to log in successfully (test them outside any company networks or LANs, or other internal networks).\\r\\nConfirm that the credentials have not expired.\\r\\nOne or more of your test cases did not produce correct results. Review all test cases carefully and rerun each one. Ensure that outputs match the expected results. Verify that there are no errors in the UI (if applicable) - for example, issues with loading content, images, or other UI issues.\\nEnsure that the returned textual output closely adheres to the user\\u2019s request, and does not offer extraneous information that is irrelevant to the request, including personal identifiers.\\nEnsure that all test cases pass on both ChatGPT web and mobile apps.\\nCompare actual outputs to clearly defined expected behavior for each tool and fix any mismatch so results are relevant to the user\\u2019s input and the app \\u201creliably does what it promises\\u201d.\\nIf required, in your resubmission, modify your test cases and expected responses to be clear and unambiguous.\\r\\nReview all test cases carefully and rerun each one. Ensure that outputs match the expected results. Verify that there are no errors in the UI (if applicable) - for example, issues with loading content, images, or other UI issues.\\r\\nEnsure that the returned textual output closely adheres to the user\\u2019s request, and does not offer extraneous information that is irrelevant to the request, including personal identifiers.\\r\\nEnsure that all test cases pass on both ChatGPT web and mobile apps.\\r\\nCompare actual outputs to clearly defined expected behavior for each tool and fix any mismatch so results are relevant to the user\\u2019s input and the app \\u201creliably does what it promises\\u201d.\\r\\nIf required, in your resubmission, modify your test cases and expected responses to be clear and unambiguous.\\r\\nYour app returns user-related data types that are not disclosed in your privacy policy. Audit your MCP tool responses in developer mode by running a few realistic example requests and listing every user-related field your app returns (including nested fields and \\u201cdebug\\u201d payloads). Ensure tools return only what\\u2019s strictly necessary for the user\\u2019s request and remove any unnecessary PII, telemetry/internal identifiers (e.g., session/trace/request IDs, timestamps, internal account IDs, logs) and/or any auth secrets (tokens/keys/passwords).\\nYou may also consider updating your published privacy policy so it clearly discloses all categories of personal data you collect/process/return and why\\u2014if a field isn\\u2019t truly needed, remove it rather than disclose it.\\nIf a user identifier is truly necessary, make it explicitly requested and clearly tied to the user\\u2019s intent (not \\u201clooked up and echoed\\u201d by default)\\r\\nAudit your MCP tool responses in developer mode by running a few realistic example requests and listing every user-related field your app returns (including nested fields and \\u201cdebug\\u201d payloads). Ensure tools return only what\\u2019s strictly necessary for the user\\u2019s request and remove any unnecessary PII, telemetry/internal identifiers (e.g., session/trace/request IDs, timestamps, internal account IDs, logs) and/or any auth secrets (tokens/keys/passwords).\\r\\nYou may also consider updating your published privacy policy so it clearly discloses all categories of personal data you collect/process/return and why\\u2014if a field isn\\u2019t truly needed, remove it rather than disclose it.\\r\\nIf a user identifier is truly necessary, make it explicitly requested and clearly tied to the user\\u2019s intent (not \\u201clooked up and echoed\\u201d by default)\\r\\nTool hint annotations do not appear to match the tool\\u2019s behavior: readOnlyHint: Set to true if it strictly fetches/looks up/lists/retrieves data and does not modify anything. Set to false if the tool can create/update/delete anything, trigger actions (send emails/messages, run jobs, enqueue tasks, write logs, start workflows), or otherwise change state.\\ndestructiveHint: Set to true if it can cause irreversible outcomes (deleting, overwriting, sending messages/transactions you can\\u2019t undo, revoking access, destructive admin actions, etc.), even in only select modes, via default parameters, or through indirect side effects. Ensure the justification provided clearly describes what is irreversible and under what conditions, including any safeguards like confirmation steps, dry-run options, or scoping constraints. Otherwise, set to false.\\nopenWorldHint: Set to true if it can write to or change publicly visible internet state (e.g., posting to social media/blogs/forums, sending emails/SMS/messages to external recipients, creating public tickets/issues, publishing pages, pushing code/content to public endpoints, submitting forms to third parties, or otherwise affecting systems outside a private/first-party context). Set to false only if it operates entirely within closed/private systems (including internal writes) and cannot change the state of the publicly visible internet.\\r\\nreadOnlyHint: Set to true if it strictly fetches/looks up/lists/retrieves data and does not modify anything. Set to false if the tool can create/update/delete anything, trigger actions (send emails/messages, run jobs, enqueue tasks, write logs, start workflows), or otherwise change state.\\r\\ndestructiveHint: Set to true if it can cause irreversible outcomes (deleting, overwriting, sending messages/transactions you can\\u2019t undo, revoking access, destructive admin actions, etc.), even in only select modes, via default parameters, or through indirect side effects. Ensure the justification provided clearly describes what is irreversible and under what conditions, including any safeguards like confirmation steps, dry-run options, or scoping constraints. Otherwise, set to false.\\r\\nopenWorldHint: Set to true if it can write to or change publicly visible internet state (e.g., posting to social media/blogs/forums, sending emails/SMS/messages to external recipients, creating public tickets/issues, publishing pages, pushing code/content to public endpoints, submitting forms to third parties, or otherwise affecting systems outside a private/first-party context). Set to false only if it operates entirely within closed/private systems (including internal writes) and cannot change the state of the publicly visible internet.\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"tool\", \"responses\", \"api\", \"require\", \"option\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Submit and maintain your app\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\"]}, \"lvl2\": {\"value\": \"App review & approval\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\", \"approval\"]}, \"lvl3\": {\"value\": \"App review & approval FAQs\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"api\", \"approval\"]}}}}, {\"url\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/#approve-or-reject-mcp-tool-calls\", \"url_without_anchor\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"anchor\": \"approve-or-reject-mcp-tool-calls\", \"content\": \"If a tool requires approval, the Realtime API inserts an mcp_approval_request item into the conversation. To continue, send a new conversation.item.create event whose item.type is mcp_approval_response.\\r\\nIf you reject the request, set approve to false and optionally include a reason.\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Realtime API with MCP\", \"lvl2\": \"Approve or reject MCP tool calls\", \"lvl3\": null, \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"10-https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 tool requires approval, the Realtime API inserts an mcp_approval \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"If a tool requires approval, the Realtime API inserts an mcp_approval_request item into the conversation. To continue, send a new conversation.item.create event whose item.type is mcp_approval_response.\\r\\nIf you reject the request, set approve to false and optionally include a reason.\", \"matchLevel\": \"full\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"tool\", \"responses\", \"api\", \"require\", \"approval\", \"option\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Realtime API with MCP\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"api\"]}, \"lvl2\": {\"value\": \"Approve or reject MCP tool calls\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"tool\"]}}}}, {\"url\": \"https://developers.openai.com/api/docs/guides/tools-connectors-mcp/#approvals\", \"url_without_anchor\": \"https://developers.openai.com/api/docs/guides/tools-connectors-mcp/\", \"anchor\": \"approvals\", \"content\": \"By default, OpenAI will request your approval before any data is shared with a connector or remote MCP server. Approvals help you maintain control and visibility over what data is being sent to an MCP server. We highly recommend that you carefully review (and optionally log) all data being shared with a remote MCP server. A request for an approval to make an MCP tool call creates a mcp_approval_request item in the Response\\u2019s output that looks like this:\\r\\nYou can then respond to this by creating a new Response object and appending an mcp_approval_response item to it.\\r\\nHere we\\u2019re using the previous_response_id parameter to chain this new Response, with the previous Response that generated the approval request. But you can also pass back the outputs from one response, as inputs into another for maximum control over what enter\\u2019s the model\\u2019s context.\\r\\nIf and when you feel comfortable trusting a remote MCP server, you can choose to skip the approvals for reduced latency. To do this, you can set the require_approval parameter of the MCP tool to an object listing just the tools you\\u2019d like to skip approvals for like shown below, or set it to the value 'never' to skip approvals for all tools in that remote MCP server.\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"MCP and Connectors\", \"lvl2\": \"How it works\", \"lvl3\": \"Step 2: Calling tools\", \"lvl4\": \"Approvals\", \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"14-https://developers.openai.com/api/docs/guides/tools-connectors-mcp/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 require_approval parameter of the MCP tool to an object \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"By default, OpenAI will request your approval before any data is shared with a connector or remote MCP server. Approvals help you maintain control and visibility over what data is being sent to an MCP server. We highly recommend that you carefully review (and optionally log) all data being shared with a remote MCP server. A request for an approval to make an MCP tool call creates a mcp_approval_request item in the Response\\u2019s output that looks like this:\\r\\nYou can then respond to this by creating a new Response object and appending an mcp_approval_response item to it.\\r\\nHere we\\u2019re using the previous_response_id parameter to chain this new Response, with the previous Response that generated the approval request. But you can also pass back the outputs from one response, as inputs into another for maximum control over what enter\\u2019s the model\\u2019s context.\\r\\nIf and when you feel comfortable trusting a remote MCP server, you can choose to skip the approvals for reduced latency. To do this, you can set the require_approval parameter of the MCP tool to an object listing just the tools you\\u2019d like to skip approvals for like shown below, or set it to the value 'never' to skip approvals for all tools in that remote MCP server.\", \"matchLevel\": \"full\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"tool\", \"responses\", \"api\", \"require\", \"approval\", \"option\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"MCP and Connectors\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\"]}, \"lvl2\": {\"value\": \"How it works\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl3\": {\"value\": \"Step 2: Calling tools\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"tool\"]}, \"lvl4\": {\"value\": \"Approvals\", \"matchLevel\": \"partial\", \"fullyHighlighted\": true, \"matchedWords\": [\"approval\"]}}}}, {\"url\": \"https://developers.openai.com/api/docs/guides/your-data/#which-models-and-features-are-eligible-for-data-residency\", \"url_without_anchor\": \"https://developers.openai.com/api/docs/guides/your-data/\", \"anchor\": \"which-models-and-features-are-eligible-for-data-residency\", \"content\": \"The following models and API services are eligible for data residency today for the regions specified below.\\r\\nTable 1: Regional data residency capabilities\\r\\nUS\\r\\n\\u2705\\r\\n\\u2705\\r\\n\\u274c\\r\\nText, Audio, Voice, Image\\r\\nus.api.openai.com (required)\\r\\nEurope (EEA + Switzerland)\\r\\n\\u2705\\r\\n\\u2705\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\neu.api.openai.com (required)\\r\\nAustralia\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nau.api.openai.com (optional)\\r\\nCanada\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nca.api.openai.com (optional)\\r\\nJapan\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\njp.api.openai.com (optional)\\r\\nIndia\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nin.api.openai.com (optional)\\r\\nSingapore\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nsg.api.openai.com (optional)\\r\\nSouth Korea\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nkr.api.openai.com (optional)\\r\\nUnited Kingdom\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\ngb.api.openai.com (required)\\r\\nUnited Arab Emirates\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nae.api.openai.com (required)\\r\\n* Image support in these regions requires approval for enhanced Zero Data Retention or enhanced Modified Abuse Monitoring.\\r\\nTable 2: API endpoint and tool support\\r\\n/v1/audio/transcriptions /v1/audio/translations /v1/audio/speech\\r\\ntts-1whisper-1gpt-4o-ttsgpt-4o-transcribegpt-4o-mini-transcribe\\r\\nAll\\r\\n/v1/batches\\r\\ngpt-5.4-pro-2026-03-05gpt-5.2-pro-2025-12-11gpt-5-pro-2025-10-06gpt-5-2025-08-07gpt-5.4-2026-03-05gpt-5.2-2025-12-11gpt-5.1-2025-11-13gpt-5-mini-2025-08-07gpt-5-nano-2025-08-07gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14gpt-4.1-nano-2025-04-14o3-2025-04-16o4-mini-2025-04-16o1-proo1-pro-2025-03-19o3-mini-2025-01-31o1-2024-12-17o1-mini-2024-09-12o1-previewgpt-4o-2024-11-20gpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4-turbo-2024-04-09gpt-4-0613gpt-3.5-turbo-0125\\r\\nAll\\r\\n/v1/chat/completions\\r\\ngpt-5-2025-08-07gpt-5.4-2026-03-05gpt-5.2-2025-12-11gpt-5.1-2025-11-13gpt-5-mini-2025-08-07gpt-5-nano-2025-08-07gpt-5-chat-latest-2025-08-07gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14gpt-4.1-nano-2025-04-14o3-mini-2025-01-31o3-2025-04-16o4-mini-2025-04-16o1-2024-12-17o1-mini-2024-09-12o1-previewgpt-4o-2024-11-20gpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4-turbo-2024-04-09gpt-4-0613gpt-3.5-turbo-0125\\r\\nAll\\r\\n/v1/embeddings\\r\\ntext-embedding-3-smalltext-embedding-3-largetext-embedding-ada-002\\r\\nAll\\r\\n/v1/evals\\r\\n\\r\\nUS and EU\\r\\n/v1/files\\r\\n\\r\\nAll\\r\\n/v1/fine_tuning/jobs\\r\\ngpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14\\r\\nAll\\r\\n/v1/images/edits\\r\\ngpt-image-1gpt-image-1.5gpt-image-1-mini\\r\\nAll\\r\\n/v1/images/generations\\r\\ndall-e-3gpt-image-1gpt-image-1.5gpt-image-1-mini\\r\\nAll\\r\\n/v1/moderations\\r\\ntext-moderation-latest*omni-moderation-latest\\r\\nAll\\r\\n/v1/realtime\\r\\ngpt-4o-realtime-preview-2025-06-03gpt-realtimegpt-realtime-1.5gpt-realtime-mini\\r\\nUS and EU\\r\\n/v1/realtime\\r\\ngpt-4o-realtime-preview-2024-12-17gpt-4o-realtime-preview-2024-10-01gpt-4o-mini-realtime-preview-2024-12-17\\r\\nUS only\\r\\n/v1/responses\\r\\ngpt-5.4-pro-2026-03-05gpt-5.2-pro-2025-12-11gpt-5-pro-2025-10-06gpt-5-2025-08-07gpt-5.4-2026-03-05gpt-5.2-2025-12-11gpt-5.1-2025-11-13gpt-5-mini-2025-08-07gpt-5-nano-2025-08-07gpt-5-chat-latest-2025-08-07gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14gpt-4.1-nano-2025-04-14o3-2025-04-16o4-mini-2025-04-16o1-proo1-pro-2025-03-19computer-use-preview*o3-mini-2025-01-31o1-2024-12-17o1-mini-2024-09-12o1-previewgpt-4o-2024-11-20gpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4-turbo-2024-04-09gpt-4-0613gpt-3.5-turbo-0125\\r\\nAll\\r\\n/v1/responses File Search\\r\\n\\r\\nAll\\r\\n/v1/responses Web Search\\r\\n\\r\\nAll\\r\\n/v1/vector_stores\\r\\n\\r\\nAll\\r\\nCode Interpreter tool\\r\\n\\r\\nAll\\r\\nFile Search\\r\\n\\r\\nAll\\r\\nFile Uploads\\r\\n\\r\\nAll, when used with base64 file uploads\\r\\nRemote MCP server tool\\r\\n\\r\\nAll, but MCP servers are third-party services, and data sent to an MCP server is subject to their data residency policies.\\r\\nScale Tier\\r\\n\\r\\nAll\\r\\nStructured Outputs (excluding schema)\\r\\n\\r\\nAll\\r\\nSupported Input Modalities\\r\\n\\r\\nText Image Audio/Voice\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Data controls in the OpenAI platform\", \"lvl2\": \"Data residency controls\", \"lvl3\": \"Which models and features are eligible for data residency?\", \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"42-https://developers.openai.com/api/docs/guides/your-data/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 file uploads\\r\\nRemote MCP server tool\\r\\n\\r\\nAll, but MCP servers \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"The following models and API services are eligible for data residency today for the regions specified below.\\r\\nTable 1: Regional data residency capabilities\\r\\nUS\\r\\n\\u2705\\r\\n\\u2705\\r\\n\\u274c\\r\\nText, Audio, Voice, Image\\r\\nus.api.openai.com (required)\\r\\nEurope (EEA + Switzerland)\\r\\n\\u2705\\r\\n\\u2705\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\neu.api.openai.com (required)\\r\\nAustralia\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nau.api.openai.com (optional)\\r\\nCanada\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nca.api.openai.com (optional)\\r\\nJapan\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\njp.api.openai.com (optional)\\r\\nIndia\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nin.api.openai.com (optional)\\r\\nSingapore\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nsg.api.openai.com (optional)\\r\\nSouth Korea\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nkr.api.openai.com (optional)\\r\\nUnited Kingdom\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\ngb.api.openai.com (required)\\r\\nUnited Arab Emirates\\r\\n\\u2705\\r\\n\\u274c\\r\\n\\u2705\\r\\nText, Audio, Voice, Image*\\r\\nae.api.openai.com (required)\\r\\n* Image support in these regions requires approval for enhanced Zero Data Retention or enhanced Modified Abuse Monitoring.\\r\\nTable 2: API endpoint and tool support\\r\\n/v1/audio/transcriptions /v1/audio/translations /v1/audio/speech\\r\\ntts-1whisper-1gpt-4o-ttsgpt-4o-transcribegpt-4o-mini-transcribe\\r\\nAll\\r\\n/v1/batches\\r\\ngpt-5.4-pro-2026-03-05gpt-5.2-pro-2025-12-11gpt-5-pro-2025-10-06gpt-5-2025-08-07gpt-5.4-2026-03-05gpt-5.2-2025-12-11gpt-5.1-2025-11-13gpt-5-mini-2025-08-07gpt-5-nano-2025-08-07gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14gpt-4.1-nano-2025-04-14o3-2025-04-16o4-mini-2025-04-16o1-proo1-pro-2025-03-19o3-mini-2025-01-31o1-2024-12-17o1-mini-2024-09-12o1-previewgpt-4o-2024-11-20gpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4-turbo-2024-04-09gpt-4-0613gpt-3.5-turbo-0125\\r\\nAll\\r\\n/v1/chat/completions\\r\\ngpt-5-2025-08-07gpt-5.4-2026-03-05gpt-5.2-2025-12-11gpt-5.1-2025-11-13gpt-5-mini-2025-08-07gpt-5-nano-2025-08-07gpt-5-chat-latest-2025-08-07gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14gpt-4.1-nano-2025-04-14o3-mini-2025-01-31o3-2025-04-16o4-mini-2025-04-16o1-2024-12-17o1-mini-2024-09-12o1-previewgpt-4o-2024-11-20gpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4-turbo-2024-04-09gpt-4-0613gpt-3.5-turbo-0125\\r\\nAll\\r\\n/v1/embeddings\\r\\ntext-embedding-3-smalltext-embedding-3-largetext-embedding-ada-002\\r\\nAll\\r\\n/v1/evals\\r\\n\\r\\nUS and EU\\r\\n/v1/files\\r\\n\\r\\nAll\\r\\n/v1/fine_tuning/jobs\\r\\ngpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14\\r\\nAll\\r\\n/v1/images/edits\\r\\ngpt-image-1gpt-image-1.5gpt-image-1-mini\\r\\nAll\\r\\n/v1/images/generations\\r\\ndall-e-3gpt-image-1gpt-image-1.5gpt-image-1-mini\\r\\nAll\\r\\n/v1/moderations\\r\\ntext-moderation-latest*omni-moderation-latest\\r\\nAll\\r\\n/v1/realtime\\r\\ngpt-4o-realtime-preview-2025-06-03gpt-realtimegpt-realtime-1.5gpt-realtime-mini\\r\\nUS and EU\\r\\n/v1/realtime\\r\\ngpt-4o-realtime-preview-2024-12-17gpt-4o-realtime-preview-2024-10-01gpt-4o-mini-realtime-preview-2024-12-17\\r\\nUS only\\r\\n/v1/responses\\r\\ngpt-5.4-pro-2026-03-05gpt-5.2-pro-2025-12-11gpt-5-pro-2025-10-06gpt-5-2025-08-07gpt-5.4-2026-03-05gpt-5.2-2025-12-11gpt-5.1-2025-11-13gpt-5-mini-2025-08-07gpt-5-nano-2025-08-07gpt-5-chat-latest-2025-08-07gpt-4.1-2025-04-14gpt-4.1-mini-2025-04-14gpt-4.1-nano-2025-04-14o3-2025-04-16o4-mini-2025-04-16o1-proo1-pro-2025-03-19computer-use-preview*o3-mini-2025-01-31o1-2024-12-17o1-mini-2024-09-12o1-previewgpt-4o-2024-11-20gpt-4o-2024-08-06gpt-4o-mini-2024-07-18gpt-4-turbo-2024-04-09gpt-4-0613gpt-3.5-turbo-0125\\r\\nAll\\r\\n/v1/responses File Search\\r\\n\\r\\nAll\\r\\n/v1/responses Web Search\\r\\n\\r\\nAll\\r\\n/v1/vector_stores\\r\\n\\r\\nAll\\r\\nCode Interpreter tool\\r\\n\\r\\nAll\\r\\nFile Search\\r\\n\\r\\nAll\\r\\nFile Uploads\\r\\n\\r\\nAll, when used with base64 file uploads\\r\\nRemote MCP server tool\\r\\n\\r\\nAll, but MCP servers are third-party services, and data sent to an MCP server is subject to their data residency policies.\\r\\nScale Tier\\r\\n\\r\\nAll\\r\\nStructured Outputs (excluding schema)\\r\\n\\r\\nAll\\r\\nSupported Input Modalities\\r\\n\\r\\nText Image Audio/Voice\", \"matchLevel\": \"full\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"tool\", \"responses\", \"api\", \"require\", \"approval\", \"option\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Data controls in the OpenAI platform\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl2\": {\"value\": \"Data residency controls\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl3\": {\"value\": \"Which models and features are eligible for data residency?\", \"matchLevel\": \"none\", \"matchedWords\": []}}}}, {\"url\": \"https://developers.openai.com/codex/config-reference/#configtoml\", \"url_without_anchor\": \"https://developers.openai.com/codex/config-reference/\", \"anchor\": \"configtoml\", \"content\": \"User-level configuration lives in ~/.codex/config.toml. You can also add project-scoped overrides in .codex/config.toml files. Codex loads project-scoped config files only when you trust the project.\\r\\nFor sandbox and approval keys (approval_policy, sandbox_mode, and sandbox_workspace_write.*), pair this reference with Sandbox and approvals, Protected paths in writable roots, and Network access.\\r\\nagents.<name>.config_file\\r\\nstring (path)\\r\\nPath to a TOML config layer for that role; relative paths resolve from the config file that declares the role.\\r\\nagents.<name>.description\\r\\nstring\\r\\nRole guidance shown to Codex when choosing and spawning that agent type.\\r\\nagents.<name>.nickname_candidates\\r\\narray<string>\\r\\nOptional pool of display nicknames for spawned agents in that role.\\r\\nagents.job_max_runtime_seconds\\r\\nnumber\\r\\nDefault per-worker timeout for spawn_agents_on_csv jobs. When unset, the tool falls back to 1800 seconds per worker.\\r\\nagents.max_depth\\r\\nnumber\\r\\nMaximum nesting depth allowed for spawned agent threads (root sessions start at depth 0; default: 1).\\r\\nagents.max_threads\\r\\nnumber\\r\\nMaximum number of agent threads that can be open concurrently. Defaults to 6 when unset.\\r\\nallow_login_shell\\r\\nboolean\\r\\nAllow shell-based tools to use login-shell semantics. Defaults to true; when false, login = true requests are rejected and omitted login defaults to non-login shells.\\r\\nanalytics.enabled\\r\\nboolean\\r\\nEnable or disable analytics for this machine/profile. When unset, the client default applies.\\r\\napproval_policy\\r\\nuntrusted | on-request | never | { reject = { sandbox_approval = bool, rules = bool, mcp_elicitations = bool } }\\r\\nControls when Codex pauses for approval before executing commands. You can also use approval_policy = { reject = { ... } } to auto-reject specific prompt categories while keeping other prompts interactive. on-failure is deprecated; use on-request for interactive runs or never for non-interactive runs.\\r\\napproval_policy.reject.mcp_elicitations\\r\\nboolean\\r\\nWhen true, MCP elicitation prompts are auto-rejected instead of shown to the user.\\r\\napproval_policy.reject.rules\\r\\nboolean\\r\\nWhen true, approvals triggered by execpolicy prompt rules are auto-rejected.\\r\\napproval_policy.reject.sandbox_approval\\r\\nboolean\\r\\nWhen true, sandbox escalation approval prompts are auto-rejected.\\r\\napps._default.destructive_enabled\\r\\nboolean\\r\\nDefault allow/deny for app tools with destructive_hint = true.\\r\\napps._default.enabled\\r\\nboolean\\r\\nDefault app enabled state for all apps unless overridden per app.\\r\\napps._default.open_world_enabled\\r\\nboolean\\r\\nDefault allow/deny for app tools with open_world_hint = true.\\r\\napps.<id>.default_tools_approval_mode\\r\\nauto | prompt | approve\\r\\nDefault approval behavior for tools in this app unless a per-tool override exists.\\r\\napps.<id>.default_tools_enabled\\r\\nboolean\\r\\nDefault enabled state for tools in this app unless a per-tool override exists.\\r\\napps.<id>.destructive_enabled\\r\\nboolean\\r\\nAllow or block tools in this app that advertise destructive_hint = true.\\r\\napps.<id>.enabled\\r\\nboolean\\r\\nEnable or disable a specific app/connector by id (default: true).\\r\\napps.<id>.open_world_enabled\\r\\nboolean\\r\\nAllow or block tools in this app that advertise open_world_hint = true.\\r\\napps.<id>.tools.<tool>.approval_mode\\r\\nauto | prompt | approve\\r\\nPer-tool approval behavior override for a single app tool.\\r\\napps.<id>.tools.<tool>.enabled\\r\\nboolean\\r\\nPer-tool enabled override for an app tool (for example repos/list).\\r\\nbackground_terminal_max_timeout\\r\\nnumber\\r\\nMaximum poll window in milliseconds for empty write_stdin polls (background terminal polling). Default: 300000 (5 minutes). Replaces the older background_terminal_timeout key.\\r\\nchatgpt_base_url\\r\\nstring\\r\\nOverride the base URL used during the ChatGPT login flow.\\r\\ncheck_for_update_on_startup\\r\\nboolean\\r\\nCheck for Codex updates on startup (set to false only when updates are centrally managed).\\r\\ncli_auth_credentials_store\\r\\nfile | keyring | auto\\r\\nControl where the CLI stores cached credentials (file-based auth.json vs OS keychain).\\r\\ncommit_attribution\\r\\nstring\\r\\nOverride the commit co-author trailer text. Set an empty string to disable automatic attribution.\\r\\ncompact_prompt\\r\\nstring\\r\\nInline override for the history compaction prompt.\\r\\ndeveloper_instructions\\r\\nstring\\r\\nAdditional developer instructions injected into the session (optional).\\r\\ndisable_paste_burst\\r\\nboolean\\r\\nDisable burst-paste detection in the TUI.\\r\\nexperimental_compact_prompt_file\\r\\nstring (path)\\r\\nLoad the compaction prompt override from a file (experimental).\\r\\nexperimental_use_unified_exec_tool\\r\\nboolean\\r\\nLegacy name for enabling unified exec; prefer [features].unified_exec or codex --enable unified_exec.\\r\\nfeatures.apps\\r\\nboolean\\r\\nEnable ChatGPT Apps/connectors support (experimental).\\r\\nfeatures.apps_mcp_gateway\\r\\nboolean\\r\\nRoute Apps MCP calls through the OpenAI connectors MCP gateway (https://api.openai.com/v1/connectors/mcp/) instead of legacy routing (experimental).\\r\\nfeatures.artifact\\r\\nboolean\\r\\nEnable native artifact tools such as slides and spreadsheets (under development).\\r\\nfeatures.child_agents_md\\r\\nboolean\\r\\nAppend AGENTS.md scope/precedence guidance even when no AGENTS.md is present (experimental).\\r\\nfeatures.collaboration_modes\\r\\nboolean\\r\\nLegacy toggle for collaboration modes. Plan and default modes are available in current builds without setting this key.\\r\\nfeatures.default_mode_request_user_input\\r\\nboolean\\r\\nAllow request_user_input in default collaboration mode (under development; off by default).\\r\\nfeatures.elevated_windows_sandbox\\r\\nboolean\\r\\nLegacy toggle for an earlier elevated Windows sandbox rollout. Current builds do not use it.\\r\\nfeatures.enable_request_compression\\r\\nboolean\\r\\nCompress streaming request bodies with zstd when supported (stable; on by default).\\r\\nfeatures.experimental_windows_sandbox\\r\\nboolean\\r\\nLegacy toggle for an earlier Windows sandbox rollout. Current builds do not use it.\\r\\nfeatures.fast_mode\\r\\nboolean\\r\\nEnable Fast mode selection and the service_tier = \\\"fast\\\" path (stable; on by default).\\r\\nfeatures.image_detail_original\\r\\nboolean\\r\\nAllow image outputs with detail = \\\"original\\\" on supported models (under development).\\r\\nfeatures.image_generation\\r\\nboolean\\r\\nEnable the built-in image generation tool (under development).\\r\\nfeatures.multi_agent\\r\\nboolean\\r\\nEnable multi-agent collaboration tools (spawn_agent, send_input, resume_agent, wait, close_agent, and spawn_agents_on_csv) (experimental; off by default).\\r\\nfeatures.personality\\r\\nboolean\\r\\nEnable personality selection controls (stable; on by default).\\r\\nfeatures.powershell_utf8\\r\\nboolean\\r\\nForce PowerShell UTF-8 output. Enabled by default on Windows and off elsewhere.\\r\\nfeatures.prevent_idle_sleep\\r\\nboolean\\r\\nPrevent the machine from sleeping while a turn is actively running (experimental; off by default).\\r\\nfeatures.remote_models\\r\\nboolean\\r\\nLegacy toggle for an older remote-model readiness flow. Current builds do not use it.\\r\\nfeatures.request_rule\\r\\nboolean\\r\\nLegacy toggle for Smart approvals. Current builds include this behavior by default, so most users can leave this unset.\\r\\nfeatures.responses_websockets\\r\\nboolean\\r\\nPrefer the Responses API WebSocket transport for supported providers (under development).\\r\\nfeatures.responses_websockets_v2\\r\\nboolean\\r\\nEnable Responses API WebSocket v2 mode (under development).\\r\\nfeatures.runtime_metrics\\r\\nboolean\\r\\nShow runtime metrics summary in TUI turn separators (experimental).\\r\\nfeatures.search_tool\\r\\nboolean\\r\\nLegacy toggle for an older Apps discovery flow. Current builds do not use it.\\r\\nfeatures.shell_snapshot\\r\\nboolean\\r\\nSnapshot shell environment to speed up repeated commands (stable; on by default).\\r\\nfeatures.shell_tool\\r\\nboolean\\r\\nEnable the default shell tool for running commands (stable; on by default).\\r\\nfeatures.skill_env_var_dependency_prompt\\r\\nboolean\\r\\nPrompt for missing skill environment-variable dependencies (under development).\\r\\nfeatures.skill_mcp_dependency_install\\r\\nboolean\\r\\nAllow prompting and installing missing MCP dependencies for skills (stable; on by default).\\r\\nfeatures.sqlite\\r\\nboolean\\r\\nEnable SQLite-backed state persistence (stable; on by default).\\r\\nfeatures.steer\\r\\nboolean\\r\\nLegacy toggle from an earlier Enter/Tab steering rollout. Current builds always use the current steering behavior.\\r\\nfeatures.undo\\r\\nboolean\\r\\nEnable undo support (stable; off by default).\\r\\nfeatures.unified_exec\\r\\nboolean\\r\\nUse the unified PTY-backed exec tool (stable; enabled by default except on Windows).\\r\\nfeatures.use_linux_sandbox_bwrap\\r\\nboolean\\r\\nUse the bubblewrap-based Linux sandbox pipeline (experimental; off by default).\\r\\nfeatures.web_search\\r\\nboolean\\r\\nDeprecated legacy toggle; prefer the top-level web_search setting.\\r\\nfeatures.web_search_cached\\r\\nboolean\\r\\nDeprecated legacy toggle. When web_search is unset, true maps to web_search = \\\"cached\\\".\\r\\nfeatures.web_search_request\\r\\nboolean\\r\\nDeprecated legacy toggle. When web_search is unset, true maps to web_search = \\\"live\\\".\\r\\nfeedback.enabled\\r\\nboolean\\r\\nEnable feedback submission via /feedback across Codex surfaces (default: true).\\r\\nfile_opener\\r\\nvscode | vscode-insiders | windsurf | cursor | none\\r\\nURI scheme used to open citations from Codex output (default: vscode).\\r\\nforced_chatgpt_workspace_id\\r\\nstring (uuid)\\r\\nLimit ChatGPT logins to a specific workspace identifier.\\r\\nforced_login_method\\r\\nchatgpt | api\\r\\nRestrict Codex to a specific authentication method.\\r\\nhide_agent_reasoning\\r\\nboolean\\r\\nSuppress reasoning events in both the TUI and codex exec output.\\r\\nhistory.max_bytes\\r\\nnumber\\r\\nIf set, caps the history file size in bytes by dropping oldest entries.\\r\\nhistory.persistence\\r\\nsave-all | none\\r\\nControl whether Codex saves session transcripts to history.jsonl.\\r\\ninstructions\\r\\nstring\\r\\nReserved for future use; prefer model_instructions_file or AGENTS.md.\\r\\nlog_dir\\r\\nstring (path)\\r\\nDirectory where Codex writes log files (for example codex-tui.log); defaults to $CODEX_HOME/log.\\r\\nmcp_oauth_callback_port\\r\\ninteger\\r\\nOptional fixed port for the local HTTP callback server used during MCP OAuth login. When unset, Codex binds to an ephemeral port chosen by the OS.\\r\\nmcp_oauth_callback_url\\r\\nstring\\r\\nOptional redirect URI override for MCP OAuth login (for example, a devbox ingress URL). mcp_oauth_callback_port still controls the callback listener port.\\r\\nmcp_oauth_credentials_store\\r\\nauto | file | keyring\\r\\nPreferred store for MCP OAuth credentials.\\r\\nmcp_servers.<id>.args\\r\\narray<string>\\r\\nArguments passed to the MCP stdio server command.\\r\\nmcp_servers.<id>.bearer_token_env_var\\r\\nstring\\r\\nEnvironment variable sourcing the bearer token for an MCP HTTP server.\\r\\nmcp_servers.<id>.command\\r\\nstring\\r\\nLauncher command for an MCP stdio server.\\r\\nmcp_servers.<id>.cwd\\r\\nstring\\r\\nWorking directory for the MCP stdio server process.\\r\\nmcp_servers.<id>.disabled_tools\\r\\narray<string>\\r\\nDeny list applied after enabled_tools for the MCP server.\\r\\nmcp_servers.<id>.enabled\\r\\nboolean\\r\\nDisable an MCP server without removing its configuration.\\r\\nmcp_servers.<id>.enabled_tools\\r\\narray<string>\\r\\nAllow list of tool names exposed by the MCP server.\\r\\nmcp_servers.<id>.env\\r\\nmap<string,string>\\r\\nEnvironment variables forwarded to the MCP stdio server.\\r\\nmcp_servers.<id>.env_http_headers\\r\\nmap<string,string>\\r\\nHTTP headers populated from environment variables for an MCP HTTP server.\\r\\nmcp_servers.<id>.env_vars\\r\\narray<string>\\r\\nAdditional environment variables to whitelist for an MCP stdio server.\\r\\nmcp_servers.<id>.http_headers\\r\\nmap<string,string>\\r\\nStatic HTTP headers included with each MCP HTTP request.\\r\\nmcp_servers.<id>.oauth_resource\\r\\nstring\\r\\nOptional RFC 8707 OAuth resource parameter to include during MCP login.\\r\\nmcp_servers.<id>.required\\r\\nboolean\\r\\nWhen true, fail startup/resume if this enabled MCP server cannot initialize.\\r\\nmcp_servers.<id>.scopes\\r\\narray<string>\\r\\nOAuth scopes to request when authenticating to that MCP server.\\r\\nmcp_servers.<id>.startup_timeout_ms\\r\\nnumber\\r\\nAlias for startup_timeout_sec in milliseconds.\\r\\nmcp_servers.<id>.startup_timeout_sec\\r\\nnumber\\r\\nOverride the default 10s startup timeout for an MCP server.\\r\\nmcp_servers.<id>.tool_timeout_sec\\r\\nnumber\\r\\nOverride the default 60s per-tool timeout for an MCP server.\\r\\nmcp_servers.<id>.url\\r\\nstring\\r\\nEndpoint for an MCP streamable HTTP server.\\r\\nmodel\\r\\nstring\\r\\nModel to use (e.g., gpt-5-codex).\\r\\nmodel_auto_compact_token_limit\\r\\nnumber\\r\\nToken threshold that triggers automatic history compaction (unset uses model defaults).\\r\\nmodel_catalog_json\\r\\nstring (path)\\r\\nOptional path to a JSON model catalog loaded on startup. Profile-level profiles.<name>.model_catalog_json can override this per profile.\\r\\nmodel_context_window\\r\\nnumber\\r\\nContext window tokens available to the active model.\\r\\nmodel_instructions_file\\r\\nstring (path)\\r\\nReplacement for built-in instructions instead of AGENTS.md.\\r\\nmodel_provider\\r\\nstring\\r\\nProvider id from model_providers (default: openai).\\r\\nmodel_providers.<id>.base_url\\r\\nstring\\r\\nAPI base URL for the model provider.\\r\\nmodel_providers.<id>.env_http_headers\\r\\nmap<string,string>\\r\\nHTTP headers populated from environment variables when present.\\r\\nmodel_providers.<id>.env_key\\r\\nstring\\r\\nEnvironment variable supplying the provider API key.\\r\\nmodel_providers.<id>.env_key_instructions\\r\\nstring\\r\\nOptional setup guidance for the provider API key.\\r\\nmodel_providers.<id>.experimental_bearer_token\\r\\nstring\\r\\nDirect bearer token for the provider (discouraged; use env_key).\\r\\nmodel_providers.<id>.http_headers\\r\\nmap<string,string>\\r\\nStatic HTTP headers added to provider requests.\\r\\nmodel_providers.<id>.name\\r\\nstring\\r\\nDisplay name for a custom model provider.\\r\\nmodel_providers.<id>.query_params\\r\\nmap<string,string>\\r\\nExtra query parameters appended to provider requests.\\r\\nmodel_providers.<id>.request_max_retries\\r\\nnumber\\r\\nRetry count for HTTP requests to the provider (default: 4).\\r\\nmodel_providers.<id>.requires_openai_auth\\r\\nboolean\\r\\nThe provider uses OpenAI authentication (defaults to false).\\r\\nmodel_providers.<id>.stream_idle_timeout_ms\\r\\nnumber\\r\\nIdle timeout for SSE streams in milliseconds (default: 300000).\\r\\nmodel_providers.<id>.stream_max_retries\\r\\nnumber\\r\\nRetry count for SSE streaming interruptions (default: 5).\\r\\nmodel_providers.<id>.supports_websockets\\r\\nboolean\\r\\nWhether that provider supports the Responses API WebSocket transport.\\r\\nmodel_providers.<id>.wire_api\\r\\nresponses\\r\\nProtocol used by the provider. responses is the only supported value, and it is the default when omitted.\\r\\nmodel_reasoning_effort\\r\\nminimal | low | medium | high | xhigh\\r\\nAdjust reasoning effort for supported models (Responses API only; xhigh is model-dependent).\\r\\nmodel_reasoning_summary\\r\\nauto | concise | detailed | none\\r\\nSelect reasoning summary detail or disable summaries entirely.\\r\\nmodel_supports_reasoning_summaries\\r\\nboolean\\r\\nForce Codex to send or not send reasoning metadata.\\r\\nmodel_verbosity\\r\\nlow | medium | high\\r\\nOptional GPT-5 Responses API verbosity override; when unset, the selected model/preset default is used.\\r\\nnotice.hide_full_access_warning\\r\\nboolean\\r\\nTrack acknowledgement of the full access warning prompt.\\r\\nnotice.hide_gpt-5.1-codex-max_migration_prompt\\r\\nboolean\\r\\nTrack acknowledgement of the gpt-5.1-codex-max migration prompt.\\r\\nnotice.hide_gpt5_1_migration_prompt\\r\\nboolean\\r\\nTrack acknowledgement of the GPT-5.1 migration prompt.\\r\\nnotice.hide_rate_limit_model_nudge\\r\\nboolean\\r\\nTrack opt-out of the rate limit model switch reminder.\\r\\nnotice.hide_world_writable_warning\\r\\nboolean\\r\\nTrack acknowledgement of the Windows world-writable directories warning.\\r\\nnotice.model_migrations\\r\\nmap<string,string>\\r\\nTrack acknowledged model migrations as old->new mappings.\\r\\nnotify\\r\\narray<string>\\r\\nCommand invoked for notifications; receives a JSON payload from Codex.\\r\\noss_provider\\r\\nlmstudio | ollama\\r\\nDefault local provider used when running with --oss (defaults to prompting if unset).\\r\\notel.environment\\r\\nstring\\r\\nEnvironment tag applied to emitted OpenTelemetry events (default: dev).\\r\\notel.exporter\\r\\nnone | otlp-http | otlp-grpc\\r\\nSelect the OpenTelemetry exporter and provide any endpoint metadata.\\r\\notel.exporter.<id>.endpoint\\r\\nstring\\r\\nExporter endpoint for OTEL logs.\\r\\notel.exporter.<id>.headers\\r\\nmap<string,string>\\r\\nStatic headers included with OTEL exporter requests.\\r\\notel.exporter.<id>.protocol\\r\\nbinary | json\\r\\nProtocol used by the OTLP/HTTP exporter.\\r\\notel.exporter.<id>.tls.ca-certificate\\r\\nstring\\r\\nCA certificate path for OTEL exporter TLS.\\r\\notel.exporter.<id>.tls.client-certificate\\r\\nstring\\r\\nClient certificate path for OTEL exporter TLS.\\r\\notel.exporter.<id>.tls.client-private-key\\r\\nstring\\r\\nClient private key path for OTEL exporter TLS.\\r\\notel.log_user_prompt\\r\\nboolean\\r\\nOpt in to exporting raw user prompts with OpenTelemetry logs.\\r\\notel.metrics_exporter\\r\\nnone | statsig | otlp-http | otlp-grpc\\r\\nSelect the OpenTelemetry metrics exporter (defaults to statsig).\\r\\notel.trace_exporter\\r\\nnone | otlp-http | otlp-grpc\\r\\nSelect the OpenTelemetry trace exporter and provide any endpoint metadata.\\r\\notel.trace_exporter.<id>.endpoint\\r\\nstring\\r\\nTrace exporter endpoint for OTEL logs.\\r\\notel.trace_exporter.<id>.headers\\r\\nmap<string,string>\\r\\nStatic headers included with OTEL trace exporter requests.\\r\\notel.trace_exporter.<id>.protocol\\r\\nbinary | json\\r\\nProtocol used by the OTLP/HTTP trace exporter.\\r\\notel.trace_exporter.<id>.tls.ca-certificate\\r\\nstring\\r\\nCA certificate path for OTEL trace exporter TLS.\\r\\notel.trace_exporter.<id>.tls.client-certificate\\r\\nstring\\r\\nClient certificate path for OTEL trace exporter TLS.\\r\\notel.trace_exporter.<id>.tls.client-private-key\\r\\nstring\\r\\nClient private key path for OTEL trace exporter TLS.\\r\\npermissions.network.admin_url\\r\\nstring\\r\\nAdmin endpoint for the managed network proxy.\\r\\npermissions.network.allow_local_binding\\r\\nboolean\\r\\nPermit local bind/listen operations through the managed proxy.\\r\\npermissions.network.allow_unix_sockets\\r\\narray<string>\\r\\nAllowlist of Unix socket paths permitted through the managed proxy.\\r\\npermissions.network.allow_upstream_proxy\\r\\nboolean\\r\\nAllow the managed proxy to chain to another upstream proxy.\\r\\npermissions.network.allowed_domains\\r\\narray<string>\\r\\nAllowlist of domains permitted through the managed proxy.\\r\\npermissions.network.dangerously_allow_all_unix_sockets\\r\\nboolean\\r\\nAllow the proxy to use arbitrary Unix sockets instead of the default restricted set.\\r\\npermissions.network.dangerously_allow_non_loopback_admin\\r\\nboolean\\r\\nPermit non-loopback bind addresses for the managed proxy admin listener.\\r\\npermissions.network.dangerously_allow_non_loopback_proxy\\r\\nboolean\\r\\nPermit non-loopback bind addresses for the managed proxy listener.\\r\\npermissions.network.denied_domains\\r\\narray<string>\\r\\nDenylist of domains blocked by the managed proxy.\\r\\npermissions.network.enable_socks5\\r\\nboolean\\r\\nExpose a SOCKS5 listener from the managed network proxy.\\r\\npermissions.network.enable_socks5_udp\\r\\nboolean\\r\\nAllow UDP over the SOCKS5 listener when enabled.\\r\\npermissions.network.enabled\\r\\nboolean\\r\\nEnable the managed network proxy configuration for subprocesses.\\r\\npermissions.network.mode\\r\\nlimited | full\\r\\nNetwork proxy mode used for subprocess traffic.\\r\\npermissions.network.proxy_url\\r\\nstring\\r\\nHTTP proxy endpoint used by the managed network proxy.\\r\\npermissions.network.socks_url\\r\\nstring\\r\\nSOCKS5 proxy endpoint used by the managed network proxy.\\r\\npersonality\\r\\nnone | friendly | pragmatic\\r\\nDefault communication style for models that advertise supportsPersonality; can be overridden per thread/turn or via /personality.\\r\\nplan_mode_reasoning_effort\\r\\nnone | minimal | low | medium | high | xhigh\\r\\nPlan-mode-specific reasoning override. When unset, Plan mode uses its built-in preset default.\\r\\nprofile\\r\\nstring\\r\\nDefault profile applied at startup (equivalent to --profile).\\r\\nprofiles.<name>.*\\r\\nvarious\\r\\nProfile-scoped overrides for any of the supported configuration keys.\\r\\nprofiles.<name>.analytics.enabled\\r\\nboolean\\r\\nProfile-scoped analytics enablement override.\\r\\nprofiles.<name>.experimental_use_unified_exec_tool\\r\\nboolean\\r\\nLegacy name for enabling unified exec; prefer [features].unified_exec.\\r\\nprofiles.<name>.model_catalog_json\\r\\nstring (path)\\r\\nProfile-scoped model catalog JSON path override (applied on startup only; overrides the top-level model_catalog_json for that profile).\\r\\nprofiles.<name>.model_instructions_file\\r\\nstring (path)\\r\\nProfile-scoped replacement for the built-in instruction file.\\r\\nprofiles.<name>.oss_provider\\r\\nlmstudio | ollama\\r\\nProfile-scoped OSS provider for --oss sessions.\\r\\nprofiles.<name>.personality\\r\\nnone | friendly | pragmatic\\r\\nProfile-scoped communication style override for supported models.\\r\\nprofiles.<name>.plan_mode_reasoning_effort\\r\\nnone | minimal | low | medium | high | xhigh\\r\\nProfile-scoped Plan-mode reasoning override.\\r\\nprofiles.<name>.service_tier\\r\\nflex | fast\\r\\nProfile-scoped service tier preference for new turns.\\r\\nprofiles.<name>.tools_view_image\\r\\nboolean\\r\\nEnable or disable the view_image tool in that profile.\\r\\nprofiles.<name>.web_search\\r\\ndisabled | cached | live\\r\\nProfile-scoped web search mode override (default: \\\"cached\\\").\\r\\nprofiles.<name>.windows.sandbox\\r\\nunelevated | elevated\\r\\nProfile-scoped Windows sandbox mode override.\\r\\nproject_doc_fallback_filenames\\r\\narray<string>\\r\\nAdditional filenames to try when AGENTS.md is missing.\\r\\nproject_doc_max_bytes\\r\\nnumber\\r\\nMaximum bytes read from AGENTS.md when building project instructions.\\r\\nproject_root_markers\\r\\narray<string>\\r\\nList of project root marker filenames; used when searching parent directories for the project root.\\r\\nprojects.<path>.trust_level\\r\\nstring\\r\\nMark a project or worktree as trusted or untrusted (\\\"trusted\\\" | \\\"untrusted\\\"). Untrusted projects skip project-scoped .codex/ layers.\\r\\nreview_model\\r\\nstring\\r\\nOptional model override used by /review (defaults to the current session model).\\r\\nsandbox_mode\\r\\nread-only | workspace-write | danger-full-access\\r\\nSandbox policy for filesystem and network access during command execution.\\r\\nsandbox_workspace_write.exclude_slash_tmp\\r\\nboolean\\r\\nExclude /tmp from writable roots in workspace-write mode.\\r\\nsandbox_workspace_write.exclude_tmpdir_env_var\\r\\nboolean\\r\\nExclude $TMPDIR from writable roots in workspace-write mode.\\r\\nsandbox_workspace_write.network_access\\r\\nboolean\\r\\nAllow outbound network access inside the workspace-write sandbox.\\r\\nsandbox_workspace_write.writable_roots\\r\\narray<string>\\r\\nAdditional writable roots when sandbox_mode = \\\"workspace-write\\\".\\r\\nservice_tier\\r\\nflex | fast\\r\\nPreferred service tier for new turns. fast is honored only when the features.fast_mode gate is enabled.\\r\\nshell_environment_policy.exclude\\r\\narray<string>\\r\\nGlob patterns for removing environment variables after the defaults.\\r\\nshell_environment_policy.experimental_use_profile\\r\\nboolean\\r\\nUse the user shell profile when spawning subprocesses.\\r\\nshell_environment_policy.ignore_default_excludes\\r\\nboolean\\r\\nKeep variables containing KEY/SECRET/TOKEN before other filters run.\\r\\nshell_environment_policy.include_only\\r\\narray<string>\\r\\nWhitelist of patterns; when set only matching variables are kept.\\r\\nshell_environment_policy.inherit\\r\\nall | core | none\\r\\nBaseline environment inheritance when spawning subprocesses.\\r\\nshell_environment_policy.set\\r\\nmap<string,string>\\r\\nExplicit environment overrides injected into every subprocess.\\r\\nshow_raw_agent_reasoning\\r\\nboolean\\r\\nSurface raw reasoning content when the active model emits it.\\r\\nskills.config\\r\\narray<object>\\r\\nPer-skill enablement overrides stored in config.toml.\\r\\nskills.config.<index>.enabled\\r\\nboolean\\r\\nEnable or disable the referenced skill.\\r\\nskills.config.<index>.path\\r\\nstring (path)\\r\\nPath to a skill folder containing SKILL.md.\\r\\nsqlite_home\\r\\nstring (path)\\r\\nDirectory where Codex stores the SQLite-backed state DB used by agent jobs and other resumable runtime state.\\r\\nsuppress_unstable_features_warning\\r\\nboolean\\r\\nSuppress the warning that appears when under-development feature flags are enabled.\\r\\ntool_output_token_limit\\r\\nnumber\\r\\nToken budget for storing individual tool/function outputs in history.\\r\\ntools.view_image\\r\\nboolean\\r\\nEnable the local-image attachment tool view_image.\\r\\ntools.web_search\\r\\nboolean\\r\\nDeprecated legacy toggle for web search; prefer the top-level web_search setting.\\r\\ntui\\r\\ntable\\r\\nTUI-specific options such as enabling inline desktop notifications.\\r\\ntui.alternate_screen\\r\\nauto | always | never\\r\\nControl alternate screen usage for the TUI (default: auto; auto skips it in Zellij to preserve scrollback).\\r\\ntui.animations\\r\\nboolean\\r\\nEnable terminal animations (welcome screen, shimmer, spinner) (default: true).\\r\\ntui.model_availability_nux.<model>\\r\\ninteger\\r\\nInternal startup-tooltip state keyed by model slug.\\r\\ntui.notification_method\\r\\nauto | osc9 | bel\\r\\nNotification method for unfocused terminal notifications (default: auto).\\r\\ntui.notifications\\r\\nboolean | array<string>\\r\\nEnable TUI notifications; optionally restrict to specific event types.\\r\\ntui.show_tooltips\\r\\nboolean\\r\\nShow onboarding tooltips in the TUI welcome screen (default: true).\\r\\ntui.status_line\\r\\narray<string> | null\\r\\nOrdered list of TUI footer status-line item identifiers. null disables the status line.\\r\\ntui.theme\\r\\nstring\\r\\nSyntax-highlighting theme override (kebab-case theme name).\\r\\nweb_search\\r\\ndisabled | cached | live\\r\\nWeb search mode (default: \\\"cached\\\"; cached uses an OpenAI-maintained index and does not fetch live pages; if you use --yolo or another full access sandbox setting, it defaults to \\\"live\\\"). Use \\\"live\\\" to fetch the most recent data from the web, or \\\"disabled\\\" to remove the tool.\\r\\nwindows_wsl_setup_acknowledged\\r\\nboolean\\r\\nTrack Windows onboarding acknowledgement (Windows only).\\r\\nwindows.sandbox\\r\\nunelevated | elevated\\r\\nWindows-only native sandbox mode when running Codex natively on Windows.\\r\\nYou can find the latest JSON schema for config.toml here.\\r\\nTo get autocompletion and diagnostics when editing config.toml in VS Code or Cursor, you can install the Even Better TOML extension and add this line to the top of your config.toml:\\r\\nNote: Rename experimental_instructions_file to model_instructions_file. Codex deprecates the old key; update existing configs to the new name.\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Configuration Reference\", \"lvl2\": \"config.toml\", \"lvl3\": null, \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"4-https://developers.openai.com/codex/config-reference/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 responses_websockets\\r\\nboolean\\r\\nPrefer the Responses API WebSocket transport \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"User-level configuration lives in ~/.codex/config.toml. You can also add project-scoped overrides in .codex/config.toml files. Codex loads project-scoped config files only when you trust the project.\\r\\nFor sandbox and approval keys (approval_policy, sandbox_mode, and sandbox_workspace_write.*), pair this reference with Sandbox and approvals, Protected paths in writable roots, and Network access.\\r\\nagents.<name>.config_file\\r\\nstring (path)\\r\\nPath to a TOML config layer for that role; relative paths resolve from the config file that declares the role.\\r\\nagents.<name>.description\\r\\nstring\\r\\nRole guidance shown to Codex when choosing and spawning that agent type.\\r\\nagents.<name>.nickname_candidates\\r\\narray<string>\\r\\nOptional pool of display nicknames for spawned agents in that role.\\r\\nagents.job_max_runtime_seconds\\r\\nnumber\\r\\nDefault per-worker timeout for spawn_agents_on_csv jobs. When unset, the tool falls back to 1800 seconds per worker.\\r\\nagents.max_depth\\r\\nnumber\\r\\nMaximum nesting depth allowed for spawned agent threads (root sessions start at depth 0; default: 1).\\r\\nagents.max_threads\\r\\nnumber\\r\\nMaximum number of agent threads that can be open concurrently. Defaults to 6 when unset.\\r\\nallow_login_shell\\r\\nboolean\\r\\nAllow shell-based tools to use login-shell semantics. Defaults to true; when false, login = true requests are rejected and omitted login defaults to non-login shells.\\r\\nanalytics.enabled\\r\\nboolean\\r\\nEnable or disable analytics for this machine/profile. When unset, the client default applies.\\r\\napproval_policy\\r\\nuntrusted | on-request | never | { reject = { sandbox_approval = bool, rules = bool, mcp_elicitations = bool } }\\r\\nControls when Codex pauses for approval before executing commands. You can also use approval_policy = { reject = { ... } } to auto-reject specific prompt categories while keeping other prompts interactive. on-failure is deprecated; use on-request for interactive runs or never for non-interactive runs.\\r\\napproval_policy.reject.mcp_elicitations\\r\\nboolean\\r\\nWhen true, MCP elicitation prompts are auto-rejected instead of shown to the user.\\r\\napproval_policy.reject.rules\\r\\nboolean\\r\\nWhen true, approvals triggered by execpolicy prompt rules are auto-rejected.\\r\\napproval_policy.reject.sandbox_approval\\r\\nboolean\\r\\nWhen true, sandbox escalation approval prompts are auto-rejected.\\r\\napps._default.destructive_enabled\\r\\nboolean\\r\\nDefault allow/deny for app tools with destructive_hint = true.\\r\\napps._default.enabled\\r\\nboolean\\r\\nDefault app enabled state for all apps unless overridden per app.\\r\\napps._default.open_world_enabled\\r\\nboolean\\r\\nDefault allow/deny for app tools with open_world_hint = true.\\r\\napps.<id>.default_tools_approval_mode\\r\\nauto | prompt | approve\\r\\nDefault approval behavior for tools in this app unless a per-tool override exists.\\r\\napps.<id>.default_tools_enabled\\r\\nboolean\\r\\nDefault enabled state for tools in this app unless a per-tool override exists.\\r\\napps.<id>.destructive_enabled\\r\\nboolean\\r\\nAllow or block tools in this app that advertise destructive_hint = true.\\r\\napps.<id>.enabled\\r\\nboolean\\r\\nEnable or disable a specific app/connector by id (default: true).\\r\\napps.<id>.open_world_enabled\\r\\nboolean\\r\\nAllow or block tools in this app that advertise open_world_hint = true.\\r\\napps.<id>.tools.<tool>.approval_mode\\r\\nauto | prompt | approve\\r\\nPer-tool approval behavior override for a single app tool.\\r\\napps.<id>.tools.<tool>.enabled\\r\\nboolean\\r\\nPer-tool enabled override for an app tool (for example repos/list).\\r\\nbackground_terminal_max_timeout\\r\\nnumber\\r\\nMaximum poll window in milliseconds for empty write_stdin polls (background terminal polling). Default: 300000 (5 minutes). Replaces the older background_terminal_timeout key.\\r\\nchatgpt_base_url\\r\\nstring\\r\\nOverride the base URL used during the ChatGPT login flow.\\r\\ncheck_for_update_on_startup\\r\\nboolean\\r\\nCheck for Codex updates on startup (set to false only when updates are centrally managed).\\r\\ncli_auth_credentials_store\\r\\nfile | keyring | auto\\r\\nControl where the CLI stores cached credentials (file-based auth.json vs OS keychain).\\r\\ncommit_attribution\\r\\nstring\\r\\nOverride the commit co-author trailer text. Set an empty string to disable automatic attribution.\\r\\ncompact_prompt\\r\\nstring\\r\\nInline override for the history compaction prompt.\\r\\ndeveloper_instructions\\r\\nstring\\r\\nAdditional developer instructions injected into the session (optional).\\r\\ndisable_paste_burst\\r\\nboolean\\r\\nDisable burst-paste detection in the TUI.\\r\\nexperimental_compact_prompt_file\\r\\nstring (path)\\r\\nLoad the compaction prompt override from a file (experimental).\\r\\nexperimental_use_unified_exec_tool\\r\\nboolean\\r\\nLegacy name for enabling unified exec; prefer [features].unified_exec or codex --enable unified_exec.\\r\\nfeatures.apps\\r\\nboolean\\r\\nEnable ChatGPT Apps/connectors support (experimental).\\r\\nfeatures.apps_mcp_gateway\\r\\nboolean\\r\\nRoute Apps MCP calls through the OpenAI connectors MCP gateway (https://api.openai.com/v1/connectors/mcp/) instead of legacy routing (experimental).\\r\\nfeatures.artifact\\r\\nboolean\\r\\nEnable native artifact tools such as slides and spreadsheets (under development).\\r\\nfeatures.child_agents_md\\r\\nboolean\\r\\nAppend AGENTS.md scope/precedence guidance even when no AGENTS.md is present (experimental).\\r\\nfeatures.collaboration_modes\\r\\nboolean\\r\\nLegacy toggle for collaboration modes. Plan and default modes are available in current builds without setting this key.\\r\\nfeatures.default_mode_request_user_input\\r\\nboolean\\r\\nAllow request_user_input in default collaboration mode (under development; off by default).\\r\\nfeatures.elevated_windows_sandbox\\r\\nboolean\\r\\nLegacy toggle for an earlier elevated Windows sandbox rollout. Current builds do not use it.\\r\\nfeatures.enable_request_compression\\r\\nboolean\\r\\nCompress streaming request bodies with zstd when supported (stable; on by default).\\r\\nfeatures.experimental_windows_sandbox\\r\\nboolean\\r\\nLegacy toggle for an earlier Windows sandbox rollout. Current builds do not use it.\\r\\nfeatures.fast_mode\\r\\nboolean\\r\\nEnable Fast mode selection and the service_tier = \\\"fast\\\" path (stable; on by default).\\r\\nfeatures.image_detail_original\\r\\nboolean\\r\\nAllow image outputs with detail = \\\"original\\\" on supported models (under development).\\r\\nfeatures.image_generation\\r\\nboolean\\r\\nEnable the built-in image generation tool (under development).\\r\\nfeatures.multi_agent\\r\\nboolean\\r\\nEnable multi-agent collaboration tools (spawn_agent, send_input, resume_agent, wait, close_agent, and spawn_agents_on_csv) (experimental; off by default).\\r\\nfeatures.personality\\r\\nboolean\\r\\nEnable personality selection controls (stable; on by default).\\r\\nfeatures.powershell_utf8\\r\\nboolean\\r\\nForce PowerShell UTF-8 output. Enabled by default on Windows and off elsewhere.\\r\\nfeatures.prevent_idle_sleep\\r\\nboolean\\r\\nPrevent the machine from sleeping while a turn is actively running (experimental; off by default).\\r\\nfeatures.remote_models\\r\\nboolean\\r\\nLegacy toggle for an older remote-model readiness flow. Current builds do not use it.\\r\\nfeatures.request_rule\\r\\nboolean\\r\\nLegacy toggle for Smart approvals. Current builds include this behavior by default, so most users can leave this unset.\\r\\nfeatures.responses_websockets\\r\\nboolean\\r\\nPrefer the Responses API WebSocket transport for supported providers (under development).\\r\\nfeatures.responses_websockets_v2\\r\\nboolean\\r\\nEnable Responses API WebSocket v2 mode (under development).\\r\\nfeatures.runtime_metrics\\r\\nboolean\\r\\nShow runtime metrics summary in TUI turn separators (experimental).\\r\\nfeatures.search_tool\\r\\nboolean\\r\\nLegacy toggle for an older Apps discovery flow. Current builds do not use it.\\r\\nfeatures.shell_snapshot\\r\\nboolean\\r\\nSnapshot shell environment to speed up repeated commands (stable; on by default).\\r\\nfeatures.shell_tool\\r\\nboolean\\r\\nEnable the default shell tool for running commands (stable; on by default).\\r\\nfeatures.skill_env_var_dependency_prompt\\r\\nboolean\\r\\nPrompt for missing skill environment-variable dependencies (under development).\\r\\nfeatures.skill_mcp_dependency_install\\r\\nboolean\\r\\nAllow prompting and installing missing MCP dependencies for skills (stable; on by default).\\r\\nfeatures.sqlite\\r\\nboolean\\r\\nEnable SQLite-backed state persistence (stable; on by default).\\r\\nfeatures.steer\\r\\nboolean\\r\\nLegacy toggle from an earlier Enter/Tab steering rollout. Current builds always use the current steering behavior.\\r\\nfeatures.undo\\r\\nboolean\\r\\nEnable undo support (stable; off by default).\\r\\nfeatures.unified_exec\\r\\nboolean\\r\\nUse the unified PTY-backed exec tool (stable; enabled by default except on Windows).\\r\\nfeatures.use_linux_sandbox_bwrap\\r\\nboolean\\r\\nUse the bubblewrap-based Linux sandbox pipeline (experimental; off by default).\\r\\nfeatures.web_search\\r\\nboolean\\r\\nDeprecated legacy toggle; prefer the top-level web_search setting.\\r\\nfeatures.web_search_cached\\r\\nboolean\\r\\nDeprecated legacy toggle. When web_search is unset, true maps to web_search = \\\"cached\\\".\\r\\nfeatures.web_search_request\\r\\nboolean\\r\\nDeprecated legacy toggle. When web_search is unset, true maps to web_search = \\\"live\\\".\\r\\nfeedback.enabled\\r\\nboolean\\r\\nEnable feedback submission via /feedback across Codex surfaces (default: true).\\r\\nfile_opener\\r\\nvscode | vscode-insiders | windsurf | cursor | none\\r\\nURI scheme used to open citations from Codex output (default: vscode).\\r\\nforced_chatgpt_workspace_id\\r\\nstring (uuid)\\r\\nLimit ChatGPT logins to a specific workspace identifier.\\r\\nforced_login_method\\r\\nchatgpt | api\\r\\nRestrict Codex to a specific authentication method.\\r\\nhide_agent_reasoning\\r\\nboolean\\r\\nSuppress reasoning events in both the TUI and codex exec output.\\r\\nhistory.max_bytes\\r\\nnumber\\r\\nIf set, caps the history file size in bytes by dropping oldest entries.\\r\\nhistory.persistence\\r\\nsave-all | none\\r\\nControl whether Codex saves session transcripts to history.jsonl.\\r\\ninstructions\\r\\nstring\\r\\nReserved for future use; prefer model_instructions_file or AGENTS.md.\\r\\nlog_dir\\r\\nstring (path)\\r\\nDirectory where Codex writes log files (for example codex-tui.log); defaults to $CODEX_HOME/log.\\r\\nmcp_oauth_callback_port\\r\\ninteger\\r\\nOptional fixed port for the local HTTP callback server used during MCP OAuth login. When unset, Codex binds to an ephemeral port chosen by the OS.\\r\\nmcp_oauth_callback_url\\r\\nstring\\r\\nOptional redirect URI override for MCP OAuth login (for example, a devbox ingress URL). mcp_oauth_callback_port still controls the callback listener port.\\r\\nmcp_oauth_credentials_store\\r\\nauto | file | keyring\\r\\nPreferred store for MCP OAuth credentials.\\r\\nmcp_servers.<id>.args\\r\\narray<string>\\r\\nArguments passed to the MCP stdio server command.\\r\\nmcp_servers.<id>.bearer_token_env_var\\r\\nstring\\r\\nEnvironment variable sourcing the bearer token for an MCP HTTP server.\\r\\nmcp_servers.<id>.command\\r\\nstring\\r\\nLauncher command for an MCP stdio server.\\r\\nmcp_servers.<id>.cwd\\r\\nstring\\r\\nWorking directory for the MCP stdio server process.\\r\\nmcp_servers.<id>.disabled_tools\\r\\narray<string>\\r\\nDeny list applied after enabled_tools for the MCP server.\\r\\nmcp_servers.<id>.enabled\\r\\nboolean\\r\\nDisable an MCP server without removing its configuration.\\r\\nmcp_servers.<id>.enabled_tools\\r\\narray<string>\\r\\nAllow list of tool names exposed by the MCP server.\\r\\nmcp_servers.<id>.env\\r\\nmap<string,string>\\r\\nEnvironment variables forwarded to the MCP stdio server.\\r\\nmcp_servers.<id>.env_http_headers\\r\\nmap<string,string>\\r\\nHTTP headers populated from environment variables for an MCP HTTP server.\\r\\nmcp_servers.<id>.env_vars\\r\\narray<string>\\r\\nAdditional environment variables to whitelist for an MCP stdio server.\\r\\nmcp_servers.<id>.http_headers\\r\\nmap<string,string>\\r\\nStatic HTTP headers included with each MCP HTTP request.\\r\\nmcp_servers.<id>.oauth_resource\\r\\nstring\\r\\nOptional RFC 8707 OAuth resource parameter to include during MCP login.\\r\\nmcp_servers.<id>.required\\r\\nboolean\\r\\nWhen true, fail startup/resume if this enabled MCP server cannot initialize.\\r\\nmcp_servers.<id>.scopes\\r\\narray<string>\\r\\nOAuth scopes to request when authenticating to that MCP server.\\r\\nmcp_servers.<id>.startup_timeout_ms\\r\\nnumber\\r\\nAlias for startup_timeout_sec in milliseconds.\\r\\nmcp_servers.<id>.startup_timeout_sec\\r\\nnumber\\r\\nOverride the default 10s startup timeout for an MCP server.\\r\\nmcp_servers.<id>.tool_timeout_sec\\r\\nnumber\\r\\nOverride the default 60s per-tool timeout for an MCP server.\\r\\nmcp_servers.<id>.url\\r\\nstring\\r\\nEndpoint for an MCP streamable HTTP server.\\r\\nmodel\\r\\nstring\\r\\nModel to use (e.g., gpt-5-codex).\\r\\nmodel_auto_compact_token_limit\\r\\nnumber\\r\\nToken threshold that triggers automatic history compaction (unset uses model defaults).\\r\\nmodel_catalog_json\\r\\nstring (path)\\r\\nOptional path to a JSON model catalog loaded on startup. Profile-level profiles.<name>.model_catalog_json can override this per profile.\\r\\nmodel_context_window\\r\\nnumber\\r\\nContext window tokens available to the active model.\\r\\nmodel_instructions_file\\r\\nstring (path)\\r\\nReplacement for built-in instructions instead of AGENTS.md.\\r\\nmodel_provider\\r\\nstring\\r\\nProvider id from model_providers (default: openai).\\r\\nmodel_providers.<id>.base_url\\r\\nstring\\r\\nAPI base URL for the model provider.\\r\\nmodel_providers.<id>.env_http_headers\\r\\nmap<string,string>\\r\\nHTTP headers populated from environment variables when present.\\r\\nmodel_providers.<id>.env_key\\r\\nstring\\r\\nEnvironment variable supplying the provider API key.\\r\\nmodel_providers.<id>.env_key_instructions\\r\\nstring\\r\\nOptional setup guidance for the provider API key.\\r\\nmodel_providers.<id>.experimental_bearer_token\\r\\nstring\\r\\nDirect bearer token for the provider (discouraged; use env_key).\\r\\nmodel_providers.<id>.http_headers\\r\\nmap<string,string>\\r\\nStatic HTTP headers added to provider requests.\\r\\nmodel_providers.<id>.name\\r\\nstring\\r\\nDisplay name for a custom model provider.\\r\\nmodel_providers.<id>.query_params\\r\\nmap<string,string>\\r\\nExtra query parameters appended to provider requests.\\r\\nmodel_providers.<id>.request_max_retries\\r\\nnumber\\r\\nRetry count for HTTP requests to the provider (default: 4).\\r\\nmodel_providers.<id>.requires_openai_auth\\r\\nboolean\\r\\nThe provider uses OpenAI authentication (defaults to false).\\r\\nmodel_providers.<id>.stream_idle_timeout_ms\\r\\nnumber\\r\\nIdle timeout for SSE streams in milliseconds (default: 300000).\\r\\nmodel_providers.<id>.stream_max_retries\\r\\nnumber\\r\\nRetry count for SSE streaming interruptions (default: 5).\\r\\nmodel_providers.<id>.supports_websockets\\r\\nboolean\\r\\nWhether that provider supports the Responses API WebSocket transport.\\r\\nmodel_providers.<id>.wire_api\\r\\nresponses\\r\\nProtocol used by the provider. responses is the only supported value, and it is the default when omitted.\\r\\nmodel_reasoning_effort\\r\\nminimal | low | medium | high | xhigh\\r\\nAdjust reasoning effort for supported models (Responses API only; xhigh is model-dependent).\\r\\nmodel_reasoning_summary\\r\\nauto | concise | detailed | none\\r\\nSelect reasoning summary detail or disable summaries entirely.\\r\\nmodel_supports_reasoning_summaries\\r\\nboolean\\r\\nForce Codex to send or not send reasoning metadata.\\r\\nmodel_verbosity\\r\\nlow | medium | high\\r\\nOptional GPT-5 Responses API verbosity override; when unset, the selected model/preset default is used.\\r\\nnotice.hide_full_access_warning\\r\\nboolean\\r\\nTrack acknowledgement of the full access warning prompt.\\r\\nnotice.hide_gpt-5.1-codex-max_migration_prompt\\r\\nboolean\\r\\nTrack acknowledgement of the gpt-5.1-codex-max migration prompt.\\r\\nnotice.hide_gpt5_1_migration_prompt\\r\\nboolean\\r\\nTrack acknowledgement of the GPT-5.1 migration prompt.\\r\\nnotice.hide_rate_limit_model_nudge\\r\\nboolean\\r\\nTrack opt-out of the rate limit model switch reminder.\\r\\nnotice.hide_world_writable_warning\\r\\nboolean\\r\\nTrack acknowledgement of the Windows world-writable directories warning.\\r\\nnotice.model_migrations\\r\\nmap<string,string>\\r\\nTrack acknowledged model migrations as old->new mappings.\\r\\nnotify\\r\\narray<string>\\r\\nCommand invoked for notifications; receives a JSON payload from Codex.\\r\\noss_provider\\r\\nlmstudio | ollama\\r\\nDefault local provider used when running with --oss (defaults to prompting if unset).\\r\\notel.environment\\r\\nstring\\r\\nEnvironment tag applied to emitted OpenTelemetry events (default: dev).\\r\\notel.exporter\\r\\nnone | otlp-http | otlp-grpc\\r\\nSelect the OpenTelemetry exporter and provide any endpoint metadata.\\r\\notel.exporter.<id>.endpoint\\r\\nstring\\r\\nExporter endpoint for OTEL logs.\\r\\notel.exporter.<id>.headers\\r\\nmap<string,string>\\r\\nStatic headers included with OTEL exporter requests.\\r\\notel.exporter.<id>.protocol\\r\\nbinary | json\\r\\nProtocol used by the OTLP/HTTP exporter.\\r\\notel.exporter.<id>.tls.ca-certificate\\r\\nstring\\r\\nCA certificate path for OTEL exporter TLS.\\r\\notel.exporter.<id>.tls.client-certificate\\r\\nstring\\r\\nClient certificate path for OTEL exporter TLS.\\r\\notel.exporter.<id>.tls.client-private-key\\r\\nstring\\r\\nClient private key path for OTEL exporter TLS.\\r\\notel.log_user_prompt\\r\\nboolean\\r\\nOpt in to exporting raw user prompts with OpenTelemetry logs.\\r\\notel.metrics_exporter\\r\\nnone | statsig | otlp-http | otlp-grpc\\r\\nSelect the OpenTelemetry metrics exporter (defaults to statsig).\\r\\notel.trace_exporter\\r\\nnone | otlp-http | otlp-grpc\\r\\nSelect the OpenTelemetry trace exporter and provide any endpoint metadata.\\r\\notel.trace_exporter.<id>.endpoint\\r\\nstring\\r\\nTrace exporter endpoint for OTEL logs.\\r\\notel.trace_exporter.<id>.headers\\r\\nmap<string,string>\\r\\nStatic headers included with OTEL trace exporter requests.\\r\\notel.trace_exporter.<id>.protocol\\r\\nbinary | json\\r\\nProtocol used by the OTLP/HTTP trace exporter.\\r\\notel.trace_exporter.<id>.tls.ca-certificate\\r\\nstring\\r\\nCA certificate path for OTEL trace exporter TLS.\\r\\notel.trace_exporter.<id>.tls.client-certificate\\r\\nstring\\r\\nClient certificate path for OTEL trace exporter TLS.\\r\\notel.trace_exporter.<id>.tls.client-private-key\\r\\nstring\\r\\nClient private key path for OTEL trace exporter TLS.\\r\\npermissions.network.admin_url\\r\\nstring\\r\\nAdmin endpoint for the managed network proxy.\\r\\npermissions.network.allow_local_binding\\r\\nboolean\\r\\nPermit local bind/listen operations through the managed proxy.\\r\\npermissions.network.allow_unix_sockets\\r\\narray<string>\\r\\nAllowlist of Unix socket paths permitted through the managed proxy.\\r\\npermissions.network.allow_upstream_proxy\\r\\nboolean\\r\\nAllow the managed proxy to chain to another upstream proxy.\\r\\npermissions.network.allowed_domains\\r\\narray<string>\\r\\nAllowlist of domains permitted through the managed proxy.\\r\\npermissions.network.dangerously_allow_all_unix_sockets\\r\\nboolean\\r\\nAllow the proxy to use arbitrary Unix sockets instead of the default restricted set.\\r\\npermissions.network.dangerously_allow_non_loopback_admin\\r\\nboolean\\r\\nPermit non-loopback bind addresses for the managed proxy admin listener.\\r\\npermissions.network.dangerously_allow_non_loopback_proxy\\r\\nboolean\\r\\nPermit non-loopback bind addresses for the managed proxy listener.\\r\\npermissions.network.denied_domains\\r\\narray<string>\\r\\nDenylist of domains blocked by the managed proxy.\\r\\npermissions.network.enable_socks5\\r\\nboolean\\r\\nExpose a SOCKS5 listener from the managed network proxy.\\r\\npermissions.network.enable_socks5_udp\\r\\nboolean\\r\\nAllow UDP over the SOCKS5 listener when enabled.\\r\\npermissions.network.enabled\\r\\nboolean\\r\\nEnable the managed network proxy configuration for subprocesses.\\r\\npermissions.network.mode\\r\\nlimited | full\\r\\nNetwork proxy mode used for subprocess traffic.\\r\\npermissions.network.proxy_url\\r\\nstring\\r\\nHTTP proxy endpoint used by the managed network proxy.\\r\\npermissions.network.socks_url\\r\\nstring\\r\\nSOCKS5 proxy endpoint used by the managed network proxy.\\r\\npersonality\\r\\nnone | friendly | pragmatic\\r\\nDefault communication style for models that advertise supportsPersonality; can be overridden per thread/turn or via /personality.\\r\\nplan_mode_reasoning_effort\\r\\nnone | minimal | low | medium | high | xhigh\\r\\nPlan-mode-specific reasoning override. When unset, Plan mode uses its built-in preset default.\\r\\nprofile\\r\\nstring\\r\\nDefault profile applied at startup (equivalent to --profile).\\r\\nprofiles.<name>.*\\r\\nvarious\\r\\nProfile-scoped overrides for any of the supported configuration keys.\\r\\nprofiles.<name>.analytics.enabled\\r\\nboolean\\r\\nProfile-scoped analytics enablement override.\\r\\nprofiles.<name>.experimental_use_unified_exec_tool\\r\\nboolean\\r\\nLegacy name for enabling unified exec; prefer [features].unified_exec.\\r\\nprofiles.<name>.model_catalog_json\\r\\nstring (path)\\r\\nProfile-scoped model catalog JSON path override (applied on startup only; overrides the top-level model_catalog_json for that profile).\\r\\nprofiles.<name>.model_instructions_file\\r\\nstring (path)\\r\\nProfile-scoped replacement for the built-in instruction file.\\r\\nprofiles.<name>.oss_provider\\r\\nlmstudio | ollama\\r\\nProfile-scoped OSS provider for --oss sessions.\\r\\nprofiles.<name>.personality\\r\\nnone | friendly | pragmatic\\r\\nProfile-scoped communication style override for supported models.\\r\\nprofiles.<name>.plan_mode_reasoning_effort\\r\\nnone | minimal | low | medium | high | xhigh\\r\\nProfile-scoped Plan-mode reasoning override.\\r\\nprofiles.<name>.service_tier\\r\\nflex | fast\\r\\nProfile-scoped service tier preference for new turns.\\r\\nprofiles.<name>.tools_view_image\\r\\nboolean\\r\\nEnable or disable the view_image tool in that profile.\\r\\nprofiles.<name>.web_search\\r\\ndisabled | cached | live\\r\\nProfile-scoped web search mode override (default: \\\"cached\\\").\\r\\nprofiles.<name>.windows.sandbox\\r\\nunelevated | elevated\\r\\nProfile-scoped Windows sandbox mode override.\\r\\nproject_doc_fallback_filenames\\r\\narray<string>\\r\\nAdditional filenames to try when AGENTS.md is missing.\\r\\nproject_doc_max_bytes\\r\\nnumber\\r\\nMaximum bytes read from AGENTS.md when building project instructions.\\r\\nproject_root_markers\\r\\narray<string>\\r\\nList of project root marker filenames; used when searching parent directories for the project root.\\r\\nprojects.<path>.trust_level\\r\\nstring\\r\\nMark a project or worktree as trusted or untrusted (\\\"trusted\\\" | \\\"untrusted\\\"). Untrusted projects skip project-scoped .codex/ layers.\\r\\nreview_model\\r\\nstring\\r\\nOptional model override used by /review (defaults to the current session model).\\r\\nsandbox_mode\\r\\nread-only | workspace-write | danger-full-access\\r\\nSandbox policy for filesystem and network access during command execution.\\r\\nsandbox_workspace_write.exclude_slash_tmp\\r\\nboolean\\r\\nExclude /tmp from writable roots in workspace-write mode.\\r\\nsandbox_workspace_write.exclude_tmpdir_env_var\\r\\nboolean\\r\\nExclude $TMPDIR from writable roots in workspace-write mode.\\r\\nsandbox_workspace_write.network_access\\r\\nboolean\\r\\nAllow outbound network access inside the workspace-write sandbox.\\r\\nsandbox_workspace_write.writable_roots\\r\\narray<string>\\r\\nAdditional writable roots when sandbox_mode = \\\"workspace-write\\\".\\r\\nservice_tier\\r\\nflex | fast\\r\\nPreferred service tier for new turns. fast is honored only when the features.fast_mode gate is enabled.\\r\\nshell_environment_policy.exclude\\r\\narray<string>\\r\\nGlob patterns for removing environment variables after the defaults.\\r\\nshell_environment_policy.experimental_use_profile\\r\\nboolean\\r\\nUse the user shell profile when spawning subprocesses.\\r\\nshell_environment_policy.ignore_default_excludes\\r\\nboolean\\r\\nKeep variables containing KEY/SECRET/TOKEN before other filters run.\\r\\nshell_environment_policy.include_only\\r\\narray<string>\\r\\nWhitelist of patterns; when set only matching variables are kept.\\r\\nshell_environment_policy.inherit\\r\\nall | core | none\\r\\nBaseline environment inheritance when spawning subprocesses.\\r\\nshell_environment_policy.set\\r\\nmap<string,string>\\r\\nExplicit environment overrides injected into every subprocess.\\r\\nshow_raw_agent_reasoning\\r\\nboolean\\r\\nSurface raw reasoning content when the active model emits it.\\r\\nskills.config\\r\\narray<object>\\r\\nPer-skill enablement overrides stored in config.toml.\\r\\nskills.config.<index>.enabled\\r\\nboolean\\r\\nEnable or disable the referenced skill.\\r\\nskills.config.<index>.path\\r\\nstring (path)\\r\\nPath to a skill folder containing SKILL.md.\\r\\nsqlite_home\\r\\nstring (path)\\r\\nDirectory where Codex stores the SQLite-backed state DB used by agent jobs and other resumable runtime state.\\r\\nsuppress_unstable_features_warning\\r\\nboolean\\r\\nSuppress the warning that appears when under-development feature flags are enabled.\\r\\ntool_output_token_limit\\r\\nnumber\\r\\nToken budget for storing individual tool/function outputs in history.\\r\\ntools.view_image\\r\\nboolean\\r\\nEnable the local-image attachment tool view_image.\\r\\ntools.web_search\\r\\nboolean\\r\\nDeprecated legacy toggle for web search; prefer the top-level web_search setting.\\r\\ntui\\r\\ntable\\r\\nTUI-specific options such as enabling inline desktop notifications.\\r\\ntui.alternate_screen\\r\\nauto | always | never\\r\\nControl alternate screen usage for the TUI (default: auto; auto skips it in Zellij to preserve scrollback).\\r\\ntui.animations\\r\\nboolean\\r\\nEnable terminal animations (welcome screen, shimmer, spinner) (default: true).\\r\\ntui.model_availability_nux.<model>\\r\\ninteger\\r\\nInternal startup-tooltip state keyed by model slug.\\r\\ntui.notification_method\\r\\nauto | osc9 | bel\\r\\nNotification method for unfocused terminal notifications (default: auto).\\r\\ntui.notifications\\r\\nboolean | array<string>\\r\\nEnable TUI notifications; optionally restrict to specific event types.\\r\\ntui.show_tooltips\\r\\nboolean\\r\\nShow onboarding tooltips in the TUI welcome screen (default: true).\\r\\ntui.status_line\\r\\narray<string> | null\\r\\nOrdered list of TUI footer status-line item identifiers. null disables the status line.\\r\\ntui.theme\\r\\nstring\\r\\nSyntax-highlighting theme override (kebab-case theme name).\\r\\nweb_search\\r\\ndisabled | cached | live\\r\\nWeb search mode (default: \\\"cached\\\"; cached uses an OpenAI-maintained index and does not fetch live pages; if you use --yolo or another full access sandbox setting, it defaults to \\\"live\\\"). Use \\\"live\\\" to fetch the most recent data from the web, or \\\"disabled\\\" to remove the tool.\\r\\nwindows_wsl_setup_acknowledged\\r\\nboolean\\r\\nTrack Windows onboarding acknowledgement (Windows only).\\r\\nwindows.sandbox\\r\\nunelevated | elevated\\r\\nWindows-only native sandbox mode when running Codex natively on Windows.\\r\\nYou can find the latest JSON schema for config.toml here.\\r\\nTo get autocompletion and diagnostics when editing config.toml in VS Code or Cursor, you can install the Even Better TOML extension and add this line to the top of your config.toml:\\r\\nNote: Rename experimental_instructions_file to model_instructions_file. Codex deprecates the old key; update existing configs to the new name.\", \"matchLevel\": \"full\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"tool\", \"responses\", \"api\", \"require\", \"approval\", \"option\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Configuration Reference\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl2\": {\"value\": \"config.toml\", \"matchLevel\": \"none\", \"matchedWords\": []}}}}, {\"url\": \"https://developers.openai.com/codex/guides/agents-sdk/#running-codex-as-an-mcp-server\", \"url_without_anchor\": \"https://developers.openai.com/codex/guides/agents-sdk/\", \"anchor\": \"running-codex-as-an-mcp-server\", \"content\": \"You can run Codex as an MCP server and connect it from other MCP clients (for example, an agent built with the OpenAI Agents SDK).\\r\\nTo start Codex as an MCP server, you can use the following command:\\r\\nYou can launch a Codex MCP server with the Model Context Protocol Inspector:\\r\\nSend a tools/list request to see two tools:\\r\\ncodex: Run a Codex session. Accepts configuration parameters that match the Codex Config struct. The codex tool takes these properties:\\r\\nprompt (required)\\r\\nstring\\r\\nThe initial user prompt to start the Codex conversation.\\r\\napproval-policy\\r\\nstring\\r\\nApproval policy for shell commands generated by the model: untrusted, on-request, and never.\\r\\nbase-instructions\\r\\nstring\\r\\nThe set of instructions to use instead of the default ones.\\r\\nconfig\\r\\nobject\\r\\nIndividual configuration settings that override what\\u2019s in $CODEX_HOME/config.toml.\\r\\ncwd\\r\\nstring\\r\\nWorking directory for the session. If relative, resolved against the server process\\u2019s current directory.\\r\\ninclude-plan-tool\\r\\nboolean\\r\\nWhether to include the plan tool in the conversation.\\r\\nmodel\\r\\nstring\\r\\nOptional override for the model name (for example, o3, o4-mini).\\r\\nprofile\\r\\nstring\\r\\nConfiguration profile from config.toml to specify default options.\\r\\nsandbox\\r\\nstring\\r\\nSandbox mode: read-only, workspace-write, or danger-full-access.\\r\\ncodex-reply: Continue a Codex session by providing the thread ID and prompt. The codex-reply tool takes these properties:\\r\\nprompt (required)\\r\\nstring\\r\\nThe next user prompt to continue the Codex conversation.\\r\\nthreadId (required)\\r\\nstring\\r\\nThe ID of the thread to continue.\\r\\nconversationId (deprecated)\\r\\nstring\\r\\nDeprecated alias for threadId (kept for compatibility).\\r\\nUse the threadId from structuredContent.threadId in the tools/call response. Approval prompts (exec/patch) also include threadId in their params payload.\\r\\nExample response payload:\\r\\nNote modern MCP clients generally report only \\\"structuredContent\\\" as the result of a tool call, if present, though the Codex MCP server also returns \\\"content\\\" for the benefit of older MCP clients.\", \"type\": \"content\", \"hierarchy\": {\"lvl0\": \"Documentation\", \"lvl1\": \"Running Codex as an MCP server\", \"lvl2\": null, \"lvl3\": null, \"lvl4\": null, \"lvl5\": null, \"lvl6\": null}, \"objectID\": \"4-https://developers.openai.com/codex/guides/agents-sdk/\", \"_snippetResult\": {\"content\": {\"value\": \"\\u2026 Content.threadId in the tools/call response. Approval prompts \\u2026\", \"matchLevel\": \"partial\"}}, \"_highlightResult\": {\"content\": {\"value\": \"You can run Codex as an MCP server and connect it from other MCP clients (for example, an agent built with the OpenAI Agents SDK).\\r\\nTo start Codex as an MCP server, you can use the following command:\\r\\nYou can launch a Codex MCP server with the Model Context Protocol Inspector:\\r\\nSend a tools/list request to see two tools:\\r\\ncodex: Run a Codex session. Accepts configuration parameters that match the Codex Config struct. The codex tool takes these properties:\\r\\nprompt (required)\\r\\nstring\\r\\nThe initial user prompt to start the Codex conversation.\\r\\napproval-policy\\r\\nstring\\r\\nApproval policy for shell commands generated by the model: untrusted, on-request, and never.\\r\\nbase-instructions\\r\\nstring\\r\\nThe set of instructions to use instead of the default ones.\\r\\nconfig\\r\\nobject\\r\\nIndividual configuration settings that override what\\u2019s in $CODEX_HOME/config.toml.\\r\\ncwd\\r\\nstring\\r\\nWorking directory for the session. If relative, resolved against the server process\\u2019s current directory.\\r\\ninclude-plan-tool\\r\\nboolean\\r\\nWhether to include the plan tool in the conversation.\\r\\nmodel\\r\\nstring\\r\\nOptional override for the model name (for example, o3, o4-mini).\\r\\nprofile\\r\\nstring\\r\\nConfiguration profile from config.toml to specify default options.\\r\\nsandbox\\r\\nstring\\r\\nSandbox mode: read-only, workspace-write, or danger-full-access.\\r\\ncodex-reply: Continue a Codex session by providing the thread ID and prompt. The codex-reply tool takes these properties:\\r\\nprompt (required)\\r\\nstring\\r\\nThe next user prompt to continue the Codex conversation.\\r\\nthreadId (required)\\r\\nstring\\r\\nThe ID of the thread to continue.\\r\\nconversationId (deprecated)\\r\\nstring\\r\\nDeprecated alias for threadId (kept for compatibility).\\r\\nUse the threadId from structuredContent.threadId in the tools/call response. Approval prompts (exec/patch) also include threadId in their params payload.\\r\\nExample response payload:\\r\\nNote modern MCP clients generally report only \\\"structuredContent\\\" as the result of a tool call, if present, though the Codex MCP server also returns \\\"content\\\" for the benefit of older MCP clients.\", \"matchLevel\": \"full\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\", \"tool\", \"responses\", \"api\", \"require\", \"approval\", \"option\"]}, \"hierarchy\": {\"lvl0\": {\"value\": \"Documentation\", \"matchLevel\": \"none\", \"matchedWords\": []}, \"lvl1\": {\"value\": \"Running Codex as an MCP server\", \"matchLevel\": \"partial\", \"fullyHighlighted\": false, \"matchedWords\": [\"mcp\"]}}}}], \"nbHits\": 9, \"page\": 0}", + "server_label": "openai_docs" + }, + { + "id": "rs_07637789c765520d0169cc4662dc388195b8951c0e84ab7424", + "type": "reasoning", + "summary": [] + }, + { + "id": "mcp_07637789c765520d0169cc466603788195b7187d1bdf523b44", + "type": "mcp_call", + "status": "completed", + "approval_request_id": null, + "arguments": "{\"url\": \"https://developers.openai.com/api/docs/guides/realtime-mcp/\", \"anchor\": \"configure-an-mcp-tool\"}", + "error": null, + "name": "fetch_openai_doc", + "output": "## Configure an MCP tool\n\nAdd MCP tools in **one of two places**:\n\n- At the **session level** with `session.tools` in [`session.update`](https://developers.openai.com/api/docs/api-reference/realtime-client-events/session/update), if you want the server available for the full session.\n- At the **response level** with `response.tools` in [`response.create`](https://developers.openai.com/api/docs/api-reference/realtime-client-events/response/create), if you only need MCP for one turn.\n\nIn Realtime, the MCP tool shape is:\n\n- `type: \"mcp\"`\n- `server_label`\n- One of `server_url` or `connector_id`\n- Optional `authorization` and `headers`\n- Optional `allowed_tools`\n- Optional `require_approval`\n- Optional `server_description`\n\nThis example makes a docs MCP server available for the full session:\n\nConfigure an MCP tool with session.update\n\n```javascript\nconst event = {\n type: \"session.update\",\n session: {\n type: \"realtime\",\n model: \"gpt-realtime-1.5\",\n output_modalities: [\"text\"],\n tools: [\n {\n type: \"mcp\",\n server_label: \"openai_docs\",\n server_url: \"https://developers.openai.com/mcp\",\n allowed_tools: [\"search_openai_docs\", \"fetch_openai_doc\"],\n require_approval: \"never\",\n },\n ],\n },\n};\n\nws.send(JSON.stringify(event));\n```\n\n```python\nevent = {\n \"type\": \"session.update\",\n \"session\": {\n \"type\": \"realtime\",\n \"model\": \"gpt-realtime-1.5\",\n \"output_modalities\": [\"text\"],\n \"tools\": [\n {\n \"type\": \"mcp\",\n \"server_label\": \"openai_docs\",\n \"server_url\": \"https://developers.openai.com/mcp\",\n \"allowed_tools\": [\"search_openai_docs\", \"fetch_openai_doc\"],\n \"require_approval\": \"never\",\n }\n ],\n },\n}\n\nws.send(json.dumps(event))\n```\n\n\nBuilt-in connectors use the same MCP tool shape, but pass `connector_id`\ninstead of `server_url`. For example, Google Calendar uses\n`connector_googlecalendar`. In Realtime, use these built-in connectors for read\nactions, such as searching or reading events or emails. Pass the user's OAuth\naccess token in `authorization`, and narrow the tool surface with\n`allowed_tools` when possible:\n\nConfigure a Google Calendar connector\n\n```javascript\nconst event = {\n type: \"session.update\",\n session: {\n type: \"realtime\",\n model: \"gpt-realtime-1.5\",\n output_modalities: [\"text\"],\n tools: [\n {\n type: \"mcp\",\n server_label: \"google_calendar\",\n connector_id: \"connector_googlecalendar\",\n authorization: \"\",\n allowed_tools: [\"search_events\", \"read_event\"],\n require_approval: \"never\",\n },\n ],\n },\n};\n\nws.send(JSON.stringify(event));\n```\n\n```python\nevent = {\n \"type\": \"session.update\",\n \"session\": {\n \"type\": \"realtime\",\n \"model\": \"gpt-realtime-1.5\",\n \"output_modalities\": [\"text\"],\n \"tools\": [\n {\n \"type\": \"mcp\",\n \"server_label\": \"google_calendar\",\n \"connector_id\": \"connector_googlecalendar\",\n \"authorization\": \"\",\n \"allowed_tools\": [\"search_events\", \"read_event\"],\n \"require_approval\": \"never\",\n }\n ],\n },\n}\n\nws.send(json.dumps(event))\n```\n\n\nRemote MCP servers{\" \"}\n do not automatically receive the full conversation context,\n but they can see any data the model sends in a tool call.\n Keep the tool surface narrow with allowed_tools,\n and require approval for any action you would not auto-run.", + "server_label": "openai_docs" + }, + { + "id": "rs_07637789c765520d0169cc466751a08195946d8336bf33dcfe", + "type": "reasoning", + "summary": [] + }, + { + "id": "mcp_07637789c765520d0169cc46687fe08195ace0cea443ad7aca", + "type": "mcp_call", + "status": "completed", + "approval_request_id": null, + "arguments": "{\"url\": \"https://developers.openai.com/api/docs/guides/tools-connectors-mcp/\", \"anchor\": \"approvals\"}", + "error": null, + "name": "fetch_openai_doc", + "output": "#### Approvals\n\nBy default, OpenAI will request your approval before any data is shared with a connector or remote MCP server. Approvals help you maintain control and visibility over what data is being sent to an MCP server. We highly recommend that you carefully review (and optionally log) all data being shared with a remote MCP server. A request for an approval to make an MCP tool call creates a `mcp_approval_request` item in the Response's output that looks like this:\n\n```json\n{\n \"id\": \"mcpr_68a619e1d82c8190b50c1ccba7ad18ef0d2d23a86136d339\",\n \"type\": \"mcp_approval_request\",\n \"arguments\": \"{\\\"diceRollExpression\\\":\\\"2d4 + 1\\\"}\",\n \"name\": \"roll\",\n \"server_label\": \"dmcp\"\n}\n```\n\nYou can then respond to this by creating a new Response object and appending an `mcp_approval_response` item to it.\n\nApproving the use of tools in an API request\n\n```bash\ncurl https://api.openai.com/v1/responses \\\\\n-H \"Content-Type: application/json\" \\\\\n-H \"Authorization: Bearer $OPENAI_API_KEY\" \\\\\n-d '{\n \"model\": \"gpt-5\",\n \"tools\": [\n {\n \"type\": \"mcp\",\n \"server_label\": \"dmcp\",\n \"server_description\": \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n \"server_url\": \"https://dmcp-server.deno.dev/sse\",\n \"require_approval\": \"always\",\n }\n ],\n \"previous_response_id\": \"resp_682d498bdefc81918b4a6aa477bfafd904ad1e533afccbfa\",\n \"input\": [{\n \"type\": \"mcp_approval_response\",\n \"approve\": true,\n \"approval_request_id\": \"mcpr_682d498e3bd4819196a0ce1664f8e77b04ad1e533afccbfa\"\n }]\n }'\n```\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst resp = await client.responses.create({\n model: \"gpt-5\",\n tools: [{\n type: \"mcp\",\n server_label: \"dmcp\",\n server_description: \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n server_url: \"https://dmcp-server.deno.dev/sse\",\n require_approval: \"always\",\n }],\n previous_response_id: \"resp_682d498bdefc81918b4a6aa477bfafd904ad1e533afccbfa\",\n input: [{\n type: \"mcp_approval_response\",\n approve: true,\n approval_request_id: \"mcpr_682d498e3bd4819196a0ce1664f8e77b04ad1e533afccbfa\"\n }],\n});\n\nconsole.log(resp.output_text);\n```\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI()\n\nresp = client.responses.create(\n model=\"gpt-5\",\n tools=[{\n \"type\": \"mcp\",\n \"server_label\": \"dmcp\",\n \"server_description\": \"A Dungeons and Dragons MCP server to assist with dice rolling.\",\n \"server_url\": \"https://dmcp-server.deno.dev/sse\",\n \"require_approval\": \"always\",\n }],\n previous_response_id=\"resp_682d498bdefc81918b4a6aa477bfafd904ad1e533afccbfa\",\n input=[{\n \"type\": \"mcp_approval_response\",\n \"approve\": True,\n \"approval_request_id\": \"mcpr_682d498e3bd4819196a0ce1664f8e77b04ad1e533afccbfa\"\n }],\n)\n\nprint(resp.output_text)\n```\n\n```csharp\nusing OpenAI.Responses;\n\nstring key = Environment.GetEnvironmentVariable(\"OPENAI_API_KEY\")!;\nOpenAIResponseClient client = new(model: \"gpt-5\", apiKey: key);\n\nResponseCreationOptions options = new();\noptions.Tools.Add(ResponseTool.CreateMcpTool(\n serverLabel: \"dmcp\",\n serverUri: new Uri(\"https://dmcp-server.deno.dev/sse\"),\n toolCallApprovalPolicy: new McpToolCallApprovalPolicy(GlobalMcpToolCallApprovalPolicy.AlwaysRequireApproval)\n));\n\n// STEP 1: Create response that requests tool call approval\nOpenAIResponse response1 = (OpenAIResponse)client.CreateResponse([\n ResponseItem.CreateUserMessageItem([\n ResponseContentPart.CreateInputTextPart(\"Roll 2d4+1\")\n ])\n], options);\n\nMcpToolCallApprovalRequestItem? approvalRequestItem = response1.OutputItems.Last() as McpToolCallApprovalRequestItem;\n\n// STEP 2: Approve the tool call request and get final response\noptions.PreviousResponseId = response1.Id;\nOpenAIResponse response2 = (OpenAIResponse)client.CreateResponse([\n ResponseItem.CreateMcpApprovalResponseItem(approvalRequestItem!.Id, approved: true),\n], options);\n\nConsole.WriteLine(response2.GetOutputText());\n```\n\n\nHere we're using the `previous_response_id` parameter to chain this new Response, with the previous Response that generated the approval request. But you can also pass back the [outputs from one response, as inputs into another](https://developers.openai.com/api/docs/guides/conversation-state#manually-manage-conversation-state) for maximum control over what enter's the model's context.\n\nIf and when you feel comfortable trusting a remote MCP server, you can choose to skip the approvals for reduced latency. To do this, you can set the `require_approval` parameter of the MCP tool to an object listing just the tools you'd like to skip approvals for like shown below, or set it to the value `'never'` to skip approvals for all tools in that remote MCP server.\n\nNever require approval for some tools\n\n```bash\ncurl https://api.openai.com/v1/responses \\\\\n-H \"Content-Type: application/json\" \\\\\n-H \"Authorization: Bearer $OPENAI_API_KEY\" \\\\\n-d '{\n \"model\": \"gpt-5\",\n \"tools\": [\n {\n \"type\": \"mcp\",\n \"server_label\": \"deepwiki\",\n \"server_url\": \"https://mcp.deepwiki.com/mcp\",\n \"require_approval\": {\n \"never\": {\n \"tool_names\": [\"ask_question\", \"read_wiki_structure\"]\n }\n }\n }\n ],\n \"input\": \"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?\"\n }'\n```\n\n```javascript\nimport OpenAI from \"openai\";\nconst client = new OpenAI();\n\nconst resp = await client.responses.create({\n model: \"gpt-5\",\n tools: [\n {\n type: \"mcp\",\n server_label: \"deepwiki\",\n server_url: \"https://mcp.deepwiki.com/mcp\",\n require_approval: {\n never: {\n tool_names: [\"ask_question\", \"read_wiki_structure\"]\n }\n }\n },\n ],\n input: \"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?\",\n});\n\nconsole.log(resp.output_text);\n```\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI()\n\nresp = client.responses.create(\n model=\"gpt-5\",\n tools=[\n {\n \"type\": \"mcp\",\n \"server_label\": \"deepwiki\",\n \"server_url\": \"https://mcp.deepwiki.com/mcp\",\n \"require_approval\": {\n \"never\": {\n \"tool_names\": [\"ask_question\", \"read_wiki_structure\"]\n }\n }\n },\n ],\n input=\"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?\",\n)\n\nprint(resp.output_text)\n```\n\n```csharp\nusing OpenAI.Responses;\n\nstring key = Environment.GetEnvironmentVariable(\"OPENAI_API_KEY\")!;\nOpenAIResponseClient client = new(model: \"gpt-5\", apiKey: key);\n\nResponseCreationOptions options = new();\noptions.Tools.Add(ResponseTool.CreateMcpTool(\n serverLabel: \"deepwiki\",\n serverUri: new Uri(\"https://mcp.deepwiki.com/mcp\"),\n allowedTools: new McpToolFilter() { ToolNames = { \"ask_question\", \"read_wiki_structure\" } },\n toolCallApprovalPolicy: new McpToolCallApprovalPolicy(GlobalMcpToolCallApprovalPolicy.NeverRequireApproval)\n));\n\nOpenAIResponse response = (OpenAIResponse)client.CreateResponse([\n ResponseItem.CreateUserMessageItem([\n ResponseContentPart.CreateInputTextPart(\"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?\")\n ])\n], options);\n\nConsole.WriteLine(response.GetOutputText());\n```", + "server_label": "openai_docs" + }, + { + "id": "rs_07637789c765520d0169cc4669df708195b0609e49af82ed33", + "type": "reasoning", + "summary": [] + }, + { + "id": "msg_07637789c765520d0169cc4682b31c8195bada61f0894f4169", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "Summary of require_approval for the Responses API MCP tool\n\nWhat it controls\n- Whether the Responses API pauses to ask for your approval before executing a tool call to a remote MCP server or connector.\n- When approval is required, the API inserts an mcp_approval_request item; you must reply with an mcp_approval_response (approve: true/false) to continue.\n\nDefault\n- By default, approval is required before any data is sent to a remote MCP server/connector.\n\nWhere to set it\n- In each MCP tool entry you pass to responses.create (or session.update for Realtime). It\u2019s a property on the MCP tool shape alongside type, server_url/connector_id, allowed_tools, etc.\n\nValid values and behavior\n- \"always\": Always require approval for tool calls (explicit form of the default).\n- \"never\": Never require approval for any tools from that MCP server (no approval prompts).\n- Per-tool skip: An object that whitelists specific tool names to bypass approval:\n - require_approval: { \"never\": { \"tool_names\": [\"tool_a\", \"tool_b\"] } }\n - Those tools won\u2019t prompt for approval; all others still will.\n\nFlow notes\n- If approval is required, you\u2019ll receive an mcp_approval_request in the response output. Approve or reject by sending an mcp_approval_response (you can chain via previous_response_id).\n\nDocs used\n- MCP and Connectors \u2013 Approvals: https://developers.openai.com/api/docs/guides/tools-connectors-mcp/#approvals\n- Realtime API with MCP \u2013 Configure an MCP tool (shows tool shape with require_approval): https://developers.openai.com/api/docs/guides/realtime-mcp/#configure-an-mcp-tool\n- Realtime API with MCP \u2013 Approve or reject MCP tool calls: https://developers.openai.com/api/docs/guides/realtime-mcp/#approve-or-reject-mcp-tool-calls" + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": "medium", + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [ + { + "type": "mcp", + "allowed_tools": null, + "headers": null, + "require_approval": "never", + "server_description": "OpenAI documentation MCP server.", + "server_label": "openai_docs", + "server_url": "https://developers.openai.com/mcp" + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 35835, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 1907, + "output_tokens_details": { + "reasoning_tokens": 1280 + }, + "total_tokens": 37742 + }, + "user": null, + "metadata": {} +}� +-- end response body --