From 538039f583ed677a572cb3504f53df5a00c5dda9 Mon Sep 17 00:00:00 2001 From: kyinhub Date: Thu, 29 Jan 2026 21:14:52 -0800 Subject: [PATCH] feat(translator): add code_execution and url_context tool passthrough Add support for Gemini's code_execution and url_context tools in the request translators, enabling: - Agentic Vision: Image analysis with Python code execution for bounding boxes, annotations, and visual reasoning - URL Context: Live web page content fetching and analysis Tools are passed through using the same pattern as google_search: - code_execution: {} -> codeExecution: {} - url_context: {} -> urlContext: {} Tested with Gemini 3 Flash Preview agentic vision successfully. Co-Authored-By: Claude Opus 4.5 --- .../antigravity_openai_request.go | 32 +++++++++++++++++-- .../gemini-cli_openai_request.go | 32 +++++++++++++++++-- .../chat-completions/gemini_openai_request.go | 32 +++++++++++++++++-- 3 files changed, 90 insertions(+), 6 deletions(-) diff --git a/internal/translator/antigravity/openai/chat-completions/antigravity_openai_request.go b/internal/translator/antigravity/openai/chat-completions/antigravity_openai_request.go index f2cb04d6..9cc809ee 100644 --- a/internal/translator/antigravity/openai/chat-completions/antigravity_openai_request.go +++ b/internal/translator/antigravity/openai/chat-completions/antigravity_openai_request.go @@ -305,12 +305,14 @@ func ConvertOpenAIRequestToAntigravity(modelName string, inputRawJSON []byte, _ } } - // tools -> request.tools[].functionDeclarations + request.tools[].googleSearch passthrough + // tools -> request.tools[].functionDeclarations + request.tools[].googleSearch/codeExecution/urlContext passthrough tools := gjson.GetBytes(rawJSON, "tools") if tools.IsArray() && len(tools.Array()) > 0 { functionToolNode := []byte(`{}`) hasFunction := false googleSearchNodes := make([][]byte, 0) + codeExecutionNodes := make([][]byte, 0) + urlContextNodes := make([][]byte, 0) for _, t := range tools.Array() { if t.Get("type").String() == "function" { fn := t.Get("function") @@ -370,8 +372,28 @@ func ConvertOpenAIRequestToAntigravity(modelName string, inputRawJSON []byte, _ } googleSearchNodes = append(googleSearchNodes, googleToolNode) } + if ce := t.Get("code_execution"); ce.Exists() { + codeToolNode := []byte(`{}`) + var errSet error + codeToolNode, errSet = sjson.SetRawBytes(codeToolNode, "codeExecution", []byte(ce.Raw)) + if errSet != nil { + log.Warnf("Failed to set codeExecution tool: %v", errSet) + continue + } + codeExecutionNodes = append(codeExecutionNodes, codeToolNode) + } + if uc := t.Get("url_context"); uc.Exists() { + urlToolNode := []byte(`{}`) + var errSet error + urlToolNode, errSet = sjson.SetRawBytes(urlToolNode, "urlContext", []byte(uc.Raw)) + if errSet != nil { + log.Warnf("Failed to set urlContext tool: %v", errSet) + continue + } + urlContextNodes = append(urlContextNodes, urlToolNode) + } } - if hasFunction || len(googleSearchNodes) > 0 { + if hasFunction || len(googleSearchNodes) > 0 || len(codeExecutionNodes) > 0 || len(urlContextNodes) > 0 { toolsNode := []byte("[]") if hasFunction { toolsNode, _ = sjson.SetRawBytes(toolsNode, "-1", functionToolNode) @@ -379,6 +401,12 @@ func ConvertOpenAIRequestToAntigravity(modelName string, inputRawJSON []byte, _ for _, googleNode := range googleSearchNodes { toolsNode, _ = sjson.SetRawBytes(toolsNode, "-1", googleNode) } + for _, codeNode := range codeExecutionNodes { + toolsNode, _ = sjson.SetRawBytes(toolsNode, "-1", codeNode) + } + for _, urlNode := range urlContextNodes { + toolsNode, _ = sjson.SetRawBytes(toolsNode, "-1", urlNode) + } out, _ = sjson.SetRawBytes(out, "request.tools", toolsNode) } } diff --git a/internal/translator/gemini-cli/openai/chat-completions/gemini-cli_openai_request.go b/internal/translator/gemini-cli/openai/chat-completions/gemini-cli_openai_request.go index 6351fa58..2351130f 100644 --- a/internal/translator/gemini-cli/openai/chat-completions/gemini-cli_openai_request.go +++ b/internal/translator/gemini-cli/openai/chat-completions/gemini-cli_openai_request.go @@ -283,12 +283,14 @@ func ConvertOpenAIRequestToGeminiCLI(modelName string, inputRawJSON []byte, _ bo } } - // tools -> request.tools[].functionDeclarations + request.tools[].googleSearch passthrough + // tools -> request.tools[].functionDeclarations + request.tools[].googleSearch/codeExecution/urlContext passthrough tools := gjson.GetBytes(rawJSON, "tools") if tools.IsArray() && len(tools.Array()) > 0 { functionToolNode := []byte(`{}`) hasFunction := false googleSearchNodes := make([][]byte, 0) + codeExecutionNodes := make([][]byte, 0) + urlContextNodes := make([][]byte, 0) for _, t := range tools.Array() { if t.Get("type").String() == "function" { fn := t.Get("function") @@ -348,8 +350,28 @@ func ConvertOpenAIRequestToGeminiCLI(modelName string, inputRawJSON []byte, _ bo } googleSearchNodes = append(googleSearchNodes, googleToolNode) } + if ce := t.Get("code_execution"); ce.Exists() { + codeToolNode := []byte(`{}`) + var errSet error + codeToolNode, errSet = sjson.SetRawBytes(codeToolNode, "codeExecution", []byte(ce.Raw)) + if errSet != nil { + log.Warnf("Failed to set codeExecution tool: %v", errSet) + continue + } + codeExecutionNodes = append(codeExecutionNodes, codeToolNode) + } + if uc := t.Get("url_context"); uc.Exists() { + urlToolNode := []byte(`{}`) + var errSet error + urlToolNode, errSet = sjson.SetRawBytes(urlToolNode, "urlContext", []byte(uc.Raw)) + if errSet != nil { + log.Warnf("Failed to set urlContext tool: %v", errSet) + continue + } + urlContextNodes = append(urlContextNodes, urlToolNode) + } } - if hasFunction || len(googleSearchNodes) > 0 { + if hasFunction || len(googleSearchNodes) > 0 || len(codeExecutionNodes) > 0 || len(urlContextNodes) > 0 { toolsNode := []byte("[]") if hasFunction { toolsNode, _ = sjson.SetRawBytes(toolsNode, "-1", functionToolNode) @@ -357,6 +379,12 @@ func ConvertOpenAIRequestToGeminiCLI(modelName string, inputRawJSON []byte, _ bo for _, googleNode := range googleSearchNodes { toolsNode, _ = sjson.SetRawBytes(toolsNode, "-1", googleNode) } + for _, codeNode := range codeExecutionNodes { + toolsNode, _ = sjson.SetRawBytes(toolsNode, "-1", codeNode) + } + for _, urlNode := range urlContextNodes { + toolsNode, _ = sjson.SetRawBytes(toolsNode, "-1", urlNode) + } out, _ = sjson.SetRawBytes(out, "request.tools", toolsNode) } } diff --git a/internal/translator/gemini/openai/chat-completions/gemini_openai_request.go b/internal/translator/gemini/openai/chat-completions/gemini_openai_request.go index 0a35cfd0..a7c20852 100644 --- a/internal/translator/gemini/openai/chat-completions/gemini_openai_request.go +++ b/internal/translator/gemini/openai/chat-completions/gemini_openai_request.go @@ -289,12 +289,14 @@ func ConvertOpenAIRequestToGemini(modelName string, inputRawJSON []byte, _ bool) } } - // tools -> tools[].functionDeclarations + tools[].googleSearch passthrough + // tools -> tools[].functionDeclarations + tools[].googleSearch/codeExecution/urlContext passthrough tools := gjson.GetBytes(rawJSON, "tools") if tools.IsArray() && len(tools.Array()) > 0 { functionToolNode := []byte(`{}`) hasFunction := false googleSearchNodes := make([][]byte, 0) + codeExecutionNodes := make([][]byte, 0) + urlContextNodes := make([][]byte, 0) for _, t := range tools.Array() { if t.Get("type").String() == "function" { fn := t.Get("function") @@ -354,8 +356,28 @@ func ConvertOpenAIRequestToGemini(modelName string, inputRawJSON []byte, _ bool) } googleSearchNodes = append(googleSearchNodes, googleToolNode) } + if ce := t.Get("code_execution"); ce.Exists() { + codeToolNode := []byte(`{}`) + var errSet error + codeToolNode, errSet = sjson.SetRawBytes(codeToolNode, "codeExecution", []byte(ce.Raw)) + if errSet != nil { + log.Warnf("Failed to set codeExecution tool: %v", errSet) + continue + } + codeExecutionNodes = append(codeExecutionNodes, codeToolNode) + } + if uc := t.Get("url_context"); uc.Exists() { + urlToolNode := []byte(`{}`) + var errSet error + urlToolNode, errSet = sjson.SetRawBytes(urlToolNode, "urlContext", []byte(uc.Raw)) + if errSet != nil { + log.Warnf("Failed to set urlContext tool: %v", errSet) + continue + } + urlContextNodes = append(urlContextNodes, urlToolNode) + } } - if hasFunction || len(googleSearchNodes) > 0 { + if hasFunction || len(googleSearchNodes) > 0 || len(codeExecutionNodes) > 0 || len(urlContextNodes) > 0 { toolsNode := []byte("[]") if hasFunction { toolsNode, _ = sjson.SetRawBytes(toolsNode, "-1", functionToolNode) @@ -363,6 +385,12 @@ func ConvertOpenAIRequestToGemini(modelName string, inputRawJSON []byte, _ bool) for _, googleNode := range googleSearchNodes { toolsNode, _ = sjson.SetRawBytes(toolsNode, "-1", googleNode) } + for _, codeNode := range codeExecutionNodes { + toolsNode, _ = sjson.SetRawBytes(toolsNode, "-1", codeNode) + } + for _, urlNode := range urlContextNodes { + toolsNode, _ = sjson.SetRawBytes(toolsNode, "-1", urlNode) + } out, _ = sjson.SetRawBytes(out, "tools", toolsNode) } }