diff --git a/agentplatform/_genai/_evals_builtin_tools.py b/agentplatform/_genai/_evals_builtin_tools.py new file mode 100644 index 0000000000..30df6994ba --- /dev/null +++ b/agentplatform/_genai/_evals_builtin_tools.py @@ -0,0 +1,228 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Built-in tool catalog for Gemini Agent evaluation display. + +The Gemini Agents API (``GET agents/{id}``) returns each tool as a bare type +discriminator (e.g. ``{"type": "code_execution"}``) with no parameter schema +or description. The authoritative, full-fidelity expansion lives server-side +in ``cloud/ai/platform/evaluation/utils/interaction_converter.py``. + +This module is a **display-only duplicate** of that server catalog, kept here +so ``show()`` can render tools with full names, descriptions, and parameter +schemas without a server round-trip. + +**If the server catalog changes, this SDK-side copy must be updated to match.** +""" + +from typing import Any, Optional + +from google.genai import types as genai_types + + +def _str_schema(description: str) -> genai_types.Schema: + return genai_types.Schema(type="STRING", description=description) + + +# Maps a built-in Gemini Agent tool type to the concrete FunctionDeclarations +# the agent actually exposes for that type. +# +# Source of truth: interaction_converter.py, _BUILTIN_TOOL_FUNCTION_DECLARATIONS +BUILTIN_TOOL_DECLARATIONS: dict[str, list[genai_types.FunctionDeclaration]] = { + "code_execution": [ + genai_types.FunctionDeclaration( + name="run_command", + description=( + "Runs a shell command on the sandbox VM. If the command does" + " not complete within WaitMsBeforeAsync, it is sent to the" + " background and a CommandId is returned for use with" + " command_status." + ), + parameters=genai_types.Schema( + type="OBJECT", + properties={ + "CommandLine": _str_schema("The shell command to run."), + "Cwd": _str_schema( + "The current working directory for the command." + ), + "WaitMsBeforeAsync": genai_types.Schema( + type="INTEGER", + description=( + "Milliseconds to wait for the command to complete" + " before sending it to the background. Default:" + " 10000." + ), + ), + "SafeToAutoRun": genai_types.Schema( + type="BOOLEAN", + description=( + "Whether the command is safe to auto-run without" + " user approval." + ), + ), + }, + required=["CommandLine", "Cwd"], + ), + ), + ], + "filesystem": [ + genai_types.FunctionDeclaration( + name="view_file", + description="Reads the content of a workspace file.", + ), + genai_types.FunctionDeclaration( + name="create_file", + description="Writes content to a new or existing file.", + ), + genai_types.FunctionDeclaration( + name="edit_file", + description="Replaces a specific block of text in a file.", + ), + genai_types.FunctionDeclaration( + name="list_dir", + description="Lists the files in a directory.", + ), + genai_types.FunctionDeclaration( + name="delete_file", + description="Removes a file from the workspace.", + ), + genai_types.FunctionDeclaration( + name="move_file", + description="Renames or moves a file.", + ), + ], +} + + +# Sandbox-environment orchestration tool declarations. +# +# Source of truth: interaction_converter.py, _SANDBOX_FUNCTION_DECLARATIONS +SANDBOX_DECLARATIONS: list[genai_types.FunctionDeclaration] = [ + genai_types.FunctionDeclaration( + name="provision_sandbox", + description="Provisions a sandbox environment.", + parameters=genai_types.Schema( + type="OBJECT", + properties={ + "display_name": _str_schema( + "The display name of the sandbox environment." + ), + "poll_creation_lro": genai_types.Schema( + type="BOOLEAN", + description=( + "Whether to poll the creation long-running operation." + ), + ), + }, + ), + ), + genai_types.FunctionDeclaration( + name="load_sandbox", + description="Loads a previously provisioned sandbox environment.", + parameters=genai_types.Schema( + type="OBJECT", + properties={ + "reasoning_engine_resource_name": _str_schema( + "The resource name of the reasoning engine. Format:" + " projects/{project}/locations/{location}/reasoningEngines/{id}" + ), + "display_name": _str_schema( + "The display name of the sandbox environment. Format: any" + " string." + ), + }, + ), + ), +] + + +def agent_tools_to_config_tools( + agent_tools: Optional[list[Any]], + has_environment: bool = False, +) -> Optional[list[genai_types.Tool]]: + """Maps Gemini Agents API tools to ``genai_types.Tool`` for display. + + Expands built-in agent tool types into their concrete function declarations + using ``BUILTIN_TOOL_DECLARATIONS`` (a display-only duplicate of the + server-side catalog in ``interaction_converter.py``). + + Mapping rules: + * ``code_execution`` is expanded to ``run_command`` with full parameter + schema. + * ``filesystem`` is expanded to ``view_file``, ``create_file``, + ``edit_file``, ``list_dir``, ``delete_file``, ``move_file``. + * ``google_search`` and ``url_context`` are mapped to their typed + ``genai_types.Tool`` variant. + * ``mcp_server`` is represented as a named declaration with a + human-readable label. + * Tools carrying explicit ``function_declarations`` are passed through. + * When ``has_environment`` is True, sandbox orchestration tools + (``provision_sandbox``, ``load_sandbox``) are appended. + + Args: + agent_tools: The ``tools`` list from a fetched Gemini agent dict. + has_environment: Whether the agent has a sandbox environment configured. + + Returns: + A list of ``genai_types.Tool``, or ``None`` if there are no mappable + tools. + """ + if not agent_tools and not has_environment: + return None + tools: list[genai_types.Tool] = [] + for tool in agent_tools or []: + if not isinstance(tool, dict): + continue + tool_type = tool.get("type") + remainder = {k: v for k, v in tool.items() if k != "type"} + + # Check the built-in catalog first (code_execution, filesystem). + catalog_decls = BUILTIN_TOOL_DECLARATIONS.get(tool_type or "") + if catalog_decls: + tools.append(genai_types.Tool(function_declarations=list(catalog_decls))) + elif tool_type == "google_search": + tools.append(genai_types.Tool(google_search=genai_types.GoogleSearch())) + elif tool_type == "url_context": + tools.append(genai_types.Tool(url_context=genai_types.UrlContext())) + elif "function_declarations" in remainder: + # Real function tool with explicit declarations. + tools.append(genai_types.Tool.model_validate(remainder)) + elif tool_type == "mcp_server": + label = remainder.get("name") or remainder.get("url") + description = f"MCP server: {label}" if label else "MCP server." + tools.append( + genai_types.Tool( + function_declarations=[ + genai_types.FunctionDeclaration( + name="mcp_server", description=description + ) + ] + ) + ) + elif tool_type: + # Unknown built-in: show by name so it isn't silently dropped. + tools.append( + genai_types.Tool( + function_declarations=[ + genai_types.FunctionDeclaration(name=tool_type) + ] + ) + ) + elif remainder: + tools.append(genai_types.Tool.model_validate(remainder)) + + if has_environment: + tools.append(genai_types.Tool(function_declarations=list(SANDBOX_DECLARATIONS))) + + return tools or None diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index f33ceb7497..42b265bbfa 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -42,6 +42,7 @@ from tqdm import tqdm from pydantic import ValidationError +from . import _evals_builtin_tools from . import _evals_constant from . import _evals_data_converters from . import _evals_metric_handlers @@ -483,6 +484,7 @@ def _get_default_prompt_template( and eval_item.evaluation_request and eval_item.evaluation_request.prompt and eval_item.evaluation_request.prompt.prompt_template_data + and eval_item.evaluation_request.prompt.prompt_template_data.values ): if ( "prompt" @@ -837,48 +839,7 @@ def _merge_text_parts_in_agent_data( content.parts = merged_parts -def _agent_tools_to_config_tools( - agent_tools: Optional[list[Any]], -) -> Optional[list[genai_types.Tool]]: - """Maps Gemini Agents API tools to ``genai_types.Tool`` for an AgentConfig. - - The Gemini Agents API returns built-in tool variants (``google_search``, - ``code_execution``, ``url_context``) whose schema differs from - ``genai_types.Tool``. Each recognised built-in variant is mapped to the - matching ``genai_types.Tool`` field. Tools with a non-empty body after - stripping the ``type`` key (e.g. ``function_declarations``) are passed - through ``model_validate``. Variants without a ``genai_types.Tool`` - equivalent (e.g. ``filesystem``, ``mcp_server``) are skipped. - - Args: - agent_tools: The ``tools`` list from a fetched Gemini agent dict. - - Returns: - A list of ``genai_types.Tool``, or ``None`` if there are no mappable - tools. - """ - if not agent_tools: - return None - tools: list[genai_types.Tool] = [] - for tool in agent_tools: - if not isinstance(tool, dict): - continue - tool_type = tool.get("type") - if tool_type == "google_search": - tools.append(genai_types.Tool(google_search=genai_types.GoogleSearch())) - elif tool_type == "code_execution": - tools.append( - genai_types.Tool(code_execution=genai_types.ToolCodeExecution()) - ) - elif tool_type == "url_context": - tools.append(genai_types.Tool(url_context=genai_types.UrlContext())) - else: - # For non-built-in tools (e.g. function_declarations), strip the - # type key and validate through genai_types.Tool. - remainder = {k: v for k, v in tool.items() if k != "type"} - if remainder: - tools.append(genai_types.Tool.model_validate(remainder)) - return tools or None +_agent_tools_to_config_tools = _evals_builtin_tools.agent_tools_to_config_tools def _fetch_agent_config_dict( @@ -916,7 +877,13 @@ def _fetch_agent_config_dict( instruction = agent_dict.get("system_instruction") or None description = agent_dict.get("description") or None agent_type = agent_dict.get("base_agent") or None - tools = _agent_tools_to_config_tools(agent_dict.get("tools")) + has_environment = bool( + agent_dict.get("environment_config") + or agent_dict.get("base_environment") + ) + tools = _agent_tools_to_config_tools( + agent_dict.get("tools"), has_environment=has_environment + ) except Exception as e: # pylint: disable=broad-exception-caught logger.warning( "Failed to fetch agent config for '%s' (continuing without it): %s", @@ -988,33 +955,6 @@ def _agent_data_response_text(agent_data: types.evals.AgentData) -> Optional[str return "".join(text_parts) or None -def _agent_resource_to_agent_info( - agent: str, api_client: BaseApiClient -) -> "types.evals.AgentInfo": - """Builds an `AgentInfo` from a Gemini Agents API agent resource name. - - Fetches the agent through the SDK's `api_client` (so replay recording is - preserved) via `_fetch_agent_config_dict` and derives a single-agent - `AgentInfo`: the agent's short name is the agents-map key and - `root_agent_id`. - - Args: - agent: The Gemini Agents API agent resource name - (`projects/{p}/locations/{l}/agents/{name}`). - api_client: The API client used to fetch the agent. - - Returns: - An `AgentInfo` describing the fetched agent. - """ - agent_config = _fetch_agent_config_dict(api_client, agent) - short_name = agent_config.agent_id - return types.evals.AgentInfo( # pytype: disable=missing-parameter - name=short_name, - agents={short_name: agent_config}, - root_agent_id=short_name, - ) - - _INTERACTION_TERMINAL_STATES = frozenset( ["completed", "failed", "cancelled", "incomplete", "budget_exceeded"] ) @@ -1108,6 +1048,11 @@ def _run_gemini_agent_inference( interactions_client = _get_interactions_client(api_client) + # Best-effort: fetch the agent config (instruction, tools, description) + # once, so every row's agent_data carries the agents map and the display + # can render the System Topology section. + agent_config = _fetch_agent_config_dict(api_client, gemini_agent) + agent_short_id = gemini_agent.split("/")[-1] prompts: list[str] = [] responses: list[Optional[str]] = [] @@ -1128,6 +1073,7 @@ def _run_gemini_agent_inference( ) interaction = _await_interaction(interactions_client, interaction) agent_data_obj = _interaction_dict_to_agent_data(interaction) + agent_data_obj.agents = {agent_config.agent_id: agent_config} _merge_text_parts_in_agent_data(agent_data_obj) responses.append(_agent_data_response_text(agent_data_obj)) interaction_ids.append(interaction.get("id")) diff --git a/agentplatform/_genai/_evals_visualization.py b/agentplatform/_genai/_evals_visualization.py index 45b50b6ebc..dc0ab0e7a1 100644 --- a/agentplatform/_genai/_evals_visualization.py +++ b/agentplatform/_genai/_evals_visualization.py @@ -372,19 +372,36 @@ def get_evaluation_html(eval_result_json: str) -> str: function formatToolDeclarations(toolDeclarations) {{ if (!toolDeclarations) return ''; let functions = []; - if (Array.isArray(toolDeclarations)) {{ - toolDeclarations.forEach(tool => {{ - if (tool.function_declarations) {{ - functions = functions.concat(tool.function_declarations); - }} else if (tool.name && tool.parameters) {{ - functions.push(tool); - }} + const builtins = []; + + function collectFromTool(tool) {{ + if (!tool || typeof tool !== 'object') return; + if (Array.isArray(tool.function_declarations) && tool.function_declarations.length > 0) {{ + functions = functions.concat(tool.function_declarations); + return; + }} + if (tool.name && tool.parameters) {{ + functions.push(tool); + return; + }} + Object.keys(tool).forEach(k => {{ + if (k === 'function_declarations') return; + if (tool[k] === null || tool[k] === undefined) return; + builtins.push(k); }}); - }} else if (typeof toolDeclarations === 'object' && toolDeclarations.function_declarations) {{ - functions = toolDeclarations.function_declarations; }} - if (functions.length === 0) {{ + if (Array.isArray(toolDeclarations)) {{ + toolDeclarations.forEach(collectFromTool); + }} else if (typeof toolDeclarations === 'object') {{ + if (toolDeclarations.function_declarations) {{ + functions = functions.concat(toolDeclarations.function_declarations); + }} else {{ + collectFromTool(toolDeclarations); + }} + }} + + if (functions.length === 0 && builtins.length === 0) {{ return `
${{DOMPurify.sanitize(JSON.stringify(toolDeclarations, null, 2))}}
`; }} @@ -402,6 +419,9 @@ def get_evaluation_html(eval_result_json: str) -> str: }}); html += ''; }}); + builtins.forEach(name => {{ + html += `
${{DOMPurify.sanitize(name)}} (built-in tool)
`; + }}); html += ''; return html; }} @@ -914,19 +934,36 @@ def get_comparison_html(eval_result_json: str) -> str: function formatToolDeclarations(toolDeclarations) {{ if (!toolDeclarations) return ''; let functions = []; - if (Array.isArray(toolDeclarations)) {{ - toolDeclarations.forEach(tool => {{ - if (tool.function_declarations) {{ - functions = functions.concat(tool.function_declarations); - }} else if (tool.name && tool.parameters) {{ - functions.push(tool); - }} + const builtins = []; + + function collectFromTool(tool) {{ + if (!tool || typeof tool !== 'object') return; + if (Array.isArray(tool.function_declarations) && tool.function_declarations.length > 0) {{ + functions = functions.concat(tool.function_declarations); + return; + }} + if (tool.name && tool.parameters) {{ + functions.push(tool); + return; + }} + Object.keys(tool).forEach(k => {{ + if (k === 'function_declarations') return; + if (tool[k] === null || tool[k] === undefined) return; + builtins.push(k); }}); - }} else if (typeof toolDeclarations === 'object' && toolDeclarations.function_declarations) {{ - functions = toolDeclarations.function_declarations; }} - if (functions.length === 0) {{ + if (Array.isArray(toolDeclarations)) {{ + toolDeclarations.forEach(collectFromTool); + }} else if (typeof toolDeclarations === 'object') {{ + if (toolDeclarations.function_declarations) {{ + functions = functions.concat(toolDeclarations.function_declarations); + }} else {{ + collectFromTool(toolDeclarations); + }} + }} + + if (functions.length === 0 && builtins.length === 0) {{ return `
${{DOMPurify.sanitize(JSON.stringify(toolDeclarations, null, 2))}}
`; }} @@ -944,6 +981,9 @@ def get_comparison_html(eval_result_json: str) -> str: }}); html += ''; }}); + builtins.forEach(name => {{ + html += `
${{DOMPurify.sanitize(name)}} (built-in tool)
`; + }}); html += ''; return html; }} diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index 63fe6d1858..c334c838bf 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -725,6 +725,11 @@ def _GenerateUserScenariosParameters_to_vertex( getv(from_object, ["allow_cross_region_model"]), ) + if getv(from_object, ["gemini_agent_config"]) is not None: + setv( + to_object, ["geminiAgentConfig"], getv(from_object, ["gemini_agent_config"]) + ) + return to_object @@ -1518,6 +1523,7 @@ def _generate_user_scenarios( ] = None, config: Optional[types.GenerateUserScenariosConfigOrDict] = None, allow_cross_region_model: Optional[bool] = None, + gemini_agent_config: Optional[types.GeminiAgentConfigOrDict] = None, ) -> types.GenerateUserScenariosResponse: """ Generates user scenarios for agent evaluation. @@ -1530,6 +1536,7 @@ def _generate_user_scenarios( user_scenario_generation_config=user_scenario_generation_config, config=config, allow_cross_region_model=allow_cross_region_model, + gemini_agent_config=gemini_agent_config, ) request_url_dict: Optional[dict[str, str]] @@ -2967,8 +2974,10 @@ def generate_conversation_scenarios( "`agent` must be a Gemini Agents API agent resource name of the" " form projects/{project}/locations/{location}/agents/{agent}." ) - parsed_agent_info = _evals_common._agent_resource_to_agent_info( - agent, self._api_client + response = self._generate_user_scenarios( + gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent), + user_scenario_generation_config=config, + allow_cross_region_model=allow_cross_region_model, ) else: parsed_agent_info = ( @@ -2976,12 +2985,12 @@ def generate_conversation_scenarios( if isinstance(agent_info, dict) else agent_info ) - response = self._generate_user_scenarios( - agents=parsed_agent_info.agents, - root_agent_id=parsed_agent_info.root_agent_id, - user_scenario_generation_config=config, - allow_cross_region_model=allow_cross_region_model, - ) + response = self._generate_user_scenarios( + agents=parsed_agent_info.agents, + root_agent_id=parsed_agent_info.root_agent_id, + user_scenario_generation_config=config, + allow_cross_region_model=allow_cross_region_model, + ) return _evals_utils._postprocess_user_scenarios_response(response) def generate_loss_clusters( @@ -3683,6 +3692,7 @@ async def _generate_user_scenarios( ] = None, config: Optional[types.GenerateUserScenariosConfigOrDict] = None, allow_cross_region_model: Optional[bool] = None, + gemini_agent_config: Optional[types.GeminiAgentConfigOrDict] = None, ) -> types.GenerateUserScenariosResponse: """ Generates user scenarios for agent evaluation. @@ -3695,6 +3705,7 @@ async def _generate_user_scenarios( user_scenario_generation_config=user_scenario_generation_config, config=config, allow_cross_region_model=allow_cross_region_model, + gemini_agent_config=gemini_agent_config, ) request_url_dict: Optional[dict[str, str]] @@ -4758,8 +4769,10 @@ async def generate_conversation_scenarios( "`agent` must be a Gemini Agents API agent resource name of the" " form projects/{project}/locations/{location}/agents/{agent}." ) - parsed_agent_info = _evals_common._agent_resource_to_agent_info( - agent, self._api_client + response = await self._generate_user_scenarios( + gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent), + user_scenario_generation_config=config, + allow_cross_region_model=allow_cross_region_model, ) else: parsed_agent_info = ( @@ -4767,12 +4780,12 @@ async def generate_conversation_scenarios( if isinstance(agent_info, dict) else agent_info ) - response = await self._generate_user_scenarios( - agents=parsed_agent_info.agents, - root_agent_id=parsed_agent_info.root_agent_id, - user_scenario_generation_config=config, - allow_cross_region_model=allow_cross_region_model, - ) + response = await self._generate_user_scenarios( + agents=parsed_agent_info.agents, + root_agent_id=parsed_agent_info.root_agent_id, + user_scenario_generation_config=config, + allow_cross_region_model=allow_cross_region_model, + ) return _evals_utils._postprocess_user_scenarios_response(response) async def generate_loss_clusters( diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index 79c577c7fd..8a7471487d 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -5501,6 +5501,11 @@ class _GenerateUserScenariosParameters(_common.BaseModel): default=None, description="""Opt-in flag to authorize cross-region routing for LLM models.""", ) + gemini_agent_config: Optional[GeminiAgentConfig] = Field( + default=None, + description="""If set, the server derives the agents map and root_agent_id + from the referenced Gemini Agent server-side.""", + ) class _GenerateUserScenariosParametersDict(TypedDict, total=False): @@ -5524,6 +5529,10 @@ class _GenerateUserScenariosParametersDict(TypedDict, total=False): allow_cross_region_model: Optional[bool] """Opt-in flag to authorize cross-region routing for LLM models.""" + gemini_agent_config: Optional[GeminiAgentConfigDict] + """If set, the server derives the agents map and root_agent_id + from the referenced Gemini Agent server-side.""" + _GenerateUserScenariosParametersOrDict = Union[ _GenerateUserScenariosParameters, _GenerateUserScenariosParametersDict diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 26cfb80e7f..9a311587e9 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -4147,11 +4147,24 @@ def test_run_inference_with_litellm_parsing( assert call_kwargs["model"] == "gpt-4o" pd.testing.assert_frame_equal(call_kwargs["prompt_dataset"], mock_df) + @mock.patch.object(_evals_common, "_fetch_agent_config_dict") @mock.patch.object(_evals_common, "_get_interactions_client") @mock.patch.object(_evals_utils, "EvalDatasetLoader") def test_run_inference_with_gemini_agent( - self, mock_eval_dataset_loader, mock_get_interactions_client + self, mock_eval_dataset_loader, mock_get_interactions_client, + mock_fetch_agent_config ): + mock_fetch_agent_config.return_value = ( + agentplatform_genai_types.evals.AgentConfig( + agent_id="test-agent", + instruction="You are helpful.", + tools=[ + genai_types.Tool( + code_execution=genai_types.ToolCodeExecution() + ), + ], + ) + ) mock_df = pd.DataFrame({"prompt": ["p1", "p2"]}) mock_eval_dataset_loader.return_value.load.return_value = mock_df.to_dict( orient="records" @@ -4210,12 +4223,20 @@ def make_interaction(interaction_id, prompt_text, output_text): assert turn_events[1]["content"]["role"] == "model" assert turn_events[1]["content"]["parts"][0]["text"] == "response 1" assert inference_result.candidate_name == "test-agent" + # agents map must be populated for System Topology rendering. + assert "agents" in agent_data_0 + assert "test-agent" in agent_data_0["agents"] + @mock.patch.object(_evals_common, "_fetch_agent_config_dict") @mock.patch.object(_evals_common, "_get_interactions_client") @mock.patch.object(_evals_utils, "EvalDatasetLoader") def test_run_inference_gemini_agent_continues_on_failure( - self, mock_eval_dataset_loader, mock_get_interactions_client + self, mock_eval_dataset_loader, mock_get_interactions_client, + mock_fetch_agent_config, ): + mock_fetch_agent_config.return_value = ( + agentplatform_genai_types.evals.AgentConfig(agent_id="test-agent") + ) mock_df = pd.DataFrame({"prompt": ["p1", "p2"]}) mock_eval_dataset_loader.return_value.load.return_value = mock_df.to_dict( orient="records" @@ -9213,19 +9234,9 @@ async def test_async_generate_conversation_scenarios(self): request_body = call_args[0][2] # Third positional arg is the request dict assert request_body.get("allowCrossRegionModel") is True - @mock.patch.object(_evals_common, "_fetch_agent_config_dict") - def test_generate_conversation_scenarios_from_gemini_agent( - self, mock_fetch_agent_config - ): - mock_fetch_agent_config.return_value = ( - agentplatform_genai_types.evals.AgentConfig( - agent_id="test-agent", - instruction="You are a helpful travel assistant.", - description="An agent that books flights.", - tools=[genai_types.Tool(google_search=genai_types.GoogleSearch())], - ) - ) - + def test_generate_conversation_scenarios_from_gemini_agent(self): + """When `agent` is a Gemini agent resource, gemini_agent_config is + forwarded to the server (no client-side synthesis).""" evals_module = evals.Evals(api_client_=self.mock_api_client) with mock.patch.object( @@ -9237,18 +9248,10 @@ def test_generate_conversation_scenarios_from_gemini_agent( config={"count": 2}, ) - mock_fetch_agent_config.assert_called_once_with( - self.mock_api_client, _TEST_GEMINI_AGENT - ) call_kwargs = mock_generate_user_scenarios.call_args.kwargs - assert call_kwargs["root_agent_id"] == "test-agent" - agents = call_kwargs["agents"] - assert "test-agent" in agents - derived_config = agents["test-agent"] - assert derived_config.instruction == "You are a helpful travel assistant." - assert derived_config.description == "An agent that books flights." - assert derived_config.tools is not None - assert derived_config.tools[0].google_search is not None + assert call_kwargs["gemini_agent_config"].gemini_agent == _TEST_GEMINI_AGENT + assert call_kwargs.get("agents") is None + assert call_kwargs.get("root_agent_id") is None def test_generate_conversation_scenarios_agent_and_agent_info_raises(self): evals_module = evals.Evals(api_client_=self.mock_api_client) @@ -10708,12 +10711,19 @@ def mock_request(method, path, *args, **kwargs): assert agent_cfg.instruction == "You are a weather assistant." assert agent_cfg.description == "Helps with weather queries." assert agent_cfg.agent_type == "gemini-2.0-flash" - # Built-in tools are mapped to typed Tool objects; function tool also included. + # code_execution is expanded via catalog to run_command; + # google_search keeps its typed variant; function tool passes through. assert agent_cfg.tools is not None assert len(agent_cfg.tools) == 3 - assert any(t.code_execution is not None for t in agent_cfg.tools) + decl_names = { + fd.name + for t in agent_cfg.tools + if t.function_declarations + for fd in t.function_declarations + } + assert "run_command" in decl_names + assert "get_weather" in decl_names assert any(t.google_search is not None for t in agent_cfg.tools) - assert any(t.function_declarations is not None for t in agent_cfg.tools) def test_consecutive_model_output_steps_merged(self): """Multiple consecutive model_output steps are merged into one event.""" @@ -11238,11 +11248,19 @@ def test_extracts_full_config(self): assert result.instruction == "You are helpful." assert result.description == "A helpful agent." assert result.agent_type == "gemini-2.0-flash" - # Built-in tools are mapped to typed Tool objects; function tool also included. + # code_execution is expanded via the catalog to run_command; + # google_search keeps its typed variant; function tool passes through. assert len(result.tools) == 3 - assert any(t.code_execution is not None for t in result.tools) + # code_execution -> run_command with full parameter schema. + decl_names = { + fd.name + for t in result.tools + if t.function_declarations + for fd in t.function_declarations + } + assert "run_command" in decl_names + assert "search" in decl_names # pass-through function tool assert any(t.google_search is not None for t in result.tools) - assert any(t.function_declarations is not None for t in result.tools) def test_fetch_failure_returns_minimal_config(self): """If the API call fails, returns just the agent_id.""" @@ -11282,3 +11300,79 @@ def test_empty_resource_name_uses_default_agent_id(self): "", ) assert result.agent_id == "agent" + + def test_code_execution_expands_to_run_command(self): + """code_execution is expanded to run_command with parameters.""" + agent_json = {"tools": [{"type": "code_execution"}]} + mock_api_client = mock.MagicMock() + mock_api_client.request.return_value = self._make_api_response(agent_json) + result = _evals_common._fetch_agent_config_dict( + mock_api_client, "projects/p/locations/l/agents/a", + ) + assert len(result.tools) == 1 + decls = result.tools[0].function_declarations + assert len(decls) == 1 + assert decls[0].name == "run_command" + assert decls[0].parameters is not None + assert "CommandLine" in decls[0].parameters.properties + + def test_filesystem_expands_to_file_tools(self): + """filesystem is expanded to view_file, create_file, etc.""" + agent_json = {"tools": [{"type": "filesystem"}]} + mock_api_client = mock.MagicMock() + mock_api_client.request.return_value = self._make_api_response(agent_json) + result = _evals_common._fetch_agent_config_dict( + mock_api_client, "projects/p/locations/l/agents/a", + ) + assert len(result.tools) == 1 + names = {fd.name for fd in result.tools[0].function_declarations} + assert names == { + "view_file", "create_file", "edit_file", + "list_dir", "delete_file", "move_file", + } + + def test_environment_adds_sandbox_tools(self): + """When agent has environment_config, sandbox tools are appended.""" + agent_json = { + "tools": [{"type": "code_execution"}], + "environment_config": {"some_field": "value"}, + } + mock_api_client = mock.MagicMock() + mock_api_client.request.return_value = self._make_api_response(agent_json) + result = _evals_common._fetch_agent_config_dict( + mock_api_client, "projects/p/locations/l/agents/a", + ) + # code_execution + sandbox tool + assert len(result.tools) == 2 + all_decl_names = { + fd.name + for t in result.tools + if t.function_declarations + for fd in t.function_declarations + } + assert "run_command" in all_decl_names + assert "provision_sandbox" in all_decl_names + assert "load_sandbox" in all_decl_names + + def test_mcp_server_kept_as_named_declaration(self): + """mcp_server entries are kept as named declarations, not dropped.""" + agent_json = { + "tools": [ + {"type": "google_search"}, + {"type": "mcp_server", "name": "my-mcp", "url": "https://x.com"}, + ], + } + mock_api_client = mock.MagicMock() + mock_api_client.request.return_value = self._make_api_response(agent_json) + result = _evals_common._fetch_agent_config_dict( + mock_api_client, "projects/p/locations/l/agents/a", + ) + assert len(result.tools) == 2 + assert any(t.google_search is not None for t in result.tools) + mcp_tool = [ + t for t in result.tools + if t.function_declarations + and t.function_declarations[0].name == "mcp_server" + ] + assert len(mcp_tool) == 1 + assert "my-mcp" in mcp_tool[0].function_declarations[0].description diff --git a/vertexai/_genai/_evals_common.py b/vertexai/_genai/_evals_common.py index 4936a60a96..4d406b7b10 100644 --- a/vertexai/_genai/_evals_common.py +++ b/vertexai/_genai/_evals_common.py @@ -469,10 +469,10 @@ def _get_default_prompt_template( and eval_item.evaluation_request.prompt and eval_item.evaluation_request.prompt.prompt_template_data ): - if ( - "prompt" - in eval_item.evaluation_request.prompt.prompt_template_data.values - ): + template_values = ( + eval_item.evaluation_request.prompt.prompt_template_data.values + ) + if template_values and "prompt" in template_values: return "{prompt}" except Exception as e: logger.warning("Failed to get prompt template from evaluation set: %s", e)