diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py
index f33ceb7497..6dce480729 100644
--- a/agentplatform/_genai/_evals_common.py
+++ b/agentplatform/_genai/_evals_common.py
@@ -840,15 +840,28 @@ def _merge_text_parts_in_agent_data(
def _agent_tools_to_config_tools(
agent_tools: Optional[list[Any]],
) -> Optional[list[genai_types.Tool]]:
- """Maps Gemini Agents API tools to ``genai_types.Tool`` for an AgentConfig.
-
- The Gemini Agents API returns built-in tool variants (``google_search``,
- ``code_execution``, ``url_context``) whose schema differs from
- ``genai_types.Tool``. Each recognised built-in variant is mapped to the
- matching ``genai_types.Tool`` field. Tools with a non-empty body after
- stripping the ``type`` key (e.g. ``function_declarations``) are passed
- through ``model_validate``. Variants without a ``genai_types.Tool``
- equivalent (e.g. ``filesystem``, ``mcp_server``) are skipped.
+ """Maps Gemini Agents API tools to ``genai_types.Tool`` for display.
+
+ NOTE: This is a low-fidelity, DISPLAY-ONLY mapping. The Gemini Agents API
+ (``GET agents/{id}``) returns each tool as a bare type discriminator
+ (e.g. ``{"type": "code_execution"}``) with no parameter schema or
+ description. The authoritative, full-fidelity expansion of built-in tool
+ types into function declarations (e.g. ``code_execution`` -> ``run_command``
+ with parameters, ``filesystem`` -> file tools, sandbox tools) is performed
+ server-side in ``interaction_converter.py`` whenever the server is handed a
+ ``gemini_agent_config``. The result of this function must therefore only be
+ used to render the local System Topology in ``show()`` and must never be
+ sent to the server-side autorater or scenario generator (which resolve the
+ agent's tools themselves).
+
+ Mapping rules:
+ * Recognised built-in variants (``google_search``, ``code_execution``,
+ ``url_context``) are mapped to the matching ``genai_types.Tool`` field.
+ * Tools that already carry ``function_declarations`` are passed through
+ via ``model_validate`` so real function tools keep their full schema.
+ * Any other typed built-in (e.g. ``filesystem``, ``mcp_server``) is
+ represented as a single named ``FunctionDeclaration`` so the tool is
+ still shown by name in the display instead of being dropped.
Args:
agent_tools: The ``tools`` list from a fetched Gemini agent dict.
@@ -864,6 +877,7 @@ def _agent_tools_to_config_tools(
if not isinstance(tool, dict):
continue
tool_type = tool.get("type")
+ remainder = {k: v for k, v in tool.items() if k != "type"}
if tool_type == "google_search":
tools.append(genai_types.Tool(google_search=genai_types.GoogleSearch()))
elif tool_type == "code_execution":
@@ -872,12 +886,31 @@ def _agent_tools_to_config_tools(
)
elif tool_type == "url_context":
tools.append(genai_types.Tool(url_context=genai_types.UrlContext()))
- else:
- # For non-built-in tools (e.g. function_declarations), strip the
- # type key and validate through genai_types.Tool.
- remainder = {k: v for k, v in tool.items() if k != "type"}
- if remainder:
- tools.append(genai_types.Tool.model_validate(remainder))
+ elif "function_declarations" in remainder:
+ # Real function tool with explicit declarations: pass through so
+ # the display shows the full name/parameter schema.
+ tools.append(genai_types.Tool.model_validate(remainder))
+ elif tool_type:
+ # Other built-in tool type with no genai_types.Tool variant (e.g.
+ # "filesystem", "mcp_server"). Represent it as a named declaration
+ # so it is shown by name rather than dropped. Full parameter
+ # schemas are only available from the server-side expansion.
+ description = None
+ if tool_type == "mcp_server":
+ label = remainder.get("name") or remainder.get("url")
+ description = f"MCP server: {label}" if label else "MCP server."
+ tools.append(
+ genai_types.Tool(
+ function_declarations=[
+ genai_types.FunctionDeclaration(
+ name=tool_type, description=description
+ )
+ ]
+ )
+ )
+ elif remainder:
+ # Untyped tool body: best-effort pass-through.
+ tools.append(genai_types.Tool.model_validate(remainder))
return tools or None
@@ -988,33 +1021,6 @@ def _agent_data_response_text(agent_data: types.evals.AgentData) -> Optional[str
return "".join(text_parts) or None
-def _agent_resource_to_agent_info(
- agent: str, api_client: BaseApiClient
-) -> "types.evals.AgentInfo":
- """Builds an `AgentInfo` from a Gemini Agents API agent resource name.
-
- Fetches the agent through the SDK's `api_client` (so replay recording is
- preserved) via `_fetch_agent_config_dict` and derives a single-agent
- `AgentInfo`: the agent's short name is the agents-map key and
- `root_agent_id`.
-
- Args:
- agent: The Gemini Agents API agent resource name
- (`projects/{p}/locations/{l}/agents/{name}`).
- api_client: The API client used to fetch the agent.
-
- Returns:
- An `AgentInfo` describing the fetched agent.
- """
- agent_config = _fetch_agent_config_dict(api_client, agent)
- short_name = agent_config.agent_id
- return types.evals.AgentInfo( # pytype: disable=missing-parameter
- name=short_name,
- agents={short_name: agent_config},
- root_agent_id=short_name,
- )
-
-
_INTERACTION_TERMINAL_STATES = frozenset(
["completed", "failed", "cancelled", "incomplete", "budget_exceeded"]
)
@@ -1109,6 +1115,12 @@ def _run_gemini_agent_inference(
interactions_client = _get_interactions_client(api_client)
agent_short_id = gemini_agent.split("/")[-1]
+
+ # Best-effort: fetch the agent config (instruction, tools, description)
+ # once, so every row's agent_data carries the agents map and the display
+ # can render the System Topology section.
+ agent_config = _fetch_agent_config_dict(api_client, gemini_agent)
+
prompts: list[str] = []
responses: list[Optional[str]] = []
interaction_ids: list[Optional[str]] = []
@@ -1128,6 +1140,7 @@ def _run_gemini_agent_inference(
)
interaction = _await_interaction(interactions_client, interaction)
agent_data_obj = _interaction_dict_to_agent_data(interaction)
+ agent_data_obj.agents = {agent_config.agent_id: agent_config}
_merge_text_parts_in_agent_data(agent_data_obj)
responses.append(_agent_data_response_text(agent_data_obj))
interaction_ids.append(interaction.get("id"))
diff --git a/agentplatform/_genai/_evals_visualization.py b/agentplatform/_genai/_evals_visualization.py
index 45b50b6ebc..164242b221 100644
--- a/agentplatform/_genai/_evals_visualization.py
+++ b/agentplatform/_genai/_evals_visualization.py
@@ -372,19 +372,42 @@ def get_evaluation_html(eval_result_json: str) -> str:
function formatToolDeclarations(toolDeclarations) {{
if (!toolDeclarations) return '';
let functions = [];
- if (Array.isArray(toolDeclarations)) {{
- toolDeclarations.forEach(tool => {{
- if (tool.function_declarations) {{
- functions = functions.concat(tool.function_declarations);
- }} else if (tool.name && tool.parameters) {{
- functions.push(tool);
- }}
+ const builtins = [];
+
+ // A built-in tool variant is a Tool object with no
+ // function_declarations, e.g. {{"google_search": {{...}}}} or
+ // {{"code_execution": {{}}}}. Collect every non-null key (other than
+ // function_declarations) as the built-in tool's name so it renders
+ // by name instead of falling through to a raw JSON dump. Null keys
+ // (from non-exclude_none serialization) are ignored.
+ function collectFromTool(tool) {{
+ if (!tool || typeof tool !== 'object') return;
+ if (Array.isArray(tool.function_declarations) && tool.function_declarations.length > 0) {{
+ functions = functions.concat(tool.function_declarations);
+ return;
+ }}
+ if (tool.name && tool.parameters) {{
+ functions.push(tool);
+ return;
+ }}
+ Object.keys(tool).forEach(k => {{
+ if (k === 'function_declarations') return;
+ if (tool[k] === null || tool[k] === undefined) return;
+ builtins.push(k);
}});
- }} else if (typeof toolDeclarations === 'object' && toolDeclarations.function_declarations) {{
- functions = toolDeclarations.function_declarations;
}}
- if (functions.length === 0) {{
+ if (Array.isArray(toolDeclarations)) {{
+ toolDeclarations.forEach(collectFromTool);
+ }} else if (typeof toolDeclarations === 'object') {{
+ if (toolDeclarations.function_declarations) {{
+ functions = functions.concat(toolDeclarations.function_declarations);
+ }} else {{
+ collectFromTool(toolDeclarations);
+ }}
+ }}
+
+ if (functions.length === 0 && builtins.length === 0) {{
return `${{DOMPurify.sanitize(JSON.stringify(toolDeclarations, null, 2))}}`;
}}
@@ -402,6 +425,9 @@ def get_evaluation_html(eval_result_json: str) -> str:
}});
html += '';
}});
+ builtins.forEach(name => {{
+ html += `${{DOMPurify.sanitize(name)}} (built-in tool)
`;
+ }});
html += '';
return html;
}}
@@ -914,19 +940,42 @@ def get_comparison_html(eval_result_json: str) -> str:
function formatToolDeclarations(toolDeclarations) {{
if (!toolDeclarations) return '';
let functions = [];
- if (Array.isArray(toolDeclarations)) {{
- toolDeclarations.forEach(tool => {{
- if (tool.function_declarations) {{
- functions = functions.concat(tool.function_declarations);
- }} else if (tool.name && tool.parameters) {{
- functions.push(tool);
- }}
+ const builtins = [];
+
+ // A built-in tool variant is a Tool object with no
+ // function_declarations, e.g. {{"google_search": {{...}}}} or
+ // {{"code_execution": {{}}}}. Collect every non-null key (other than
+ // function_declarations) as the built-in tool's name so it renders
+ // by name instead of falling through to a raw JSON dump. Null keys
+ // (from non-exclude_none serialization) are ignored.
+ function collectFromTool(tool) {{
+ if (!tool || typeof tool !== 'object') return;
+ if (Array.isArray(tool.function_declarations) && tool.function_declarations.length > 0) {{
+ functions = functions.concat(tool.function_declarations);
+ return;
+ }}
+ if (tool.name && tool.parameters) {{
+ functions.push(tool);
+ return;
+ }}
+ Object.keys(tool).forEach(k => {{
+ if (k === 'function_declarations') return;
+ if (tool[k] === null || tool[k] === undefined) return;
+ builtins.push(k);
}});
- }} else if (typeof toolDeclarations === 'object' && toolDeclarations.function_declarations) {{
- functions = toolDeclarations.function_declarations;
}}
- if (functions.length === 0) {{
+ if (Array.isArray(toolDeclarations)) {{
+ toolDeclarations.forEach(collectFromTool);
+ }} else if (typeof toolDeclarations === 'object') {{
+ if (toolDeclarations.function_declarations) {{
+ functions = functions.concat(toolDeclarations.function_declarations);
+ }} else {{
+ collectFromTool(toolDeclarations);
+ }}
+ }}
+
+ if (functions.length === 0 && builtins.length === 0) {{
return `${{DOMPurify.sanitize(JSON.stringify(toolDeclarations, null, 2))}}`;
}}
@@ -944,6 +993,9 @@ def get_comparison_html(eval_result_json: str) -> str:
}});
html += '';
}});
+ builtins.forEach(name => {{
+ html += `${{DOMPurify.sanitize(name)}} (built-in tool)
`;
+ }});
html += '';
return html;
}}
diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py
index 63fe6d1858..9e119b09f1 100644
--- a/agentplatform/_genai/evals.py
+++ b/agentplatform/_genai/evals.py
@@ -706,6 +706,13 @@ def _GenerateUserScenariosParameters_to_vertex(
if getv(from_object, ["root_agent_id"]) is not None:
setv(to_object, ["rootAgentId"], getv(from_object, ["root_agent_id"]))
+ if getv(from_object, ["gemini_agent_config", "gemini_agent"]) is not None:
+ setv(
+ to_object,
+ ["geminiAgentConfig", "geminiAgent"],
+ getv(from_object, ["gemini_agent_config", "gemini_agent"]),
+ )
+
if getv(from_object, ["user_scenario_generation_config"]) is not None:
setv(
to_object,
@@ -1513,6 +1520,7 @@ def _generate_user_scenarios(
location: Optional[str] = None,
agents: Optional[dict[str, evals_types.AgentConfigOrDict]] = None,
root_agent_id: Optional[str] = None,
+ gemini_agent_config: Optional[types.GeminiAgentConfigOrDict] = None,
user_scenario_generation_config: Optional[
evals_types.UserScenarioGenerationConfigOrDict
] = None,
@@ -1527,6 +1535,7 @@ def _generate_user_scenarios(
location=location,
agents=agents,
root_agent_id=root_agent_id,
+ gemini_agent_config=gemini_agent_config,
user_scenario_generation_config=user_scenario_generation_config,
config=config,
allow_cross_region_model=allow_cross_region_model,
@@ -2938,16 +2947,17 @@ def generate_conversation_scenarios(
and the agent under test.
Exactly one of `agent_info` or `agent` must be provided. When `agent` is
- a Gemini Agents API agent resource name, the agent is fetched and an
- `AgentInfo` is derived from it.
+ a Gemini Agents API agent resource name, the agent resource is passed to
+ the server, which resolves its configuration (instruction and tools,
+ including built-in tool expansion) into the agents map.
Args:
agent_info: The agent info to generate user scenarios for. Mutually
exclusive with `agent`.
agent: A Gemini Agents API agent resource name
(`projects/{p}/locations/{l}/agents/{name}`). When provided, the
- agent is fetched and its configuration is used to build the agent
- info. Mutually exclusive with `agent_info`.
+ agent resource is sent to the server, which resolves its
+ configuration server-side. Mutually exclusive with `agent_info`.
config: Configuration for generating user scenarios.
allow_cross_region_model: Opt-in flag to authorize cross-region
routing for model inference.
@@ -2967,8 +2977,13 @@ def generate_conversation_scenarios(
"`agent` must be a Gemini Agents API agent resource name of the"
" form projects/{project}/locations/{location}/agents/{agent}."
)
- parsed_agent_info = _evals_common._agent_resource_to_agent_info(
- agent, self._api_client
+ # Pass the agent resource to the server so it resolves the agent's
+ # tools/instruction (with full built-in tool expansion) itself,
+ # rather than synthesizing a low-fidelity AgentConfig client-side.
+ response = self._generate_user_scenarios(
+ gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent),
+ user_scenario_generation_config=config,
+ allow_cross_region_model=allow_cross_region_model,
)
else:
parsed_agent_info = (
@@ -2976,12 +2991,12 @@ def generate_conversation_scenarios(
if isinstance(agent_info, dict)
else agent_info
)
- response = self._generate_user_scenarios(
- agents=parsed_agent_info.agents,
- root_agent_id=parsed_agent_info.root_agent_id,
- user_scenario_generation_config=config,
- allow_cross_region_model=allow_cross_region_model,
- )
+ response = self._generate_user_scenarios(
+ agents=parsed_agent_info.agents,
+ root_agent_id=parsed_agent_info.root_agent_id,
+ user_scenario_generation_config=config,
+ allow_cross_region_model=allow_cross_region_model,
+ )
return _evals_utils._postprocess_user_scenarios_response(response)
def generate_loss_clusters(
@@ -3678,6 +3693,7 @@ async def _generate_user_scenarios(
location: Optional[str] = None,
agents: Optional[dict[str, evals_types.AgentConfigOrDict]] = None,
root_agent_id: Optional[str] = None,
+ gemini_agent_config: Optional[types.GeminiAgentConfigOrDict] = None,
user_scenario_generation_config: Optional[
evals_types.UserScenarioGenerationConfigOrDict
] = None,
@@ -3692,6 +3708,7 @@ async def _generate_user_scenarios(
location=location,
agents=agents,
root_agent_id=root_agent_id,
+ gemini_agent_config=gemini_agent_config,
user_scenario_generation_config=user_scenario_generation_config,
config=config,
allow_cross_region_model=allow_cross_region_model,
@@ -4729,16 +4746,17 @@ async def generate_conversation_scenarios(
and the agent under test.
Exactly one of `agent_info` or `agent` must be provided. When `agent` is
- a Gemini Agents API agent resource name, the agent is fetched and an
- `AgentInfo` is derived from it.
+ a Gemini Agents API agent resource name, the agent resource is passed to
+ the server, which resolves its configuration (instruction and tools,
+ including built-in tool expansion) into the agents map.
Args:
agent_info: The agent info to generate user scenarios for. Mutually
exclusive with `agent`.
agent: A Gemini Agents API agent resource name
(`projects/{p}/locations/{l}/agents/{name}`). When provided, the
- agent is fetched and its configuration is used to build the agent
- info. Mutually exclusive with `agent_info`.
+ agent resource is sent to the server, which resolves its
+ configuration server-side. Mutually exclusive with `agent_info`.
config: Configuration for generating user scenarios.
allow_cross_region_model: Opt-in flag to authorize cross-region
routing for model inference.
@@ -4758,8 +4776,13 @@ async def generate_conversation_scenarios(
"`agent` must be a Gemini Agents API agent resource name of the"
" form projects/{project}/locations/{location}/agents/{agent}."
)
- parsed_agent_info = _evals_common._agent_resource_to_agent_info(
- agent, self._api_client
+ # Pass the agent resource to the server so it resolves the agent's
+ # tools/instruction (with full built-in tool expansion) itself,
+ # rather than synthesizing a low-fidelity AgentConfig client-side.
+ response = await self._generate_user_scenarios(
+ gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent),
+ user_scenario_generation_config=config,
+ allow_cross_region_model=allow_cross_region_model,
)
else:
parsed_agent_info = (
@@ -4767,12 +4790,12 @@ async def generate_conversation_scenarios(
if isinstance(agent_info, dict)
else agent_info
)
- response = await self._generate_user_scenarios(
- agents=parsed_agent_info.agents,
- root_agent_id=parsed_agent_info.root_agent_id,
- user_scenario_generation_config=config,
- allow_cross_region_model=allow_cross_region_model,
- )
+ response = await self._generate_user_scenarios(
+ agents=parsed_agent_info.agents,
+ root_agent_id=parsed_agent_info.root_agent_id,
+ user_scenario_generation_config=config,
+ allow_cross_region_model=allow_cross_region_model,
+ )
return _evals_utils._postprocess_user_scenarios_response(response)
async def generate_loss_clusters(
diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py
index 79c577c7fd..d0b5d60c31 100644
--- a/agentplatform/_genai/types/common.py
+++ b/agentplatform/_genai/types/common.py
@@ -5491,6 +5491,13 @@ class _GenerateUserScenariosParameters(_common.BaseModel):
default=None, description=""""""
)
root_agent_id: Optional[str] = Field(default=None, description="""""")
+ gemini_agent_config: Optional[GeminiAgentConfig] = Field(
+ default=None,
+ description="""If set, the server derives the `agents` map and `root_agent_id`
+ from the referenced Gemini Agent, resolving its tools (including
+ built-in tool expansion) server-side. Mutually exclusive with
+ `agents`/`root_agent_id`.""",
+ )
user_scenario_generation_config: Optional[
evals_types.UserScenarioGenerationConfig
] = Field(default=None, description="""""")
@@ -5515,6 +5522,11 @@ class _GenerateUserScenariosParametersDict(TypedDict, total=False):
root_agent_id: Optional[str]
""""""
+ gemini_agent_config: Optional[GeminiAgentConfigDict]
+ """If set, the server derives the `agents` map and `root_agent_id` from the
+ referenced Gemini Agent, resolving its tools (including built-in tool
+ expansion) server-side. Mutually exclusive with `agents`/`root_agent_id`."""
+
user_scenario_generation_config: Optional[evals_types.UserScenarioGenerationConfig]
""""""
diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py
index 26cfb80e7f..f614607256 100644
--- a/tests/unit/agentplatform/genai/test_evals.py
+++ b/tests/unit/agentplatform/genai/test_evals.py
@@ -4147,11 +4147,27 @@ def test_run_inference_with_litellm_parsing(
assert call_kwargs["model"] == "gpt-4o"
pd.testing.assert_frame_equal(call_kwargs["prompt_dataset"], mock_df)
+ @mock.patch.object(_evals_common, "_fetch_agent_config_dict")
@mock.patch.object(_evals_common, "_get_interactions_client")
@mock.patch.object(_evals_utils, "EvalDatasetLoader")
def test_run_inference_with_gemini_agent(
- self, mock_eval_dataset_loader, mock_get_interactions_client
+ self, mock_eval_dataset_loader, mock_get_interactions_client,
+ mock_fetch_agent_config,
):
+ mock_fetch_agent_config.return_value = (
+ agentplatform_genai_types.evals.AgentConfig(
+ agent_id="test-agent",
+ instruction="You are helpful.",
+ tools=[
+ genai_types.Tool(
+ code_execution=genai_types.ToolCodeExecution()
+ ),
+ genai_types.Tool(
+ google_search=genai_types.GoogleSearch()
+ ),
+ ],
+ )
+ )
mock_df = pd.DataFrame({"prompt": ["p1", "p2"]})
mock_eval_dataset_loader.return_value.load.return_value = mock_df.to_dict(
orient="records"
@@ -4211,11 +4227,28 @@ def make_interaction(interaction_id, prompt_text, output_text):
assert turn_events[1]["content"]["parts"][0]["text"] == "response 1"
assert inference_result.candidate_name == "test-agent"
+ # The agents map must be populated so the display renders the
+ # System Topology section (the real bug that was reported).
+ assert "agents" in agent_data_0, (
+ "agent_data must contain an 'agents' map so the System Topology "
+ "section renders in show(). Keys present: "
+ + str(list(agent_data_0.keys()))
+ )
+ assert "test-agent" in agent_data_0["agents"]
+ agent_cfg = agent_data_0["agents"]["test-agent"]
+ assert agent_cfg["instruction"] == "You are helpful."
+ assert len(agent_cfg["tools"]) == 2
+
+ @mock.patch.object(_evals_common, "_fetch_agent_config_dict")
@mock.patch.object(_evals_common, "_get_interactions_client")
@mock.patch.object(_evals_utils, "EvalDatasetLoader")
def test_run_inference_gemini_agent_continues_on_failure(
- self, mock_eval_dataset_loader, mock_get_interactions_client
+ self, mock_eval_dataset_loader, mock_get_interactions_client,
+ mock_fetch_agent_config,
):
+ mock_fetch_agent_config.return_value = (
+ agentplatform_genai_types.evals.AgentConfig(agent_id="test-agent")
+ )
mock_df = pd.DataFrame({"prompt": ["p1", "p2"]})
mock_eval_dataset_loader.return_value.load.return_value = mock_df.to_dict(
orient="records"
@@ -9213,19 +9246,10 @@ async def test_async_generate_conversation_scenarios(self):
request_body = call_args[0][2] # Third positional arg is the request dict
assert request_body.get("allowCrossRegionModel") is True
- @mock.patch.object(_evals_common, "_fetch_agent_config_dict")
- def test_generate_conversation_scenarios_from_gemini_agent(
- self, mock_fetch_agent_config
- ):
- mock_fetch_agent_config.return_value = (
- agentplatform_genai_types.evals.AgentConfig(
- agent_id="test-agent",
- instruction="You are a helpful travel assistant.",
- description="An agent that books flights.",
- tools=[genai_types.Tool(google_search=genai_types.GoogleSearch())],
- )
- )
-
+ def test_generate_conversation_scenarios_from_gemini_agent(self):
+ """When `agent` is a Gemini agent resource, the resource is forwarded to
+ the server via `gemini_agent_config` and no client-side agent config is
+ synthesized (the server resolves the tools/instruction itself)."""
evals_module = evals.Evals(api_client_=self.mock_api_client)
with mock.patch.object(
@@ -9237,18 +9261,28 @@ def test_generate_conversation_scenarios_from_gemini_agent(
config={"count": 2},
)
- mock_fetch_agent_config.assert_called_once_with(
- self.mock_api_client, _TEST_GEMINI_AGENT
- )
call_kwargs = mock_generate_user_scenarios.call_args.kwargs
- assert call_kwargs["root_agent_id"] == "test-agent"
- agents = call_kwargs["agents"]
- assert "test-agent" in agents
- derived_config = agents["test-agent"]
- assert derived_config.instruction == "You are a helpful travel assistant."
- assert derived_config.description == "An agent that books flights."
- assert derived_config.tools is not None
- assert derived_config.tools[0].google_search is not None
+ assert call_kwargs["gemini_agent_config"].gemini_agent == _TEST_GEMINI_AGENT
+ # The client must not synthesize a low-fidelity agents map / root_agent_id;
+ # the server resolves them from the referenced agent resource.
+ assert call_kwargs.get("agents") is None
+ assert call_kwargs.get("root_agent_id") is None
+
+ def test_generate_conversation_scenarios_from_gemini_agent_request_body(self):
+ """The generated request forwards geminiAgentConfig and omits the
+ client-side agents map."""
+ evals_module = evals.Evals(api_client_=self.mock_api_client)
+
+ evals_module.generate_conversation_scenarios(
+ agent=_TEST_GEMINI_AGENT,
+ config={"count": 2},
+ )
+
+ self.mock_api_client.request.assert_called_once()
+ request_body = self.mock_api_client.request.call_args[0][2]
+ assert request_body["geminiAgentConfig"]["geminiAgent"] == _TEST_GEMINI_AGENT
+ assert "agents" not in request_body
+ assert "rootAgentId" not in request_body
def test_generate_conversation_scenarios_agent_and_agent_info_raises(self):
evals_module = evals.Evals(api_client_=self.mock_api_client)
@@ -11282,3 +11316,248 @@ def test_empty_resource_name_uses_default_agent_id(self):
"",
)
assert result.agent_id == "agent"
+
+ def test_builtin_tools_without_variant_are_kept_as_named_declarations(self):
+ """Built-in tool types with no genai_types.Tool variant (e.g.
+ `filesystem`, `mcp_server`) must be represented by name rather than
+ dropped, so the display can still show them."""
+ agent_json = {
+ "tools": [
+ {"type": "filesystem"},
+ {"type": "url_context"},
+ {
+ "type": "mcp_server",
+ "name": "my-mcp",
+ "url": "https://example.com/mcp",
+ },
+ ],
+ }
+ mock_api_client = mock.MagicMock()
+ mock_api_client.request.return_value = self._make_api_response(agent_json)
+
+ result = _evals_common._fetch_agent_config_dict(
+ mock_api_client,
+ "projects/p/locations/l/agents/my-agent",
+ )
+ # None of the three tools are dropped.
+ assert len(result.tools) == 3
+ # url_context keeps its typed variant.
+ assert any(t.url_context is not None for t in result.tools)
+ # filesystem and mcp_server become named function declarations.
+ named = {
+ fd.name: fd
+ for t in result.tools
+ if t.function_declarations
+ for fd in t.function_declarations
+ }
+ assert "filesystem" in named
+ assert "mcp_server" in named
+ # mcp_server carries a human-readable label in its description.
+ assert "my-mcp" in (named["mcp_server"].description or "")
+
+ def test_mcp_server_tool_does_not_drop_other_tools(self):
+ """An `mcp_server` entry must not raise and wipe out the whole tools
+ list (regression: model_validate on the mcp body used to fail)."""
+ agent_json = {
+ "tools": [
+ {"type": "google_search"},
+ {"type": "mcp_server", "url": "https://example.com/mcp"},
+ ],
+ }
+ mock_api_client = mock.MagicMock()
+ mock_api_client.request.return_value = self._make_api_response(agent_json)
+
+ result = _evals_common._fetch_agent_config_dict(
+ mock_api_client,
+ "projects/p/locations/l/agents/my-agent",
+ )
+ assert len(result.tools) == 2
+ assert any(t.google_search is not None for t in result.tools)
+
+
+class TestEvalsVisualizationSystemTopology:
+ """Tests that the System Topology section renders agent tools end-to-end."""
+
+ def _build_dataset_with_tools(self):
+ agent_config = agentplatform_genai_types.evals.AgentConfig(
+ agent_id="travel-agent",
+ instruction="You are a travel agent.",
+ tools=[
+ genai_types.Tool(google_search=genai_types.GoogleSearch()),
+ # Built-in type with no genai variant, kept as a named decl.
+ genai_types.Tool(
+ function_declarations=[
+ genai_types.FunctionDeclaration(name="filesystem")
+ ]
+ ),
+ # Real function tool with a parameter schema.
+ genai_types.Tool(
+ function_declarations=[
+ genai_types.FunctionDeclaration(
+ name="run_command",
+ description="Runs a shell command.",
+ parameters=genai_types.Schema(
+ type="OBJECT",
+ properties={
+ "CommandLine": genai_types.Schema(
+ type="STRING",
+ description="The command to run.",
+ )
+ },
+ required=["CommandLine"],
+ ),
+ )
+ ]
+ ),
+ ],
+ )
+ agent_data = agentplatform_genai_types.evals.AgentData(
+ agents={"travel-agent": agent_config},
+ turns=[
+ agentplatform_genai_types.evals.ConversationTurn(
+ turn_index=0,
+ turn_id="turn_0",
+ events=[
+ agentplatform_genai_types.evals.AgentEvent(
+ author="user",
+ content=genai_types.Content(
+ parts=[genai_types.Part(text="hi")]
+ ),
+ ),
+ agentplatform_genai_types.evals.AgentEvent(
+ author="model",
+ content=genai_types.Content(
+ parts=[genai_types.Part(text="hello")]
+ ),
+ ),
+ ],
+ )
+ ],
+ )
+ case = agentplatform_genai_types.EvalCase(agent_data=agent_data)
+ return agentplatform_genai_types.EvaluationDataset(eval_cases=[case])
+
+ def _decode_payload(self, html):
+ match = re.search(r'atob\("([A-Za-z0-9+/=]+)"\)', html)
+ assert match, "Could not find embedded base64 payload in HTML."
+ return json.loads(base64.b64decode(match.group(1)).decode("utf-8"))
+
+ def test_system_topology_includes_tools(self):
+ dataset = self._build_dataset_with_tools()
+ rows = _evals_visualization._extract_dataset_rows(dataset)
+ result_dump = {
+ "summary_metrics": [],
+ "eval_case_results": [
+ {
+ "eval_case_index": 0,
+ "response_candidate_results": [{}],
+ }
+ ],
+ "metadata": {"dataset": rows},
+ }
+
+ html = _evals_visualization.get_evaluation_html(json.dumps(result_dump))
+
+ # The section and its renderer must be present in the template.
+ assert "System Topology" in html
+ assert "formatSystemTopology" in html
+
+ # The embedded payload must carry the agents map with all three tools
+ # (none dropped), so the client renders them by name.
+ payload = self._decode_payload(html)
+ agent_data = json.loads(payload["metadata"]["dataset"][0]["agent_data"])
+ tools = agent_data["agents"]["travel-agent"]["tools"]
+ assert len(tools) == 3
+
+ # google_search is preserved as a built-in variant (rendered by name).
+ assert any("google_search" in t for t in tools)
+ # filesystem and run_command are present as named function declarations.
+ decl_names = {
+ fd["name"]
+ for t in tools
+ if t.get("function_declarations")
+ for fd in t["function_declarations"]
+ }
+ assert "filesystem" in decl_names
+ assert "run_command" in decl_names
+
+ def test_builtin_tool_serialization_roundtrip(self):
+ """Tests that built-in Tool variants survive the exclude_none=True
+ serialization that _extract_dataset_rows uses, so they don't become
+ empty objects that the JS renderer ignores."""
+ # These are the three Tool variants the user's agent has.
+ builtin_tools = [
+ genai_types.Tool(code_execution=genai_types.ToolCodeExecution()),
+ genai_types.Tool(google_search=genai_types.GoogleSearch()),
+ genai_types.Tool(url_context=genai_types.UrlContext()),
+ ]
+ agent_config = agentplatform_genai_types.evals.AgentConfig(
+ agent_id="test-agent",
+ tools=builtin_tools,
+ )
+ agent_data = agentplatform_genai_types.evals.AgentData(
+ agents={"test-agent": agent_config},
+ turns=[],
+ )
+ # This is the serialization path used by _extract_dataset_rows (line 205)
+ dumped = agent_data.model_dump(mode="json", exclude_none=True)
+ agent_tools = dumped["agents"]["test-agent"].get("tools", [])
+
+ # Each tool must still have at least one non-null key so the JS
+ # renderer can identify it by name.
+ for i, t in enumerate(agent_tools):
+ non_null_keys = [k for k, v in t.items() if v is not None]
+ assert non_null_keys, (
+ f"Tool[{i}] serialized as empty object — the JS renderer "
+ f"will not display it. Full dump: {json.dumps(t)}"
+ )
+
+ def test_builtin_tools_via_extract_dataset_rows(self):
+ """End-to-end: tools from _agent_tools_to_config_tools survive through
+ _extract_dataset_rows into the HTML payload agent_data JSON."""
+ # Simulate what _fetch_agent_config_dict returns for the user's agent:
+ # the agent API returns [code_execution, google_search, url_context]
+ mapped = _evals_common._agent_tools_to_config_tools([
+ {"type": "code_execution"},
+ {"type": "google_search"},
+ {"type": "url_context"},
+ ])
+ assert mapped is not None, "Mapped tools should not be None"
+ assert len(mapped) == 3, f"Expected 3 tools, got {len(mapped)}"
+
+ agent_config = agentplatform_genai_types.evals.AgentConfig(
+ agent_id="test-agent",
+ tools=mapped,
+ )
+ agent_data = agentplatform_genai_types.evals.AgentData(
+ agents={"test-agent": agent_config},
+ turns=[
+ agentplatform_genai_types.evals.ConversationTurn(
+ turn_index=0, turn_id="t0", events=[
+ agentplatform_genai_types.evals.AgentEvent(
+ author="user",
+ content=genai_types.Content(
+ parts=[genai_types.Part(text="hi")]
+ ),
+ ),
+ ],
+ ),
+ ],
+ )
+ case = agentplatform_genai_types.EvalCase(agent_data=agent_data)
+ ds = agentplatform_genai_types.EvaluationDataset(eval_cases=[case])
+ rows = _evals_visualization._extract_dataset_rows(ds)
+ assert rows, "Expected at least one row"
+
+ ad_json = rows[0].get("agent_data")
+ assert ad_json, "agent_data should be a non-empty JSON string"
+ ad = json.loads(ad_json)
+ agents = ad.get("agents", {})
+ assert "test-agent" in agents, f"Missing agent. keys: {list(agents.keys())}"
+ tools = agents["test-agent"].get("tools", [])
+ assert len(tools) == 3, (
+ f"Expected 3 tools in serialized payload, got {len(tools)}. "
+ f"Serialized tools: {json.dumps(tools)}"
+ )
+
+