Skip to content

Commit bddd51f

Browse files
rogerbarretorogerbarreto
authored
Merge branch 'main' into bugfix/webfiledownload-plugin
2 parents 56a2f67 + e500f72 commit bddd51f

16 files changed

Lines changed: 550 additions & 28 deletions

File tree

‎python/samples/concepts/chat_completion/simple_chatbot_with_image.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
# make sure to use a service that supports image input from a URI.
3535
chat_completion_service, request_settings = get_chat_completion_service_and_request_settings(Services.AZURE_OPENAI)
3636

37-
IMAGE_URI = "https://upload.wikimedia.org/wikipedia/commons/d/d5/Half-timbered_mansion%2C_Zirkel%2C_East_view.jpg"
37+
IMAGE_URI = "https://raw.githubusercontent.com/microsoft/semantic-kernel/main/python/tests/assets/sample_image.jpg"
3838
IMAGE_PATH = "samples/concepts/resources/sample_image.jpg"
3939

4040
# Create an image content with the image URI.

‎python/semantic_kernel/connectors/ai/google/google_ai/services/google_ai_chat_completion.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,13 +303,18 @@ def _create_chat_message_content(
303303
if part.text:
304304
items.append(TextContent(text=part.text, inner_content=response, metadata=response_metadata))
305305
elif part.function_call:
306+
fc_metadata: dict[str, Any] = {}
307+
thought_sig = getattr(part, "thought_signature", None)
308+
if thought_sig:
309+
fc_metadata["thought_signature"] = thought_sig
306310
items.append(
307311
FunctionCallContent(
308312
id=f"{part.function_call.name}_{idx!s}",
309313
name=format_gemini_function_name_to_kernel_function_fully_qualified_name(
310314
part.function_call.name # type: ignore[arg-type]
311315
),
312316
arguments={k: v for k, v in part.function_call.args.items()}, # type: ignore
317+
metadata=fc_metadata if fc_metadata else None,
313318
)
314319
)
315320

@@ -360,13 +365,18 @@ def _create_streaming_chat_message_content(
360365
)
361366
)
362367
elif part.function_call:
368+
fc_metadata: dict[str, Any] = {}
369+
thought_sig = getattr(part, "thought_signature", None)
370+
if thought_sig:
371+
fc_metadata["thought_signature"] = thought_sig
363372
items.append(
364373
FunctionCallContent(
365374
id=f"{part.function_call.name}_{idx!s}",
366375
name=format_gemini_function_name_to_kernel_function_fully_qualified_name(
367376
part.function_call.name # type: ignore[arg-type]
368377
),
369378
arguments={k: v for k, v in part.function_call.args.items()}, # type: ignore
379+
metadata=fc_metadata if fc_metadata else None,
370380
)
371381
)
372382

‎python/semantic_kernel/connectors/ai/google/google_ai/services/utils.py‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -91,12 +91,24 @@ def format_assistant_message(message: ChatMessageContent) -> list[Part]:
9191
if item.text:
9292
parts.append(Part.from_text(text=item.text))
9393
elif isinstance(item, FunctionCallContent):
94-
parts.append(
95-
Part.from_function_call(
96-
name=item.name, # type: ignore[arg-type]
97-
args=json.loads(item.arguments) if isinstance(item.arguments, str) else item.arguments, # type: ignore[arg-type]
94+
thought_signature = item.metadata.get("thought_signature") if item.metadata else None
95+
if thought_signature:
96+
parts.append(
97+
Part(
98+
function_call={
99+
"name": item.name, # type: ignore[arg-type]
100+
"args": json.loads(item.arguments) if isinstance(item.arguments, str) else item.arguments,
101+
},
102+
thought_signature=thought_signature,
103+
)
104+
)
105+
else:
106+
parts.append(
107+
Part.from_function_call(
108+
name=item.name, # type: ignore[arg-type]
109+
args=json.loads(item.arguments) if isinstance(item.arguments, str) else item.arguments, # type: ignore[arg-type]
110+
)
98111
)
99-
)
100112
elif isinstance(item, ImageContent):
101113
parts.append(_create_image_part(item))
102114
else:

‎python/semantic_kernel/connectors/ai/google/vertex_ai/services/utils.py‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import json
44
import logging
5-
from typing import TYPE_CHECKING
5+
from typing import TYPE_CHECKING, Any
66

77
from google.cloud.aiplatform_v1beta1.types.content import Candidate
88
from vertexai.generative_models import FunctionDeclaration, Part, Tool, ToolConfig
@@ -89,14 +89,16 @@ def format_assistant_message(message: ChatMessageContent) -> list[Part]:
8989
if item.text:
9090
parts.append(Part.from_text(item.text))
9191
elif isinstance(item, FunctionCallContent):
92-
parts.append(
93-
Part.from_dict({
94-
"function_call": {
95-
"name": item.name,
96-
"args": json.loads(item.arguments) if isinstance(item.arguments, str) else item.arguments,
97-
}
98-
})
99-
)
92+
part_dict: dict[str, Any] = {
93+
"function_call": {
94+
"name": item.name, # type: ignore[arg-type]
95+
"args": json.loads(item.arguments) if isinstance(item.arguments, str) else item.arguments,
96+
}
97+
}
98+
thought_signature = item.metadata.get("thought_signature") if item.metadata else None
99+
if thought_signature:
100+
part_dict["thought_signature"] = thought_signature
101+
parts.append(Part.from_dict(part_dict))
100102
elif isinstance(item, ImageContent):
101103
parts.append(_create_image_part(item))
102104
else:

‎python/semantic_kernel/connectors/ai/google/vertex_ai/services/vertex_ai_chat_completion.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,13 +252,18 @@ def _create_chat_message_content(self, response: GenerationResponse, candidate:
252252
if "text" in part_dict:
253253
items.append(TextContent(text=part.text, inner_content=response, metadata=response_metadata))
254254
elif "function_call" in part_dict:
255+
fc_metadata: dict[str, Any] = {}
256+
thought_sig = part_dict.get("thought_signature")
257+
if thought_sig:
258+
fc_metadata["thought_signature"] = thought_sig
255259
items.append(
256260
FunctionCallContent(
257261
id=f"{part.function_call.name}_{idx!s}",
258262
name=format_gemini_function_name_to_kernel_function_fully_qualified_name(
259263
part.function_call.name
260264
),
261265
arguments={k: v for k, v in part.function_call.args.items()},
266+
metadata=fc_metadata if fc_metadata else None,
262267
)
263268
)
264269

@@ -309,13 +314,18 @@ def _create_streaming_chat_message_content(
309314
)
310315
)
311316
elif "function_call" in part_dict:
317+
fc_metadata_s: dict[str, Any] = {}
318+
thought_sig_s = part_dict.get("thought_signature")
319+
if thought_sig_s:
320+
fc_metadata_s["thought_signature"] = thought_sig_s
312321
items.append(
313322
FunctionCallContent(
314323
id=f"{part.function_call.name}_{idx!s}",
315324
name=format_gemini_function_name_to_kernel_function_fully_qualified_name(
316325
part.function_call.name
317326
),
318327
arguments={k: v for k, v in part.function_call.args.items()},
328+
metadata=fc_metadata_s if fc_metadata_s else None,
319329
)
320330
)
321331

‎python/semantic_kernel/contents/chat_history.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,23 @@ def from_rendered_prompt(cls: type[_T], rendered_prompt: str) -> _T:
353353
elif item.tag == CHAT_HISTORY_TAG:
354354
for message in item:
355355
messages.append(ChatMessageContent.from_element(message))
356+
else:
357+
# Unknown XML tags (e.g. HTML tags like <p>, <div>) are not SK
358+
# template tags. Serialize them back to text and append to the
359+
# previous message so that the original content is preserved.
360+
saved_tail = item.tail
361+
item.tail = None
362+
raw = unescape(tostring(item, encoding="unicode", short_empty_elements=False))
363+
item.tail = saved_tail
364+
if messages:
365+
messages[-1].content = (messages[-1].content or "") + raw
366+
else:
367+
messages.append(ChatMessageContent(role=AuthorRole.USER, content=raw))
368+
# For unknown tags the tail is part of the surrounding text,
369+
# so keep it in the same message instead of starting a new one.
370+
if item.tail:
371+
messages[-1].content = (messages[-1].content or "") + unescape(item.tail)
372+
continue
356373
if item.tail and item.tail.strip():
357374
messages.append(ChatMessageContent(role=AuthorRole.USER, content=unescape(item.tail.strip())))
358375
if len(messages) == 1 and messages[0].role == AuthorRole.SYSTEM:

‎python/tests/integration/agents/chat_completion_agent/test_chat_completion_agent_integration.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ async def test_image_content_stream(
251251
):
252252
"""Test function calling streaming."""
253253
IMAGE_URI = (
254-
"https://upload.wikimedia.org/wikipedia/commons/d/d5/Half-timbered_mansion%2C_Zirkel%2C_East_view.jpg"
254+
"https://raw.githubusercontent.com/microsoft/semantic-kernel/main/python/tests/assets/sample_image.jpg"
255255
)
256256
image_content_remote = ImageContent(uri=IMAGE_URI)
257257

‎python/tests/integration/completions/test_chat_completion_with_image_input_text_output.py‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,12 @@
2626
from tests.integration.completions.completion_test_base import ServiceType
2727
from tests.utils import retry
2828

29+
# Use the repo's own sample image via raw GitHub URL for URI-based tests.
30+
# Previously this pointed to a 17.5 MB Wikimedia image that got blocked by
31+
# Wikimedia's User-Agent policy (Phabricator T400119), causing Azure's
32+
# server-side image fetcher to fail with HTTP 403.
33+
IMAGE_TEST_URL = "https://raw.githubusercontent.com/microsoft/semantic-kernel/main/python/tests/assets/sample_image.jpg"
34+
2935
pytestmark = pytest.mark.parametrize(
3036
"service_id, execution_settings_kwargs, inputs, kwargs",
3137
[
@@ -37,15 +43,12 @@
3743
role=AuthorRole.USER,
3844
items=[
3945
TextContent(text="What is in this image?"),
40-
ImageContent(
41-
uri="https://upload.wikimedia.org/wikipedia/commons/d/d5/Half-timbered_mansion%2C_Zirkel%2C_East_view.jpg"
42-
),
46+
ImageContent(uri=IMAGE_TEST_URL),
4347
],
4448
),
4549
ChatMessageContent(role=AuthorRole.USER, items=[TextContent(text="Where was it made?")]),
4650
],
4751
{},
48-
marks=pytest.mark.xfail(reason="OpenAI service raise error for downloading image from URL"),
4952
id="openai_image_input_uri",
5053
),
5154
pytest.param(
@@ -74,9 +77,7 @@
7477
role=AuthorRole.USER,
7578
items=[
7679
TextContent(text="What is in this image?"),
77-
ImageContent(
78-
uri="https://upload.wikimedia.org/wikipedia/commons/d/d5/Half-timbered_mansion%2C_Zirkel%2C_East_view.jpg"
79-
),
80+
ImageContent(uri=IMAGE_TEST_URL),
8081
],
8182
),
8283
ChatMessageContent(role=AuthorRole.USER, items=[TextContent(text="Where was it made?")]),
@@ -134,9 +135,7 @@
134135
role=AuthorRole.USER,
135136
items=[
136137
TextContent(text="What is in this image?"),
137-
ImageContent(
138-
uri="https://upload.wikimedia.org/wikipedia/commons/d/d5/Half-timbered_mansion%2C_Zirkel%2C_East_view.jpg"
139-
),
138+
ImageContent(uri=IMAGE_TEST_URL),
140139
],
141140
),
142141
ChatMessageContent(role=AuthorRole.USER, items=[TextContent(text="Where was it made?")]),

0 commit comments

Comments
 (0)