diff --git a/python/pyproject.toml b/python/pyproject.toml index 7b563b4c86e0..a0122a2a2f29 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ [project.optional-dependencies] azure = [ "azure-ai-inference >= 1.0.0b6", + "azure-ai-projects >= 1.0.0b5", "azure-core-tracing-opentelemetry >= 1.0.0b11", "azure-search-documents >= 11.6.0b4", "azure-identity ~= 1.13", diff --git a/python/samples/concepts/agents/azure_ai_agent/.env.example b/python/samples/concepts/agents/azure_ai_agent/.env.example new file mode 100644 index 000000000000..c2d16cea26aa --- /dev/null +++ b/python/samples/concepts/agents/azure_ai_agent/.env.example @@ -0,0 +1,6 @@ +AZURE_AI_AGENT_PROJECT_CONNECTION_STRING = "" +AZURE_AI_AGENT_MODEL_DEPLOYMENT_NAME = "" +AZURE_AI_AGENT_ENDPOINT = "" +AZURE_AI_AGENT_SUBSCRIPTION_ID = "" +AZURE_AI_AGENT_RESOURCE_GROUP_NAME = "" +AZURE_AI_AGENT_PROJECT_NAME = "" \ No newline at end of file diff --git a/python/samples/concepts/agents/azure_ai_agent/README.md b/python/samples/concepts/agents/azure_ai_agent/README.md new file mode 100644 index 000000000000..0ba20d3a2b09 --- /dev/null +++ b/python/samples/concepts/agents/azure_ai_agent/README.md @@ -0,0 +1,3 @@ +## Azure AI Agents + +For details on using Azure AI Agents within Semantic Kernel, please see the [README](../../../getting_started_with_agents/azure_ai_agent/README.md) located in the `getting_started_with_agents/azure_ai_agent` directory. \ No newline at end of file diff --git a/python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_file_manipulation.py b/python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_file_manipulation.py new file mode 100644 index 000000000000..a7d9d2f349ad --- /dev/null +++ b/python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_file_manipulation.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import os + +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import CodeInterpreterTool, FilePurpose +from azure.identity.aio import DefaultAzureCredential + +from semantic_kernel.agents.azure_ai import AzureAIAgent, AzureAIAgentSettings +from semantic_kernel.contents.annotation_content import AnnotationContent +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole + +################################################################### +# The following sample demonstrates how to create a simple, # +# Azure AI agent that uses the code interpreter tool to answer # +# a coding question. # +################################################################### + + +async def main() -> None: + ai_agent_settings = AzureAIAgentSettings.create() + + async with ( + DefaultAzureCredential() as creds, + AIProjectClient.from_connection_string( + credential=creds, + conn_str=ai_agent_settings.project_connection_string.get_secret_value(), + ) as client, + ): + csv_file_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "resources", + "agent_assistant_file_manipulation", + "sales.csv", + ) + + file = await client.agents.upload_file_and_poll(file_path=csv_file_path, purpose=FilePurpose.AGENTS) + + code_interpreter = CodeInterpreterTool(file_ids=[file.id]) + + # Create agent definition + agent_definition = await client.agents.create_agent( + model=ai_agent_settings.model_deployment_name, + tools=code_interpreter.definitions, + tool_resources=code_interpreter.resources, + ) + + # Create the AzureAI Agent + agent = AzureAIAgent( + client=client, + definition=agent_definition, + ) + + # Create a new thread + thread = await client.agents.create_thread() + + user_inputs = [ + "Which segment had the most sales?", + "List the top 5 countries that generated the most profit.", + "Create a tab delimited file report of profit by each country per month.", + ] + + try: + for user_input in user_inputs: + # Add the user input as a chat message + await agent.add_chat_message( + thread_id=thread.id, + message=ChatMessageContent(role=AuthorRole.USER, content=user_input), + ) + print(f"# User: '{user_input}'") + # Invoke the agent for the specified thread + async for content in agent.invoke(thread_id=thread.id): + if content.role != AuthorRole.TOOL: + print(f"# Agent: {content.content}") + if len(content.items) > 0: + for item in content.items: + if isinstance(item, AnnotationContent): + print(f"\n`{item.quote}` => {item.file_id}") + response_content = await client.agents.get_file_content(file_id=item.file_id) + content_bytes = bytearray() + async for chunk in response_content: + content_bytes.extend(chunk) + tab_delimited_text = content_bytes.decode("utf-8") + print(tab_delimited_text) + finally: + await client.agents.delete_thread(thread.id) + await client.agents.delete_agent(agent.id) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_streaming.py b/python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_streaming.py new file mode 100644 index 000000000000..235c0efc95ba --- /dev/null +++ b/python/samples/concepts/agents/azure_ai_agent/azure_ai_agent_streaming.py @@ -0,0 +1,94 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +from typing import Annotated + +from azure.ai.projects.aio import AIProjectClient +from azure.identity.aio import DefaultAzureCredential + +from semantic_kernel.agents.azure_ai import AzureAIAgent, AzureAIAgentSettings +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.functions.kernel_function_decorator import kernel_function + + +# Define a sample plugin for the sample +class MenuPlugin: + """A sample Menu Plugin used for the concept sample.""" + + @kernel_function(description="Provides a list of specials from the menu.") + def get_specials(self) -> Annotated[str, "Returns the specials from the menu."]: + return """ + Special Soup: Clam Chowder + Special Salad: Cobb Salad + Special Drink: Chai Tea + """ + + @kernel_function(description="Provides the price of the requested menu item.") + def get_item_price( + self, menu_item: Annotated[str, "The name of the menu item."] + ) -> Annotated[str, "Returns the price of the menu item."]: + return "$9.99" + + +async def main() -> None: + ai_agent_settings = AzureAIAgentSettings.create() + + async with ( + DefaultAzureCredential() as creds, + AIProjectClient.from_connection_string( + credential=creds, + conn_str=ai_agent_settings.project_connection_string.get_secret_value(), + ) as client, + ): + AGENT_NAME = "Host" + AGENT_INSTRUCTIONS = "Answer questions about the menu." + + # Create agent definition + agent_definition = await client.agents.create_agent( + model=ai_agent_settings.model_deployment_name, + name=AGENT_NAME, + instructions=AGENT_INSTRUCTIONS, + ) + + # Create the AzureAI Agent + agent = AzureAIAgent( + client=client, + definition=agent_definition, + ) + + # Add the sample plugin to the kernel + agent.kernel.add_plugin(MenuPlugin(), plugin_name="menu") + + # Create a new thread + thread = await client.agents.create_thread() + + user_inputs = [ + "Hello", + "What is the special soup?", + "How much does that cost?", + "Thank you", + ] + + try: + for user_input in user_inputs: + # Add the user input as a chat message + await agent.add_chat_message( + thread_id=thread.id, message=ChatMessageContent(role=AuthorRole.USER, content=user_input) + ) + print(f"# User: '{user_input}'") + first_chunk = True + async for content in agent.invoke_stream(thread_id=thread.id): + if content.role != AuthorRole.TOOL: + if first_chunk: + print(f"# {content.role}: ", end="", flush=True) + first_chunk = False + print(content.content, end="", flush=True) + print() + finally: + await client.agents.delete_thread(thread.id) + await client.agents.delete_agent(agent.id) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/getting_started_with_agents/README.md b/python/samples/getting_started_with_agents/README.md index b34d1d56c05d..5a250bb75c04 100644 --- a/python/samples/getting_started_with_agents/README.md +++ b/python/samples/getting_started_with_agents/README.md @@ -16,20 +16,36 @@ This project contains a step by step guide to get started with _Semantic Kernel The getting started with agents examples include: +## Chat Completion + +Example|Description +---|--- +[step1_agent](../getting_started_with_agents/chat_completion/step1_agent.py)|How to create and use an agent. +[step2_plugins](../getting_started_with_agents/chat_completion/step2_plugins.py)|How to associate plugins with an agent. +[step3_chat](../getting_started_with_agents/chat_completion/step3_chat.py)|How to create a conversation between agents. +[step4_kernel_function_strategies](../getting_started_with_agents/chat_completion/step4_kernel_function_strategies.py)|How to utilize a `KernelFunction` as a chat strategy. +[step5_json_result](../getting_started_with_agents/chat_completion/step5_json_result.py)|How to have an agent produce JSON. +[step6_logging](../getting_started_with_agents/chat_completion/step6_logging.py)|How to enable logging for agents. + +## OpenAI Assistant + Example|Description ---|--- -[step1_agent](../getting_started_with_agents/step1_agent.py)|How to create and use an agent. -[step2_plugins](../getting_started_with_agents/step2_plugins.py)|How to associate plugins with an agent. -[step3_chat](../getting_started_with_agents/step3_chat.py)|How to create a conversation between agents. -[step4_kernel_function_strategies](../getting_started_with_agents/step4_kernel_function_strategies.py)|How to utilize a `KernelFunction` as a chat strategy. -[step5_json_result](../getting_started_with_agents/step5_json_result.py)|How to have an agent produce JSON. -[step6_logging](../getting_started_with_agents/step6_logging.py)|How to enable logging for agents. -[step7_assistant](../getting_started_with_agents/step7_assistant.py)|How to create and use an OpenAI Assistant agent. -[step8_assistant_vision](../getting_started_with_agents/step8_assistant_vision.py)|How to provide an image as input to an Open AI Assistant agent. -[step9_assistant_tool_code_interpreter](../getting_started_with_agents/step9_assistant_tool_code_interpreter.py)|How to use the code-interpreter tool for an Open AI Assistant agent. -[step10_assistant_tool_file_search](../getting_started_with_agents/step10_assistant_tool_file_search.py)|How to use the file-search tool for an Open AI Assistant agent. - -*Note: As we strive for parity with .NET, more getting_started_with_agent samples will be added. The current steps and names may be revised to further align with our .NET counterpart.* +[step1_assistant](../getting_started_with_agents/openai_assistant/step1_assistant.py)|How to create and use an OpenAI Assistant agent. +[step2_assistant_vision](../getting_started_with_agents/openai_assistant/step2_assistant_vision.py)|How to provide an image as input to an Open AI Assistant agent. +[step3_assistant_tool_code_interpreter](../getting_started_with_agents/openai_assistant/step3_assistant_tool_code_interpreter.py)|How to use the code-interpreter tool for an Open AI Assistant agent. +[step4_assistant_tool_file_search](../getting_started_with_agents/openai_assistant/step4_assistant_tool_file_search.py)|How to use the file-search tool for an Open AI Assistant agent. + +## Azure AI Agent +Example|Description +---|--- +[step1_azure_ai_agent](../getting_started_with_agents/azure_ai_agent/step1_azure_ai_agent.py)|How to create an Azure AI Agent and invoke a Semantic Kernel plugin. +[step2_azure_ai_agent_chat](../getting_started_with_agents/azure_ai_agent/step2_azure_ai_agent_chat.py)|How to an agent group chat with Azure AI Agents. +[step3_azure_ai_agent_code_interpreter](../getting_started_with_agents/azure_ai_agent/step3_azure_ai_agent_code_interpreter.py)|How to use the code-interpreter tool for an Azure AI agent. +[step4_azure_ai_agent_file_search](../getting_started_with_agents/azure_ai_agent/step4_azure_ai_agent_file_search.py)|How to use the file-search tool for an Azure AI agent. +[step5_azure_ai_agent_openapi](../getting_started_with_agents/azure_ai_agent/step5_azure_ai_agent_openapi.py)|How to use the Open API tool for an Azure AI agent. + +_Note: For details on configuring an Azure AI Agent, please see [here](../getting_started_with_agents/azure_ai_agent/README.md)._ ## Configuring the Kernel diff --git a/python/samples/getting_started_with_agents/azure_ai_agent/.env.example b/python/samples/getting_started_with_agents/azure_ai_agent/.env.example new file mode 100644 index 000000000000..c2d16cea26aa --- /dev/null +++ b/python/samples/getting_started_with_agents/azure_ai_agent/.env.example @@ -0,0 +1,6 @@ +AZURE_AI_AGENT_PROJECT_CONNECTION_STRING = "" +AZURE_AI_AGENT_MODEL_DEPLOYMENT_NAME = "" +AZURE_AI_AGENT_ENDPOINT = "" +AZURE_AI_AGENT_SUBSCRIPTION_ID = "" +AZURE_AI_AGENT_RESOURCE_GROUP_NAME = "" +AZURE_AI_AGENT_PROJECT_NAME = "" \ No newline at end of file diff --git a/python/samples/getting_started_with_agents/azure_ai_agent/README.md b/python/samples/getting_started_with_agents/azure_ai_agent/README.md new file mode 100644 index 000000000000..65293eaa3eb8 --- /dev/null +++ b/python/samples/getting_started_with_agents/azure_ai_agent/README.md @@ -0,0 +1,65 @@ +## Azure AI Agents + +The following getting started samples show how to use Azure AI Agents with Semantic Kernel. + +To set up the required resources, follow the "Quickstart: Create a new agent" guide [here](https://learn.microsoft.com/en-us/azure/ai-services/agents/quickstart?pivots=programming-language-python-azure). + +You will need to install the optional Semantic Kernel `azure` dependencies if you haven't already via: + +```bash +pip install semantic-kernel[azure] +``` + +Before running an Azure AI Agent, modify your .env file to include: + +```bash +AZURE_AI_AGENT_PROJECT_CONNECTION_STRING = "" +AZURE_AI_AGENT_MODEL_DEPLOYMENT_NAME = "" +``` + +or + +```bash +AZURE_AI_AGENT_MODEL_DEPLOYMENT_NAME = "" +AZURE_AI_AGENT_ENDPOINT = "" +AZURE_AI_AGENT_SUBSCRIPTION_ID = "" +AZURE_AI_AGENT_RESOURCE_GROUP_NAME = "" +AZURE_AI_AGENT_PROJECT_NAME = "" +``` + +The .env should be placed in the root directory. + +### Configuring the AI Project Client + +This can be done in one of two ways: + +```python +ai_agent_settings = AzureAIAgentSettings.create() + +async with ( + DefaultAzureCredential() as creds, + AIProjectClient.from_connection_string( + credential=creds, + conn_str=ai_agent_settings.project_connection_string.get_secret_value(), + ) as client, +): +# code +``` + +or + +```python +ai_agent_settings = AzureAIAgentSettings.create() + +async with ( + DefaultAzureCredential() as creds, + AIProjectClient( + credential=creds, + endpoint=ai_agent_settings.endpoint, + subscription_id=ai_agent_settings.subscription_id, + resource_group_name=ai_agent_settings.resource_group_name, + project_name=ai_agent_settings.project_name + ) as client, +): +# code +``` \ No newline at end of file diff --git a/python/samples/getting_started_with_agents/azure_ai_agent/step1_azure_ai_agent.py b/python/samples/getting_started_with_agents/azure_ai_agent/step1_azure_ai_agent.py new file mode 100644 index 000000000000..4a6b148044ab --- /dev/null +++ b/python/samples/getting_started_with_agents/azure_ai_agent/step1_azure_ai_agent.py @@ -0,0 +1,99 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +from typing import Annotated + +from azure.ai.projects.aio import AIProjectClient +from azure.identity.aio import DefaultAzureCredential + +from semantic_kernel.agents.azure_ai import AzureAIAgent, AzureAIAgentSettings +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.functions.kernel_function_decorator import kernel_function + +################################################################### +# The following sample demonstrates how to create a simple, # +# Azure AI agent that answers questions about a sample menu # +# using a Semantic Kernel Plugin. # +################################################################### + + +# Define a sample plugin for the sample +class MenuPlugin: + """A sample Menu Plugin used for the concept sample.""" + + @kernel_function(description="Provides a list of specials from the menu.") + def get_specials(self) -> Annotated[str, "Returns the specials from the menu."]: + return """ + Special Soup: Clam Chowder + Special Salad: Cobb Salad + Special Drink: Chai Tea + """ + + @kernel_function(description="Provides the price of the requested menu item.") + def get_item_price( + self, menu_item: Annotated[str, "The name of the menu item."] + ) -> Annotated[str, "Returns the price of the menu item."]: + return "$9.99" + + +async def main() -> None: + ai_agent_settings = AzureAIAgentSettings.create() + + async with ( + DefaultAzureCredential() as creds, + AIProjectClient.from_connection_string( + credential=creds, + conn_str=ai_agent_settings.project_connection_string.get_secret_value(), + ) as client, + ): + AGENT_NAME = "Host" + AGENT_INSTRUCTIONS = "Answer questions about the menu." + + # Create agent definition + agent_definition = await client.agents.create_agent( + model=ai_agent_settings.model_deployment_name, + name=AGENT_NAME, + instructions=AGENT_INSTRUCTIONS, + ) + + # Create the AzureAI Agent + agent = AzureAIAgent( + client=client, + definition=agent_definition, + ) + + # Add the sample plugin to the kernel + agent.kernel.add_plugin(MenuPlugin(), plugin_name="menu") + + # Create a new thread + thread = await client.agents.create_thread() + + user_inputs = [ + "Hello", + "What is the special soup?", + "How much does that cost?", + "Thank you", + ] + + try: + for user_input in user_inputs: + # Add the user input as a chat message + await agent.add_chat_message( + thread_id=thread.id, message=ChatMessageContent(role=AuthorRole.USER, content=user_input) + ) + print(f"# User: '{user_input}'") + # Invoke the agent for the specified thread + async for content in agent.invoke( + thread_id=thread.id, + temperature=0.2, # override the agent-level temperature setting with a run-time value + ): + if content.role != AuthorRole.TOOL: + print(f"# Agent: {content.content}") + finally: + await client.agents.delete_thread(thread.id) + await client.agents.delete_agent(agent.id) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/getting_started_with_agents/azure_ai_agent/step2_azure_ai_agent_chat.py b/python/samples/getting_started_with_agents/azure_ai_agent/step2_azure_ai_agent_chat.py new file mode 100644 index 000000000000..46012d754431 --- /dev/null +++ b/python/samples/getting_started_with_agents/azure_ai_agent/step2_azure_ai_agent_chat.py @@ -0,0 +1,110 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio + +from azure.ai.projects.aio import AIProjectClient +from azure.identity.aio import DefaultAzureCredential + +from semantic_kernel.agents import AgentGroupChat +from semantic_kernel.agents.azure_ai import AzureAIAgent, AzureAIAgentSettings +from semantic_kernel.agents.strategies.termination.termination_strategy import TerminationStrategy +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole + +##################################################################### +# The following sample demonstrates how to create an OpenAI # +# assistant using either Azure OpenAI or OpenAI, a chat completion # +# agent and have them participate in a group chat to work towards # +# the user's requirement. # +##################################################################### + + +class ApprovalTerminationStrategy(TerminationStrategy): + """A strategy for determining when an agent should terminate.""" + + async def should_agent_terminate(self, agent, history): + """Check if the agent should terminate.""" + return "approved" in history[-1].content.lower() + + +REVIEWER_NAME = "ArtDirector" +REVIEWER_INSTRUCTIONS = """ +You are an art director who has opinions about copywriting born of a love for David Ogilvy. +The goal is to determine if the given copy is acceptable to print. +If so, state that it is approved. Do not use the word "approve" unless you are giving approval. +If not, provide insight on how to refine suggested copy without example. +""" + +COPYWRITER_NAME = "CopyWriter" +COPYWRITER_INSTRUCTIONS = """ +You are a copywriter with ten years of experience and are known for brevity and a dry humor. +The goal is to refine and decide on the single best copy as an expert in the field. +Only provide a single proposal per response. +You're laser focused on the goal at hand. +Don't waste time with chit chat. +Consider suggestions when refining an idea. +""" + + +async def main(): + ai_agent_settings = AzureAIAgentSettings.create() + + async with ( + DefaultAzureCredential() as creds, + AIProjectClient.from_connection_string( + credential=creds, + conn_str=ai_agent_settings.project_connection_string.get_secret_value(), + ) as client, + ): + # Create the reviewer agent definition + reviewer_agent_definition = await client.agents.create_agent( + model=ai_agent_settings.model_deployment_name, + name=REVIEWER_NAME, + instructions=REVIEWER_INSTRUCTIONS, + ) + # Create the reviewer Azure AI Agent + agent_reviewer = AzureAIAgent( + client=client, + definition=reviewer_agent_definition, + ) + + # Create the copy writer agent definition + copy_writer_agent_definition = await client.agents.create_agent( + model=ai_agent_settings.model_deployment_name, + name=COPYWRITER_NAME, + instructions=COPYWRITER_INSTRUCTIONS, + ) + # Create the copy writer Azure AI Agent + agent_writer = AzureAIAgent( + client=client, + definition=copy_writer_agent_definition, + ) + + chat = AgentGroupChat( + agents=[agent_writer, agent_reviewer], + termination_strategy=ApprovalTerminationStrategy(agents=[agent_reviewer], maximum_iterations=10), + ) + + input = "a slogan for a new line of electric cars." + + try: + await chat.add_chat_message(ChatMessageContent(role=AuthorRole.USER, content=input)) + print(f"# {AuthorRole.USER}: '{input}'") + + async for content in chat.invoke(): + print(f"# {content.role} - {content.name or '*'}: '{content.content}'") + + print(f"# IS COMPLETE: {chat.is_complete}") + + print("*" * 60) + print("Chat History (In Descending Order):\n") + async for message in chat.get_chat_messages(agent=agent_reviewer): + print(f"# {message.role} - {message.name or '*'}: '{message.content}'") + finally: + await chat.reset() + await client.agents.delete_agent(agent_reviewer.id) + await client.agents.delete_agent(agent_writer.id) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/getting_started_with_agents/azure_ai_agent/step3_azure_ai_agent_code_interpreter.py b/python/samples/getting_started_with_agents/azure_ai_agent/step3_azure_ai_agent_code_interpreter.py new file mode 100644 index 000000000000..1431c36bb9bf --- /dev/null +++ b/python/samples/getting_started_with_agents/azure_ai_agent/step3_azure_ai_agent_code_interpreter.py @@ -0,0 +1,69 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio + +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import CodeInterpreterTool +from azure.identity.aio import DefaultAzureCredential + +from semantic_kernel.agents.azure_ai import AzureAIAgent, AzureAIAgentSettings +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole + +################################################################### +# The following sample demonstrates how to create a simple, # +# Azure AI agent that uses the code interpreter tool to answer # +# a coding question. # +################################################################### + + +async def main() -> None: + ai_agent_settings = AzureAIAgentSettings.create() + + async with ( + DefaultAzureCredential() as creds, + AIProjectClient.from_connection_string( + credential=creds, + conn_str=ai_agent_settings.project_connection_string.get_secret_value(), + ) as client, + ): + code_interpreter = CodeInterpreterTool() + + # Create agent definition + agent_definition = await client.agents.create_agent( + model=ai_agent_settings.model_deployment_name, + tools=code_interpreter.definitions, + tool_resources=code_interpreter.resources, + ) + + # Create the AzureAI Agent + agent = AzureAIAgent( + client=client, + definition=agent_definition, + ) + + # Create a new thread + thread = await client.agents.create_thread() + + user_inputs = [ + "Use code to determine the values in the Fibonacci sequence that that are less then the value of 101." + ] + + try: + for user_input in user_inputs: + # Add the user input as a chat message + await agent.add_chat_message( + thread_id=thread.id, message=ChatMessageContent(role=AuthorRole.USER, content=user_input) + ) + print(f"# User: '{user_input}'") + # Invoke the agent for the specified thread + async for content in agent.invoke(thread_id=thread.id): + if content.role != AuthorRole.TOOL: + print(f"# Agent: {content.content}") + finally: + await client.agents.delete_thread(thread.id) + await client.agents.delete_agent(agent.id) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/getting_started_with_agents/azure_ai_agent/step4_azure_ai_agent_file_search.py b/python/samples/getting_started_with_agents/azure_ai_agent/step4_azure_ai_agent_file_search.py new file mode 100644 index 000000000000..9ed92123bce1 --- /dev/null +++ b/python/samples/getting_started_with_agents/azure_ai_agent/step4_azure_ai_agent_file_search.py @@ -0,0 +1,83 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import os + +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import FileSearchTool, OpenAIFile, VectorStore +from azure.identity.aio import DefaultAzureCredential + +from semantic_kernel.agents.azure_ai import AzureAIAgent, AzureAIAgentSettings +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole + +################################################################### +# The following sample demonstrates how to create a simple, # +# Azure AI agent that uses the code interpreter tool to answer # +# a coding question. # +################################################################### + + +async def main() -> None: + ai_agent_settings = AzureAIAgentSettings.create() + + async with ( + DefaultAzureCredential() as creds, + AIProjectClient.from_connection_string( + credential=creds, + conn_str=ai_agent_settings.project_connection_string.get_secret_value(), + ) as client, + ): + pdf_file_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "resources", "employees.pdf" + ) + + file: OpenAIFile = await client.agents.upload_file_and_poll(file_path=pdf_file_path, purpose="assistants") + + vector_store: VectorStore = await client.agents.create_vector_store_and_poll( + file_ids=[file.id], name="my_vectorstore" + ) + + # Create file search tool with resources followed by creating agent + file_search = FileSearchTool(vector_store_ids=[vector_store.id]) + + # Create agent definition + agent_definition = await client.agents.create_agent( + model=ai_agent_settings.model_deployment_name, + tools=file_search.definitions, + tool_resources=file_search.resources, + ) + + # Create the AzureAI Agent + agent = AzureAIAgent( + client=client, + definition=agent_definition, + ) + + # Create a new thread + thread = await client.agents.create_thread() + + user_inputs = [ + "Who is the youngest employee?", + "Who works in sales?", + "I have a customer request, who can help me?", + ] + + try: + for user_input in user_inputs: + # Add the user input as a chat message + await agent.add_chat_message( + thread_id=thread.id, message=ChatMessageContent(role=AuthorRole.USER, content=user_input) + ) + print(f"# User: '{user_input}'") + # Invoke the agent for the specified thread + async for content in agent.invoke(thread_id=thread.id): + if content.role != AuthorRole.TOOL: + print(f"# Agent: {content.content}") + finally: + await client.agents.delete_thread(thread.id) + await client.agents.delete_agent(agent.id) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/getting_started_with_agents/azure_ai_agent/step5_azure_ai_agent_openapi.py b/python/samples/getting_started_with_agents/azure_ai_agent/step5_azure_ai_agent_openapi.py new file mode 100644 index 000000000000..16144d0db0ca --- /dev/null +++ b/python/samples/getting_started_with_agents/azure_ai_agent/step5_azure_ai_agent_openapi.py @@ -0,0 +1,100 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import json +import os + +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import ( + OpenApiAnonymousAuthDetails, + OpenApiTool, +) +from azure.identity.aio import DefaultAzureCredential + +from semantic_kernel.agents.azure_ai import AzureAIAgent, AzureAIAgentSettings +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole + +################################################################### +# The following sample demonstrates how to create a simple, # +# Azure AI agent that uses the code interpreter tool to answer # +# a coding question. # +################################################################### + + +async def main() -> None: + ai_agent_settings = AzureAIAgentSettings.create() + + async with ( + DefaultAzureCredential() as creds, + AIProjectClient.from_connection_string( + credential=creds, + conn_str=ai_agent_settings.project_connection_string.get_secret_value(), + ) as client, + ): + openapi_spec_file_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), + "resources", + ) + + # Create Auth object for the OpenApiTool (note that connection or managed identity + # auth setup requires additional setup in Azure) + auth = OpenApiAnonymousAuthDetails() + + # Initialize agent OpenApi tool using the read in OpenAPI spec + + with open(os.path.join(openapi_spec_file_path, "weather.json")) as weather_file: + weather_openapi_spec = json.loads(weather_file.read()) + + openapi_weather = OpenApiTool( + name="get_weather", + spec=weather_openapi_spec, + description="Retrieve weather information for a location", + auth=auth, + ) + + with open(os.path.join(openapi_spec_file_path, "countries.json")) as countries_file: + countries_openapi_spec = json.loads(countries_file.read()) + + openapi_countries = OpenApiTool( + name="get_country", spec=countries_openapi_spec, description="Retrieve country information", auth=auth + ) + + # Create agent definition + agent_definition = await client.agents.create_agent( + model=ai_agent_settings.model_deployment_name, + tools=openapi_weather.definitions + openapi_countries.definitions, + ) + + # Create the AzureAI Agent + agent = AzureAIAgent( + client=client, + definition=agent_definition, + ) + + # Create a new thread + thread = await client.agents.create_thread() + + user_inputs = [ + "What is the name and population of the country that uses currency with abbreviation THB", + "What is the capital city of the country?", + ] + + try: + for user_input in user_inputs: + # Add the user input as a chat message + await agent.add_chat_message( + thread_id=thread.id, message=ChatMessageContent(role=AuthorRole.USER, content=user_input) + ) + print(f"# User: '{user_input}'") + # Invoke the agent for the specified thread + async for content in agent.invoke(thread_id=thread.id): + if content.role != AuthorRole.TOOL: + print(f"# Agent: {content.content}") + finally: + await client.agents.delete_thread(thread.id) + await client.agents.delete_agent(agent.id) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/getting_started_with_agents/chat_completion/README.md b/python/samples/getting_started_with_agents/chat_completion/README.md new file mode 100644 index 000000000000..353ac42583b6 --- /dev/null +++ b/python/samples/getting_started_with_agents/chat_completion/README.md @@ -0,0 +1,3 @@ +## Chat Completion Agents + +The following getting started samples show how to use Chat Completion agents with Semantic Kernel. \ No newline at end of file diff --git a/python/samples/getting_started_with_agents/step1_agent.py b/python/samples/getting_started_with_agents/chat_completion/step1_agent.py similarity index 100% rename from python/samples/getting_started_with_agents/step1_agent.py rename to python/samples/getting_started_with_agents/chat_completion/step1_agent.py diff --git a/python/samples/getting_started_with_agents/step2_plugins.py b/python/samples/getting_started_with_agents/chat_completion/step2_plugins.py similarity index 100% rename from python/samples/getting_started_with_agents/step2_plugins.py rename to python/samples/getting_started_with_agents/chat_completion/step2_plugins.py diff --git a/python/samples/getting_started_with_agents/step3_chat.py b/python/samples/getting_started_with_agents/chat_completion/step3_chat.py similarity index 100% rename from python/samples/getting_started_with_agents/step3_chat.py rename to python/samples/getting_started_with_agents/chat_completion/step3_chat.py diff --git a/python/samples/getting_started_with_agents/step4_kernel_function_strategies.py b/python/samples/getting_started_with_agents/chat_completion/step4_kernel_function_strategies.py similarity index 100% rename from python/samples/getting_started_with_agents/step4_kernel_function_strategies.py rename to python/samples/getting_started_with_agents/chat_completion/step4_kernel_function_strategies.py diff --git a/python/samples/getting_started_with_agents/step5_json_result.py b/python/samples/getting_started_with_agents/chat_completion/step5_json_result.py similarity index 100% rename from python/samples/getting_started_with_agents/step5_json_result.py rename to python/samples/getting_started_with_agents/chat_completion/step5_json_result.py diff --git a/python/samples/getting_started_with_agents/step6_logging.py b/python/samples/getting_started_with_agents/chat_completion/step6_logging.py similarity index 100% rename from python/samples/getting_started_with_agents/step6_logging.py rename to python/samples/getting_started_with_agents/chat_completion/step6_logging.py diff --git a/python/samples/getting_started_with_agents/openai_assistant/README.md b/python/samples/getting_started_with_agents/openai_assistant/README.md new file mode 100644 index 000000000000..7deba8c7f3e8 --- /dev/null +++ b/python/samples/getting_started_with_agents/openai_assistant/README.md @@ -0,0 +1,3 @@ +## OpenAI Assistant Agents + +The following getting started samples show how to use OpenAI Assistant agents with Semantic Kernel. \ No newline at end of file diff --git a/python/samples/getting_started_with_agents/step7_assistant.py b/python/samples/getting_started_with_agents/openai_assistant/step1_assistant.py similarity index 100% rename from python/samples/getting_started_with_agents/step7_assistant.py rename to python/samples/getting_started_with_agents/openai_assistant/step1_assistant.py diff --git a/python/samples/getting_started_with_agents/step8_assistant_vision.py b/python/samples/getting_started_with_agents/openai_assistant/step2_assistant_vision.py similarity index 97% rename from python/samples/getting_started_with_agents/step8_assistant_vision.py rename to python/samples/getting_started_with_agents/openai_assistant/step2_assistant_vision.py index 22cb0c305258..c73a2f4e2d08 100644 --- a/python/samples/getting_started_with_agents/step8_assistant_vision.py +++ b/python/samples/getting_started_with_agents/openai_assistant/step2_assistant_vision.py @@ -31,9 +31,7 @@ async def main(): ) cat_image_file_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "resources", - "cat.jpg", + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "resources", "cat.jpg" ) # Upload the file for use with the assistant diff --git a/python/samples/getting_started_with_agents/step9_assistant_tool_code_interpreter.py b/python/samples/getting_started_with_agents/openai_assistant/step3_assistant_tool_code_interpreter.py similarity index 100% rename from python/samples/getting_started_with_agents/step9_assistant_tool_code_interpreter.py rename to python/samples/getting_started_with_agents/openai_assistant/step3_assistant_tool_code_interpreter.py diff --git a/python/samples/getting_started_with_agents/step10_assistant_tool_file_search.py b/python/samples/getting_started_with_agents/openai_assistant/step4_assistant_tool_file_search.py similarity index 94% rename from python/samples/getting_started_with_agents/step10_assistant_tool_file_search.py rename to python/samples/getting_started_with_agents/openai_assistant/step4_assistant_tool_file_search.py index c2ff3afe8483..8685889f19d9 100644 --- a/python/samples/getting_started_with_agents/step10_assistant_tool_file_search.py +++ b/python/samples/getting_started_with_agents/openai_assistant/step4_assistant_tool_file_search.py @@ -23,7 +23,9 @@ async def main(): # Get the path to the employees.pdf file - pdf_file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "resources", "employees.pdf") + pdf_file_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "resources", "employees.pdf" + ) # Define a service_id for the sample service_id = "agent" diff --git a/python/samples/getting_started_with_agents/resources/countries.json b/python/samples/getting_started_with_agents/resources/countries.json new file mode 100644 index 000000000000..b88d5040750a --- /dev/null +++ b/python/samples/getting_started_with_agents/resources/countries.json @@ -0,0 +1,46 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "RestCountries.NET API", + "description": "Web API version 3.1 for managing country items, based on previous implementations from restcountries.eu and restcountries.com.", + "version": "v3.1" + }, + "servers": [ + { "url": "https://restcountries.net" } + ], + "auth": [], + "paths": { + "/v3.1/currency": { + "get": { + "description": "Search by currency.", + "operationId": "LookupCountryByCurrency", + "parameters": [ + { + "name": "currency", + "in": "query", + "description": "The currency to search for.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + } + }, + "components": { + "schemes": {} + } +} \ No newline at end of file diff --git a/python/samples/getting_started_with_agents/resources/weather.json b/python/samples/getting_started_with_agents/resources/weather.json new file mode 100644 index 000000000000..c3009f417de4 --- /dev/null +++ b/python/samples/getting_started_with_agents/resources/weather.json @@ -0,0 +1,62 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "get weather data", + "description": "Retrieves current weather data for a location based on wttr.in.", + "version": "v1.0.0" + }, + "servers": [ + { + "url": "https://wttr.in" + } + ], + "auth": [], + "paths": { + "/{location}": { + "get": { + "description": "Get weather information for a specific location", + "operationId": "GetCurrentWeather", + "parameters": [ + { + "name": "location", + "in": "path", + "description": "City or location to retrieve the weather for", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "format", + "in": "query", + "description": "Always use j1 value for this parameter", + "required": true, + "schema": { + "type": "string", + "default": "j1" + } + } + ], + "responses": { + "200": { + "description": "Successful response", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + }, + "404": { + "description": "Location not found" + } + }, + "deprecated": false + } + } + }, + "components": { + "schemes": {} + } +} \ No newline at end of file diff --git a/python/semantic_kernel/agents/azure_ai/__init__.py b/python/semantic_kernel/agents/azure_ai/__init__.py new file mode 100644 index 000000000000..bb074ae1499c --- /dev/null +++ b/python/semantic_kernel/agents/azure_ai/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent +from semantic_kernel.agents.azure_ai.azure_ai_agent_settings import AzureAIAgentSettings + +__all__ = ["AzureAIAgent", "AzureAIAgentSettings"] diff --git a/python/semantic_kernel/agents/azure_ai/agent_content_generation.py b/python/semantic_kernel/agents/azure_ai/agent_content_generation.py new file mode 100644 index 000000000000..2d4f33d0903a --- /dev/null +++ b/python/semantic_kernel/agents/azure_ai/agent_content_generation.py @@ -0,0 +1,414 @@ +# Copyright (c) Microsoft. All rights reserved. + +from typing import TYPE_CHECKING, Any, cast + +from azure.ai.projects.models import ( + MessageDeltaImageFileContent, + MessageDeltaImageFileContentObject, + MessageDeltaTextContent, + MessageDeltaTextFileCitationAnnotation, + MessageDeltaTextFilePathAnnotation, + MessageImageFileContent, + MessageTextContent, + MessageTextFileCitationAnnotation, + MessageTextFilePathAnnotation, + RunStep, + RunStepDeltaCodeInterpreterDetailItemObject, + RunStepDeltaCodeInterpreterImageOutput, + RunStepDeltaCodeInterpreterLogOutput, + RunStepDeltaCodeInterpreterToolCall, + RunStepDeltaFileSearchToolCall, + RunStepDeltaFunctionToolCall, + RunStepFunctionToolCall, + ThreadMessage, + ThreadRun, +) + +from semantic_kernel.contents.annotation_content import AnnotationContent +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.file_reference_content import FileReferenceContent +from semantic_kernel.contents.function_call_content import FunctionCallContent +from semantic_kernel.contents.function_result_content import FunctionResultContent +from semantic_kernel.contents.image_content import ImageContent +from semantic_kernel.contents.streaming_annotation_content import StreamingAnnotationContent +from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent +from semantic_kernel.contents.streaming_file_reference_content import StreamingFileReferenceContent +from semantic_kernel.contents.streaming_text_content import StreamingTextContent +from semantic_kernel.contents.text_content import TextContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.utils.experimental_decorator import experimental_function + +if TYPE_CHECKING: + from azure.ai.projects.models import ( + MessageDeltaChunk, + RunStepDeltaToolCallObject, + ) + +################################################################### +# The methods in this file are used with Azure AI Agent # +# related code. They are used to invoke, create chat messages, # +# or generate message content. # +################################################################### + + +@experimental_function +def get_message_contents(message: "ChatMessageContent") -> list[dict[str, Any]]: + """Get the message contents. + + Args: + message: The message. + """ + contents: list[dict[str, Any]] = [] + for content in message.items: + match content: + case TextContent(): + # Make sure text is a string + final_text = content.text + if not isinstance(final_text, str): + if isinstance(final_text, (list, tuple)): + final_text = " ".join(map(str, final_text)) + else: + final_text = str(final_text) + + contents.append({"type": "text", "text": final_text}) + + case ImageContent(): + if content.uri: + contents.append(content.to_dict()) + + case FileReferenceContent(): + contents.append({ + "type": "image_file", + "image_file": {"file_id": content.file_id}, + }) + + case FunctionResultContent(): + final_result = content.result + match final_result: + case str(): + contents.append({"type": "text", "text": final_result}) + case list() | tuple(): + contents.append({"type": "text", "text": " ".join(map(str, final_result))}) + case _: + contents.append({"type": "text", "text": str(final_result)}) + + return contents + + +@experimental_function +def generate_message_content( + assistant_name: str, message: "ThreadMessage", completed_step: "RunStep | None" = None +) -> ChatMessageContent: + """Generate message content.""" + role = AuthorRole(message.role) + + metadata = ( + { + "created_at": completed_step.created_at, + "message_id": message.id, # message needs to be defined in context + "step_id": completed_step.id, + "run_id": completed_step.run_id, + "thread_id": completed_step.thread_id, + "assistant_id": completed_step.assistant_id, + "usage": completed_step.usage, + } + if completed_step is not None + else None + ) + + content: ChatMessageContent = ChatMessageContent(role=role, name=assistant_name, metadata=metadata) # type: ignore + + messages: list[MessageImageFileContent | MessageTextContent] = cast( + list[MessageImageFileContent | MessageTextContent], message.content or [] + ) + for item_content in messages: + if item_content.type == "text": + content.items.append( + TextContent( + text=item_content.text.value, + ) + ) + for annotation in item_content.text.annotations: + content.items.append(generate_annotation_content(annotation)) + elif item_content.type == "image_file": + content.items.append( + FileReferenceContent( + file_id=item_content.image_file.file_id, + ) + ) + return content + + +@experimental_function +def generate_streaming_message_content( + assistant_name: str, message_delta_event: "MessageDeltaChunk" +) -> StreamingChatMessageContent: + """Generate streaming message content from a MessageDeltaEvent.""" + delta = message_delta_event.delta + + # Determine the role + role = AuthorRole(delta.role) if delta.role is not None else AuthorRole("assistant") + + items: list[StreamingTextContent | StreamingAnnotationContent | StreamingFileReferenceContent] = [] + + delta_chunks: list[MessageDeltaImageFileContent | MessageDeltaTextContent] = cast( + list[MessageDeltaImageFileContent | MessageDeltaTextContent], delta.content or [] + ) + + for delta_block in delta_chunks: + if delta_block.type == "text": + if delta_block.text and delta_block.text.value: # Ensure text is not None + text_value = delta_block.text.value + items.append( + StreamingTextContent( + text=text_value, + choice_index=delta_block.index, + ) + ) + # Process annotations if any + if delta_block.text.annotations: + for annotation in delta_block.text.annotations or []: + if isinstance( + annotation, + ( + MessageDeltaTextFileCitationAnnotation, + MessageDeltaTextFilePathAnnotation, + ), + ): + items.append(generate_streaming_annotation_content(annotation)) + elif delta_block.type == "image_file": + assert isinstance(delta_block, MessageDeltaImageFileContent) # nosec + if delta_block.image_file and isinstance(delta_block.image_file, MessageDeltaImageFileContentObject): + file_id = delta_block.image_file.file_id + items.append( + StreamingFileReferenceContent( + file_id=file_id, + ) + ) + + return StreamingChatMessageContent(role=role, name=assistant_name, items=items, choice_index=0) # type: ignore + + +@experimental_function +def get_function_call_contents( + run: "ThreadRun", function_steps: dict[str, FunctionCallContent] +) -> list[FunctionCallContent]: + """Extract function call contents from the run. + + Args: + run: The run. + function_steps: The function steps + + Returns: + The list of function call contents. + """ + function_call_contents: list[FunctionCallContent] = [] + required_action = getattr(run, "required_action", None) + if not required_action or not getattr(required_action, "submit_tool_outputs", False): + return function_call_contents + for tool_call in required_action.submit_tool_outputs.tool_calls: + tool: RunStepFunctionToolCall = tool_call + fcc = FunctionCallContent( + id=tool.id, + index=getattr(tool, "index", None), + name=tool.function.name, + arguments=tool.function.arguments, + ) + function_call_contents.append(fcc) + function_steps[tool.id] = fcc + return function_call_contents + + +@experimental_function +def generate_function_call_content(agent_name: str, fccs: list[FunctionCallContent]) -> ChatMessageContent: + """Generate function call content. + + Args: + agent_name: The agent name. + fccs: The function call contents. + + Returns: + ChatMessageContent: The chat message content containing the function call content as the items. + """ + return ChatMessageContent(role=AuthorRole.ASSISTANT, name=agent_name, items=fccs) # type: ignore + + +@experimental_function +def generate_function_result_content( + agent_name: str, function_step: FunctionCallContent, tool_call: "RunStepFunctionToolCall" +) -> ChatMessageContent: + """Generate function result content.""" + function_call_content: ChatMessageContent = ChatMessageContent(role=AuthorRole.TOOL, name=agent_name) # type: ignore + function_call_content.items.append( + FunctionResultContent( + function_name=function_step.function_name, + plugin_name=function_step.plugin_name, + id=function_step.id, + result=tool_call.function.output, # type: ignore + ) + ) + return function_call_content + + +@experimental_function +def generate_code_interpreter_content(agent_name: str, code: str) -> "ChatMessageContent": + """Generate code interpreter content. + + Args: + agent_name: The agent name. + code: The code. + + Returns: + ChatMessageContent: The chat message content. + """ + return ChatMessageContent( + role=AuthorRole.ASSISTANT, + content=code, + name=agent_name, + metadata={"code": True}, + ) + + +@experimental_function +def generate_streaming_function_content( + agent_name: str, step_details: "RunStepDeltaToolCallObject" +) -> "StreamingChatMessageContent | None": + """Generate streaming function content. + + Args: + agent_name: The agent name. + step_details: The function step. + + Returns: + StreamingChatMessageContent: The chat message content. + """ + if not step_details.tool_calls: + return None + + items: list[FunctionCallContent] = [] + + tool_calls: list[ + RunStepDeltaCodeInterpreterToolCall | RunStepDeltaFileSearchToolCall | RunStepDeltaFunctionToolCall + ] = cast( + list[RunStepDeltaCodeInterpreterToolCall | RunStepDeltaFileSearchToolCall | RunStepDeltaFunctionToolCall], + step_details.tool_calls or [], + ) + + for tool in tool_calls: + if tool.type == "function" and tool.function: + items.append( + FunctionCallContent( + id=tool.id, + index=getattr(tool, "index", None), + name=tool.function.name, + arguments=tool.function.arguments, + ) + ) + + return ( + StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + name=agent_name, + items=items, # type: ignore + choice_index=0, + ) + if len(items) > 0 + else None + ) + + +@experimental_function +def generate_streaming_code_interpreter_content( + agent_name: str, step_details: "RunStepDeltaToolCallObject" +) -> "StreamingChatMessageContent | None": + """Generate code interpreter content. + + Args: + agent_name: The agent name. + step_details: The current step details. + + Returns: + StreamingChatMessageContent: The chat message content. + """ + items: list[StreamingTextContent | StreamingFileReferenceContent] = [] + + if not step_details.tool_calls: + return None + + metadata: dict[str, bool] = {} + for index, tool in enumerate(step_details.tool_calls): + if isinstance(tool.type, RunStepDeltaCodeInterpreterToolCall): + code_interpreter_tool_call: RunStepDeltaCodeInterpreterDetailItemObject = tool + if code_interpreter_tool_call.input: + items.append( + StreamingTextContent( + choice_index=index, + text=code_interpreter_tool_call.input, + ) + ) + metadata["code"] = True + if code_interpreter_tool_call.outputs: + for output in code_interpreter_tool_call.outputs: + if isinstance(output, RunStepDeltaCodeInterpreterImageOutput) and output.image.file_id: + items.append( + StreamingFileReferenceContent( + file_id=output.image.file_id, + ) + ) + if isinstance(output, RunStepDeltaCodeInterpreterLogOutput) and output.logs: + items.append( + StreamingTextContent( + choice_index=index, + text=output.logs, + ) + ) + + return ( + StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + name=agent_name, + items=items, # type: ignore + choice_index=0, + metadata=metadata if metadata else None, + ) + if len(items) > 0 + else None + ) + + +@experimental_function +def generate_annotation_content( + annotation: MessageTextFilePathAnnotation | MessageTextFileCitationAnnotation, +) -> AnnotationContent: + """Generate annotation content.""" + file_id = None + if isinstance(annotation, MessageTextFilePathAnnotation): + file_id = annotation.file_path.file_id + elif isinstance(annotation, MessageTextFileCitationAnnotation): + file_id = annotation.file_citation.file_id + + return AnnotationContent( + file_id=file_id, + quote=annotation.text, + start_index=annotation.start_index, + end_index=annotation.end_index, + ) + + +@experimental_function +def generate_streaming_annotation_content( + annotation: MessageDeltaTextFilePathAnnotation | MessageDeltaTextFileCitationAnnotation, +) -> StreamingAnnotationContent: + """Generate streaming annotation content.""" + file_id = None + if isinstance(annotation, MessageDeltaTextFilePathAnnotation) and annotation.file_path: + file_id = annotation.file_path.file_id if annotation.file_path.file_id else None + elif isinstance(annotation, MessageDeltaTextFileCitationAnnotation) and annotation.file_citation: + file_id = annotation.file_citation.file_id if annotation.file_citation.file_id else None + + return StreamingAnnotationContent( + file_id=file_id, + quote=annotation.text, + start_index=annotation.start_index, + end_index=annotation.end_index, + ) diff --git a/python/semantic_kernel/agents/azure_ai/agent_thread_actions.py b/python/semantic_kernel/agents/azure_ai/agent_thread_actions.py new file mode 100644 index 000000000000..8abbcbb1eca4 --- /dev/null +++ b/python/semantic_kernel/agents/azure_ai/agent_thread_actions.py @@ -0,0 +1,865 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import logging +from collections.abc import AsyncIterable +from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, cast + +from azure.ai.projects.models import ( + AgentsApiResponseFormat, + AgentsApiResponseFormatMode, + AgentsNamedToolChoiceType, + AgentStreamEvent, + AsyncAgentEventHandler, + AsyncAgentRunStream, + BaseAsyncAgentEventHandler, + OpenAIPageableListOfThreadMessage, + ResponseFormatJsonSchemaType, + RunStep, + RunStepCodeInterpreterToolCall, + RunStepDeltaChunk, + RunStepDeltaToolCallObject, + RunStepMessageCreationDetails, + RunStepToolCallDetails, + RunStepType, + SubmitToolOutputsAction, + ThreadMessage, + ThreadRun, + ToolDefinition, + TruncationObject, +) +from azure.ai.projects.models._enums import MessageRole + +from semantic_kernel.agents.azure_ai.agent_content_generation import ( + generate_code_interpreter_content, + generate_function_call_content, + generate_function_result_content, + generate_message_content, + generate_streaming_code_interpreter_content, + generate_streaming_function_content, + generate_streaming_message_content, + get_function_call_contents, +) +from semantic_kernel.agents.azure_ai.azure_ai_agent_utils import AzureAIAgentUtils +from semantic_kernel.agents.open_ai.function_action_result import FunctionActionResult +from semantic_kernel.connectors.ai.function_calling_utils import ( + kernel_function_metadata_to_function_call_format, + merge_function_results, +) +from semantic_kernel.contents.function_call_content import FunctionCallContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.exceptions.agent_exceptions import AgentInvokeException +from semantic_kernel.functions import KernelArguments +from semantic_kernel.utils.experimental_decorator import experimental_class + +if TYPE_CHECKING: + from azure.ai.projects.aio import AIProjectClient + + from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent + from semantic_kernel.contents import ChatHistory, ChatMessageContent + from semantic_kernel.kernel import Kernel + +_T = TypeVar("_T", bound="AgentThreadActions") + +logger: logging.Logger = logging.getLogger(__name__) + + +@experimental_class +class AgentThreadActions: + """AzureAI Agent Thread Actions.""" + + polling_status: ClassVar[list[str]] = ["queued", "in_progress", "cancelling"] + error_message_states: ClassVar[list[str]] = ["failed", "cancelled", "expired", "incomplete"] + + # region Invocation Methods + + @classmethod + async def invoke( + cls: type[_T], + *, + agent: "AzureAIAgent", + thread_id: str, + arguments: KernelArguments | None = None, + kernel: "Kernel | None" = None, + # Run-level parameters: + model: str | None = None, + instructions_override: str | None = None, + additional_instructions: str | None = None, + additional_messages: "list[ChatMessageContent] | None" = None, + tools: list[ToolDefinition] | None = None, + temperature: float | None = None, + top_p: float | None = None, + max_prompt_tokens: int | None = None, + max_completion_tokens: int | None = None, + truncation_strategy: TruncationObject | None = None, + response_format: AgentsApiResponseFormat + | AgentsApiResponseFormatMode + | ResponseFormatJsonSchemaType + | None = None, + parallel_tool_calls: bool | None = None, + metadata: dict[str, str] | None = None, + **kwargs: Any, + ) -> AsyncIterable[tuple[bool, "ChatMessageContent"]]: + """Invoke the message in the thread. + + Args: + agent: The agent to invoke. + thread_id: The thread id. + arguments: The kernel arguments. + kernel: The kernel. + model: The model. + instructions_override: The instructions override. + additional_instructions: The additional instructions. + additional_messages: The additional messages to add to the thread. Only supports messages with + role = User or Assistant. + https://platform.openai.com/docs/api-reference/runs/createRun#runs-createrun-additional_messages + tools: The tools. + temperature: The temperature. + top_p: The top p. + max_prompt_tokens: The max prompt tokens. + max_completion_tokens: The max completion tokens. + truncation_strategy: The truncation strategy. + response_format: The response format. + parallel_tool_calls: The parallel tool calls. + metadata: The metadata. + kwargs: Additional keyword arguments. + + Returns: + A tuple of the visibility flag and the invoked message. + """ + arguments = KernelArguments() if arguments is None else KernelArguments(**arguments, **kwargs) + kernel = kernel or agent.kernel + + tools = cls._get_tools(agent=agent, kernel=kernel) # type: ignore + + base_instructions = await agent.format_instructions(kernel=kernel, arguments=arguments) + + merged_instructions: str = "" + if instructions_override is not None: + merged_instructions = instructions_override + elif base_instructions and additional_instructions: + merged_instructions = f"{base_instructions}\n\n{additional_instructions}" + else: + merged_instructions = base_instructions or additional_instructions or "" + + run_options = cls._generate_options( + agent=agent, + model=model, + additional_messages=additional_messages, + max_completion_tokens=max_completion_tokens, + max_prompt_tokens=max_prompt_tokens, + temperature=temperature, + top_p=top_p, + metadata=metadata, + truncation_strategy=truncation_strategy, + response_format=response_format, + parallel_tool_calls=parallel_tool_calls, + ) + # Remove keys with None values. + run_options = {k: v for k, v in run_options.items() if v is not None} + + run: ThreadRun = await agent.client.agents.create_run( + assistant_id=agent.id, + thread_id=thread_id, + instructions=merged_instructions or agent.instructions, + tools=tools, + **run_options, + ) + + processed_step_ids = set() + function_steps: dict[str, "FunctionCallContent"] = {} + + while run.status != "completed": + run = await cls._poll_run_status(agent=agent, run=run, thread_id=thread_id) + + if run.status in cls.error_message_states: + error_message = "" + if run.last_error and run.last_error.message: + error_message = run.last_error.message + raise AgentInvokeException( + f"Run failed with status: `{run.status}` for agent `{agent.name}` and thread `{thread_id}` " + f"with error: {error_message}" + ) + + # Check if function calling is required + if run.status == "requires_action" and isinstance(run.required_action, SubmitToolOutputsAction): + logger.debug(f"Run [{run.id}] requires tool action for agent `{agent.name}` and thread `{thread_id}`") + fccs = get_function_call_contents(run, function_steps) + if fccs: + logger.debug( + f"Yielding generate_function_call_content for agent `{agent.name}` and " + f"thread `{thread_id}`, visibility False" + ) + yield False, generate_function_call_content(agent_name=agent.name, fccs=fccs) + + from semantic_kernel.contents.chat_history import ChatHistory + + chat_history = ChatHistory() if kwargs.get("chat_history") is None else kwargs["chat_history"] + _ = await cls._invoke_function_calls(kernel=kernel, fccs=fccs, chat_history=chat_history) + + tool_outputs = cls._format_tool_outputs(fccs, chat_history) + await agent.client.agents.submit_tool_outputs_to_run( + run_id=run.id, + thread_id=thread_id, + tool_outputs=tool_outputs, # type: ignore + ) + logger.debug(f"Submitted tool outputs for agent `{agent.name}` and thread `{thread_id}`") + + steps_response = await agent.client.agents.list_run_steps(run_id=run.id, thread_id=thread_id) + logger.debug(f"Called for steps_response for run [{run.id}] agent `{agent.name}` and thread `{thread_id}`") + steps: list[RunStep] = steps_response.data + + def sort_key(step: RunStep): + # Put tool_calls first, then message_creation. + # If multiple steps share a type, break ties by completed_at. + return (0 if step.type == "tool_calls" else 1, step.completed_at) + + completed_steps_to_process = sorted( + [s for s in steps if s.completed_at is not None and s.id not in processed_step_ids], + key=sort_key, + ) + + logger.debug( + f"Completed steps to process for run [{run.id}] agent `{agent.name}` and thread `{thread_id}` " + f"with length `{len(completed_steps_to_process)}`" + ) + + message_count = 0 + for completed_step in completed_steps_to_process: + match completed_step.type: + case RunStepType.TOOL_CALLS: + logger.debug( + f"Entering step type tool_calls for run [{run.id}], agent `{agent.name}` and " + f"thread `{thread_id}`" + ) + tool_call_details: RunStepToolCallDetails = cast( + RunStepToolCallDetails, completed_step.step_details + ) + for tool_call in tool_call_details.tool_calls: + is_visible = False + content: "ChatMessageContent | None" = None + match tool_call.type: + case AgentsNamedToolChoiceType.CODE_INTERPRETER: + logger.debug( + f"Entering tool_calls (code_interpreter) for run [{run.id}], agent " + f"`{agent.name}` and thread `{thread_id}`" + ) + code_call: RunStepCodeInterpreterToolCall = cast( + RunStepCodeInterpreterToolCall, tool_call + ) + content = generate_code_interpreter_content( + agent.name, + code_call.code_interpreter.input, + ) + is_visible = True + case AgentsNamedToolChoiceType.FUNCTION: + logger.debug( + f"Entering tool_calls (function) for run [{run.id}], agent `{agent.name}` " + f"and thread `{thread_id}`" + ) + function_step = function_steps.get(tool_call.id) + assert function_step is not None # nosec + content = generate_function_result_content( + agent_name=agent.name, function_step=function_step, tool_call=tool_call + ) + + if content: + message_count += 1 + logger.debug( + f"Yielding tool_message for run [{run.id}], agent `{agent.name}`, " + f"thread `{thread_id}`, message count `{message_count}`, " + f"is_visible `{is_visible}`" + ) + yield is_visible, content + case RunStepType.MESSAGE_CREATION: + logger.debug( + f"Entering message_creation for run [{run.id}], agent `{agent.name}` and thread " + f"`{thread_id}`" + ) + message_call_details: RunStepMessageCreationDetails = cast( + RunStepMessageCreationDetails, completed_step.step_details + ) + message = await cls._retrieve_message( + agent=agent, + thread_id=thread_id, + message_id=message_call_details.message_creation.message_id, # type: ignore + ) + if message: + content = generate_message_content(agent.name, message) + if content and len(content.items) > 0: + message_count += 1 + logger.debug( + f"Yielding message_creation for run [{run.id}], agent `{agent.name}`, " + f"thread `{thread_id}`, message count `{message_count}`, is_visible `True`" + ) + yield True, content + processed_step_ids.add(completed_step.id) + + @classmethod + async def invoke_stream( + cls: type[_T], + *, + agent: "AzureAIAgent", + thread_id: str, + messages: "list[ChatMessageContent] | None" = None, + arguments: KernelArguments | None = None, + kernel: "Kernel | None" = None, + # Run-level parameters: + model: str | None = None, + instructions_override: str | None = None, + additional_instructions: str | None = None, + additional_messages: "list[ChatMessageContent] | None" = None, + tools: list[ToolDefinition] | None = None, + temperature: float | None = None, + top_p: float | None = None, + max_prompt_tokens: int | None = None, + max_completion_tokens: int | None = None, + truncation_strategy: TruncationObject | None = None, + response_format: AgentsApiResponseFormat + | AgentsApiResponseFormatMode + | ResponseFormatJsonSchemaType + | None = None, + parallel_tool_calls: bool | None = None, + metadata: dict[str, str] | None = None, + **kwargs: Any, + ) -> AsyncIterable["ChatMessageContent"]: + """Invoke the agent stream and yield ChatMessageContent continuously. + + Args: + agent: The agent to invoke. + thread_id: The thread id. + messages: The messages. + arguments: The kernel arguments. + kernel: The kernel. + model: The model. + instructions_override: The instructions override. + additional_instructions: The additional instructions. + additional_messages: The additional messages to add to the thread. Only supports messages with + role = User or Assistant. + https://platform.openai.com/docs/api-reference/runs/createRun#runs-createrun-additional_messages + tools: The tools. + temperature: The temperature. + top_p: The top p. + max_prompt_tokens: The max prompt tokens. + max_completion_tokens: The max completion tokens. + truncation_strategy: The truncation strategy. + response_format: The response format. + parallel_tool_calls: The parallel tool calls. + metadata: The metadata. + kwargs: Additional keyword arguments. + + Returns: + An async iterable of streamed content. + """ + arguments = KernelArguments() if arguments is None else KernelArguments(**arguments, **kwargs) + kernel = kernel or agent.kernel + arguments = agent.merge_arguments(arguments) + + tools = cls._get_tools(agent=agent, kernel=kernel) # type: ignore + + base_instructions = await agent.format_instructions(kernel=kernel, arguments=arguments) + + merged_instructions: str = "" + if instructions_override is not None: + merged_instructions = instructions_override + elif base_instructions and additional_instructions: + merged_instructions = f"{base_instructions}\n\n{additional_instructions}" + else: + merged_instructions = base_instructions or additional_instructions or "" + + run_options = cls._generate_options( + agent=agent, + model=model, + additional_messages=additional_messages, + max_completion_tokens=max_completion_tokens, + max_prompt_tokens=max_prompt_tokens, + temperature=temperature, + top_p=top_p, + metadata=metadata, + truncation_strategy=truncation_strategy, + response_format=response_format, + parallel_tool_calls=parallel_tool_calls, + ) + run_options = {k: v for k, v in run_options.items() if v is not None} + + stream: AsyncAgentRunStream = await agent.client.agents.create_stream( + assistant_id=agent.id, + thread_id=thread_id, + instructions=merged_instructions or agent.instructions, + tools=tools, + **run_options, + ) + + function_steps: dict[str, FunctionCallContent] = {} + active_messages: dict[str, RunStep] = {} + + async for content in cls._process_stream_events( + stream=stream, + agent=agent, + thread_id=thread_id, + messages=messages, + kernel=kernel, + function_steps=function_steps, + active_messages=active_messages, + ): + if content: + yield content + + @classmethod + async def _process_stream_events( + cls: type[_T], + stream: AsyncAgentRunStream, + agent: "AzureAIAgent", + thread_id: str, + kernel: "Kernel", + function_steps: dict[str, FunctionCallContent], + active_messages: dict[str, RunStep], + messages: "list[ChatMessageContent] | None" = None, + ) -> AsyncIterable["ChatMessageContent"]: + """Process events from the main stream and delegate tool output handling as needed.""" + while True: + async with stream as response_stream: + async for event_type, event_data, _ in response_stream: + if event_type == AgentStreamEvent.THREAD_RUN_CREATED: + run = event_data + logger.info(f"Assistant run created with ID: {run.id}") + + elif event_type == AgentStreamEvent.THREAD_RUN_IN_PROGRESS: + run_step = cast(RunStep, event_data) + logger.info(f"Assistant run in progress with ID: {run_step.id}") + + elif event_type == AgentStreamEvent.THREAD_MESSAGE_DELTA: + yield generate_streaming_message_content(agent.name, event_data) + + elif event_type == AgentStreamEvent.THREAD_RUN_STEP_COMPLETED: + step_completed = cast(RunStep, event_data) + logger.info(f"Run step completed with ID: {step_completed.id}") + if isinstance(step_completed.step_details, RunStepMessageCreationDetails): + msg_id = step_completed.step_details.message_creation.message_id + active_messages.setdefault(msg_id, step_completed) + + elif event_type == AgentStreamEvent.THREAD_RUN_STEP_DELTA: + run_step_event: RunStepDeltaChunk = event_data + details = run_step_event.delta.step_details + if not details: + continue + if isinstance(details, RunStepDeltaToolCallObject) and details.tool_calls: + for tool_call in details.tool_calls: + content = None + if tool_call.type == "function": + content = generate_streaming_function_content(agent.name, details) + elif tool_call.type == "code_interpreter": + content = generate_streaming_code_interpreter_content(agent.name, details) + if content: + yield content + + elif event_type == AgentStreamEvent.THREAD_RUN_REQUIRES_ACTION: + run = cast(ThreadRun, event_data) + action_result = await cls._handle_streaming_requires_action( + agent_name=agent.name, + kernel=kernel, + run=run, + function_steps=function_steps, + ) + if action_result is None: + raise RuntimeError( + f"Function call required but no function steps found for agent `{agent.name}` " + f"thread: {thread_id}." + ) + + if action_result.function_result_content: + yield action_result.function_result_content + if messages: + messages.append(action_result.function_result_content) + + if action_result.function_call_content: + if messages: + messages.append(action_result.function_call_content) + async for sub_content in cls._stream_tool_outputs( + agent=agent, + thread_id=thread_id, + run=run, + action_result=action_result, + active_messages=active_messages, + messages=messages, + ): + if sub_content: + yield sub_content + break + + elif event_type == AgentStreamEvent.THREAD_RUN_COMPLETED: + run = cast(ThreadRun, event_data) + logger.info(f"Run completed with ID: {run.id}") + if active_messages: + for msg_id, step in active_messages.items(): + message = await cls._retrieve_message( + agent=agent, thread_id=thread_id, message_id=msg_id + ) + if message and hasattr(message, "content"): + final_content = generate_message_content(agent.name, message, step) + if messages: + messages.append(final_content) + return + + elif event_type == AgentStreamEvent.THREAD_RUN_FAILED: + run_failed = cast(ThreadRun, event_data) + error_message = ( + run_failed.last_error.message + if run_failed.last_error and run_failed.last_error.message + else "" + ) + raise RuntimeError( + f"Run failed with status: `{run_failed.status}` for agent `{agent.name}` " + f"thread `{thread_id}` with error: {error_message}" + ) + else: + break + return + + @classmethod + async def _stream_tool_outputs( + cls: type[_T], + agent: "AzureAIAgent", + thread_id: str, + run: ThreadRun, + action_result: FunctionActionResult, + active_messages: dict[str, RunStep], + messages: "list[ChatMessageContent] | None" = None, + ) -> AsyncIterable["ChatMessageContent"]: + """Wrap the tool outputs stream as an async generator. + + This allows downstream consumers to iterate over the yielded content. + """ + handler: BaseAsyncAgentEventHandler = AsyncAgentEventHandler() + await agent.client.agents.submit_tool_outputs_to_stream( + run_id=run.id, + thread_id=thread_id, + tool_outputs=action_result.tool_outputs, # type: ignore + event_handler=handler, + ) + async for sub_event_type, sub_event_data, _ in handler: + if sub_event_type == AgentStreamEvent.THREAD_MESSAGE_DELTA: + yield generate_streaming_message_content(agent.name, sub_event_data) + elif sub_event_type == AgentStreamEvent.THREAD_RUN_COMPLETED: + logger.info(f"Run completed with ID: {sub_event_data.id}") + if active_messages: + for msg_id, step in active_messages.items(): + message = await cls._retrieve_message(agent=agent, thread_id=thread_id, message_id=msg_id) + if message and hasattr(message, "content"): + final_content = generate_message_content(agent.name, message, step) + if messages: + messages.append(final_content) + return + elif sub_event_type == AgentStreamEvent.THREAD_RUN_FAILED: + run_failed: ThreadRun = sub_event_data + error_message = ( + run_failed.last_error.message if run_failed.last_error and run_failed.last_error.message else "" + ) + raise RuntimeError( + f"Run failed with status: `{run_failed.status}` for agent `{agent.name}` " + f"thread `{thread_id}` with error: {error_message}" + ) + elif sub_event_type == AgentStreamEvent.DONE: + break + + # endregion + + # region Messaging Handling Methods + + @classmethod + async def create_thread( + cls: type[_T], + client: "AIProjectClient", + **kwargs: Any, + ) -> str: + """Create a thread. + + Args: + client: The client to use to create the thread. + kwargs: Additional keyword arguments. + + Returns: + The ID of the created thread. + """ + thread = await client.agents.create_thread(**kwargs) + return thread.id + + @classmethod + async def create_message( + cls: type[_T], + client: "AIProjectClient", + thread_id: str, + message: "ChatMessageContent", + **kwargs: Any, + ) -> "ThreadMessage | None": + """Create a message in the thread. + + Args: + client: The client to use to create the message. + thread_id: The ID of the thread to create the message in. + message: The message to create. + kwargs: Additional keyword arguments. + + Returns: + The created message. + """ + if any(isinstance(item, FunctionCallContent) for item in message.items): + return None + + if not message.content.strip(): + return None + + return await client.agents.create_message( + thread_id=thread_id, + role=MessageRole.USER if message.role == AuthorRole.USER else MessageRole.AGENT, + content=message.content, + attachments=AzureAIAgentUtils.get_attachments(message), + metadata=AzureAIAgentUtils.get_metadata(message), + **kwargs, + ) + + @classmethod + async def get_messages( + cls: type[_T], + client: "AIProjectClient", + thread_id: str, + ) -> AsyncIterable["ChatMessageContent"]: + """Get messages from a thread. + + Args: + client: The client to use to get the messages. + thread_id: The ID of the thread to get the messages from. + + Yields: + The messages from the thread. + """ + agent_names: dict[str, Any] = {} + last_id: str | None = None + messages: OpenAIPageableListOfThreadMessage + + while True: + messages = await client.agents.list_messages( + thread_id=thread_id, + run_id=None, + limit=None, + order="desc", + after=last_id, + before=None, + ) + + if not messages: + break + + for message in messages.data: + last_id = message.id + assistant_name: str | None = None + + if message.assistant_id and message.assistant_id.strip() and message.assistant_id not in agent_names: + assistant = await client.agents.get_agent(message.assistant_id) + if assistant.name and assistant.name.strip(): + agent_names[assistant.id] = assistant.name + + assistant_name = agent_names.get(message.assistant_id) or message.assistant_id + + content = generate_message_content(assistant_name, message) + + if len(content.items) > 0: + yield content + + if not messages.has_more: + break + + # endregion + + # region Internal Methods + + @classmethod + def _merge_options( + cls: type[_T], + *, + agent: "AzureAIAgent", + model: str | None = None, + response_format: AgentsApiResponseFormat + | AgentsApiResponseFormatMode + | ResponseFormatJsonSchemaType + | None = None, + temperature: float | None = None, + top_p: float | None = None, + metadata: dict[str, str] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Merge run-time options with the agent-level options. + + Run-level parameters take precedence. + """ + return { + "model": model if model is not None else agent.definition.model, + "response_format": response_format if response_format is not None else agent.definition.response_format, + "temperature": temperature if temperature is not None else agent.definition.temperature, + "top_p": top_p if top_p is not None else agent.definition.top_p, + "metadata": metadata if metadata is not None else agent.definition.metadata, + **kwargs, + } + + @classmethod + def _generate_options(cls: type[_T], **kwargs: Any) -> dict[str, Any]: + """Generate a dictionary of options that can be passed directly to create_run.""" + merged = cls._merge_options(**kwargs) + trunc_count = merged.get("truncation_message_count", None) + max_completion_tokens = merged.get("max_completion_tokens", None) + max_prompt_tokens = merged.get("max_prompt_tokens", None) + parallel_tool_calls = merged.get("parallel_tool_calls_enabled", None) + additional_messages = cls._translate_additional_messages(merged.get("additional_messages", None)) + return { + "model": merged.get("model"), + "top_p": merged.get("top_p"), + "response_format": merged.get("response_format"), + "temperature": merged.get("temperature"), + "truncation_strategy": trunc_count, + "metadata": merged.get("metadata"), + "max_completion_tokens": max_completion_tokens, + "max_prompt_tokens": max_prompt_tokens, + "parallel_tool_calls": parallel_tool_calls, + "additional_messages": additional_messages, + } + + @classmethod + def _translate_additional_messages( + cls: type[_T], messages: "list[ChatMessageContent] | None" + ) -> list[ThreadMessage] | None: + """Translate additional messages to the required format.""" + if not messages: + return None + return AzureAIAgentUtils.get_thread_messages(messages) + + @classmethod + def _prepare_tool_definition(cls: type[_T], tool: dict | ToolDefinition) -> dict | ToolDefinition: + """Prepare the tool definition.""" + if tool.get("type") == "openapi" and "openapi" in tool: + openapi_data = dict(tool["openapi"]) + openapi_data.pop("functions", None) + tool = dict(tool) + tool["openapi"] = openapi_data + return tool + + @classmethod + def _get_tools(cls: type[_T], agent: "AzureAIAgent", kernel: "Kernel") -> list[dict[str, Any] | ToolDefinition]: + """Get the tools for the agent.""" + tools: list[Any] = list(agent.definition.tools) + funcs = kernel.get_full_list_of_function_metadata() + dict_defs = [kernel_function_metadata_to_function_call_format(f) for f in funcs] + tools.extend(dict_defs) + return [cls._prepare_tool_definition(tool) for tool in tools] + + @classmethod + async def _poll_run_status(cls: type[_T], agent: "AzureAIAgent", run: ThreadRun, thread_id: str) -> ThreadRun: + """Poll the run status.""" + logger.info(f"Polling run status: {run.id}, threadId: {thread_id}") + try: + run = await asyncio.wait_for( + cls._poll_loop(agent=agent, run=run, thread_id=thread_id), + timeout=agent.polling_options.run_polling_timeout.total_seconds(), + ) + except asyncio.TimeoutError: + timeout_duration = agent.polling_options.run_polling_timeout + error_message = ( + f"Polling timed out for run id: `{run.id}` and thread id: `{thread_id}` " + f"after waiting {timeout_duration}." + ) + logger.error(error_message) + raise AgentInvokeException(error_message) + logger.info(f"Polled run status: {run.status}, {run.id}, threadId: {thread_id}") + return run + + @classmethod + async def _poll_loop(cls: type[_T], agent: "AzureAIAgent", run: ThreadRun, thread_id: str) -> ThreadRun: + """Continuously poll the run status until it is no longer pending.""" + count = 0 + while True: + await asyncio.sleep(agent.polling_options.get_polling_interval(count).total_seconds()) + count += 1 + try: + run = await agent.client.agents.get_run(run_id=run.id, thread_id=thread_id) + except Exception as e: + logger.warning(f"Failed to retrieve run for run id: `{run.id}` and thread id: `{thread_id}`: {e}") + if run.status not in cls.polling_status: + break + return run + + @classmethod + async def _retrieve_message( + cls: type[_T], agent: "AzureAIAgent", thread_id: str, message_id: str + ) -> ThreadMessage | None: + """Retrieve a message from a thread.""" + message: ThreadMessage | None = None + count = 0 + max_retries = 3 + while count < max_retries: + try: + message = await agent.client.agents.get_message(thread_id=thread_id, message_id=message_id) + break + except Exception as ex: + logger.error(f"Failed to retrieve message {message_id} from thread {thread_id}: {ex}") + count += 1 + if count >= max_retries: + logger.error( + f"Max retries reached. Unable to retrieve message {message_id} from thread {thread_id}." + ) + break + backoff_time: float = agent.polling_options.message_synchronization_delay.total_seconds() * (2**count) + await asyncio.sleep(backoff_time) + return message + + @classmethod + async def _invoke_function_calls( + cls: type[_T], kernel: "Kernel", fccs: list["FunctionCallContent"], chat_history: "ChatHistory" + ) -> list[Any]: + """Invoke the function calls.""" + tasks = [ + kernel.invoke_function_call(function_call=function_call, chat_history=chat_history) + for function_call in fccs + ] + return await asyncio.gather(*tasks) + + @classmethod + def _format_tool_outputs( + cls: type[_T], fccs: list["FunctionCallContent"], chat_history: "ChatHistory" + ) -> list[dict[str, str]]: + """Format the tool outputs for submission.""" + from semantic_kernel.contents.function_result_content import FunctionResultContent + + tool_call_lookup = { + tool_call.id: tool_call + for message in chat_history.messages + for tool_call in message.items + if isinstance(tool_call, FunctionResultContent) + } + return [ + {"tool_call_id": fcc.id, "output": str(tool_call_lookup[fcc.id].result)} + for fcc in fccs + if fcc.id in tool_call_lookup + ] + + @classmethod + async def _handle_streaming_requires_action( + cls: type[_T], + agent_name: str, + kernel: "Kernel", + run: ThreadRun, + function_steps: dict[str, "FunctionCallContent"], + **kwargs: Any, + ) -> FunctionActionResult | None: + """Handle the requires action event for a streaming run.""" + fccs = get_function_call_contents(run, function_steps) + if fccs: + function_call_content = generate_function_call_content(agent_name=agent_name, fccs=fccs) + from semantic_kernel.contents.chat_history import ChatHistory + + chat_history = ChatHistory() if kwargs.get("chat_history") is None else kwargs["chat_history"] + _ = await cls._invoke_function_calls(kernel=kernel, fccs=fccs, chat_history=chat_history) + function_result_content = merge_function_results(chat_history.messages)[0] + tool_outputs = cls._format_tool_outputs(fccs, chat_history) + return FunctionActionResult(function_call_content, function_result_content, tool_outputs) + return None + + # endregion diff --git a/python/semantic_kernel/agents/azure_ai/azure_ai_agent.py b/python/semantic_kernel/agents/azure_ai/azure_ai_agent.py new file mode 100644 index 000000000000..d4835af19fd6 --- /dev/null +++ b/python/semantic_kernel/agents/azure_ai/azure_ai_agent.py @@ -0,0 +1,267 @@ +# Copyright (c) Microsoft. All rights reserved. + +import logging +from collections.abc import AsyncIterable, Iterable +from typing import TYPE_CHECKING, Any, ClassVar + +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import Agent as AzureAIAgentModel +from azure.ai.projects.models import ( + AgentsApiResponseFormat, + AgentsApiResponseFormatMode, + ResponseFormatJsonSchemaType, + ThreadMessage, + ThreadMessageOptions, + ToolDefinition, + TruncationObject, +) +from pydantic import Field + +from semantic_kernel.agents.agent import Agent +from semantic_kernel.agents.azure_ai.agent_thread_actions import AgentThreadActions +from semantic_kernel.agents.azure_ai.azure_ai_channel import AzureAIChannel +from semantic_kernel.agents.channels.agent_channel import AgentChannel +from semantic_kernel.agents.open_ai.run_polling_options import RunPollingOptions +from semantic_kernel.functions import KernelArguments +from semantic_kernel.functions.kernel_function import TEMPLATE_FORMAT_MAP +from semantic_kernel.kernel import Kernel +from semantic_kernel.prompt_template.prompt_template_config import PromptTemplateConfig +from semantic_kernel.utils.experimental_decorator import experimental_class +from semantic_kernel.utils.naming import generate_random_ascii_name +from semantic_kernel.utils.telemetry.agent_diagnostics.decorators import trace_agent_invocation + +logger: logging.Logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from semantic_kernel.contents.chat_message_content import ChatMessageContent + +AgentsApiResponseFormatOption = ( + str | AgentsApiResponseFormatMode | AgentsApiResponseFormat | ResponseFormatJsonSchemaType +) + + +@experimental_class +class AzureAIAgent(Agent): + """Azure AI Agent class.""" + + client: AIProjectClient + definition: AzureAIAgentModel + polling_options: RunPollingOptions = Field(default_factory=RunPollingOptions) + + channel_type: ClassVar[type[AgentChannel]] = AzureAIChannel + + def __init__( + self, + *, + client: AIProjectClient, + definition: AzureAIAgentModel, + kernel: "Kernel | None" = None, + arguments: "KernelArguments | None" = None, + prompt_template_config: "PromptTemplateConfig | None" = None, + **kwargs: Any, + ) -> None: + """Initialize the Azure AI Agent. + + Args: + client: The AzureAI Project client. See "Quickstart: Create a new agent" guide + https://learn.microsoft.com/en-us/azure/ai-services/agents/quickstart?pivots=programming-language-python-azure + for details on how to create a new agent. + definition: The AzureAI Agent model created via the AzureAI Project client. + kernel: The Kernel instance used if invoking plugins + arguments: The KernelArguments instance + prompt_template_config: The prompt template configuration. If this is provided along with + instructions, the prompt template will be used in place of the instructions. + **kwargs: Additional keyword arguments + """ + args: dict[str, Any] = { + "client": client, + "definition": definition, + "name": definition.name or f"azure_agent_{generate_random_ascii_name(length=8)}", + "description": definition.description, + } + + if definition.id is not None: + args["id"] = definition.id + if kernel is not None: + args["kernel"] = kernel + if arguments is not None: + args["arguments"] = arguments + if ( + definition.instructions + and prompt_template_config + and definition.instructions != prompt_template_config.template + ): + logger.info( + f"Both `instructions` ({definition.instructions}) and `prompt_template_config` " + f"({prompt_template_config.template}) were provided. Using template in `prompt_template_config` " + "and ignoring `instructions`." + ) + + if definition.instructions is not None: + args["instructions"] = definition.instructions + if prompt_template_config is not None: + args["prompt_template"] = TEMPLATE_FORMAT_MAP[prompt_template_config.template_format]( + prompt_template_config=prompt_template_config + ) + if prompt_template_config.template is not None: + # Use the template from the prompt_template_config if it is provided + args["instructions"] = prompt_template_config.template + if kwargs: + args.update(kwargs) + + super().__init__(**args) + + async def add_chat_message(self, thread_id: str, message: "ChatMessageContent") -> "ThreadMessage | None": + """Add a chat message to the thread. + + Args: + thread_id: The ID of the thread + message: The chat message to add + + Returns: + ThreadMessage | None: The thread message + """ + return await AgentThreadActions.create_message(client=self.client, thread_id=thread_id, message=message) + + @trace_agent_invocation + async def invoke( + self, + thread_id: str, + arguments: KernelArguments | None = None, + kernel: Kernel | None = None, + # Run-level parameters: + *, + model: str | None = None, + instructions_override: str | None = None, + additional_instructions: str | None = None, + additional_messages: list[ThreadMessageOptions] | None = None, + tools: list[ToolDefinition] | None = None, + temperature: float | None = None, + top_p: float | None = None, + max_prompt_tokens: int | None = None, + max_completion_tokens: int | None = None, + truncation_strategy: TruncationObject | None = None, + response_format: AgentsApiResponseFormatOption | None = None, + parallel_tool_calls: bool | None = None, + metadata: dict[str, str] | None = None, + **kwargs: Any, + ) -> AsyncIterable["ChatMessageContent"]: + """Invoke the agent on the specified thread.""" + if arguments is None: + arguments = KernelArguments(**kwargs) + else: + arguments.update(kwargs) + + kernel = kernel or self.kernel + arguments = self.merge_arguments(arguments) + + run_level_params = { + "model": model, + "instructions_override": instructions_override, + "additional_instructions": additional_instructions, + "additional_messages": additional_messages, + "tools": tools, + "temperature": temperature, + "top_p": top_p, + "max_prompt_tokens": max_prompt_tokens, + "max_completion_tokens": max_completion_tokens, + "truncation_strategy": truncation_strategy, + "response_format": response_format, + "parallel_tool_calls": parallel_tool_calls, + "metadata": metadata, + } + run_level_params = {k: v for k, v in run_level_params.items() if v is not None} + + async for is_visible, message in AgentThreadActions.invoke( + agent=self, + thread_id=thread_id, + kernel=kernel, + arguments=arguments, + **run_level_params, # type: ignore + ): + if is_visible: + yield message + + @trace_agent_invocation + async def invoke_stream( + self, + thread_id: str, + messages: list["ChatMessageContent"] | None = None, + kernel: Kernel | None = None, + arguments: KernelArguments | None = None, + # Run-level parameters: + *, + model: str | None = None, + instructions_override: str | None = None, + additional_instructions: str | None = None, + additional_messages: list[ThreadMessageOptions] | None = None, + tools: list[ToolDefinition] | None = None, + temperature: float | None = None, + top_p: float | None = None, + max_prompt_tokens: int | None = None, + max_completion_tokens: int | None = None, + truncation_strategy: TruncationObject | None = None, + response_format: AgentsApiResponseFormatOption | None = None, + parallel_tool_calls: bool | None = None, + metadata: dict[str, str] | None = None, + **kwargs: Any, + ) -> AsyncIterable["ChatMessageContent"]: + """Invoke the agent on the specified thread with a stream of messages.""" + if arguments is None: + arguments = KernelArguments(**kwargs) + else: + arguments.update(kwargs) + + kernel = kernel or self.kernel + arguments = self.merge_arguments(arguments) + + run_level_params = { + "model": model, + "instructions_override": instructions_override, + "additional_instructions": additional_instructions, + "additional_messages": additional_messages, + "tools": tools, + "temperature": temperature, + "top_p": top_p, + "max_prompt_tokens": max_prompt_tokens, + "max_completion_tokens": max_completion_tokens, + "truncation_strategy": truncation_strategy, + "response_format": response_format, + "parallel_tool_calls": parallel_tool_calls, + "metadata": metadata, + } + run_level_params = {k: v for k, v in run_level_params.items() if v is not None} + + async for message in AgentThreadActions.invoke_stream( + agent=self, + thread_id=thread_id, + messages=messages, + kernel=kernel, + arguments=arguments, + **run_level_params, # type: ignore + ): + yield message + + def get_channel_keys(self) -> Iterable[str]: + """Get the channel keys. + + Returns: + Iterable[str]: The channel keys. + """ + # Distinguish from other channel types. + yield f"{AzureAIAgent.__name__}" + + # Distinguish between different agent IDs + yield self.id + + # Distinguish between agent names + yield self.name + + # Distinguish between different scopes + yield str(self.client.scope) + + async def create_channel(self) -> AgentChannel: + """Create a channel.""" + thread_id = await AgentThreadActions.create_thread(self.client) + + return AzureAIChannel(client=self.client, thread_id=thread_id) diff --git a/python/semantic_kernel/agents/azure_ai/azure_ai_agent_settings.py b/python/semantic_kernel/agents/azure_ai/azure_ai_agent_settings.py new file mode 100644 index 000000000000..d4348678d8a0 --- /dev/null +++ b/python/semantic_kernel/agents/azure_ai/azure_ai_agent_settings.py @@ -0,0 +1,32 @@ +# Copyright (c) Microsoft. All rights reserved. + +from typing import ClassVar + +from pydantic import SecretStr + +from semantic_kernel.kernel_pydantic import KernelBaseSettings +from semantic_kernel.utils.experimental_decorator import experimental_class + + +@experimental_class +class AzureAIAgentSettings(KernelBaseSettings): + """Azure AI Agent settings currently used by the AzureAIAgent. + + Args: + model_deployment_name: Azure AI Agent (Env var AZURE_AI_AGENT_MODEL_DEPLOYMENT_NAME) + project_connection_string: Azure AI Agent Project Connection String + (Env var AZURE_AI_AGENT_PROJECT_CONNECTION_STRING) + endpoint: Azure AI Agent Endpoint (Env var AZURE_AI_AGENT_ENDPOINT) + subscription_id: Azure AI Agent Subscription ID (Env var AZURE_AI_AGENT_SUBSCRIPTION_ID) + resource_group_name: Azure AI Agent Resource Group Name (Env var AZURE_AI_AGENT_RESOURCE_GROUP_NAME) + project_name: Azure AI Agent Project Name (Env var AZURE_AI_AGENT_PROJECT_NAME) + """ + + env_prefix: ClassVar[str] = "AZURE_AI_AGENT_" + + model_deployment_name: str + project_connection_string: SecretStr | None = None + endpoint: str | None = None + subscription_id: str | None = None + resource_group_name: str | None = None + project_name: str | None = None diff --git a/python/semantic_kernel/agents/azure_ai/azure_ai_agent_utils.py b/python/semantic_kernel/agents/azure_ai/azure_ai_agent_utils.py new file mode 100644 index 000000000000..3f2a86972c24 --- /dev/null +++ b/python/semantic_kernel/agents/azure_ai/azure_ai_agent_utils.py @@ -0,0 +1,87 @@ +# Copyright (c) Microsoft. All rights reserved. + +from collections.abc import Iterable, Sequence +from typing import TYPE_CHECKING, Any, ClassVar, TypeVar + +from azure.ai.projects.models import ( + CodeInterpreterTool, + FileSearchTool, + MessageAttachment, + MessageRole, + ThreadMessageOptions, + ToolDefinition, +) + +from semantic_kernel.contents.file_reference_content import FileReferenceContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.utils.experimental_decorator import experimental_class + +if TYPE_CHECKING: + from semantic_kernel.contents import ChatMessageContent + +_T = TypeVar("_T", bound="AzureAIAgentUtils") + + +@experimental_class +class AzureAIAgentUtils: + """AzureAI Agent Utility Methods.""" + + tool_metadata: ClassVar[dict[str, Sequence[ToolDefinition]]] = { + "file_search": FileSearchTool().definitions, + "code_interpreter": CodeInterpreterTool().definitions, + } + + @classmethod + def get_thread_messages(cls: type[_T], messages: list["ChatMessageContent"]) -> Any: + """Get the thread messages for an agent message.""" + if not messages: + return None + + thread_messages: list[ThreadMessageOptions] = [] + + for message in messages: + if not message.content: + continue + + thread_msg = ThreadMessageOptions( + content=message.content, + role=MessageRole.USER if message.role == AuthorRole.USER else MessageRole.AGENT, + attachments=cls.get_attachments(message), + metadata=cls.get_metadata(message) if message.metadata else None, + ) + thread_messages.append(thread_msg) + + return thread_messages + + @classmethod + def get_metadata(cls: type[_T], message: "ChatMessageContent") -> dict[str, str]: + """Get the metadata for an agent message.""" + return {k: str(v) if v is not None else "" for k, v in (message.metadata or {}).items()} + + @classmethod + def get_attachments(cls: type[_T], message: "ChatMessageContent") -> list[MessageAttachment]: + """Get the attachments for an agent message. + + Args: + message: The ChatMessageContent + + Returns: + A list of MessageAttachment + """ + return [ + MessageAttachment( + file_id=file_content.file_id, + tools=list(cls._get_tool_definition(file_content.tools)), # type: ignore + data_source=file_content.data_source if file_content.data_source else None, + ) + for file_content in message.items + if isinstance(file_content, FileReferenceContent) + ] + + @classmethod + def _get_tool_definition(cls: type[_T], tools: list[Any]) -> Iterable[ToolDefinition]: + if not tools: + return + for tool in tools: + if tool_definition := cls.tool_metadata.get(tool): + yield from tool_definition diff --git a/python/semantic_kernel/agents/azure_ai/azure_ai_channel.py b/python/semantic_kernel/agents/azure_ai/azure_ai_channel.py new file mode 100644 index 000000000000..00042074e1b7 --- /dev/null +++ b/python/semantic_kernel/agents/azure_ai/azure_ai_channel.py @@ -0,0 +1,121 @@ +# Copyright (c) Microsoft. All rights reserved. + +import sys +from collections.abc import AsyncIterable +from typing import TYPE_CHECKING + +if sys.version_info >= (3, 12): + from typing import override # pragma: no cover +else: + from typing_extensions import override # pragma: no cover + +from semantic_kernel.agents.azure_ai.agent_thread_actions import AgentThreadActions +from semantic_kernel.agents.channels.agent_channel import AgentChannel +from semantic_kernel.exceptions.agent_exceptions import AgentChatException +from semantic_kernel.utils.experimental_decorator import experimental_class + +if TYPE_CHECKING: + from azure.ai.projects.aio import AIProjectClient + + from semantic_kernel.agents.agent import Agent + from semantic_kernel.contents.chat_message_content import ChatMessageContent + + +@experimental_class +class AzureAIChannel(AgentChannel): + """AzureAI Channel.""" + + def __init__(self, client: "AIProjectClient", thread_id: str) -> None: + """Initialize the AzureAI Channel. + + Args: + client: The AzureAI Project client. + thread_id: The thread ID. + """ + self.client = client + self.thread_id = thread_id + + @override + async def receive(self, history: list["ChatMessageContent"]) -> None: + """Receive the conversation messages. + + Args: + history: The conversation messages. + """ + for message in history: + await AgentThreadActions.create_message(self.client, self.thread_id, message) + + @override + async def invoke(self, agent: "Agent", **kwargs) -> AsyncIterable[tuple[bool, "ChatMessageContent"]]: + """Invoke the agent. + + Args: + agent: The agent to invoke. + kwargs: The keyword arguments. + + Yields: + tuple[bool, ChatMessageContent]: The conversation messages. + """ + from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent + + if not isinstance(agent, AzureAIAgent): + raise AgentChatException(f"Agent is not of the expected type {type(AzureAIAgent)}.") + + async for is_visible, message in AgentThreadActions.invoke( + agent=agent, + thread_id=self.thread_id, + arguments=agent.arguments, + kernel=agent.kernel, + **kwargs, + ): + yield is_visible, message + + @override + async def invoke_stream( + self, + agent: "Agent", + messages: list["ChatMessageContent"], + **kwargs, + ) -> AsyncIterable["ChatMessageContent"]: + """Invoke the agent stream. + + Args: + agent: The agent to invoke. + messages: The conversation messages. + kwargs: The keyword arguments. + + Yields: + tuple[bool, StreamingChatMessageContent]: The conversation messages. + """ + from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent + + if not isinstance(agent, AzureAIAgent): + raise AgentChatException(f"Agent is not of the expected type {type(AzureAIAgent)}.") + + async for message in AgentThreadActions.invoke_stream( + agent=agent, + thread_id=self.thread_id, + messages=messages, + arguments=agent.arguments, + kernel=agent.kernel, + **kwargs, + ): + yield message + + @override + async def get_history(self) -> AsyncIterable["ChatMessageContent"]: + """Get the conversation history. + + Yields: + ChatMessageContent: The conversation history. + """ + async for message in AgentThreadActions.get_messages(self.client, thread_id=self.thread_id): + yield message + + @override + async def reset(self) -> None: + """Reset the agent's thread.""" + try: + await self.client.agents.delete_thread(thread_id=self.thread_id) + except Exception as e: + raise AgentChatException(f"Failed to delete thread: {e}") diff --git a/python/semantic_kernel/agents/channels/agent_channel.py b/python/semantic_kernel/agents/channels/agent_channel.py index b7a56d1f4a32..bc30c08fb650 100644 --- a/python/semantic_kernel/agents/channels/agent_channel.py +++ b/python/semantic_kernel/agents/channels/agent_channel.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from collections.abc import AsyncIterable -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from semantic_kernel.utils.experimental_decorator import experimental_class @@ -36,11 +36,13 @@ async def receive( def invoke( self, agent: "Agent", + **kwargs: Any, ) -> AsyncIterable[tuple[bool, "ChatMessageContent"]]: """Perform a discrete incremental interaction between a single Agent and AgentChat. Args: agent: The agent to interact with. + kwargs: The keyword arguments. Returns: An async iterable of a bool, ChatMessageContent. @@ -51,13 +53,15 @@ def invoke( def invoke_stream( self, agent: "Agent", - history: "list[ChatMessageContent]", + messages: "list[ChatMessageContent]", + **kwargs: Any, ) -> AsyncIterable["ChatMessageContent"]: """Perform a discrete incremental stream interaction between a single Agent and AgentChat. Args: agent: The agent to interact with. - history: The history of messages in the conversation. + messages: The history of messages in the conversation. + kwargs: The keyword arguments. Returns: An async iterable ChatMessageContent. diff --git a/python/semantic_kernel/agents/channels/bedrock_agent_channel.py b/python/semantic_kernel/agents/channels/bedrock_agent_channel.py index fca83b266d25..ca295eafb803 100644 --- a/python/semantic_kernel/agents/channels/bedrock_agent_channel.py +++ b/python/semantic_kernel/agents/channels/bedrock_agent_channel.py @@ -37,11 +37,12 @@ class BedrockAgentChannel(AgentChannel, ChatHistory): MESSAGE_PLACEHOLDER: ClassVar[str] = "[SILENCE]" @override - async def invoke(self, agent: "Agent") -> AsyncIterable[tuple[bool, ChatMessageContent]]: + async def invoke(self, agent: "Agent", **kwargs: Any) -> AsyncIterable[tuple[bool, ChatMessageContent]]: """Perform a discrete incremental interaction between a single Agent and AgentChat. Args: agent: The agent to interact with. + kwargs: Additional keyword arguments. Returns: An async iterable of ChatMessageContent with a boolean indicating if the @@ -75,12 +76,14 @@ async def invoke_stream( self, agent: "Agent", messages: list[ChatMessageContent], + **kwargs: Any, ) -> AsyncIterable[ChatMessageContent]: """Perform a streaming interaction between a single Agent and AgentChat. Args: agent: The agent to interact with. messages: The history of messages in the conversation. + kwargs: Additional keyword arguments. Returns: An async iterable of ChatMessageContent. diff --git a/python/semantic_kernel/agents/channels/chat_history_channel.py b/python/semantic_kernel/agents/channels/chat_history_channel.py index 2df31a838927..77da09d17924 100644 --- a/python/semantic_kernel/agents/channels/chat_history_channel.py +++ b/python/semantic_kernel/agents/channels/chat_history_channel.py @@ -15,7 +15,7 @@ from typing_extensions import override # pragma: no cover from abc import abstractmethod -from typing import TYPE_CHECKING, ClassVar, Deque, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Any, ClassVar, Deque, Protocol, runtime_checkable from semantic_kernel.agents.channels.agent_channel import AgentChannel from semantic_kernel.contents import ChatMessageContent @@ -62,11 +62,13 @@ class ChatHistoryChannel(AgentChannel, ChatHistory): async def invoke( self, agent: "Agent", + **kwargs: Any, ) -> AsyncIterable[tuple[bool, ChatMessageContent]]: """Perform a discrete incremental interaction between a single Agent and AgentChat. Args: agent: The agent to interact with. + kwargs: The keyword arguments. Returns: An async iterable of ChatMessageContent. @@ -113,15 +115,14 @@ async def invoke( @override async def invoke_stream( - self, - agent: "Agent", - messages: list[ChatMessageContent], + self, agent: "Agent", messages: list[ChatMessageContent], **kwargs: Any ) -> AsyncIterable[ChatMessageContent]: """Perform a discrete incremental stream interaction between a single Agent and AgentChat. Args: agent: The agent to interact with. messages: The history of messages in the conversation. + kwargs: The keyword arguments Returns: An async iterable of ChatMessageContent. diff --git a/python/semantic_kernel/agents/channels/open_ai_assistant_channel.py b/python/semantic_kernel/agents/channels/open_ai_assistant_channel.py index 7ba31b598827..bc06cf40df39 100644 --- a/python/semantic_kernel/agents/channels/open_ai_assistant_channel.py +++ b/python/semantic_kernel/agents/channels/open_ai_assistant_channel.py @@ -44,11 +44,12 @@ async def receive(self, history: list["ChatMessageContent"]) -> None: await create_chat_message(self.client, self.thread_id, message) @override - async def invoke(self, agent: "Agent") -> AsyncIterable[tuple[bool, "ChatMessageContent"]]: + async def invoke(self, agent: "Agent", **kwargs: Any) -> AsyncIterable[tuple[bool, "ChatMessageContent"]]: """Invoke the agent. Args: agent: The agent to invoke. + kwargs: The keyword arguments. Yields: tuple[bool, ChatMessageContent]: The conversation messages. @@ -61,18 +62,19 @@ async def invoke(self, agent: "Agent") -> AsyncIterable[tuple[bool, "ChatMessage if agent._is_deleted: raise AgentChatException("Agent is deleted.") - async for is_visible, message in agent._invoke_internal(thread_id=self.thread_id): + async for is_visible, message in agent._invoke_internal(thread_id=self.thread_id, **kwargs): yield is_visible, message @override async def invoke_stream( - self, agent: "Agent", messages: list[ChatMessageContent] + self, agent: "Agent", messages: list[ChatMessageContent], **kwargs: Any ) -> AsyncIterable["ChatMessageContent"]: """Invoke the agent stream. Args: agent: The agent to invoke. messages: The conversation messages. + kwargs: The keyword arguments. Yields: tuple[bool, StreamingChatMessageContent]: The conversation messages. @@ -85,7 +87,7 @@ async def invoke_stream( if agent._is_deleted: raise AgentChatException("Agent is deleted.") - async for message in agent._invoke_internal_stream(thread_id=self.thread_id, messages=messages): + async for message in agent._invoke_internal_stream(thread_id=self.thread_id, messages=messages, **kwargs): yield message @override diff --git a/python/semantic_kernel/contents/file_reference_content.py b/python/semantic_kernel/contents/file_reference_content.py index 99cd15f341ef..a1aaa2805948 100644 --- a/python/semantic_kernel/contents/file_reference_content.py +++ b/python/semantic_kernel/contents/file_reference_content.py @@ -22,6 +22,8 @@ class FileReferenceContent(KernelContent): content_type: Literal[ContentTypes.FILE_REFERENCE_CONTENT] = Field(FILE_REFERENCE_CONTENT_TAG, init=False) # type: ignore tag: ClassVar[str] = FILE_REFERENCE_CONTENT_TAG file_id: str | None = None + tools: list[Any] = Field(default_factory=list) + data_source: Any | None = None def __str__(self) -> str: """Return the string representation of the file reference content.""" diff --git a/python/semantic_kernel/contents/streaming_file_reference_content.py b/python/semantic_kernel/contents/streaming_file_reference_content.py index 4f934848e174..43c6e7a0fa20 100644 --- a/python/semantic_kernel/contents/streaming_file_reference_content.py +++ b/python/semantic_kernel/contents/streaming_file_reference_content.py @@ -25,6 +25,8 @@ class StreamingFileReferenceContent(KernelContent): ) tag: ClassVar[str] = STREAMING_FILE_REFERENCE_CONTENT_TAG file_id: str | None = None + tools: list[Any] = Field(default_factory=list) + data_source: Any | None = None def __str__(self) -> str: """Return the string representation of the file reference content.""" diff --git a/python/tests/samples/test_concepts.py b/python/tests/samples/test_concepts.py index d2455d4f4d6d..eb69b65db622 100644 --- a/python/tests/samples/test_concepts.py +++ b/python/tests/samples/test_concepts.py @@ -53,10 +53,10 @@ from samples.concepts.rag.rag_with_text_memory_plugin import main as rag_with_text_memory_plugin from samples.concepts.search.bing_search_plugin import main as bing_search_plugin from samples.concepts.service_selector.custom_service_selector import main as custom_service_selector -from samples.getting_started_with_agents.step1_agent import main as step1_agent -from samples.getting_started_with_agents.step2_plugins import main as step2_plugins -from samples.getting_started_with_agents.step3_chat import main as step3_chat -from samples.getting_started_with_agents.step7_assistant import main as step7_assistant +from samples.getting_started_with_agents.chat_completion.step1_agent import main as step1_agent +from samples.getting_started_with_agents.chat_completion.step2_plugins import main as step2_plugins +from samples.getting_started_with_agents.chat_completion.step3_chat import main as step3_chat +from samples.getting_started_with_agents.openai_assistant.step1_assistant import main as step7_assistant from tests.utils import retry # These environment variable names are used to control which samples are run during integration testing. diff --git a/python/tests/unit/agents/azure_ai_agent/test_agent_content_generation.py b/python/tests/unit/agents/azure_ai_agent/test_agent_content_generation.py new file mode 100644 index 000000000000..b792575c7737 --- /dev/null +++ b/python/tests/unit/agents/azure_ai_agent/test_agent_content_generation.py @@ -0,0 +1,279 @@ +# Copyright (c) Microsoft. All rights reserved. + + +from azure.ai.projects.models import ( + MessageDelta, + MessageDeltaChunk, + MessageDeltaImageFileContent, + MessageDeltaImageFileContentObject, + MessageDeltaTextContent, + MessageDeltaTextContentObject, + MessageDeltaTextFileCitationAnnotation, + MessageDeltaTextFileCitationAnnotationObject, + MessageDeltaTextFilePathAnnotation, + MessageDeltaTextFilePathAnnotationObject, + MessageImageFileContent, + MessageImageFileDetails, + MessageTextContent, + MessageTextDetails, + MessageTextFileCitationAnnotation, + MessageTextFileCitationDetails, + MessageTextFilePathAnnotation, + MessageTextFilePathDetails, + RunStep, + RunStepDeltaFunction, + RunStepDeltaFunctionToolCall, + RunStepDeltaToolCallObject, + ThreadMessage, +) + +from semantic_kernel.agents.azure_ai.agent_content_generation import ( + generate_annotation_content, + generate_code_interpreter_content, + generate_function_call_content, + generate_function_result_content, + generate_message_content, + generate_streaming_code_interpreter_content, + generate_streaming_function_content, + generate_streaming_message_content, + get_function_call_contents, + get_message_contents, +) +from semantic_kernel.contents.annotation_content import AnnotationContent +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.file_reference_content import FileReferenceContent +from semantic_kernel.contents.function_call_content import FunctionCallContent +from semantic_kernel.contents.function_result_content import FunctionResultContent +from semantic_kernel.contents.image_content import ImageContent +from semantic_kernel.contents.streaming_annotation_content import StreamingAnnotationContent +from semantic_kernel.contents.streaming_file_reference_content import StreamingFileReferenceContent +from semantic_kernel.contents.streaming_text_content import StreamingTextContent +from semantic_kernel.contents.text_content import TextContent +from semantic_kernel.contents.utils.author_role import AuthorRole + + +def test_get_message_contents_all_types(): + chat_msg = ChatMessageContent(role=AuthorRole.USER) + chat_msg.items.append(TextContent(text="hello world")) + chat_msg.items.append(ImageContent(uri="http://example.com/image.png")) + chat_msg.items.append(FileReferenceContent(file_id="file123")) + chat_msg.items.append(FunctionResultContent(id="func1", result={"a": 1})) + results = get_message_contents(chat_msg) + assert len(results) == 4 + assert results[0]["type"] == "text" + assert results[1]["type"] == "image_url" + assert results[2]["type"] == "image_file" + assert results[3]["type"] == "text" + + +def test_generate_message_content_text_and_image(): + thread_msg = ThreadMessage( + content=[], + role="user", + ) + + image = MessageImageFileContent(image_file=MessageImageFileDetails(file_id="test_file_id")) + + text = MessageTextContent( + text=MessageTextDetails( + value="some text", + annotations=[ + MessageTextFileCitationAnnotation( + text="text", + file_citation=MessageTextFileCitationDetails(file_id="file_id", quote="some quote"), + start_index=0, + end_index=9, + ), + MessageTextFilePathAnnotation( + text="text again", + file_path=MessageTextFilePathDetails(file_id="file_id_2"), + start_index=1, + end_index=10, + ), + ], + ) + ) + + thread_msg.content = [image, text] + step = RunStep(id="step_id", run_id="run_id", thread_id="thread_id", assistant_id="assistant_id") + out = generate_message_content("assistant", thread_msg, step) + assert len(out.items) == 4 + assert isinstance(out.items[0], FileReferenceContent) + assert isinstance(out.items[1], TextContent) + assert isinstance(out.items[2], AnnotationContent) + assert isinstance(out.items[3], AnnotationContent) + + assert out.items[0].file_id == "test_file_id" + + assert out.items[1].text == "some text" + + assert out.items[2].file_id == "file_id" + assert out.items[2].quote == "text" + assert out.items[2].start_index == 0 + assert out.items[2].end_index == 9 + + assert out.items[3].file_id == "file_id_2" + assert out.items[3].quote == "text again" + assert out.items[3].start_index == 1 + assert out.items[3].end_index == 10 + + assert out.metadata["step_id"] == "step_id" + assert out.role == AuthorRole.USER + + +def test_generate_annotation_content(): + message_text_file_path_ann = MessageTextFilePathAnnotation( + text="some text", + file_path=MessageTextFilePathDetails(file_id="file123"), + start_index=0, + end_index=9, + ) + + message_text_file_citation_ann = MessageTextFileCitationAnnotation( + text="some text", + file_citation=MessageTextFileCitationDetails(file_id="file123"), + start_index=0, + end_index=9, + ) + + for fake_ann in [message_text_file_path_ann, message_text_file_citation_ann]: + out = generate_annotation_content(fake_ann) + assert out.file_id == "file123" + assert out.quote == "some text" + assert out.start_index == 0 + assert out.end_index == 9 + + +def test_generate_streaming_message_content_text_annotations(): + message_delta_image_file_content = MessageDeltaImageFileContent( + index=0, + image_file=MessageDeltaImageFileContentObject(file_id="image_file"), + ) + + MessageDeltaTextFileCitationAnnotation, MessageDeltaTextFilePathAnnotation + + message_delta_text_content = MessageDeltaTextContent( + index=0, + text=MessageDeltaTextContentObject( + value="some text", + annotations=[ + MessageDeltaTextFileCitationAnnotation( + index=0, + file_citation=MessageDeltaTextFileCitationAnnotationObject(file_id="file123"), + start_index=0, + end_index=9, + text="some text", + ), + MessageDeltaTextFilePathAnnotation( + index=0, + file_path=MessageDeltaTextFilePathAnnotationObject(file_id="file123"), + start_index=0, + end_index=9, + text="some text", + ), + ], + ), + ) + + delta = MessageDeltaChunk( + id="chunk123", + delta=MessageDelta(role="user", content=[message_delta_image_file_content, message_delta_text_content]), + ) + + out = generate_streaming_message_content("assistant", delta) + assert out is not None + assert out.content == "some text" + assert len(out.items) == 4 + assert out.items[0].file_id == "image_file" + assert isinstance(out.items[0], StreamingFileReferenceContent) + assert isinstance(out.items[1], StreamingTextContent) + assert isinstance(out.items[2], StreamingAnnotationContent) + + assert out.items[2].file_id == "file123" + assert out.items[2].quote == "some text" + assert out.items[2].start_index == 0 + assert out.items[2].end_index == 9 + + assert isinstance(out.items[3], StreamingAnnotationContent) + assert out.items[3].file_id == "file123" + assert out.items[3].quote == "some text" + assert out.items[3].start_index == 0 + assert out.items[3].end_index == 9 + + +def test_generate_streaming_function_content_with_function(): + step_details = RunStepDeltaToolCallObject( + tool_calls=[ + RunStepDeltaFunctionToolCall( + index=0, id="tool123", function=RunStepDeltaFunction(name="some_func", arguments={"arg": "val"}) + ) + ] + ) + + out = generate_streaming_function_content("my_agent", step_details) + assert out is not None + assert len(out.items) == 1 + assert isinstance(out.items[0], FunctionCallContent) + assert out.items[0].function_name == "some_func" + assert out.items[0].arguments == "{'arg': 'val'}" + + +def test_get_function_call_contents_no_action(): + run = type("ThreadRunFake", (), {"required_action": None})() + fc = get_function_call_contents(run, {}) + assert fc == [] + + +def test_get_function_call_contents_submit_tool_outputs(): + class FakeFunction: + name = "test_function" + arguments = {"arg": "val"} + + class FakeToolCall: + id = "tool_id" + function = FakeFunction() + + run = type( + "ThreadRunFake", + (), + { + "required_action": type( + "RequiredAction", (), {"submit_tool_outputs": type("FakeSubmit", (), {"tool_calls": [FakeToolCall()]})} + ) + }, + )() + function_steps = {} + fc = get_function_call_contents(run, function_steps) + assert len(fc) == 1 + assert function_steps["tool_id"].function_name == "test_function" + + +def test_generate_function_call_content(): + fcc = FunctionCallContent(id="id123", name="func_name", arguments={"x": 1}) + msg = generate_function_call_content("my_agent", [fcc]) + assert len(msg.items) == 1 + assert msg.role == AuthorRole.ASSISTANT + + +def test_generate_function_result_content(): + step = FunctionCallContent(id="123", name="func_name", arguments={"k": "v"}) + + class FakeToolCall: + function = type("Function", (), {"output": "result_data"}) + + tool_call = FakeToolCall() + msg = generate_function_result_content("my_agent", step, tool_call) + assert len(msg.items) == 1 + assert msg.items[0].result == "result_data" + assert msg.role == AuthorRole.TOOL + + +def test_generate_code_interpreter_content(): + msg = generate_code_interpreter_content("my_agent", "some_code()") + assert msg.content == "some_code()" + assert msg.metadata["code"] is True + + +def test_generate_streaming_code_interpreter_content_no_calls(): + step_details = type("Details", (), {"tool_calls": None}) + assert generate_streaming_code_interpreter_content("my_agent", step_details) is None diff --git a/python/tests/unit/agents/azure_ai_agent/test_agent_thread_actions.py b/python/tests/unit/agents/azure_ai_agent/test_agent_thread_actions.py new file mode 100644 index 000000000000..1c836657a323 --- /dev/null +++ b/python/tests/unit/agents/azure_ai_agent/test_agent_thread_actions.py @@ -0,0 +1,351 @@ +# Copyright (c) Microsoft. All rights reserved. + +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock, patch + +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import Agent as AzureAIAgentModel +from azure.ai.projects.models import ( + MessageTextContent, + MessageTextDetails, + OpenAIPageableListOfRunStep, + RequiredFunctionToolCall, + RequiredFunctionToolCallDetails, + RunStep, + RunStepCodeInterpreterToolCall, + RunStepCodeInterpreterToolCallDetails, + RunStepFunctionToolCall, + RunStepFunctionToolCallDetails, + RunStepMessageCreationDetails, + RunStepMessageCreationReference, + RunStepToolCallDetails, + SubmitToolOutputsAction, + SubmitToolOutputsDetails, + ThreadMessage, + ThreadRun, +) + +from semantic_kernel.agents.azure_ai.agent_thread_actions import AgentThreadActions +from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent +from semantic_kernel.contents import ( + FunctionCallContent, + FunctionResultContent, + TextContent, +) +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.kernel import Kernel + + +async def test_agent_thread_actions_create_thread(): + class FakeAgentClient: + create_thread = AsyncMock(return_value=type("FakeThread", (), {"id": "thread123"})) + + class FakeClient: + agents = FakeAgentClient() + + client = FakeClient() + thread_id = await AgentThreadActions.create_thread(client) + assert thread_id == "thread123" + + +async def test_agent_thread_actions_create_message(): + class FakeAgentClient: + create_message = AsyncMock(return_value="someMessage") + + class FakeClient: + agents = FakeAgentClient() + + msg = ChatMessageContent(role=AuthorRole.USER, content="some content") + out = await AgentThreadActions.create_message(FakeClient(), "threadXYZ", msg) + assert out == "someMessage" + + +async def test_agent_thread_actions_create_message_no_content(): + class FakeAgentClient: + create_message = AsyncMock(return_value="should_not_be_called") + + class FakeClient: + agents = FakeAgentClient() + + message = ChatMessageContent(role=AuthorRole.USER, content=" ") + out = await AgentThreadActions.create_message(FakeClient(), "threadXYZ", message) + assert out is None + assert FakeAgentClient.create_message.await_count == 0 + + +async def test_agent_thread_actions_invoke(): + client = AsyncMock(spec=AIProjectClient) + definition = AsyncMock(spec=AzureAIAgentModel) + definition.id = "agent123" + definition.name = "agentName" + definition.description = "desc" + definition.instructions = "test agent" + agent = AzureAIAgent(client=client, definition=definition) + + agent.client.agents = MagicMock() + + mock_thread_run = ThreadRun( + id="run123", + thread_id="thread123", + status="running", + instructions="test agent", + created_at=int(datetime.now(timezone.utc).timestamp()), + model="model", + ) + + agent.client.agents.create_run = AsyncMock(return_value=mock_thread_run) + + mock_run_steps = OpenAIPageableListOfRunStep( + data=[ + RunStep( + type="message_creation", + id="msg123", + thread_id="thread123", + run_id="run123", + created_at=int(datetime.now(timezone.utc).timestamp()), + completed_at=int(datetime.now(timezone.utc).timestamp()), + status="completed", + assistant_id="assistant123", + step_details=RunStepMessageCreationDetails( + message_creation=RunStepMessageCreationReference( + message_id="msg123", + ), + ), + ), + ] + ) + + agent.client.agents.list_run_steps = AsyncMock(return_value=mock_run_steps) + + mock_message = ThreadMessage( + id="msg123", + thread_id="thread123", + run_id="run123", + created_at=int(datetime.now(timezone.utc).timestamp()), + completed_at=int(datetime.now(timezone.utc).timestamp()), + status="completed", + assistant_id="assistant123", + role="assistant", + content=[MessageTextContent(text=MessageTextDetails(value="some message", annotations=[]))], + ) + + agent.client.agents.get_message = AsyncMock(return_value=mock_message) + + async for message in AgentThreadActions.invoke(agent=agent, thread_id="thread123", kernel=AsyncMock(spec=Kernel)): + assert message is not None + break + + +async def test_agent_thread_actions_invoke_with_requires_action(): + client = AsyncMock(spec=AIProjectClient) + definition = AsyncMock(spec=AzureAIAgentModel) + definition.id = "agent123" + definition.name = "agentName" + definition.description = "desc" + definition.instructions = "test agent" + + agent = AzureAIAgent(client=client, definition=definition) + agent.client.agents = MagicMock() + + mock_thread_run = ThreadRun( + id="run123", + thread_id="thread123", + status="running", + instructions="test agent", + created_at=int(datetime.now(timezone.utc).timestamp()), + model="model", + ) + + agent.client.agents.create_run = AsyncMock(return_value=mock_thread_run) + + poll_count = 0 + + async def mock_poll_run_status(*args, **kwargs): + nonlocal poll_count + if poll_count == 0: + mock_thread_run.status = "requires_action" + mock_thread_run.required_action = SubmitToolOutputsAction( + submit_tool_outputs=SubmitToolOutputsDetails( + tool_calls=[ + RequiredFunctionToolCall( + id="tool_call_id", + function=RequiredFunctionToolCallDetails( + name="mock_function_call", arguments={"arg": "value"} + ), + ) + ] + ) + ) + else: + mock_thread_run.status = "completed" + poll_count += 1 + return mock_thread_run + + def mock_get_function_call_contents(run: ThreadRun, function_steps: dict): + function_call_content = FunctionCallContent( + name="mock_function_call", + arguments={"arg": "value"}, + id="tool_call_id", + ) + function_steps[function_call_content.id] = function_call_content + return [function_call_content] + + mock_run_step_tool_calls = RunStep( + type="tool_calls", + id="tool_step123", + thread_id="thread123", + run_id="run123", + created_at=int(datetime.now(timezone.utc).timestamp()), + completed_at=int(datetime.now(timezone.utc).timestamp()), + status="completed", + assistant_id="assistant123", + step_details=RunStepToolCallDetails( + tool_calls=[ + RunStepCodeInterpreterToolCall( + id="tool_call_id", + code_interpreter=RunStepCodeInterpreterToolCallDetails( + input="some code", + ), + ), + RunStepFunctionToolCall( + id="tool_call_id", + function=RunStepFunctionToolCallDetails( + name="mock_function_call", + arguments={"arg": "value"}, + output="some output", + ), + ), + ] + ), + ) + + mock_run_step_message_creation = RunStep( + type="message_creation", + id="msg_step123", + thread_id="thread123", + run_id="run123", + created_at=int(datetime.now(timezone.utc).timestamp()), + completed_at=int(datetime.now(timezone.utc).timestamp()), + status="completed", + assistant_id="assistant123", + step_details=RunStepMessageCreationDetails( + message_creation=RunStepMessageCreationReference(message_id="msg123") + ), + ) + + mock_run_steps = OpenAIPageableListOfRunStep(data=[mock_run_step_tool_calls, mock_run_step_message_creation]) + agent.client.agents.list_run_steps = AsyncMock(return_value=mock_run_steps) + + mock_message = ThreadMessage( + id="msg123", + thread_id="thread123", + run_id="run123", + created_at=int(datetime.now(timezone.utc).timestamp()), + completed_at=int(datetime.now(timezone.utc).timestamp()), + status="completed", + assistant_id="assistant123", + role="assistant", + content=[MessageTextContent(text=MessageTextDetails(value="some message", annotations=[]))], + ) + agent.client.agents.get_message = AsyncMock(return_value=mock_message) + + agent.client.agents.submit_tool_outputs_to_run = AsyncMock() + + with ( + patch.object(AgentThreadActions, "_poll_run_status", side_effect=mock_poll_run_status), + patch( + "semantic_kernel.agents.azure_ai.agent_thread_actions.get_function_call_contents", + side_effect=mock_get_function_call_contents, + ), + ): + messages = [] + async for is_visible, content in AgentThreadActions.invoke( + agent=agent, + thread_id="thread123", + kernel=AsyncMock(spec=Kernel), + ): + messages.append((is_visible, content)) + + assert len(messages) == 4, "There should be four yields in total." + + assert isinstance(messages[0][1].items[0], FunctionCallContent) + assert isinstance(messages[1][1].items[0], TextContent) + assert messages[1][1].items[0].metadata.get("code") is True + assert isinstance(messages[2][1].items[0], FunctionResultContent) + assert isinstance(messages[3][1].items[0], TextContent) + + agent.client.agents.submit_tool_outputs_to_run.assert_awaited_once() + + +class MockEvent: + def __init__(self, event, data): + self.event = event + self.data = data + + def __iter__(self): + return iter((self.event, self.data, None)) + + +class MockRunData: + def __init__(self, id, status): + self.id = id + self.status = status + + +class MockAsyncIterable: + def __init__(self, items): + self.items = items.copy() + + def __aiter__(self): + self._iter = iter(self.items) + return self + + async def __anext__(self): + try: + return next(self._iter) + except StopIteration: + raise StopAsyncIteration + + +class MockStream: + def __init__(self, events): + self.events = events + + async def __aenter__(self): + return MockAsyncIterable(self.events) + + async def __aexit__(self, exc_type, exc_val, exc_tb): + pass + + +async def test_agent_thread_actions_invoke_stream(): + client = AsyncMock(spec=AIProjectClient) + definition = AsyncMock(spec=AzureAIAgentModel) + definition.id = "agent123" + definition.name = "agentName" + definition.description = "desc" + definition.instructions = "test agent" + agent = AzureAIAgent(client=client, definition=definition) + agent.client.agents = AsyncMock() + + events = [ + MockEvent("thread.run.created", MockRunData(id="run_1", status="queued")), + MockEvent("thread.run.in_progress", MockRunData(id="run_1", status="in_progress")), + MockEvent("thread.run.completed", MockRunData(id="run_1", status="completed")), + ] + + main_run_stream = MockStream(events) + agent.client.agents.create_stream.return_value = main_run_stream + + with ( + patch.object(AgentThreadActions, "_invoke_function_calls", return_value=None), + patch.object(AgentThreadActions, "_format_tool_outputs", return_value=[{"type": "mock_tool_output"}]), + ): + collected_messages = [] + async for content in AgentThreadActions.invoke_stream( + agent=agent, + thread_id="thread123", + kernel=AsyncMock(spec=Kernel), + ): + collected_messages.append(content) diff --git a/python/tests/unit/agents/azure_ai_agent/test_azure_ai_agent.py b/python/tests/unit/agents/azure_ai_agent/test_azure_ai_agent.py new file mode 100644 index 000000000000..f19bc9c0a74e --- /dev/null +++ b/python/tests/unit/agents/azure_ai_agent/test_azure_ai_agent.py @@ -0,0 +1,113 @@ +# Copyright (c) Microsoft. All rights reserved. + +from unittest.mock import AsyncMock, patch + +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import Agent as AzureAIAgentModel + +from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent +from semantic_kernel.agents.channels.agent_channel import AgentChannel +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole + + +async def test_azure_ai_agent_init(): + client = AsyncMock(spec=AIProjectClient) + definition = AsyncMock(spec=AzureAIAgentModel) + definition.id = "agent123" + definition.name = "agentName" + definition.description = "desc" + definition.instructions = "test agent" + agent = AzureAIAgent(client=client, definition=definition) + assert agent.id == "agent123" + assert agent.name == "agentName" + assert agent.description == "desc" + + +async def test_azure_ai_agent_add_chat_message(): + client = AsyncMock(spec=AIProjectClient) + definition = AsyncMock(spec=AzureAIAgentModel) + definition.id = "agent123" + definition.name = "agentName" + definition.description = "desc" + definition.instructions = "test agent" + agent = AzureAIAgent(client=client, definition=definition) + with patch( + "semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.create_message", + ): + await agent.add_chat_message("threadId", ChatMessageContent(role="user", content="text")) # pass anything + + +async def test_azure_ai_agent_invoke(): + client = AsyncMock(spec=AIProjectClient) + definition = AsyncMock(spec=AzureAIAgentModel) + definition.id = "agent123" + definition.name = "agentName" + definition.description = "desc" + definition.instructions = "test agent" + agent = AzureAIAgent(client=client, definition=definition) + results = [] + + async def fake_invoke(*args, **kwargs): + yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content") + + with patch( + "semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke", + side_effect=fake_invoke, + ): + async for item in agent.invoke("thread_id"): + results.append(item) + + assert len(results) == 1 + + +async def test_azure_ai_agent_invoke_stream(): + client = AsyncMock(spec=AIProjectClient) + definition = AsyncMock(spec=AzureAIAgentModel) + definition.id = "agent123" + definition.name = "agentName" + definition.description = "desc" + definition.instructions = "test agent" + agent = AzureAIAgent(client=client, definition=definition) + results = [] + + async def fake_invoke(*args, **kwargs): + yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content") + + with patch( + "semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke_stream", + side_effect=fake_invoke, + ): + async for item in agent.invoke_stream("thread_id"): + results.append(item) + + assert len(results) == 1 + + +def test_azure_ai_agent_get_channel_keys(): + client = AsyncMock(spec=AIProjectClient) + definition = AsyncMock(spec=AzureAIAgentModel) + definition.id = "agent123" + definition.name = "agentName" + definition.description = "desc" + definition.instructions = "test agent" + agent = AzureAIAgent(client=client, definition=definition) + keys = list(agent.get_channel_keys()) + assert len(keys) >= 3 + + +async def test_azure_ai_agent_create_channel(): + client = AsyncMock(spec=AIProjectClient) + definition = AsyncMock(spec=AzureAIAgentModel) + definition.id = "agent123" + definition.name = "agentName" + definition.description = "desc" + definition.instructions = "test agent" + agent = AzureAIAgent(client=client, definition=definition) + with patch( + "semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.create_thread", + side_effect="t", + ): + ch = await agent.create_channel() + assert isinstance(ch, AgentChannel) + assert ch.thread_id == "t" diff --git a/python/tests/unit/agents/azure_ai_agent/test_azure_ai_agent_settings.py b/python/tests/unit/agents/azure_ai_agent/test_azure_ai_agent_settings.py new file mode 100644 index 000000000000..a2ca8cdc077e --- /dev/null +++ b/python/tests/unit/agents/azure_ai_agent/test_azure_ai_agent_settings.py @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft. All rights reserved. + +import pytest +from pydantic import Field, SecretStr, ValidationError + +from semantic_kernel.kernel_pydantic import KernelBaseSettings +from semantic_kernel.utils.experimental_decorator import experimental_class + + +@experimental_class +class AzureAIAgentSettings(KernelBaseSettings): + """Slightly modified to ensure invalid data raises ValidationError.""" + + env_prefix = "AZURE_AI_AGENT_" + model_deployment_name: str = Field(min_length=1) + project_connection_string: SecretStr = Field(..., min_length=1) + + +def test_azure_ai_agent_settings_valid(): + settings = AzureAIAgentSettings( + model_deployment_name="test_model", + project_connection_string="secret_value", + ) + assert settings.model_deployment_name == "test_model" + assert settings.project_connection_string.get_secret_value() == "secret_value" + + +def test_azure_ai_agent_settings_invalid(): + with pytest.raises(ValidationError): + # Should fail due to min_length=1 constraints + AzureAIAgentSettings( + model_deployment_name="", # empty => invalid + project_connection_string="", + ) diff --git a/python/tests/unit/agents/azure_ai_agent/test_azure_ai_agent_utils.py b/python/tests/unit/agents/azure_ai_agent/test_azure_ai_agent_utils.py new file mode 100644 index 000000000000..74237e1e0b33 --- /dev/null +++ b/python/tests/unit/agents/azure_ai_agent/test_azure_ai_agent_utils.py @@ -0,0 +1,51 @@ +# Copyright (c) Microsoft. All rights reserved. + +from azure.ai.projects.models import MessageAttachment, MessageRole + +from semantic_kernel.agents.azure_ai.azure_ai_agent_utils import AzureAIAgentUtils +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.file_reference_content import FileReferenceContent +from semantic_kernel.contents.utils.author_role import AuthorRole + + +def test_azure_ai_agent_utils_get_thread_messages_none(): + msgs = AzureAIAgentUtils.get_thread_messages([]) + assert msgs is None + + +def test_azure_ai_agent_utils_get_thread_messages(): + msg1 = ChatMessageContent(role=AuthorRole.USER, content="Hello!") + msg1.items.append(FileReferenceContent(file_id="file123")) + results = AzureAIAgentUtils.get_thread_messages([msg1]) + assert len(results) == 1 + assert results[0].content == "Hello!" + assert results[0].role == MessageRole.USER + assert len(results[0].attachments) == 1 + assert isinstance(results[0].attachments[0], MessageAttachment) + + +def test_azure_ai_agent_utils_get_attachments_empty(): + msg1 = ChatMessageContent(role=AuthorRole.USER, content="No file items") + atts = AzureAIAgentUtils.get_attachments(msg1) + assert atts == [] + + +def test_azure_ai_agent_utils_get_attachments_file(): + msg1 = ChatMessageContent(role=AuthorRole.USER, content="One file item") + msg1.items.append(FileReferenceContent(file_id="file123")) + atts = AzureAIAgentUtils.get_attachments(msg1) + assert len(atts) == 1 + assert atts[0].file_id == "file123" + + +def test_azure_ai_agent_utils_get_metadata(): + msg1 = ChatMessageContent(role=AuthorRole.USER, content="has meta", metadata={"k": 123}) + meta = AzureAIAgentUtils.get_metadata(msg1) + assert meta["k"] == "123" + + +def test_azure_ai_agent_utils_get_tool_definition(): + gen = AzureAIAgentUtils._get_tool_definition(["file_search", "code_interpreter", "non_existent"]) + # file_search & code_interpreter exist, non_existent yields nothing + tools_list = list(gen) + assert len(tools_list) == 2 diff --git a/python/tests/unit/agents/azure_ai_agent/test_azure_ai_channel.py b/python/tests/unit/agents/azure_ai_agent/test_azure_ai_channel.py new file mode 100644 index 000000000000..e97a96c0d6c4 --- /dev/null +++ b/python/tests/unit/agents/azure_ai_agent/test_azure_ai_channel.py @@ -0,0 +1,125 @@ +# Copyright (c) Microsoft. All rights reserved. + +from unittest.mock import AsyncMock, patch + +import pytest +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import Agent as AzureAIAgentModel + +from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent +from semantic_kernel.agents.azure_ai.azure_ai_channel import AzureAIChannel +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.exceptions.agent_exceptions import AgentChatException + + +async def test_azure_ai_channel_receive(): + class FakeAgentClient: + create_message = AsyncMock() + + class FakeClient: + agents = FakeAgentClient() + + channel = AzureAIChannel(FakeClient(), "thread123") + await channel.receive([ChatMessageContent(role=AuthorRole.USER, content="Hello")]) + FakeAgentClient.create_message.assert_awaited_once() + + +async def test_azure_ai_channel_invoke_invalid_agent(): + channel = AzureAIChannel(AsyncMock(spec=AIProjectClient), "thread123") + with pytest.raises(AgentChatException): + async for _ in channel.invoke(object()): + pass + + +async def test_azure_ai_channel_invoke_valid_agent(): + client = AsyncMock(spec=AIProjectClient) + definition = AsyncMock(spec=AzureAIAgentModel) + definition.id = "agent123" + definition.name = "agentName" + definition.description = "desc" + definition.instructions = "test agent" + agent = AzureAIAgent(client=client, definition=definition) + + async def fake_invoke(*args, **kwargs): + yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content") + + with patch( + "semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke", + side_effect=fake_invoke, + ): + channel = AzureAIChannel(client, "thread123") + results = [] + async for is_visible, msg in channel.invoke(agent): + results.append((is_visible, msg)) + + assert len(results) == 1 + + +async def test_azure_ai_channel_invoke_stream_valid_agent(): + client = AsyncMock(spec=AIProjectClient) + definition = AsyncMock(spec=AzureAIAgentModel) + definition.id = "agent123" + definition.name = "agentName" + definition.description = "desc" + definition.instructions = "test agent" + agent = AzureAIAgent(client=client, definition=definition) + + async def fake_invoke(*args, **kwargs): + yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content") + + with patch( + "semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke_stream", + side_effect=fake_invoke, + ): + channel = AzureAIChannel(client, "thread123") + results = [] + async for is_visible, msg in channel.invoke_stream(agent, messages=[]): + results.append((is_visible, msg)) + + assert len(results) == 1 + + +async def test_azure_ai_channel_get_history(): + # We need to return an async iterable, so let's do an AsyncMock returning an _async_gen + class FakeAgentClient: + delete_thread = AsyncMock() + # We'll patch get_messages directly below + + class FakeClient: + agents = FakeAgentClient() + + channel = AzureAIChannel(FakeClient(), "threadXYZ") + + async def fake_get_messages(client, thread_id): + # Must produce an async iterable + yield ChatMessageContent(role=AuthorRole.ASSISTANT, content="Previous msg") + + with patch( + "semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.get_messages", + new=fake_get_messages, # direct replacement with a coroutine + ): + results = [] + async for item in channel.get_history(): + results.append(item) + + assert len(results) == 1 + assert results[0].content == "Previous msg" + + +async def test_azure_ai_channel_reset(): + class FakeAgentClient: + delete_thread = AsyncMock() + + class FakeClient: + agents = FakeAgentClient() + + channel = AzureAIChannel(FakeClient(), "threadXYZ") + await channel.reset() + FakeAgentClient.delete_thread.assert_awaited_once_with(thread_id="threadXYZ") + + +# Helper for returning an async generator +async def _async_gen(items): + for i in items: + yield i diff --git a/python/uv.lock b/python/uv.lock index 76db163c5764..854a5926a379 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -253,6 +253,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3c/5f/5d09fdef2a67a646bdc1f39027b2d5b134d5403d39be3043b03b945c3e67/azure_ai_inference-1.0.0b8-py3-none-any.whl", hash = "sha256:9bfcfe6ef5b1699fed6c70058027c253bcbc88f4730e7409fbfc675636ec05e4", size = 123426 }, ] +[[package]] +name = "azure-ai-projects" +version = "1.0.0b5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "isodate", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/0f/47600c1c3bb1b92913350c9878c517ccbd2630f4c5393cd4054dc71d5fbd/azure_ai_projects-1.0.0b5.tar.gz", hash = "sha256:7bb068b38a8810c5a59a628e18f196c76af2a06cb774e725b97bfc8b61b3aaf0", size = 303846 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/3b/5ef0717685e4d3e3b537d7538e430199ab4c16665766ab3de90d5fedc8ef/azure_ai_projects-1.0.0b5-py3-none-any.whl", hash = "sha256:7ed7a44e8d76bf108be7de6f33b9c54a63fc47193cd259770006a6862ce25ccd", size = 193481 }, +] + [[package]] name = "azure-common" version = "1.1.28" @@ -1250,7 +1264,7 @@ wheels = [ [[package]] name = "google-cloud-aiplatform" -version = "1.60.0" +version = "1.79.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "docstring-parser", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -1264,10 +1278,11 @@ dependencies = [ { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "shapely", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/db/57/41a6e447bb486c2da67629b5399be78952711d21a400b70a9208109c02a9/google-cloud-aiplatform-1.60.0.tar.gz", hash = "sha256:782c7f1ec0e77a7c7daabef3b65bfd506ed2b4b1dc2186753c43cd6faf8dd04e", size = 6129755 } +sdist = { url = "https://files.pythonhosted.org/packages/a5/8e/93e9f5a7059883c21a82adf8687248c6615d4b833b3bf665631a768b8ebd/google_cloud_aiplatform-1.79.0.tar.gz", hash = "sha256:362bfd16716dcfb6c131736f25246790002b29c99a246fcf4c08a7c71bd2301f", size = 8455732 } wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/75/b055676eb1bff2fe276566d5f8b593a61a82feb6d2a3a8dcd370768e254c/google_cloud_aiplatform-1.60.0-py2.py3-none-any.whl", hash = "sha256:5f14159c9575f4b46335027e3ceb8fa57bd5eaa76a07f858105b8c6c034ec0d6", size = 5129635 }, + { url = "https://files.pythonhosted.org/packages/d9/df/a7629fc1c405ead82249a70903068992932cc5a8c494c396e22995b4429d/google_cloud_aiplatform-1.79.0-py2.py3-none-any.whl", hash = "sha256:e52d518c386ce2b4ce57f1b73b46c57531d9a6ccd70c21a37b349f428bfc1c3f", size = 7086167 }, ] [[package]] @@ -2764,6 +2779,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/f7/97a9ea26ed4bbbfc2d470994b8b4f338ef663be97b8f677519ac195e113d/nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ea4f11a2904e2a8dc4b1833cc1b5181cde564edd0d5cd33e3c168eff2d1863f1", size = 207454763 }, ] +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/a8/bcbb63b53a4b1234feeafb65544ee55495e1bb37ec31b999b963cbccfd1d/nvidia_cusparselt_cu12-0.6.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:df2c24502fd76ebafe7457dbc4716b2fec071aabaed4fb7691a201cde03704d9", size = 150057751 }, +] + [[package]] name = "nvidia-nccl-cu12" version = "2.21.5" @@ -4717,6 +4740,7 @@ aws = [ ] azure = [ { name = "azure-ai-inference", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "azure-ai-projects", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "azure-core-tracing-opentelemetry", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "azure-cosmos", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "azure-identity", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -4805,6 +4829,7 @@ requires-dist = [ { name = "aiohttp", specifier = "~=3.8" }, { name = "anthropic", marker = "extra == 'anthropic'", specifier = "~=0.32" }, { name = "azure-ai-inference", marker = "extra == 'azure'", specifier = ">=1.0.0b6" }, + { name = "azure-ai-projects", marker = "extra == 'azure'", specifier = ">=1.0.0b5" }, { name = "azure-core-tracing-opentelemetry", marker = "extra == 'azure'", specifier = ">=1.0.0b11" }, { name = "azure-cosmos", marker = "extra == 'azure'", specifier = "~=4.7" }, { name = "azure-identity", specifier = "~=1.13" }, @@ -4817,7 +4842,7 @@ requires-dist = [ { name = "dapr-ext-fastapi", marker = "extra == 'dapr'", specifier = ">=1.14.0" }, { name = "defusedxml", specifier = "~=0.7" }, { name = "flask-dapr", marker = "extra == 'dapr'", specifier = ">=1.14.0" }, - { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = "==1.60" }, + { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = "==1.79.0" }, { name = "google-generativeai", marker = "extra == 'google'", specifier = "==0.7" }, { name = "ipykernel", marker = "extra == 'notebooks'", specifier = "~=6.29" }, { name = "jinja2", specifier = "~=3.1" }, @@ -4842,12 +4867,12 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.0,!=2.10.0,!=2.10.1,!=2.10.2,!=2.10.3,<2.11" }, { name = "pydantic-settings", specifier = "~=2.0" }, { name = "pymilvus", marker = "extra == 'milvus'", specifier = ">=2.3,<2.6" }, - { name = "pymongo", marker = "extra == 'mongo'", specifier = ">=4.8.0,<4.11" }, + { name = "pymongo", marker = "extra == 'mongo'", specifier = ">=4.8.0,<4.12" }, { name = "qdrant-client", marker = "extra == 'qdrant'", specifier = "~=1.9" }, { name = "redis", extras = ["hiredis"], marker = "extra == 'redis'", specifier = "~=5.0" }, { name = "redisvl", marker = "extra == 'redis'", specifier = ">=0.3.6" }, { name = "sentence-transformers", marker = "extra == 'hugging-face'", specifier = ">=2.2,<4.0" }, - { name = "torch", marker = "extra == 'hugging-face'", specifier = "==2.5.1" }, + { name = "torch", marker = "extra == 'hugging-face'", specifier = "==2.6.0" }, { name = "transformers", extras = ["torch"], marker = "extra == 'hugging-face'", specifier = "~=4.28" }, { name = "types-redis", marker = "extra == 'redis'", specifier = "~=4.6.0.20240425" }, { name = "usearch", marker = "extra == 'usearch'", specifier = "~=2.9" }, @@ -5202,7 +5227,7 @@ wheels = [ [[package]] name = "torch" -version = "2.5.1" +version = "2.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -5218,28 +5243,32 @@ dependencies = [ { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "setuptools", marker = "(python_full_version >= '3.12' and sys_platform == 'darwin') or (python_full_version >= '3.12' and sys_platform == 'linux') or (python_full_version >= '3.12' and sys_platform == 'win32')" }, { name = "sympy", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "triton", marker = "python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/ef/834af4a885b31a0b32fff2d80e1e40f771e1566ea8ded55347502440786a/torch-2.5.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:71328e1bbe39d213b8721678f9dcac30dfc452a46d586f1d514a6aa0a99d4744", size = 906446312 }, - { url = "https://files.pythonhosted.org/packages/69/f0/46e74e0d145f43fa506cb336eaefb2d240547e4ce1f496e442711093ab25/torch-2.5.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:34bfa1a852e5714cbfa17f27c49d8ce35e1b7af5608c4bc6e81392c352dbc601", size = 91919522 }, - { url = "https://files.pythonhosted.org/packages/a5/13/1eb674c8efbd04d71e4a157ceba991904f633e009a584dd65dccbafbb648/torch-2.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:32a037bd98a241df6c93e4c789b683335da76a2ac142c0973675b715102dc5fa", size = 203088048 }, - { url = "https://files.pythonhosted.org/packages/a9/9d/e0860474ee0ff8f6ef2c50ec8f71a250f38d78a9b9df9fd241ad3397a65b/torch-2.5.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:23d062bf70776a3d04dbe74db950db2a5245e1ba4f27208a87f0d743b0d06e86", size = 63877046 }, - { url = "https://files.pythonhosted.org/packages/d1/35/e8b2daf02ce933e4518e6f5682c72fd0ed66c15910ea1fb4168f442b71c4/torch-2.5.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:de5b7d6740c4b636ef4db92be922f0edc425b65ed78c5076c43c42d362a45457", size = 906474467 }, - { url = "https://files.pythonhosted.org/packages/40/04/bd91593a4ca178ece93ca55f27e2783aa524aaccbfda66831d59a054c31e/torch-2.5.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:340ce0432cad0d37f5a31be666896e16788f1adf8ad7be481196b503dad675b9", size = 91919450 }, - { url = "https://files.pythonhosted.org/packages/0d/4a/e51420d46cfc90562e85af2fee912237c662ab31140ab179e49bd69401d6/torch-2.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:603c52d2fe06433c18b747d25f5c333f9c1d58615620578c326d66f258686f9a", size = 203098237 }, - { url = "https://files.pythonhosted.org/packages/d0/db/5d9cbfbc7968d79c5c09a0bc0bc3735da079f2fd07cc10498a62b320a480/torch-2.5.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:31f8c39660962f9ae4eeec995e3049b5492eb7360dd4f07377658ef4d728fa4c", size = 63884466 }, - { url = "https://files.pythonhosted.org/packages/8b/5c/36c114d120bfe10f9323ed35061bc5878cc74f3f594003854b0ea298942f/torch-2.5.1-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:ed231a4b3a5952177fafb661213d690a72caaad97d5824dd4fc17ab9e15cec03", size = 906389343 }, - { url = "https://files.pythonhosted.org/packages/6d/69/d8ada8b6e0a4257556d5b4ddeb4345ea8eeaaef3c98b60d1cca197c7ad8e/torch-2.5.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:3f4b7f10a247e0dcd7ea97dc2d3bfbfc90302ed36d7f3952b0008d0df264e697", size = 91811673 }, - { url = "https://files.pythonhosted.org/packages/5f/ba/607d013b55b9fd805db2a5c2662ec7551f1910b4eef39653eeaba182c5b2/torch-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:73e58e78f7d220917c5dbfad1a40e09df9929d3b95d25e57d9f8558f84c9a11c", size = 203046841 }, - { url = "https://files.pythonhosted.org/packages/57/6c/bf52ff061da33deb9f94f4121fde7ff3058812cb7d2036c97bc167793bd1/torch-2.5.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:8c712df61101964eb11910a846514011f0b6f5920c55dbf567bff8a34163d5b1", size = 63858109 }, - { url = "https://files.pythonhosted.org/packages/69/72/20cb30f3b39a9face296491a86adb6ff8f1a47a897e4d14667e6cf89d5c3/torch-2.5.1-cp313-cp313-manylinux1_x86_64.whl", hash = "sha256:9b61edf3b4f6e3b0e0adda8b3960266b9009d02b37555971f4d1c8f7a05afed7", size = 906393265 }, + { url = "https://files.pythonhosted.org/packages/37/81/aa9ab58ec10264c1abe62c8b73f5086c3c558885d6beecebf699f0dbeaeb/torch-2.6.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:6860df13d9911ac158f4c44031609700e1eba07916fff62e21e6ffa0a9e01961", size = 766685561 }, + { url = "https://files.pythonhosted.org/packages/86/86/e661e229df2f5bfc6eab4c97deb1286d598bbeff31ab0cdb99b3c0d53c6f/torch-2.6.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:c4f103a49830ce4c7561ef4434cc7926e5a5fe4e5eb100c19ab36ea1e2b634ab", size = 95751887 }, + { url = "https://files.pythonhosted.org/packages/20/e0/5cb2f8493571f0a5a7273cd7078f191ac252a402b5fb9cb6091f14879109/torch-2.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:56eeaf2ecac90da5d9e35f7f35eb286da82673ec3c582e310a8d1631a1c02341", size = 204165139 }, + { url = "https://files.pythonhosted.org/packages/e5/16/ea1b7842413a7b8a5aaa5e99e8eaf3da3183cc3ab345ad025a07ff636301/torch-2.6.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:09e06f9949e1a0518c5b09fe95295bc9661f219d9ecb6f9893e5123e10696628", size = 66520221 }, + { url = "https://files.pythonhosted.org/packages/78/a9/97cbbc97002fff0de394a2da2cdfa859481fdca36996d7bd845d50aa9d8d/torch-2.6.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:7979834102cd5b7a43cc64e87f2f3b14bd0e1458f06e9f88ffa386d07c7446e1", size = 766715424 }, + { url = "https://files.pythonhosted.org/packages/6d/fa/134ce8f8a7ea07f09588c9cc2cea0d69249efab977707cf67669431dcf5c/torch-2.6.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:ccbd0320411fe1a3b3fec7b4d3185aa7d0c52adac94480ab024b5c8f74a0bf1d", size = 95759416 }, + { url = "https://files.pythonhosted.org/packages/11/c5/2370d96b31eb1841c3a0883a492c15278a6718ccad61bb6a649c80d1d9eb/torch-2.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:46763dcb051180ce1ed23d1891d9b1598e07d051ce4c9d14307029809c4d64f7", size = 204164970 }, + { url = "https://files.pythonhosted.org/packages/0b/fa/f33a4148c6fb46ca2a3f8de39c24d473822d5774d652b66ed9b1214da5f7/torch-2.6.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:94fc63b3b4bedd327af588696559f68c264440e2503cc9e6954019473d74ae21", size = 66530713 }, + { url = "https://files.pythonhosted.org/packages/e5/35/0c52d708144c2deb595cd22819a609f78fdd699b95ff6f0ebcd456e3c7c1/torch-2.6.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:2bb8987f3bb1ef2675897034402373ddfc8f5ef0e156e2d8cfc47cacafdda4a9", size = 766624563 }, + { url = "https://files.pythonhosted.org/packages/01/d6/455ab3fbb2c61c71c8842753b566012e1ed111e7a4c82e0e1c20d0c76b62/torch-2.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b789069020c5588c70d5c2158ac0aa23fd24a028f34a8b4fcb8fcb4d7efcf5fb", size = 95607867 }, + { url = "https://files.pythonhosted.org/packages/18/cf/ae99bd066571656185be0d88ee70abc58467b76f2f7c8bfeb48735a71fe6/torch-2.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:7e1448426d0ba3620408218b50aa6ada88aeae34f7a239ba5431f6c8774b1239", size = 204120469 }, + { url = "https://files.pythonhosted.org/packages/81/b4/605ae4173aa37fb5aa14605d100ff31f4f5d49f617928c9f486bb3aaec08/torch-2.6.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:9a610afe216a85a8b9bc9f8365ed561535c93e804c2a317ef7fabcc5deda0989", size = 66532538 }, + { url = "https://files.pythonhosted.org/packages/24/85/ead1349fc30fe5a32cadd947c91bda4a62fbfd7f8c34ee61f6398d38fb48/torch-2.6.0-cp313-cp313-manylinux1_x86_64.whl", hash = "sha256:4874a73507a300a5d089ceaff616a569e7bb7c613c56f37f63ec3ffac65259cf", size = 766626191 }, + { url = "https://files.pythonhosted.org/packages/dd/b0/26f06f9428b250d856f6d512413e9e800b78625f63801cbba13957432036/torch-2.6.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a0d5e1b9874c1a6c25556840ab8920569a7a4137afa8a63a32cee0bc7d89bd4b", size = 95611439 }, + { url = "https://files.pythonhosted.org/packages/c2/9c/fc5224e9770c83faed3a087112d73147cd7c7bfb7557dcf9ad87e1dda163/torch-2.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:510c73251bee9ba02ae1cb6c9d4ee0907b3ce6020e62784e2d7598e0cfa4d6cc", size = 204126475 }, + { url = "https://files.pythonhosted.org/packages/88/8b/d60c0491ab63634763be1537ad488694d316ddc4a20eaadd639cedc53971/torch-2.6.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:ff96f4038f8af9f7ec4231710ed4549da1bdebad95923953a25045dcf6fd87e2", size = 66536783 }, ] [[package]] @@ -5310,15 +5339,13 @@ torch = [ [[package]] name = "triton" -version = "3.1.0" +version = "3.2.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "filelock", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, -] wheels = [ - { url = "https://files.pythonhosted.org/packages/98/29/69aa56dc0b2eb2602b553881e34243475ea2afd9699be042316842788ff5/triton-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b0dd10a925263abbe9fa37dcde67a5e9b2383fc269fdf59f5657cac38c5d1d8", size = 209460013 }, - { url = "https://files.pythonhosted.org/packages/86/17/d9a5cf4fcf46291856d1e90762e36cbabd2a56c7265da0d1d9508c8e3943/triton-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f34f6e7885d1bf0eaaf7ba875a5f0ce6f3c13ba98f9503651c1e6dc6757ed5c", size = 209506424 }, - { url = "https://files.pythonhosted.org/packages/78/eb/65f5ba83c2a123f6498a3097746607e5b2f16add29e36765305e4ac7fdd8/triton-3.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8182f42fd8080a7d39d666814fa36c5e30cc00ea7eeeb1a2983dbb4c99a0fdc", size = 209551444 }, + { url = "https://files.pythonhosted.org/packages/01/65/3ffa90e158a2c82f0716eee8d26a725d241549b7d7aaf7e4f44ac03ebd89/triton-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3e54983cd51875855da7c68ec05c05cf8bb08df361b1d5b69e05e40b0c9bd62", size = 253090354 }, + { url = "https://files.pythonhosted.org/packages/a7/2e/757d2280d4fefe7d33af7615124e7e298ae7b8e3bc4446cdb8e88b0f9bab/triton-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8009a1fb093ee8546495e96731336a33fb8856a38e45bb4ab6affd6dbc3ba220", size = 253157636 }, + { url = "https://files.pythonhosted.org/packages/06/00/59500052cb1cf8cf5316be93598946bc451f14072c6ff256904428eaf03c/triton-3.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d9b215efc1c26fa7eefb9a157915c92d52e000d2bf83e5f69704047e63f125c", size = 253159365 }, + { url = "https://files.pythonhosted.org/packages/c7/30/37a3384d1e2e9320331baca41e835e90a3767303642c7a80d4510152cbcf/triton-3.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5dfa23ba84541d7c0a531dfce76d8bcd19159d50a4a8b14ad01e91734a5c1b0", size = 253154278 }, ] [[package]]