python/notebooks/11-streaming-completions.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "id": "68e1c158",
      "metadata": {},
      "source": [
        "# Streaming Results\n",
        "\n",
        "Here is an example pattern if you want to stream your multiple results. Note that this is not supported for Hugging Face text completions at this time.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a77bdf89",
      "metadata": {},
      "outputs": [],
      "source": [
        "!python -m pip install semantic-kernel==0.5.1.dev0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from services import Service\n",
        "\n",
        "# Select a service to use for this notebook (available services: OpenAI, AzureOpenAI, HuggingFace)\n",
        "selectedService = Service.AzureOpenAI"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "508ad44f",
      "metadata": {},
      "outputs": [],
      "source": [
        "import semantic_kernel as sk\n",
        "\n",
        "if selectedService == Service.OpenAI or selectedService == Service.AzureOpenAI:\n",
        "    from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import (\n",
        "        OpenAITextPromptExecutionSettings,\n",
        "        OpenAIChatPromptExecutionSettings,\n",
        "    )\n",
        "    from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.azure_chat_prompt_execution_settings import (\n",
        "        AzureChatPromptExecutionSettings,\n",
        "    )\n",
        "    from semantic_kernel.connectors.ai.open_ai import (\n",
        "        AzureTextCompletion,\n",
        "        AzureChatCompletion,\n",
        "        OpenAITextCompletion,\n",
        "        OpenAIChatCompletion,\n",
        "    )\n",
        "if selectedService == Service.HuggingFace:\n",
        "    from semantic_kernel.connectors.ai.hugging_face import HuggingFaceTextCompletion"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "id": "d8ddffc1",
      "metadata": {},
      "source": [
        "First, we will set up the text and chat services we will be submitting prompts to.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8f8dcbc6",
      "metadata": {},
      "outputs": [],
      "source": [
        "kernel = sk.Kernel()\n",
        "\n",
        "# Configure Azure LLM service\n",
        "if selectedService == Service.AzureOpenAI:\n",
        "    deployment, api_key, endpoint = sk.azure_openai_settings_from_dot_env()\n",
        "    azure_text_service = AzureTextCompletion(\n",
        "        deployment_name=\"text-davinci-003\", endpoint=endpoint, api_key=api_key\n",
        "    )  # set the deployment name to the value of your text model (e.g. gpt-35-turbo-instruct or text-davinci-003)\n",
        "    azure_chat_service = AzureChatCompletion(\n",
        "        deployment_name=\"gpt-35-turbo\", endpoint=endpoint, api_key=api_key\n",
        "    )  # set the deployment name to the value of your chat model\n",
        "\n",
        "# Configure OpenAI service\n",
        "if selectedService == Service.OpenAI:\n",
        "    api_key, org_id = sk.openai_settings_from_dot_env()\n",
        "    oai_text_service = OpenAITextCompletion(ai_model_id=\"gpt-3.5-turbo-instruct\", api_key=api_key, org_id=org_id)\n",
        "    oai_chat_service = OpenAIChatCompletion(ai_model_id=\"gpt-3.5-turbo\", api_key=api_key, org_id=org_id)\n",
        "\n",
        "# Configure Hugging Face service\n",
        "if selectedService == Service.HuggingFace:\n",
        "    hf_text_service = HuggingFaceTextCompletion(ai_model_id=\"distilgpt2\", task=\"text-generation\")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "id": "50561d82",
      "metadata": {},
      "source": [
        "Next, we'll set up the completion request settings for text completion services.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "628c843e",
      "metadata": {},
      "outputs": [],
      "source": [
        "oai_prompt_execution_settings = OpenAITextPromptExecutionSettings(\n",
        "    max_tokens=150,\n",
        "    temperature=0.7,\n",
        "    top_p=1,\n",
        "    frequency_penalty=0.5,\n",
        "    presence_penalty=0.5,\n",
        ")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "id": "857a9c89",
      "metadata": {},
      "source": [
        "## Streaming Open AI Text Completion\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e2979db8",
      "metadata": {},
      "outputs": [],
      "source": [
        "if selectedService == Service.OpenAI:\n",
        "    prompt = \"what is the purpose of a rubber duck?\"\n",
        "    stream = oai_text_service.complete_stream(prompt=prompt, settings=oai_prompt_execution_settings)\n",
        "    async for text in stream:\n",
        "        print(text, end=\"\")  # end = \"\" to avoid newlines"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "id": "4288d09f",
      "metadata": {},
      "source": [
        "## Streaming Azure Open AI Text Completion\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5319f14d",
      "metadata": {},
      "outputs": [],
      "source": [
        "if selectedService == Service.AzureOpenAI:\n",
        "    prompt = \"provide me a list of possible meanings for the acronym 'ORLD'\"\n",
        "    stream = azure_text_service.complete_stream(prompt=prompt, settings=oai_prompt_execution_settings)\n",
        "    async for text in stream:\n",
        "        print(text, end=\"\")  # end = \"\" to avoid newlines"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "id": "eb548f9c",
      "metadata": {},
      "source": [
        "## Streaming Hugging Face Text Completion\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "be7b1c2e",
      "metadata": {},
      "outputs": [],
      "source": [
        "if selectedService == Service.HuggingFace:\n",
        "    from semantic_kernel.connectors.ai.hugging_face.hf_prompt_execution_settings import (\n",
        "        HuggingFacePromptExecutionSettings,\n",
        "    )\n",
        "\n",
        "    hf_prompt_execution_settings = HuggingFacePromptExecutionSettings(\n",
        "        extension_data={\n",
        "            \"max_new_tokens\": 80,\n",
        "            \"top_p\": 1,\n",
        "            \"eos_token_id\": 11,\n",
        "            \"pad_token_id\": 0,\n",
        "        }\n",
        "    )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9525e4f3",
      "metadata": {},
      "outputs": [],
      "source": [
        "if selectedService == Service.HuggingFace:\n",
        "    prompt = \"The purpose of a rubber duck is\"\n",
        "    stream = hf_text_service.complete_stream(prompt=prompt, prompt_execution_settings=hf_prompt_execution_settings)\n",
        "    async for text in stream:\n",
        "        print(text, end=\"\")  # end = \"\" to avoid newlines"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "id": "da632e12",
      "metadata": {},
      "source": [
        "Here, we're setting up the settings for Chat completions.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e5f11e46",
      "metadata": {},
      "outputs": [],
      "source": [
        "oai_chat_prompt_execution_settings = OpenAIChatPromptExecutionSettings(\n",
        "    max_tokens=150,\n",
        "    temperature=0.7,\n",
        "    top_p=1,\n",
        "    frequency_penalty=0.5,\n",
        "    presence_penalty=0.5,\n",
        ")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "id": "d6bf238e",
      "metadata": {},
      "source": [
        "## Streaming OpenAI Chat Completion\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dabc6a4c",
      "metadata": {},
      "outputs": [],
      "source": [
        "if selectedService == Service.OpenAI:\n",
        "    role = \"system\"\n",
        "    content = \"You are an AI assistant that helps people find information.\"\n",
        "    message = {\"role\": role, \"content\": content}\n",
        "    stream = oai_chat_service.complete_chat_stream(messages=[message], settings=oai_chat_prompt_execution_settings)\n",
        "    async for text in stream:\n",
        "        print(text, end=\"\")  # end = \"\" to avoid newlines"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "id": "cdb8f740",
      "metadata": {},
      "source": [
        "## Streaming Azure OpenAI Chat Completion\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "da1e9f59",
      "metadata": {},
      "outputs": [],
      "source": [
        "az_oai_chat_prompt_execution_settings = AzureChatPromptExecutionSettings(\n",
        "    max_tokens=150,\n",
        "    temperature=0.7,\n",
        "    top_p=1,\n",
        "    frequency_penalty=0.5,\n",
        "    presence_penalty=0.5,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b74a64a9",
      "metadata": {},
      "outputs": [],
      "source": [
        "if selectedService == Service.AzureOpenAI:\n",
        "    role = \"system\"\n",
        "    content = \"You are an AI assistant that helps people find information.\"\n",
        "    message = {\"role\": role, \"content\": content}\n",
        "    stream = azure_chat_service.complete_chat_stream(messages=[message], settings=az_oai_chat_prompt_execution_settings)\n",
        "    async for text in stream:\n",
        "        print(text, end=\"\")  # end = \"\" to avoid newlines"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.12"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}