From 9348eb6255d2a1b6ba9f71a8901af5379ae4d46e Mon Sep 17 00:00:00 2001 From: Rob Chambers Date: Sat, 28 Oct 2023 20:05:56 -0700 Subject: [PATCH] update sdk and other requirements; make changes as needed for that (#72) * update sdk and other requirements; make changes as needed for that * udpate evaluation --- .devcontainer/devcontainer.json | 38 +++++++++++++ dockerfiles/debian10.Dockerfile | 2 +- dockerfiles/debian11.Dockerfile | 2 +- dockerfiles/debian12.Dockerfile | 2 +- dockerfiles/ubuntu2004.Dockerfile | 2 +- dockerfiles/ubuntu2204.Dockerfile | 2 +- ideas/azure-ai-cli-installation-spec.md | 2 +- requirements.txt | 12 ++-- ...python.script.api_key_connection_create.py | 57 ++++++++++++++----- ...de.python.script.api_key_connection_get.py | 4 +- ...include.python.script.connection_delete.py | 4 +- .../include.python.script.connection_list.py | 4 +- ...de.python.script.function_call_evaluate.py | 21 +++---- .../help/include.python.script.hub_create.py | 26 +++++---- .../help/include.python.script.hub_delete.py | 16 +++--- .../.x/help/include.python.script.hub_list.py | 13 +++-- .../include.python.script.ml_index_update.py | 18 +++--- .../include.python.script.project_create.py | 18 +++--- .../include.python.script.project_delete.py | 12 ++-- .../include.python.script.project_list.py | 13 +++-- src/ai/.x/help/include.python.script.temp.py | 41 ------------- src/ai/commands/search_command.cs | 7 +++ src/ai/helpers/config_environment_helpers.cs | 4 ++ 23 files changed, 175 insertions(+), 145 deletions(-) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..d0cc9e75 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,38 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/miniconda +{ + "name": "AI CLI Dev Container", + "build": { + "context": "..", + "dockerfile": "../dockerfiles/debian12.Dockerfile" + }, + + "hostRequirements": { + "cpus": 8, + "memory": "4gb", + "storage": "64gb" + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-toolsai.jupyter" + ] + } + } + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "python --version", + + // Configure tool-specific properties. + // "customizations": {}, + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/dockerfiles/debian10.Dockerfile b/dockerfiles/debian10.Dockerfile index 5deb64ea..cf566313 100644 --- a/dockerfiles/debian10.Dockerfile +++ b/dockerfiles/debian10.Dockerfile @@ -20,7 +20,7 @@ COPY ./scripts/InstallAzureAICLIDeb.sh /_scratch/ COPY ./scripts/InstallAzureAICLIDeb-UpdateVersion.sh /_scratch/ # Feature flags/arguments -ARG AZURE_CLI_VERSION=1.0.0-alpha1010.2 +ARG AZURE_CLI_VERSION=1.0.0-alpha1027.1 ARG DOWNLOAD_SCRIPT=false # If we're downloading the script, do so diff --git a/dockerfiles/debian11.Dockerfile b/dockerfiles/debian11.Dockerfile index 29cee15d..c489e65e 100644 --- a/dockerfiles/debian11.Dockerfile +++ b/dockerfiles/debian11.Dockerfile @@ -20,7 +20,7 @@ COPY ./scripts/InstallAzureAICLIDeb.sh /_scratch/ COPY ./scripts/InstallAzureAICLIDeb-UpdateVersion.sh /_scratch/ # Feature flags/arguments -ARG AZURE_CLI_VERSION=1.0.0-alpha1010.2 +ARG AZURE_CLI_VERSION=1.0.0-alpha1027.1 ARG DOWNLOAD_SCRIPT=false # If we're downloading the script, do so diff --git a/dockerfiles/debian12.Dockerfile b/dockerfiles/debian12.Dockerfile index 9214700e..776ae6a5 100644 --- a/dockerfiles/debian12.Dockerfile +++ b/dockerfiles/debian12.Dockerfile @@ -20,7 +20,7 @@ COPY ./scripts/InstallAzureAICLIDeb.sh /_scratch/ COPY ./scripts/InstallAzureAICLIDeb-UpdateVersion.sh /_scratch/ # Feature flags/arguments -ARG AZURE_CLI_VERSION=1.0.0-alpha1010.2 +ARG AZURE_CLI_VERSION=1.0.0-alpha1027.1 ARG DOWNLOAD_SCRIPT=false # If we're downloading the script, do so diff --git a/dockerfiles/ubuntu2004.Dockerfile b/dockerfiles/ubuntu2004.Dockerfile index 3568e922..1b2ac7eb 100644 --- a/dockerfiles/ubuntu2004.Dockerfile +++ b/dockerfiles/ubuntu2004.Dockerfile @@ -22,7 +22,7 @@ COPY ./scripts/InstallAzureAICLIDeb.sh /_scratch/ COPY ./scripts/InstallAzureAICLIDeb-UpdateVersion.sh /_scratch/ # Feature flags/arguments -ARG AZURE_CLI_VERSION=1.0.0-alpha1010.2 +ARG AZURE_CLI_VERSION=1.0.0-alpha1027.1 ARG DOWNLOAD_SCRIPT=false # If we're downloading the script, do so diff --git a/dockerfiles/ubuntu2204.Dockerfile b/dockerfiles/ubuntu2204.Dockerfile index cd52b4aa..467d1e60 100644 --- a/dockerfiles/ubuntu2204.Dockerfile +++ b/dockerfiles/ubuntu2204.Dockerfile @@ -22,7 +22,7 @@ COPY ./scripts/InstallAzureAICLIDeb.sh /_scratch/ COPY ./scripts/InstallAzureAICLIDeb-UpdateVersion.sh /_scratch/ # Feature flags/arguments -ARG AZURE_CLI_VERSION=1.0.0-alpha1010.2 +ARG AZURE_CLI_VERSION=1.0.0-alpha1027.1 ARG DOWNLOAD_SCRIPT=false # If we're downloading the script, do so diff --git a/ideas/azure-ai-cli-installation-spec.md b/ideas/azure-ai-cli-installation-spec.md index 5925aadb..9ea2f8cd 100644 --- a/ideas/azure-ai-cli-installation-spec.md +++ b/ideas/azure-ai-cli-installation-spec.md @@ -42,7 +42,7 @@ Customer Requirements: - Support Debian 10, 11, and 12, and Ubunutu 20.04 and 22.04 - Uses VS Code base images (e.g. `mcr.microsoft.com/devcontainers/base:bookworm`) - Tagged similarly to VS Code base images (e.g. `${REGISTRY}/azure-ai-cli:bookworm`) -- Tagged with versions as well (e.g. `${REGISTRY}/azure-ai-cli:bookworm-1.0.0-alpha924.3`) +- Tagged with versions as well (e.g. `${REGISTRY}/azure-ai-cli:bookworm-1.0.0-alpha1027.1`) - `${REGISTRY}` is currently `acrbn.azurecr.io` - `${REGISTRY}` likely (???) should be `mcr.microsoft.com` - `latest` tag points to `${REGISTRY}/azure-ai-cli:bookworm` diff --git a/requirements.txt b/requirements.txt index 72ba51cf..278bda00 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,22 +5,22 @@ ipykernel openai pydantic<2 azure-search-documents==11.4.0b11 -azureml-metrics[generative-ai] @https://azuremlsdktestpypi.blob.core.windows.net/repo/Create-Dev-Index/108397413/azureml_metrics-0.1.0.108397413-py3-none-any.whl -azure-ai-generative[evaluate,index] @https://azuremlsdktestpypi.blob.core.windows.net/repo/test-azure-ai-generative/azure_ai_generative-0.8.3-py3-none-any.whl +azure-ai-resources @https://pkgs.dev.azure.com/azure-sdk/29ec6040-b234-4e31-b139-33dc4287b756/_packaging/3572dbf9-b5ef-433b-9137-fc4d7768e7cc/pypi/download/azure-ai-resources/1a20231026010/azure_ai_resources-1.0.0a20231026010-py3-none-any.whl +azure-ai-generative[evaluate,index,document_parsing] @https://pkgs.dev.azure.com/azure-sdk/29ec6040-b234-4e31-b139-33dc4287b756/_packaging/3572dbf9-b5ef-433b-9137-fc4d7768e7cc/pypi/download/azure-ai-generative/1a20231026010/azure_ai_generative-1.0.0a20231026010-py3-none-any.whl +azure-ai-ml @https://pkgs.dev.azure.com/azure-sdk/29ec6040-b234-4e31-b139-33dc4287b756/_packaging/3572dbf9-b5ef-433b-9137-fc4d7768e7cc/pypi/download/azure-ai-ml/1.12a20231026005/azure_ai_ml-1.12.0a20231026005-py3-none-any.whl # promptflow dependencies, these should be optional in the future # hardcoded the version of azureml-mlflow here for faster Docker image building speed azureml-mlflow==1.53.0 -promptflow[azure] -# python-dotenv # promptflow[azure] depends on this package -promptflow-tools +promptflow[azure]==0.1.0b8 +promptflow-tools==0.1.0b10 keyring pytest # langchain dependencies, these should be optional in the future # langchain moved Embeddings from langchain.embeddings.base to langchain.schema.embeddings while azureml-rag is still referencing it. # once azureml-rag fixes, we should remove the langchain reference from this file -langchain==0.0.317 +langchain==0.0.324 semantic-kernel # inference server dependencies diff --git a/src/ai/.x/help/include.python.script.api_key_connection_create.py b/src/ai/.x/help/include.python.script.api_key_connection_create.py index 49c02b12..58276916 100644 --- a/src/ai/.x/help/include.python.script.api_key_connection_create.py +++ b/src/ai/.x/help/include.python.script.api_key_connection_create.py @@ -2,13 +2,12 @@ import json import time from datetime import datetime, timedelta -from azure.ai.generative import AIClient -from azure.ai.generative.entities import Connection +from azure.ai.resources.client import AIClient +from azure.ai.resources.entities import BaseConnection, AzureOpenAIConnection, AzureAISearchConnection, AzureAIServiceConnection from azure.ai.ml.entities._credentials import ApiKeyConfiguration from azure.identity import DefaultAzureCredential -def create_api_key_connection(subscription_id, resource_group_name, project_name, connection_name, connection_type, endpoint, key): - +def create_api_key_connection(subscription_id, resource_group_name, project_name, connection_name, connection_type, endpoint, key, api_version, kind): client = AIClient( credential=DefaultAzureCredential(), subscription_id=subscription_id, @@ -17,14 +16,42 @@ def create_api_key_connection(subscription_id, resource_group_name, project_name user_agent="ai-cli 0.0.1" ) - conn = Connection( - name=connection_name, - type=connection_type, - credentials=ApiKeyConfiguration(key=key), - target=endpoint, - metadata={"Kind": "dummy", "ApiVersion": "dummy", "ApiType": "dummy"} - ) + conn_class = BaseConnection._get_ai_connection_class_from_type(connection_type) + if conn_class == BaseConnection: + # TODO print warning, users shouldn't run into this unless dealing with odd legacy stuff. + conn = conn_class( + name=connection_name, + type=connection_type, + credentials=ApiKeyConfiguration(key=key), + target=endpoint, + ) + elif conn_class == AzureOpenAIConnection: + conn = conn_class( + name=connection_name, + credentials=ApiKeyConfiguration(key=key), + target=endpoint, + api_version = api_version, + ) + elif conn_class == AzureAISearchConnection: + conn = conn_class( + name=connection_name, + credentials=ApiKeyConfiguration(key=key), + target=endpoint, + api_version=api_version, + ) + elif conn_class == AzureAIServiceConnection: + if kind is None: + print("Error: --kind argument is required for Cognitive Service connection.") + return {} + conn = conn_class( + name=connection_name, + credentials=ApiKeyConfiguration(key=key), + target=endpoint, + api_version=api_version, + kind=kind, + ) + conn = client.connections.create_or_update(conn) conn2 = client.connections.get(conn.name) @@ -45,9 +72,11 @@ def main(): parser.add_argument("--group", required=False, help="Azure resource group name") parser.add_argument("--project-name", required=True, help="Azure AI project project name.") parser.add_argument("--connection-name", required=True, help="Azure AI project connection name.") - parser.add_argument("--connection-type", required=True, help="Azure AI project connection type.") + parser.add_argument("--connection-type", required=True, help="Azure AI project connection type. Accepted types are 'azure-open-ai', 'cognitive-search', and 'cognitive-service'.") parser.add_argument("--endpoint", required=True, help="Azure AI Project connection endpoint.") parser.add_argument("--key", required=True, help="Azure AI Project connection key.") + parser.add_argument("--api-version", required=False, help="The expected api version of the service this connection will link to.", default="unset") + parser.add_argument("--kind", required=False, help="Kind of AI Service being connected to. Required for Cognitive Service connections.", default=None) args = parser.parse_args() subscription_id = args.subscription @@ -57,6 +86,8 @@ def main(): connection_type = args.connection_type endpoint = args.endpoint key = args.key + api_version = args.api_version + kind = args.kind timeout_seconds = 10 @@ -66,7 +97,7 @@ def main(): while datetime.now() - start_time < timeout: try: - connection = create_api_key_connection(subscription_id, resource_group_name, project_name, connection_name, connection_type, endpoint, key) + connection = create_api_key_connection(subscription_id, resource_group_name, project_name, connection_name, connection_type, endpoint, key, api_version, kind)) if connection is not None: success = True break diff --git a/src/ai/.x/help/include.python.script.api_key_connection_get.py b/src/ai/.x/help/include.python.script.api_key_connection_get.py index 34348897..eab98eaa 100644 --- a/src/ai/.x/help/include.python.script.api_key_connection_get.py +++ b/src/ai/.x/help/include.python.script.api_key_connection_get.py @@ -2,9 +2,7 @@ import json import time from datetime import datetime, timedelta -from azure.ai.generative import AIClient -from azure.ai.generative.entities import Connection -from azure.ai.ml.entities._credentials import ApiKeyConfiguration +from azure.ai.resources.client import AIClient from azure.identity import DefaultAzureCredential def get_api_key_connection(subscription_id, resource_group_name, project_name, connection_name): diff --git a/src/ai/.x/help/include.python.script.connection_delete.py b/src/ai/.x/help/include.python.script.connection_delete.py index 6ab14ebd..dc7612b1 100644 --- a/src/ai/.x/help/include.python.script.connection_delete.py +++ b/src/ai/.x/help/include.python.script.connection_delete.py @@ -1,8 +1,6 @@ import argparse import json -from azure.ai.generative import AIClient -from azure.ai.generative.entities import Connection -from azure.ai.ml.entities._credentials import ApiKeyConfiguration +from azure.ai.resources.client import AIClient from azure.identity import DefaultAzureCredential def delete_connection(subscription_id, resource_group_name, project_name, connection_name): diff --git a/src/ai/.x/help/include.python.script.connection_list.py b/src/ai/.x/help/include.python.script.connection_list.py index 5be5ac1f..dbe71fb8 100644 --- a/src/ai/.x/help/include.python.script.connection_list.py +++ b/src/ai/.x/help/include.python.script.connection_list.py @@ -1,8 +1,6 @@ import argparse import json -from azure.ai.generative import AIClient -from azure.ai.generative.entities import Connection -from azure.ai.ml.entities._credentials import ApiKeyConfiguration +from azure.ai.resources.client import AIClient from azure.identity import DefaultAzureCredential def list_connections(subscription_id, resource_group_name, project_name): diff --git a/src/ai/.x/help/include.python.script.function_call_evaluate.py b/src/ai/.x/help/include.python.script.function_call_evaluate.py index 61a4d632..d5012a53 100755 --- a/src/ai/.x/help/include.python.script.function_call_evaluate.py +++ b/src/ai/.x/help/include.python.script.function_call_evaluate.py @@ -1,21 +1,11 @@ import asyncio -import pathlib -import platform -import json -import os -import sys -from azure.identity import DefaultAzureCredential -from azure.ai.generative import AIClient -from azure.ai.generative.operations._index_data_source import LocalSource, ACSOutputConfig -from azure.ai.generative.functions.build_mlindex import build_mlindex -from azure.ai.generative.entities.mlindex import MLIndex -import asyncio -import argparse import importlib -import inspect import json import os +import pathlib import sys +from azure.identity import DefaultAzureCredential +from azure.ai.resources.client import AIClient from typing import Any, List, Dict, Generator class AutoFlushingStream: @@ -257,7 +247,10 @@ def run_and_or_evaluate(subscription_id, resource_group_name, project_name, modu print("Evaluating... Done!") print(eval_results) - return eval_results + return { + "metrics_summary": eval_results.metrics_summary, + "artifacts": eval_results.artifacts + } def main(): diff --git a/src/ai/.x/help/include.python.script.hub_create.py b/src/ai/.x/help/include.python.script.hub_create.py index 48b24596..39c80897 100644 --- a/src/ai/.x/help/include.python.script.hub_create.py +++ b/src/ai/.x/help/include.python.script.hub_create.py @@ -1,20 +1,21 @@ import argparse import json -from azure.ai.ml import MLClient -from azure.ai.ml.entities import WorkspaceHub, ManagedNetwork, WorkspaceHubConfig +from azure.ai.resources.client import AIClient +from azure.ai.resources.entities import AIResource +from azure.ai.ml.entities import ManagedNetwork from azure.identity import DefaultAzureCredential -def create_hub(subscription_id, resource_group_name, resource_name, location, display_name, description): - """Create Azure ML hub.""" - ml_client = MLClient( +def create_hub(subscription_id, resource_group_name, ai_resource_name, location, display_name, description): + """Create Azure AI hub.""" + ai_client = AIClient( credential=DefaultAzureCredential(), subscription_id=subscription_id, resource_group_name=resource_group_name, user_agent="ai-cli 0.0.1" ) - wshub = WorkspaceHub( - name=resource_name, + resource = AIResource( + name=ai_resource_name, location=location, display_name=display_name, description=description, @@ -24,12 +25,13 @@ def create_hub(subscription_id, resource_group_name, resource_name, location, di ) ) - result = ml_client.workspace_hubs.begin_create(workspace_hub=wshub).result() - return result._to_dict() + # TODO allow setting of optional bool update_dependent_resources? + result = ai_client.ai_resources.begin_create(ai_resource=resource).result() + return result._workspace_hub._to_dict() def main(): """Parse command line arguments and print created hub.""" - parser = argparse.ArgumentParser(description="Create Azure ML hub") + parser = argparse.ArgumentParser(description="Create Azure AI hub") parser.add_argument("--subscription", required=True, help="Azure subscription ID") parser.add_argument("--group", required=True, help="Azure resource group name") parser.add_argument("--name", required=True, help="Azure AI resource display name. This is non-unique within the resource group.") @@ -40,12 +42,12 @@ def main(): subscription_id = args.subscription resource_group_name = args.group - resource_name = args.name + ai_resource_name = args.name location = args.location display_name = args.display_name description = args.description - hub = create_hub(subscription_id, resource_group_name, resource_name, location, display_name, description) + hub = create_hub(subscription_id, resource_group_name, ai_resource_name, location, display_name, description) formatted = json.dumps({"hub": hub}, indent=2) print("---") diff --git a/src/ai/.x/help/include.python.script.hub_delete.py b/src/ai/.x/help/include.python.script.hub_delete.py index b342476e..f83ce51b 100644 --- a/src/ai/.x/help/include.python.script.hub_delete.py +++ b/src/ai/.x/help/include.python.script.hub_delete.py @@ -1,23 +1,23 @@ import argparse import json -from azure.ai.ml import MLClient +from azure.ai.resources.client import AIClient from azure.identity import DefaultAzureCredential -def delete_hub(subscription_id, resource_group_name, resource_name, delete_dependent_resources): - """Delete Azure ML hubs.""" - ml_client = MLClient( +def delete_hub(subscription_id, resource_group_name, ai_resource_name, delete_dependent_resources): + """Delete Azure AI hubs.""" + ai_client = AIClient( credential=DefaultAzureCredential(), subscription_id=subscription_id, resource_group_name=resource_group_name, user_agent="ai-cli 0.0.1" ) - result = ml_client.workspace_hubs.begin_delete(resource_name, delete_dependent_resources=delete_dependent_resources).result() + result = ai_client.ai_resources.begin_delete(name=ai_resource_name, delete_dependent_resources=delete_dependent_resources).result() return result def main(): """Parse command line arguments and delete's the hub.""" - parser = argparse.ArgumentParser(description="Delete Azure ML hub") + parser = argparse.ArgumentParser(description="Delete Azure AI hub") parser.add_argument("--subscription", required=True, help="Azure subscription ID") parser.add_argument("--group", required=True, help="Azure resource group name") parser.add_argument("--name", required=True, help="Azure resource hub name") @@ -26,10 +26,10 @@ def main(): subscription_id = args.subscription resource_group_name = args.group - resource_name = args.name; + ai_resource_name = args.name; delete_dependent_resources = args.delete_dependent_resources - result = delete_hub(subscription_id, resource_group_name, resource_name, delete_dependent_resources) + result = delete_hub(subscription_id, resource_group_name, ai_resource_name, delete_dependent_resources) formatted = json.dumps(result, indent=2) print("---") diff --git a/src/ai/.x/help/include.python.script.hub_list.py b/src/ai/.x/help/include.python.script.hub_list.py index 243ef340..d2a9cbea 100644 --- a/src/ai/.x/help/include.python.script.hub_list.py +++ b/src/ai/.x/help/include.python.script.hub_list.py @@ -1,28 +1,29 @@ import argparse import json -from azure.ai.ml import MLClient +from azure.ai.resources.client import AIClient from azure.identity import DefaultAzureCredential +from azure.ai.ml.constants._common import Scope def list_hubs(subscription_id, resource_group_name): - """List Azure ML hubs.""" - ml_client = MLClient( + """List Azure AI hubs.""" + ai_client = AIClient( credential=DefaultAzureCredential(), subscription_id=subscription_id, resource_group_name=resource_group_name, user_agent="ai-cli 0.0.1" ) - items = ml_client.workspace_hubs.list(scope="subscription") + items = ai_client.ai_resources.list(scope=Scope.SUBSCRIPTION) results = [] for item in items: - results.append(item._to_dict()) + results.append(item._workspace_hub._to_dict()) return results def main(): """Parse command line arguments and print hubs.""" - parser = argparse.ArgumentParser(description="List Azure ML hubs") + parser = argparse.ArgumentParser(description="List Azure AI hubs") parser.add_argument("--subscription", required=True, help="Azure subscription ID") parser.add_argument("--group", required=False, help="Azure resource group name") args = parser.parse_args() diff --git a/src/ai/.x/help/include.python.script.ml_index_update.py b/src/ai/.x/help/include.python.script.ml_index_update.py index b0ec708c..c60770eb 100644 --- a/src/ai/.x/help/include.python.script.ml_index_update.py +++ b/src/ai/.x/help/include.python.script.ml_index_update.py @@ -3,10 +3,10 @@ import os import sys from azure.identity import DefaultAzureCredential -from azure.ai.generative import AIClient -from azure.ai.generative.operations._index_data_source import LocalSource, ACSOutputConfig -from azure.ai.generative.functions.build_mlindex import build_mlindex -from azure.ai.generative.entities.mlindex import MLIndex +from azure.ai.resources.client import AIClient +from azure.ai.resources.operations._index_data_source import LocalSource, ACSOutputConfig +from azure.ai.generative.index import build_index +from azure.ai.resources.entities import Index class AutoFlushingStream: def __init__(self, stream): @@ -22,9 +22,9 @@ def flush(self): sys.stdout = AutoFlushingStream(sys.stdout) sys.stderr = AutoFlushingStream(sys.stderr) -class MLIndexEncoder(json.JSONEncoder): +class IndexEncoder(json.JSONEncoder): def default(self, obj): - if isinstance(obj, MLIndex): + if isinstance(obj, Index): return { "name": obj.name, "path": obj.path, @@ -80,7 +80,7 @@ def search_index_update( print(f"Data files path: {data_files_path}") print(f"Data files glob pattern: {data_files_glob_pattern}") - index = build_mlindex( + index = build_index( output_index_name=index_name, vector_store="azure_cognitive_search", embeddings_model = f"azure_open_ai://deployment/{embedding_model_deployment}/model/{embedding_model_name}", @@ -92,7 +92,7 @@ def search_index_update( ), ) - return client.mlindexes.create_or_update(index) + return client.indexes.create_or_update(index) def main(): """Parse command line arguments and build MLIndex.""" @@ -118,7 +118,7 @@ def main(): external_source_url = args.external_source_url index = search_index_update(subscription_id, resource_group_name, project_name, index_name, embedding_model_deployment, embedding_model_name, data_files, external_source_url) - formatted = json.dumps({"index": index}, indent=2, cls=MLIndexEncoder) + formatted = json.dumps({"index": index}, indent=2, cls=IndexEncoder) print("---") print(formatted) diff --git a/src/ai/.x/help/include.python.script.project_create.py b/src/ai/.x/help/include.python.script.project_create.py index c42e28e1..1401253d 100644 --- a/src/ai/.x/help/include.python.script.project_create.py +++ b/src/ai/.x/help/include.python.script.project_create.py @@ -1,32 +1,32 @@ import argparse import json -from azure.ai.ml import MLClient -from azure.ai.ml.entities import Workspace +from azure.ai.resources.client import AIClient +from azure.ai.resources.entities import Project from azure.identity import DefaultAzureCredential def create_project(subscription_id, resource_id, resource_group_name, project_name, location, display_name, description, openai_resource_id): - """Create Azure ML project.""" - ml_client = MLClient( + """Create Azure AI project.""" + ai_client = AIClient( credential=DefaultAzureCredential(), subscription_id=subscription_id, resource_group_name=resource_group_name, user_agent="ai-cli 0.0.1" ) - project = Workspace( + project = Project( name=project_name, location=location, display_name=display_name, description=description, - workspace_hub=resource_id + ai_resource=resource_id, ) - result = ml_client.workspaces.begin_create(project, byo_open_ai_resource_id=openai_resource_id).result() - return result._to_dict() + result = ai_client.projects.begin_create(project=project, byo_open_ai_resource_id=openai_resource_id).result() + return result._workspace._to_dict() def main(): """Parse command line arguments and print created project.""" - parser = argparse.ArgumentParser(description="Create Azure ML project") + parser = argparse.ArgumentParser(description="Create Azure AI project") parser.add_argument("--subscription", required=True, help="Azure subscription ID") parser.add_argument("--resource-id", required=True, help="Azure AI resource ID") parser.add_argument("--group", required=True, help="Azure resource group name") diff --git a/src/ai/.x/help/include.python.script.project_delete.py b/src/ai/.x/help/include.python.script.project_delete.py index c51bb15d..189ad1d0 100644 --- a/src/ai/.x/help/include.python.script.project_delete.py +++ b/src/ai/.x/help/include.python.script.project_delete.py @@ -1,23 +1,23 @@ import argparse import json -from azure.ai.ml import MLClient +from azure.ai.resources.client import AIClient from azure.identity import DefaultAzureCredential def delete_project(subscription_id, resource_group_name, project_name, delete_dependent_resources): - """Delete Azure ML projects.""" - ml_client = MLClient( + """Delete Azure AI projects.""" + ai_client = AIClient( credential=DefaultAzureCredential(), subscription_id=subscription_id, resource_group_name=resource_group_name, user_agent="ai-cli 0.0.1" ) - - result = ml_client.workspaces.begin_delete(project_name, delete_dependent_resources=delete_dependent_resources).result() + # TODO should we allow assigning optional permanently_delete bool? + result = ai_client.projects.begin_delete(name=project_name, delete_dependent_resources=delete_dependent_resources).result() return result def main(): """Parse command line arguments and delete's the project.""" - parser = argparse.ArgumentParser(description="Delete Azure ML project") + parser = argparse.ArgumentParser(description="Delete Azure AI project") parser.add_argument("--subscription", required=True, help="Azure subscription ID") parser.add_argument("--group", required=True, help="Azure resource group name") parser.add_argument("--name", required=True, help="Azure resource project name") diff --git a/src/ai/.x/help/include.python.script.project_list.py b/src/ai/.x/help/include.python.script.project_list.py index 1b16511b..aa103e51 100644 --- a/src/ai/.x/help/include.python.script.project_list.py +++ b/src/ai/.x/help/include.python.script.project_list.py @@ -1,28 +1,29 @@ import argparse import json -from azure.ai.ml import MLClient +from azure.ai.resources.client import AIClient from azure.identity import DefaultAzureCredential +from azure.ai.ml.constants._common import Scope def list_projects(subscription_id, resource_group_name): - """List Azure ML projects.""" - ml_client = MLClient( + """List Azure AI projects.""" + ai_client = AIClient( credential=DefaultAzureCredential(), subscription_id=subscription_id, resource_group_name=resource_group_name, user_agent="ai-cli 0.0.1" ) - items = ml_client.workspaces.list(scope="subscription") + items = ai_client.projects.list(scope=Scope.SUBSCRIPTION) results = [] for item in items: - results.append(item._to_dict()) + results.append(item._workspace._to_dict()) return results def main(): """Parse command line arguments and print projects.""" - parser = argparse.ArgumentParser(description="List Azure ML projects") + parser = argparse.ArgumentParser(description="List Azure AI projects") parser.add_argument("--subscription", required=True, help="Azure subscription ID") parser.add_argument("--group", required=False, help="Azure resource group name") args = parser.parse_args() diff --git a/src/ai/.x/help/include.python.script.temp.py b/src/ai/.x/help/include.python.script.temp.py index d43e09f9..e69de29b 100644 --- a/src/ai/.x/help/include.python.script.temp.py +++ b/src/ai/.x/help/include.python.script.temp.py @@ -1,41 +0,0 @@ -from azure.ai.generative import AIClient -from azure.ai.generative.entities import Connection -from azure.ai.ml.entities._credentials import ApiKeyConfiguration -from azure.identity import DefaultAzureCredential -from azure.identity import AzureCliCredential - -aad_credential = AzureCliCredential() - -client = AIClient( - credential=aad_credential, # DefaultAzureCredential(), - subscription_id="e72e5254-f265-4e95-9bd2-9ee8e7329051", - resource_group_name="robch-hub-0823-2-rg", - project_name="robch-hub-0823-2-project", -) - -print("--before--") -connections = client.connections.list() -for conn in connections: - print(conn.target) - print(conn.credentials.values()) - -conn = Connection( - name="Default_CognitiveSearch", - type="cognitive_search", - credentials=ApiKeyConfiguration(key="12345"), - target="https://robch-cogsearch-westus2.search.windows.net", - metadata={"Kind": "dummy", "ApiVersion": "dummy", "ApiType": "dummy"} -) - -print("--creating--") -conn = client.connections.create_or_update(conn) - -print("--created--") -print(conn.target) -print(conn.credentials.values()) - -print("--after--") -connections = client.connections.list() -for conn in connections: - print(conn.target) - print(conn.credentials.values()) diff --git a/src/ai/commands/search_command.cs b/src/ai/commands/search_command.cs index 525c8cdb..2e6bdbea 100644 --- a/src/ai/commands/search_command.cs +++ b/src/ai/commands/search_command.cs @@ -105,6 +105,13 @@ private void DoIndexUpdate() var externalSourceUrl = ExternalSourceToken.Data().GetOrDefault(_values); output = DoIndexUpdateWithGenAi(subscription, group, project, searchIndexName, embeddingModelDeployment, embeddingModelName, pattern, externalSourceUrl); + + var parsed = !string.IsNullOrEmpty(output) ? JToken.Parse(output) : null; + var index = parsed?.Type == JTokenType.Object ? parsed["index"] : null; + if (index == null) + { + _values.AddThrowError("ERROR:", $"Failed to update search index '{searchIndexName}'"); + } } if (!_quiet) Console.WriteLine($"{message} Done!\n"); diff --git a/src/ai/helpers/config_environment_helpers.cs b/src/ai/helpers/config_environment_helpers.cs index cf673d3d..908e7b86 100644 --- a/src/ai/helpers/config_environment_helpers.cs +++ b/src/ai/helpers/config_environment_helpers.cs @@ -34,6 +34,10 @@ public static Dictionary GetEnvironment(INamedValues values) env.Add("AZURE_AI_SEARCH_INDEX_NAME", ReadConfig(values, "search.index.name")); env.Add("AZURE_AI_SEARCH_KEY", ReadConfig(values, "search.key")); + env.Add("AZURE_AI_SPEECH_ENDPOINT", ReadConfig(values, "speech.endpoint")); + env.Add("AZURE_AI_SPEECH_KEY", ReadConfig(values, "speech.key")); + env.Add("AZURE_AI_SPEECH_REGION", ReadConfig(values, "speech.region")); + // Add "non-standard" AZURE_AI_" prefixed env variables to interop with various SDKs // OpenAI's SDK